forge-sql-orm 2.1.14 → 2.1.16

package/src/utils/cacheUtils.ts

@@ -6,6 +6,7 @@ import { getTableName } from "drizzle-orm/table";
 import { Filter, FilterConditions, kvs, WhereConditions } from "@forge/kvs";
 import { ForgeSqlOrmOptions } from "../core/ForgeSQLQueryBuilder";
 import { cacheApplicationContext, isTableContainsTableInCacheContext } from "./cacheContextUtils";
+import { extractBacktickedValues } from "./cacheTableUtils";
 
 // Constants for better maintainability
 const CACHE_CONSTANTS = {
@@ -47,29 +48,6 @@ function nowPlusSeconds(secondsToAdd: number): number {
   return Math.floor(dt.toSeconds());
 }
 
-/**
- * Extracts all table/column names between backticks from SQL query and returns them as comma-separated string.
- *
- * @param sql - SQL query string
- * @returns Comma-separated string of unique backticked values
- */
-function extractBacktickedValues(sql: string): string {
-  const regex = /`([^`]+)`/g;
-  const matches = new Set<string>();
-  let match;
-
-  while ((match = regex.exec(sql.toLowerCase())) !== null) {
-    if (!match[1].startsWith("a_")) {
-      matches.add(`\`${match[1]}\``);
-    }
-  }
-
-  // Sort to ensure consistent order for the same input
-  return Array.from(matches)
-    .sort((a, b) => a.localeCompare(b, undefined, { sensitivity: "base", numeric: true }))
-    .join(",");
-}
-
 /**
  * Generates a hash key for a query based on its SQL and parameters.
  *
@@ -362,7 +340,7 @@ export async function getFromCache<T>(
     if (
       cacheResult &&
       (cacheResult[expirationName] as number) >= getCurrentTime() &&
-      extractBacktickedValues(sqlQuery.sql) === cacheResult[entityQueryName]
+      extractBacktickedValues(sqlQuery.sql, options) === cacheResult[entityQueryName]
     ) {
       if (options.logCache) {
         // eslint-disable-next-line no-console
@@ -423,7 +401,7 @@ export async function setCacheResult(
       .set(
         key,
         {
-          [entityQueryName]: extractBacktickedValues(sqlQuery.sql),
+          [entityQueryName]: extractBacktickedValues(sqlQuery.sql, options),
           [expirationName]: nowPlusSeconds(cacheTtl),
           [dataName]: JSON.stringify(results),
         },

package/src/utils/forgeDriver.ts

@@ -2,6 +2,7 @@ import { sql, UpdateQueryResponse } from "@forge/sql";
 import { saveMetaDataToContext } from "./metadataContextUtils";
 import { getOperationType } from "./requestTypeContextUtils";
 import { withTimeout } from "./sqlUtils";
+import { SQL_API_ENDPOINTS } from "@forge/sql/out/sql";
 
 const timeoutMs = 10000;
 const timeoutMessage = `Atlassian @forge/sql did not return a response within ${timeoutMs}ms (${timeoutMs / 1000} seconds), so the request is blocked. Possible causes: slow query, network issues, or exceeding Forge SQL limits.`;
@@ -66,25 +67,23 @@ export function isUpdateQueryResponse(obj: unknown): obj is UpdateQueryResponse
   );
 }
 
-function inlineParams(sql: string, params: unknown[]): string {
-  let i = 0;
-  return sql.replace(/\?/g, () => {
-    const val = params[i++];
-    if (val === null) return "NULL";
-    if (typeof val === "number") return val.toString();
-    return `'${String(val).replace(/'/g, "''")}'`;
-  });
-}
-
 /**
- * Processes DDL query results and saves metadata.
+ * Processes DDL query results and saves metadata to the execution context.
  *
+ * @param query - The SQL query string
+ * @param params - Query parameters
+ * @param method - Execution method ("all" or "execute")
  * @param result - The DDL query result
  * @returns Processed result for Drizzle ORM
  */
-async function processDDLResult(method: QueryMethod, result: any): Promise<ForgeDriverResult> {
+async function processDDLResult(
+  query: string,
+  params: unknown[],
+  method: QueryMethod,
+  result: any,
+): Promise<ForgeDriverResult> {
   if (result.metadata) {
-    await saveMetaDataToContext(result.metadata as ForgeSQLMetadata);
+    await saveMetaDataToContext(query, params, result.metadata as ForgeSQLMetadata);
   }
 
   if (!result?.rows) {
@@ -109,13 +108,16 @@ async function processDDLResult(method: QueryMethod, result: any): Promise<Forge
 }
 
 /**
- * Processes execute method results (UPDATE, INSERT, DELETE).
+ * Processes execute method results (UPDATE, INSERT, DELETE) and saves metadata to the execution context.
  *
- * @param query - The SQL query
- * @param params - Query parameters
+ * @param query - The SQL query string
+ * @param params - Query parameters (may be undefined)
  * @returns Processed result for Drizzle ORM
  */
-async function processExecuteMethod(query: string, params: unknown[]): Promise<ForgeDriverResult> {
+async function processExecuteMethod(
+  query: string,
+  params: unknown[] | undefined,
+): Promise<ForgeDriverResult> {
   const sqlStatement = sql.prepare<UpdateQueryResponse>(query);
 
   if (params) {
@@ -123,7 +125,7 @@ async function processExecuteMethod(query: string, params: unknown[]): Promise<F
   }
 
   const result = await withTimeout(sqlStatement.execute(), timeoutMessage, timeoutMs);
-  await saveMetaDataToContext(result.metadata as ForgeSQLMetadata);
+  await saveMetaDataToContext(query, params ?? [], result.metadata as ForgeSQLMetadata);
   if (!result.rows) {
     return { rows: [[]] };
   }
@@ -132,13 +134,16 @@ async function processExecuteMethod(query: string, params: unknown[]): Promise<F
 }
 
 /**
- * Processes all method results (SELECT queries).
+ * Processes all method results (SELECT queries) and saves metadata to the execution context.
  *
- * @param query - The SQL query
- * @param params - Query parameters
+ * @param query - The SQL query string
+ * @param params - Query parameters (may be undefined)
  * @returns Processed result for Drizzle ORM
  */
-async function processAllMethod(query: string, params: unknown[]): Promise<ForgeDriverResult> {
+async function processAllMethod(
+  query: string,
+  params: unknown[] | undefined,
+): Promise<ForgeDriverResult> {
   const sqlStatement = await sql.prepare<unknown>(query);
 
   if (params) {
@@ -146,7 +151,7 @@ async function processAllMethod(query: string, params: unknown[]): Promise<Forge
   }
 
   const result = await withTimeout(sqlStatement.execute(), timeoutMessage, timeoutMs);
-  await saveMetaDataToContext(result.metadata as ForgeSQLMetadata);
+  await saveMetaDataToContext(query, params ?? [], result.metadata as ForgeSQLMetadata);
 
   if (!result.rows) {
     return { rows: [] };
@@ -160,9 +165,10 @@ async function processAllMethod(query: string, params: unknown[]): Promise<Forge
 /**
  * Main Forge SQL driver function for Drizzle ORM integration.
  * Handles DDL operations, execute operations (UPDATE/INSERT/DELETE), and select operations.
+ * Automatically saves query execution metadata to the context for performance monitoring.
  *
  * @param query - The SQL query to execute
- * @param params - Query parameters
+ * @param params - Query parameters (may be undefined or empty array)
  * @param method - Execution method ("all" for SELECT, "execute" for UPDATE/INSERT/DELETE)
  * @returns Promise with query results compatible with Drizzle ORM
  *
@@ -182,25 +188,28 @@ async function processAllMethod(query: string, params: unknown[]): Promise<Forge
  */
 export const forgeDriver = async (
   query: string,
-  params: unknown[],
+  params: unknown[] | undefined,
   method: QueryMethod,
 ): Promise<ForgeDriverResult> => {
   const operationType = await getOperationType();
   // Handle DDL operations
   if (operationType === "DDL") {
     const result = await withTimeout(
-      sql.executeDDL(inlineParams(query, params)),
+      sql
+        .prepare(query, SQL_API_ENDPOINTS.EXECUTE_DDL)
+        .bindParams(params ?? [])
+        .execute(),
       timeoutMessage,
       timeoutMs,
     );
-    return await processDDLResult(method, result);
+    return await processDDLResult(query, params ?? [], method, result);
   }
 
   // Handle execute method (UPDATE, INSERT, DELETE)
   if (method === "execute") {
-    return await processExecuteMethod(query, params ?? []);
+    return await processExecuteMethod(query, params);
   }
 
   // Handle all method (SELECT)
-  return await processAllMethod(query, params ?? []);
+  return await processAllMethod(query, params);
 };

package/src/utils/metadataContextUtils.ts

@@ -1,32 +1,312 @@
 import { AsyncLocalStorage } from "node:async_hooks";
 import { ForgeSQLMetadata } from "./forgeDriver";
 import { ForgeSqlOperation } from "../core/ForgeSQLQueryBuilder";
+import { ExplainAnalyzeRow } from "../core/SystemTables";
 import { printQueriesWithPlan } from "./sqlUtils";
+import { Parser } from "node-sql-parser";
+
+const DEFAULT_WINDOW_SIZE = 15 * 1000;
+
+type Statistic = { query: string; params: unknown[]; metadata: ForgeSQLMetadata };
+
+export type QueryPlanMode = "TopSlowest" | "SummaryTable";
+
+export type MetadataQueryOptions = {
+  mode?: QueryPlanMode;
+  summaryTableWindowTime?: number;
+  topQueries?: number;
+  showSlowestPlans?: boolean;
+  normalizeQuery?: boolean;
+};
 
 export type MetadataQueryContext = {
   totalDbExecutionTime: number;
   totalResponseSize: number;
   beginTime: Date;
+  options?: MetadataQueryOptions;
+  statistics: Statistic[];
   printQueriesWithPlan: () => Promise<void>;
   forgeSQLORM: ForgeSqlOperation;
 };
+
 export const metadataQueryContext = new AsyncLocalStorage<MetadataQueryContext>();
 
-export async function saveMetaDataToContext(metadata?: ForgeSQLMetadata): Promise<void> {
+/**
+ * Creates default options for metadata query context.
+ * @returns Default options object
+ */
+function createDefaultOptions(): Required<MetadataQueryOptions> {
+  return {
+    mode: "TopSlowest",
+    topQueries: 1,
+    summaryTableWindowTime: DEFAULT_WINDOW_SIZE,
+    showSlowestPlans: true,
+    normalizeQuery: true,
+  };
+}
+
+/**
+ * Merges provided options with defaults, using defaults only for undefined fields.
+ * @param options - Optional partial options to merge
+ * @returns Complete options object with all fields set
+ */
+function mergeOptionsWithDefaults(options?: MetadataQueryOptions): Required<MetadataQueryOptions> {
+  const defaults = createDefaultOptions();
+  return {
+    mode: options?.mode ?? defaults.mode,
+    topQueries: options?.topQueries ?? defaults.topQueries,
+    summaryTableWindowTime: options?.summaryTableWindowTime ?? defaults.summaryTableWindowTime,
+    showSlowestPlans: options?.showSlowestPlans ?? defaults.showSlowestPlans,
+    normalizeQuery: options?.normalizeQuery ?? defaults.normalizeQuery,
+  };
+}
+
+/**
+ * Normalizes SQL query using regex fallback by replacing parameter values with placeholders.
+ * Replaces string literals, numeric values, and boolean values with '?' for logging.
+ *
+ * Note: This is a fallback function used when node-sql-parser fails.
+ * It uses simple, safe regex patterns to avoid ReDoS (Regular Expression Denial of Service) vulnerabilities.
+ * For proper handling of escaped quotes and complex SQL, use the main normalizeSqlForLogging function
+ * which uses node-sql-parser.
+ *
+ * @param sql - SQL query string to normalize
+ * @returns Normalized SQL string with parameters replaced by '?'
+ */
+function normalizeSqlForLoggingRegex(sql: string): string {
+  let normalized = sql;
+
+  // Replace string literals (single quotes) - using simple greedy match
+  // This avoids catastrophic backtracking by using a simple [^']* pattern
+  // Note: This does not handle SQL-style escaped quotes (doubled quotes: '')
+  // For proper handling, use the main normalizeSqlForLogging function with node-sql-parser
+  normalized = normalized.replace(/'[^']*'/g, "?");
+
+  // Replace string literals (double quotes) - using simple greedy match
+  // Same safety considerations as above
+  normalized = normalized.replace(/"[^"]*"/g, "?");
+
+  // Replace numeric literals - simplified pattern to avoid backtracking
+  // Match: optional minus, digits, optional decimal point and more digits
+  // Using word boundaries (\b) for safety - avoids complex lookahead/lookbehind
+  normalized = normalized.replace(/\b-?\d+\.?\d*\b/g, "?");
+
+  // Replace boolean literals - safe pattern with word boundaries
+  // Simple alternation with word boundaries - no nested quantifiers
+  normalized = normalized.replace(/\b(true|false)\b/gi, "?");
+
+  // Replace NULL values (but be careful not to replace in identifiers)
+  // Simple word boundary match - safe from backtracking
+  normalized = normalized.replace(/\bNULL\b/gi, "?");
+
+  return normalized;
+}
+
+/**
+ * Normalizes SQL query by replacing parameter values with placeholders.
+ * First attempts to use node-sql-parser for structure normalization, then applies regex for value replacement.
+ * Falls back to regex-based normalization if parsing fails.
+ * @param sql - SQL query string to normalize
+ * @returns Normalized SQL string with parameters replaced by '?'
+ */
+function normalizeSqlForLogging(sql: string): string {
+  try {
+    const parser = new Parser();
+    const ast = parser.astify(sql.trim());
+
+    // Convert AST back to SQL (this normalizes structure and formatting)
+    const normalized = parser.sqlify(Array.isArray(ast) ? ast[0] : ast);
+
+    // Apply regex-based value replacement to the normalized SQL
+    // This handles the case where sqlify might preserve some literal values
+    let result = normalizeSqlForLoggingRegex(normalized.trim());
+
+    // Remove backticks added by sqlify for cleaner logging (optional - can be removed if backticks are preferred)
+    result = result.replace(/`/g, "");
+
+    return result;
+    //eslint-disable-next-line @typescript-eslint/no-unused-vars
+  } catch (e) {
+    // If parsing fails, fall back to regex-based normalization
+    return normalizeSqlForLoggingRegex(sql);
+  }
+}
+
+/**
+ * Formats row information (estRows, actRows) into a string.
+ * @param row - ExplainAnalyzeRow object
+ * @returns Formatted row info string or null if no row info available
+ */
+function formatRowInfo(row: ExplainAnalyzeRow): string | null {
+  const rowInfo: string[] = [];
+  if (row.estRows) rowInfo.push(`estRows:${row.estRows}`);
+  if (row.actRows) rowInfo.push(`actRows:${row.actRows}`);
+  return rowInfo.length > 0 ? `[${rowInfo.join(", ")}]` : null;
+}
+
+/**
+ * Formats resource information (memory, disk) into a string.
+ * @param row - ExplainAnalyzeRow object
+ * @returns Formatted resource info string or null if no resource info available
+ */
+function formatResourceInfo(row: ExplainAnalyzeRow): string | null {
+  const resourceInfo: string[] = [];
+  if (row.memory) resourceInfo.push(`memory:${row.memory}`);
+  if (row.disk) resourceInfo.push(`disk:${row.disk}`);
+  return resourceInfo.length > 0 ? `(${resourceInfo.join(", ")})` : null;
+}
+
+/**
+ * Formats a single execution plan row into a string.
+ * @param row - ExplainAnalyzeRow object
+ * @returns Formatted string representation of the row
+ */
+function formatPlanRow(row: ExplainAnalyzeRow): string {
+  const parts: string[] = [];
+
+  if (row.id) parts.push(row.id);
+  if (row.task) parts.push(`task:${row.task}`);
+  if (row.operatorInfo) parts.push(row.operatorInfo);
+
+  const rowInfo = formatRowInfo(row);
+  if (rowInfo) parts.push(rowInfo);
+
+  if (row.executionInfo) parts.push(`execution info:${row.executionInfo}`);
+
+  const resourceInfo = formatResourceInfo(row);
+  if (resourceInfo) parts.push(resourceInfo);
+
+  if (row.accessObject) parts.push(`access object:${row.accessObject}`);
+
+  return parts.join(" | ");
+}
+
+/**
+ * Formats an execution plan array into a readable string representation.
+ * @param planRows - Array of ExplainAnalyzeRow objects representing the execution plan
+ * @returns Formatted string representation of the execution plan
+ */
+function formatExplainPlan(planRows: ExplainAnalyzeRow[]): string {
+  if (!planRows || planRows.length === 0) {
+    return "No execution plan available";
+  }
+
+  return planRows.map(formatPlanRow).join("\n");
+}
+
+/**
+ * Prints query plans using summary tables if mode is SummaryTable and within time window.
+ * @param context - The metadata query context
+ * @param options - The merged options with defaults
+ * @returns Promise that resolves when plans are printed
+ */
+async function printPlansUsingSummaryTables(
+  context: MetadataQueryContext,
+  options: Required<MetadataQueryOptions>,
+): Promise<boolean> {
+  const timeDiff = Date.now() - context.beginTime.getTime();
+
+  if (options.mode !== "SummaryTable") {
+    return false;
+  }
+
+  if (timeDiff <= options.summaryTableWindowTime) {
+    await new Promise((resolve) => setTimeout(resolve, 200));
+    const summaryTableDiffMs = Date.now() - context.beginTime.getTime();
+    await printQueriesWithPlan(context.forgeSQLORM, summaryTableDiffMs);
+    return true;
+  }
+  // eslint-disable-next-line no-console
+  console.warn("Summary table window expired — showing query plans instead");
+  return false;
+}
+
+/**
+ * Prints query plans for the top slowest queries.
+ * @param context - The metadata query context
+ * @param options - The merged options with defaults
+ * @returns Promise that resolves when plans are printed
+ */
+async function printTopQueriesPlans(
+  context: MetadataQueryContext,
+  options: Required<MetadataQueryOptions>,
+): Promise<void> {
+  const topQueries = context.statistics
+    .toSorted((a, b) => b.metadata.dbExecutionTime - a.metadata.dbExecutionTime)
+    .slice(0, options.topQueries);
+
+  for (const query of topQueries) {
+    const normalizedQuery = options.normalizeQuery
+      ? normalizeSqlForLogging(query.query)
+      : query.query;
+    if (options.showSlowestPlans) {
+      const explainAnalyzeRows = await context.forgeSQLORM
+        .analyze()
+        .explainAnalyzeRaw(query.query, query.params);
+      const formattedPlan = formatExplainPlan(explainAnalyzeRows);
+      // eslint-disable-next-line no-console
+      console.warn(
+        `SQL: ${normalizedQuery} | Time: ${query.metadata.dbExecutionTime} ms\n Plan:\n${formattedPlan}`,
+      );
+    } else {
+      // eslint-disable-next-line no-console
+      console.warn(`SQL: ${normalizedQuery} | Time: ${query.metadata.dbExecutionTime} ms`);
+    }
+  }
+}
+
+/**
+ * Saves query metadata to the current context and sets up the printQueriesWithPlan function.
+ * @param stringQuery - The SQL query string
+ * @param params - Query parameters
+ * @param metadata - Query execution metadata
+ */
+export async function saveMetaDataToContext(
+  stringQuery: string,
+  params: unknown[],
+  metadata: ForgeSQLMetadata,
+): Promise<void> {
   const context = metadataQueryContext.getStore();
-  if (context) {
-    context.printQueriesWithPlan = async () => {
-      await new Promise((r) => setTimeout(r, 200));
-      await printQueriesWithPlan(context.forgeSQLORM, Date.now() - context.beginTime.getTime());
-    };
-    if (metadata) {
-      context.totalResponseSize += metadata.responseSize;
-      context.totalDbExecutionTime += metadata.dbExecutionTime;
+  if (!context) {
+    return;
+  }
+
+  // Initialize statistics array if needed
+  if (!context.statistics) {
+    context.statistics = [];
+  }
+
+  // Merge options with defaults
+  context.options = mergeOptionsWithDefaults(context.options);
+
+  // Add query statistics
+  context.statistics.push({ query: stringQuery, params, metadata });
+
+  // Set up printQueriesWithPlan function
+  context.printQueriesWithPlan = async () => {
+    const options = mergeOptionsWithDefaults(context.options);
+
+    // Try to use summary tables first if enabled
+    const usedSummaryTables = await printPlansUsingSummaryTables(context, options);
+    if (usedSummaryTables) {
+      return;
     }
-    // Log the results to console
+
+    // Fall back to printing top queries plans
+    await printTopQueriesPlans(context, options);
+  };
+
+  // Update aggregated metrics
+  if (metadata) {
+    context.totalResponseSize += metadata.responseSize;
+    context.totalDbExecutionTime += metadata.dbExecutionTime;
   }
 }
 
+/**
+ * Gets the latest metadata from the current context.
+ * @returns The current metadata context or undefined if not in a context
+ */
 export async function getLastestMetadata(): Promise<MetadataQueryContext | undefined> {
   return metadataQueryContext.getStore();
 }