forge-sql-orm 2.1.15 → 2.1.17

package/src/utils/cacheUtils.ts

@@ -6,6 +6,7 @@ import { getTableName } from "drizzle-orm/table";
 import { Filter, FilterConditions, kvs, WhereConditions } from "@forge/kvs";
 import { ForgeSqlOrmOptions } from "../core/ForgeSQLQueryBuilder";
 import { cacheApplicationContext, isTableContainsTableInCacheContext } from "./cacheContextUtils";
+import { extractBacktickedValues } from "./cacheTableUtils";
 
 // Constants for better maintainability
 const CACHE_CONSTANTS = {
@@ -47,29 +48,6 @@ function nowPlusSeconds(secondsToAdd: number): number {
   return Math.floor(dt.toSeconds());
 }
 
-/**
- * Extracts all table/column names between backticks from SQL query and returns them as comma-separated string.
- *
- * @param sql - SQL query string
- * @returns Comma-separated string of unique backticked values
- */
-function extractBacktickedValues(sql: string): string {
-  const regex = /`([^`]+)`/g;
-  const matches = new Set<string>();
-  let match;
-
-  while ((match = regex.exec(sql.toLowerCase())) !== null) {
-    if (!match[1].startsWith("a_")) {
-      matches.add(`\`${match[1]}\``);
-    }
-  }
-
-  // Sort to ensure consistent order for the same input
-  return Array.from(matches)
-    .sort((a, b) => a.localeCompare(b, undefined, { sensitivity: "base", numeric: true }))
-    .join(",");
-}
-
 /**
  * Generates a hash key for a query based on its SQL and parameters.
  *
@@ -362,7 +340,7 @@ export async function getFromCache<T>(
     if (
       cacheResult &&
       (cacheResult[expirationName] as number) >= getCurrentTime() &&
-      extractBacktickedValues(sqlQuery.sql) === cacheResult[entityQueryName]
+      extractBacktickedValues(sqlQuery.sql, options) === cacheResult[entityQueryName]
     ) {
       if (options.logCache) {
         // eslint-disable-next-line no-console
@@ -423,7 +401,7 @@ export async function setCacheResult(
       .set(
         key,
         {
-          [entityQueryName]: extractBacktickedValues(sqlQuery.sql),
+          [entityQueryName]: extractBacktickedValues(sqlQuery.sql, options),
           [expirationName]: nowPlusSeconds(cacheTtl),
           [dataName]: JSON.stringify(results),
         },

package/src/utils/forgeDriver.ts

@@ -2,6 +2,7 @@ import { sql, UpdateQueryResponse } from "@forge/sql";
 import { saveMetaDataToContext } from "./metadataContextUtils";
 import { getOperationType } from "./requestTypeContextUtils";
 import { withTimeout } from "./sqlUtils";
+import { SQL_API_ENDPOINTS } from "@forge/sql/out/sql";
 
 const timeoutMs = 10000;
 const timeoutMessage = `Atlassian @forge/sql did not return a response within ${timeoutMs}ms (${timeoutMs / 1000} seconds), so the request is blocked. Possible causes: slow query, network issues, or exceeding Forge SQL limits.`;
@@ -66,16 +67,6 @@ export function isUpdateQueryResponse(obj: unknown): obj is UpdateQueryResponse
   );
 }
 
-function inlineParams(sql: string, params: unknown[]): string {
-  let i = 0;
-  return sql.replace(/\?/g, () => {
-    const val = params[i++];
-    if (val === null) return "NULL";
-    if (typeof val === "number") return val.toString();
-    return `'${String(val).replace(/'/g, "''")}'`;
-  });
-}
-
 /**
  * Processes DDL query results and saves metadata to the execution context.
  *
@@ -204,7 +195,10 @@ export const forgeDriver = async (
   // Handle DDL operations
   if (operationType === "DDL") {
     const result = await withTimeout(
-      sql.executeDDL(inlineParams(query, params ?? [])),
+      sql
+        .prepare(query, SQL_API_ENDPOINTS.EXECUTE_DDL)
+        .bindParams(params ?? [])
+        .execute(),
       timeoutMessage,
       timeoutMs,
     );

package/src/utils/forgeDriverProxy.ts

@@ -1,7 +1,7 @@
 import { forgeDriver } from "./forgeDriver";
 import { injectSqlHints, SqlHints } from "./sqlHints";
 import { ForgeSqlOperation } from "../core/ForgeSQLQueryBuilder";
-import { printQueriesWithPlan } from "./sqlUtils";
+import { handleErrorsWithPlan } from "./sqlUtils";
 
 /**
  * Error codes and constants for query analysis
@@ -57,22 +57,22 @@ export function createForgeDriverProxy(
         error?.context?.debug?.errno === QUERY_ERROR_CODES.OUT_OF_MEMORY_ERRNO;
 
       if (isTimeoutError || isOutOfMemoryError) {
+        // Wait for CLUSTER_STATEMENTS_SUMMARY to be populated with our failed query data
+        await new Promise((resolve) => setTimeout(resolve, STATEMENTS_SUMMARY_DELAY_MS));
+
+        const queryEndTime = Date.now();
+        const queryDuration = queryEndTime - queryStartTime;
+        let errorType: "OOM" | "TIMEOUT" = "TIMEOUT";
         if (isTimeoutError) {
           // eslint-disable-next-line no-console
           console.error(` TIMEOUT detected - Query exceeded time limit`);
         } else {
           // eslint-disable-next-line no-console
           console.error(`OUT OF MEMORY detected - Query exceeded memory limit`);
+          errorType = "OOM";
         }
-
-        // Wait for CLUSTER_STATEMENTS_SUMMARY to be populated with our failed query data
-        await new Promise((resolve) => setTimeout(resolve, STATEMENTS_SUMMARY_DELAY_MS));
-
-        const queryEndTime = Date.now();
-        const queryDuration = queryEndTime - queryStartTime;
-
         // Analyze the failed query using CLUSTER_STATEMENTS_SUMMARY
-        await printQueriesWithPlan(forgeSqlOperation, queryDuration);
+        await handleErrorsWithPlan(forgeSqlOperation, queryDuration, errorType);
       }
 
       // Log SQL error details if requested

package/src/utils/metadataContextUtils.ts

@@ -2,12 +2,15 @@ import { AsyncLocalStorage } from "node:async_hooks";
 import { ForgeSQLMetadata } from "./forgeDriver";
 import { ForgeSqlOperation } from "../core/ForgeSQLQueryBuilder";
 import { ExplainAnalyzeRow } from "../core/SystemTables";
-import { printQueriesWithPlan } from "./sqlUtils";
+import { printQueriesWithPlan, withTimeout } from "./sqlUtils";
 import { Parser } from "node-sql-parser";
+import { PushResult, Queue } from "@forge/events";
+import { AsyncEventPrintQuery } from "../async/PrintQueryConsumer";
 
+const TIMEOUT_ASYNC_EVENT_SENT = 1200;
 const DEFAULT_WINDOW_SIZE = 15 * 1000;
 
-type Statistic = { query: string; params: unknown[]; metadata: ForgeSQLMetadata };
+export type Statistic = { query: string; params: unknown[]; metadata: ForgeSQLMetadata };
 
 export type QueryPlanMode = "TopSlowest" | "SummaryTable";
 
@@ -17,6 +20,7 @@ export type MetadataQueryOptions = {
   topQueries?: number;
   showSlowestPlans?: boolean;
   normalizeQuery?: boolean;
+  asyncQueueName?: string;
 };
 
 export type MetadataQueryContext = {
@@ -42,6 +46,7 @@ function createDefaultOptions(): Required<MetadataQueryOptions> {
     summaryTableWindowTime: DEFAULT_WINDOW_SIZE,
     showSlowestPlans: true,
     normalizeQuery: true,
+    asyncQueueName: "",
   };
 }
 
@@ -58,6 +63,7 @@ function mergeOptionsWithDefaults(options?: MetadataQueryOptions): Required<Meta
     summaryTableWindowTime: options?.summaryTableWindowTime ?? defaults.summaryTableWindowTime,
     showSlowestPlans: options?.showSlowestPlans ?? defaults.showSlowestPlans,
     normalizeQuery: options?.normalizeQuery ?? defaults.normalizeQuery,
+    asyncQueueName: options?.asyncQueueName ?? defaults.asyncQueueName,
   };
 }
 
@@ -133,56 +139,84 @@ function normalizeSqlForLogging(sql: string): string {
 }
 
 /**
- * Formats an execution plan array into a readable string representation.
- * @param planRows - Array of ExplainAnalyzeRow objects representing the execution plan
- * @returns Formatted string representation of the execution plan
+ * Formats row information (estRows, actRows) into a string.
+ * @param row - ExplainAnalyzeRow object
+ * @returns Formatted row info string or null if no row info available
  */
-function formatExplainPlan(planRows: ExplainAnalyzeRow[]): string {
-  if (!planRows || planRows.length === 0) {
-    return "No execution plan available";
-  }
+function formatRowInfo(row: ExplainAnalyzeRow): string | null {
+  const rowInfo: string[] = [];
+  if (row.estRows) rowInfo.push(`estRows:${row.estRows}`);
+  if (row.actRows) rowInfo.push(`actRows:${row.actRows}`);
+  return rowInfo.length > 0 ? `[${rowInfo.join(", ")}]` : null;
+}
 
-  const lines: string[] = [];
+/**
+ * Formats resource information (memory, disk) into a string.
+ * @param row - ExplainAnalyzeRow object
+ * @returns Formatted resource info string or null if no resource info available
+ */
+function formatResourceInfo(row: ExplainAnalyzeRow): string | null {
+  const resourceInfo: string[] = [];
+  if (row.memory) resourceInfo.push(`memory:${row.memory}`);
+  if (row.disk) resourceInfo.push(`disk:${row.disk}`);
+  return resourceInfo.length > 0 ? `(${resourceInfo.join(", ")})` : null;
+}
 
-  for (const row of planRows) {
-    const parts: string[] = [];
+/**
+ * Formats a single execution plan row into a string.
+ * @param row - ExplainAnalyzeRow object
+ * @returns Formatted string representation of the row
+ */
+function formatPlanRow(row: ExplainAnalyzeRow): string {
+  const parts: string[] = [];
+
+  if (row.id) parts.push(row.id);
+  if (row.task) parts.push(`task:${row.task}`);
+  if (row.operatorInfo) parts.push(row.operatorInfo);
 
-    if (row.id) parts.push(row.id);
-    if (row.task) parts.push(`task:${row.task}`);
-    if (row.operatorInfo) parts.push(row.operatorInfo);
+  const rowInfo = formatRowInfo(row);
+  if (rowInfo) parts.push(rowInfo);
 
-    const rowInfo: string[] = [];
-    if (row.estRows) rowInfo.push(`estRows:${row.estRows}`);
-    if (row.actRows) rowInfo.push(`actRows:${row.actRows}`);
-    if (rowInfo.length > 0) parts.push(`[${rowInfo.join(", ")}]`);
+  if (row.executionInfo) parts.push(`execution info:${row.executionInfo}`);
 
-    if (row.executionInfo) parts.push(`execution info:${row.executionInfo}`);
+  const resourceInfo = formatResourceInfo(row);
+  if (resourceInfo) parts.push(resourceInfo);
 
-    const resourceInfo: string[] = [];
-    if (row.memory) resourceInfo.push(`memory:${row.memory}`);
-    if (row.disk) resourceInfo.push(`disk:${row.disk}`);
-    if (resourceInfo.length > 0) parts.push(`(${resourceInfo.join(", ")})`);
+  if (row.accessObject) parts.push(`access object:${row.accessObject}`);
 
-    if (row.accessObject) parts.push(`access object:${row.accessObject}`);
+  return parts.join(" | ");
+}
 
-    lines.push(parts.join(" | "));
+/**
+ * Formats an execution plan array into a readable string representation.
+ * @param planRows - Array of ExplainAnalyzeRow objects representing the execution plan
+ * @returns Formatted string representation of the execution plan
+ */
+function formatExplainPlan(planRows: ExplainAnalyzeRow[]): string {
+  if (!planRows || planRows.length === 0) {
+    return "No execution plan available";
   }
 
-  return lines.join("\n");
+  return planRows.map(formatPlanRow).join("\n");
 }
 
 /**
  * Prints query plans using summary tables if mode is SummaryTable and within time window.
- * @param context - The metadata query context
- * @param options - The merged options with defaults
- * @returns Promise that resolves when plans are printed
+ *
+ * Attempts to use CLUSTER_STATEMENTS_SUMMARY table for query analysis if:
+ * - Mode is set to "SummaryTable"
+ * - Time since query execution start is within the configured window
+ *
+ * @param context - The async event payload containing query statistics and options
+ * @param forgeSQLORM - The ForgeSQL operation instance for database access
+ * @returns Promise that resolves to true if summary tables were used, false otherwise
  */
 async function printPlansUsingSummaryTables(
-  context: MetadataQueryContext,
-  options: Required<MetadataQueryOptions>,
+  context: AsyncEventPrintQuery,
+  forgeSQLORM: ForgeSqlOperation,
 ): Promise<boolean> {
   const timeDiff = Date.now() - context.beginTime.getTime();
-
+  const options = context.options;
   if (options.mode !== "SummaryTable") {
     return false;
   }
@@ -190,7 +224,7 @@ async function printPlansUsingSummaryTables(
   if (timeDiff <= options.summaryTableWindowTime) {
     await new Promise((resolve) => setTimeout(resolve, 200));
     const summaryTableDiffMs = Date.now() - context.beginTime.getTime();
-    await printQueriesWithPlan(context.forgeSQLORM, summaryTableDiffMs);
+    await printQueriesWithPlan(forgeSQLORM, summaryTableDiffMs);
     return true;
   }
   // eslint-disable-next-line no-console
@@ -199,17 +233,22 @@ async function printPlansUsingSummaryTables(
 }
 
 /**
- * Prints query plans for the top slowest queries.
- * @param context - The metadata query context
- * @param options - The merged options with defaults
- * @returns Promise that resolves when plans are printed
+ * Prints query plans for the top slowest queries from the statistics.
+ *
+ * Sorts queries by execution time and prints the top N queries (based on topQueries option).
+ * For each query, it can optionally print the execution plan using EXPLAIN ANALYZE.
+ *
+ * @param context - The async event payload containing query statistics and options
+ * @param forgeSQLORM - The ForgeSQL operation instance for database access
+ * @returns Promise that resolves when all query plans are printed
  */
 async function printTopQueriesPlans(
-  context: MetadataQueryContext,
-  options: Required<MetadataQueryOptions>,
+  context: AsyncEventPrintQuery,
+  forgeSQLORM: ForgeSqlOperation,
 ): Promise<void> {
+  const options = context.options;
   const topQueries = context.statistics
-    .sort((a, b) => b.metadata.dbExecutionTime - a.metadata.dbExecutionTime)
+    .toSorted((a, b) => b.metadata.dbExecutionTime - a.metadata.dbExecutionTime)
     .slice(0, options.topQueries);
 
   for (const query of topQueries) {
@@ -217,7 +256,7 @@ async function printTopQueriesPlans(
       ? normalizeSqlForLogging(query.query)
       : query.query;
     if (options.showSlowestPlans) {
-      const explainAnalyzeRows = await context.forgeSQLORM
+      const explainAnalyzeRows = await forgeSQLORM
         .analyze()
         .explainAnalyzeRaw(query.query, query.params);
       const formattedPlan = formatExplainPlan(explainAnalyzeRows);
@@ -234,9 +273,33 @@ async function printTopQueriesPlans(
 
 /**
  * Saves query metadata to the current context and sets up the printQueriesWithPlan function.
+ *
+ * This function accumulates query statistics in the async context. When printQueriesWithPlan
+ * is called, it can either:
+ * - Queue the analysis for async processing (if asyncQueueName is provided)
+ * - Execute the analysis synchronously (fallback or if asyncQueueName is not set)
+ *
+ * For async processing, the function sends an event to the specified queue with a timeout.
+ * If the event cannot be sent within the timeout, it falls back to synchronous execution.
+ *
  * @param stringQuery - The SQL query string
- * @param params - Query parameters
- * @param metadata - Query execution metadata
+ * @param params - Query parameters used in the query
+ * @param metadata - Query execution metadata including execution time and response size
+ *
+ * @example
+ * ```typescript
+ * await FORGE_SQL_ORM.executeWithMetadata(
+ *   async () => {
+ *     // ... queries ...
+ *   },
+ *   async (totalDbExecutionTime, totalResponseSize, printQueries) => {
+ *     if (totalDbExecutionTime > threshold) {
+ *       await printQueries(); // Will use async queue if configured
+ *     }
+ *   },
+ *   { asyncQueueName: "degradationQueue" }
+ * );
+ * ```
  */
 export async function saveMetaDataToContext(
   stringQuery: string,
@@ -262,15 +325,41 @@ export async function saveMetaDataToContext(
   // Set up printQueriesWithPlan function
   context.printQueriesWithPlan = async () => {
     const options = mergeOptionsWithDefaults(context.options);
-
-    // Try to use summary tables first if enabled
-    const usedSummaryTables = await printPlansUsingSummaryTables(context, options);
-    if (usedSummaryTables) {
-      return;
+    const param: AsyncEventPrintQuery = {
+      statistics: context.statistics,
+      totalDbExecutionTime: context.totalDbExecutionTime,
+      totalResponseSize: context.totalResponseSize,
+      beginTime: context.beginTime,
+      options,
+    };
+    if (options.asyncQueueName) {
+      const queue = new Queue({ key: options.asyncQueueName });
+      try {
+        const eventInfo = await withTimeout<PushResult>(
+          queue.push({
+            body: param,
+            concurrency: {
+              key: "orm_" + options.asyncQueueName,
+              limit: 2,
+            },
+          }),
+          `Event was not sent within ${TIMEOUT_ASYNC_EVENT_SENT}ms`,
+          TIMEOUT_ASYNC_EVENT_SENT,
+        );
+        // eslint-disable-next-line no-console
+        console.warn(
+          `[Performance Analysis] Query degradation event queued for async processing | Job ID: ${eventInfo.jobId} | Total DB time: ${context.totalDbExecutionTime}ms | Queries: ${context.statistics.length} | Look for consumer log with jobId: ${eventInfo.jobId}`,
+        );
+        return;
+      } catch (e: any) {
+        // eslint-disable-next-line no-console
+        console.warn(
+          "Async printing failed — falling back to synchronous execution: " + e.message,
+          e,
+        );
+      }
     }
-
-    // Fall back to printing top queries plans
-    await printTopQueriesPlans(context, options);
+    await printDegradationQueries(context.forgeSQLORM, param);
   };
 
   // Update aggregated metrics
@@ -280,6 +369,34 @@ export async function saveMetaDataToContext(
   }
 }
 
+/**
+ * Prints query degradation analysis for the provided event payload.
+ *
+ * This function processes query degradation events (either from async queue or synchronous call).
+ * It first attempts to use summary tables (CLUSTER_STATEMENTS_SUMMARY) if configured and within
+ * the time window. Otherwise, it falls back to printing execution plans for the top slowest queries.
+ *
+ * @param forgeSQLORM - The ForgeSQL operation instance for database access
+ * @param params - The async event payload containing query statistics, options, and metadata
+ * @returns Promise that resolves when query analysis is complete
+ *
+ * @see printPlansUsingSummaryTables - For summary table analysis
+ * @see printTopQueriesPlans - For top slowest queries analysis
+ */
+export async function printDegradationQueries(
+  forgeSQLORM: ForgeSqlOperation,
+  params: AsyncEventPrintQuery,
+): Promise<void> {
+  // Try to use summary tables first if enabled
+  const usedSummaryTables = await printPlansUsingSummaryTables(params, forgeSQLORM);
+  if (usedSummaryTables) {
+    return;
+  }
+
+  // Fall back to printing top queries plans
+  await printTopQueriesPlans(params, forgeSQLORM);
+}
+
 /**
  * Gets the latest metadata from the current context.
  * @returns The current metadata context or undefined if not in a context