forge-sql-orm 2.1.23 → 2.1.25

package/src/utils/cacheTableUtils.ts

@@ -1,6 +1,17 @@
+// qlty-ignore: +qlty:file-complexity
 import { Parser } from "node-sql-parser";
 import { ForgeSqlOrmOptions } from "../core/ForgeSQLQueryBuilder";
 
+/**
+ * Extracts table name from backticks_quote_string type.
+ */
+function extractTableNameFromBackticks(value: any): string | null {
+  if (value.type === "backticks_quote_string" && typeof value.value === "string") {
+    return value.value === "dual" ? null : value.value.toLowerCase();
+  }
+  return null;
+}
+
 /**
  * Extracts table name from object value.
  */
@@ -9,22 +20,25 @@ function extractTableNameFromObject(value: any, context?: string): string | null
   if (Array.isArray(value)) {
     return null;
   }
+
   // Handle backticks_quote_string type only for node.table context
-  if (
-    context?.includes("node.table") &&
-    value.type === "backticks_quote_string" &&
-    typeof value.value === "string"
-  ) {
-    return value.value === "dual" ? null : value.value.toLowerCase();
+  if (context?.includes("node.table")) {
+    const fromBackticks = extractTableNameFromBackticks(value);
+    if (fromBackticks !== null) {
+      return fromBackticks;
+    }
   }
+
   // Try value.name first (most common)
   if (typeof value.name === "string") {
     return value.name === "dual" ? null : value.name.toLowerCase();
   }
+
   // Try value.table if it's a nested structure
   if (value.table) {
     return normalizeTableName(value.table, context);
   }
+
   // Log when we encounter an object that we can't extract table name from
   // eslint-disable-next-line no-console
   console.warn(
@@ -93,6 +107,22 @@ function isLikelyAlias(node: any): boolean {
   return isColumnRefAlias(node) || isExplicitAlias(node) || isShortNameAlias(node);
 }
 
+/**
+ * Extracts table name from table/dual node.
+ */
+function extractTableNameFromTableNode(node: any): string | null {
+  const fromTable = node.table
+    ? normalizeTableName(node.table, `node.type=${node.type}, node.table`)
+    : null;
+  if (fromTable) {
+    return fromTable;
+  }
+  const fromName = node.name
+    ? normalizeTableName(node.name, `node.type=${node.type}, node.name`)
+    : null;
+  return fromName;
+}
+
 /**
  * Extracts table name from table node.
  *
@@ -100,38 +130,18 @@ function isLikelyAlias(node: any): boolean {
  * @returns Table name in lowercase or null if not applicable
  */
 function extractTableName(node: any): string | null {
-  if (!node) {
-    return null;
-  }
-
-  // Early return for likely aliases
-  if (isLikelyAlias(node)) {
+  if (!node || isLikelyAlias(node)) {
     return null;
   }
 
   // Handle table node directly
   if (node.type === "table" || node.type === "dual") {
-    const fromTable = node.table
-      ? normalizeTableName(node.table, `node.type=${node.type}, node.table`)
-      : null;
-    if (fromTable) {
-      return fromTable;
-    }
-    const fromName = node.name
-      ? normalizeTableName(node.name, `node.type=${node.type}, node.name`)
-      : null;
-    if (fromName) {
-      return fromName;
-    }
-    return null;
+    return extractTableNameFromTableNode(node);
   }
 
   // Handle table reference in different formats
   if (node.table) {
-    const tableName = normalizeTableName(node.table, `node.table (type: ${node.type})`);
-    if (tableName) {
-      return tableName;
-    }
+    return normalizeTableName(node.table, `node.table (type: ${node.type})`);
   }
 
   return null;
@@ -205,6 +215,30 @@ function processFromAndJoin(node: any, tables: Set<string>): void {
   }
 }
 
+/**
+ * Processes a single column for table extraction.
+ */
+function processSingleColumn(col: any, tables: Set<string>): void {
+  if (!col) {
+    return;
+  }
+
+  // If the column itself is a subquery
+  if (col.type === "subquery" || col.type === "select") {
+    extractTablesFromNode(col, tables);
+  }
+
+  // Process expression (may contain subqueries)
+  if (col.expr) {
+    extractTablesFromNode(col.expr, tables);
+  }
+
+  // Process AST (alternative structure for subqueries)
+  if (col.ast) {
+    extractTablesFromNode(col.ast, tables);
+  }
+}
+
 /**
  * Processes SELECT columns that may contain subqueries.
  */
@@ -215,24 +249,7 @@ function processSelectColumns(node: any, tables: Set<string>): void {
   }
 
   if (Array.isArray(columns)) {
-    columns.forEach((col: any) => {
-      if (!col) return;
-
-      // If the column itself is a subquery
-      if (col.type === "subquery" || col.type === "select") {
-        extractTablesFromNode(col, tables);
-      }
-
-      // Process expression (may contain subqueries)
-      if (col.expr) {
-        extractTablesFromNode(col.expr, tables);
-      }
-
-      // Process AST (alternative structure for subqueries)
-      if (col.ast) {
-        extractTablesFromNode(col.ast, tables);
-      }
-    });
+    columns.forEach((col: any) => processSingleColumn(col, tables));
   } else if (typeof columns === "object") {
     extractTablesFromNode(columns, tables);
   }
@@ -265,14 +282,16 @@ function processUnionNode(unionNode: any, tables: Set<string>): void {
     return;
   }
 
-  const isUnionType =
-    unionNode.type === "select" ||
-    unionNode.type === "union" ||
-    unionNode.type === "union_all" ||
-    unionNode.type === "union_distinct" ||
-    unionNode.type === "intersect" ||
-    unionNode.type === "except" ||
-    unionNode.type === "minus";
+  const unionTypes = [
+    "select",
+    "union",
+    "union_all",
+    "union_distinct",
+    "intersect",
+    "except",
+    "minus",
+  ];
+  const isUnionType = unionTypes.includes(unionNode.type);
 
   if (isUnionType) {
     extractTablesFromNode(unionNode, tables);
@@ -304,13 +323,15 @@ function processUnion(node: any, tables: Set<string>): void {
  * Processes UNION/INTERSECT/EXCEPT operation nodes.
  */
 function processUnionOperation(node: any, tables: Set<string>): void {
-  const isUnionOperation =
-    node.type === "union" ||
-    node.type === "union_all" ||
-    node.type === "union_distinct" ||
-    node.type === "intersect" ||
-    node.type === "except" ||
-    node.type === "minus";
+  const unionOperationTypes = [
+    "union",
+    "union_all",
+    "union_distinct",
+    "intersect",
+    "except",
+    "minus",
+  ];
+  const isUnionOperation = unionOperationTypes.includes(node.type);
 
   if (!isUnionOperation) {
     return;
@@ -342,27 +363,25 @@ function processNext(node: any, tables: Set<string>): void {
 /**
  * Recursively processes all object properties for any remaining nested structures.
  */
-function processRecursively(node: any, tables: Set<string>): void {
-  const isLikelyAlias =
-    (node.type === "column_ref" && !node.table) ||
-    (node.name &&
-      !node.table &&
-      node.type !== "table" &&
-      node.type !== "dual" &&
-      node.name.length <= 2);
-
-  if (isLikelyAlias || Array.isArray(node)) {
-    return;
-  }
+/**
+ * Processes array values recursively.
+ */
+function processArrayValues(values: any[], tables: Set<string>): void {
+  values.forEach((item: any) => {
+    if (item && typeof item === "object") {
+      extractTablesFromNode(item, tables);
+    }
+  });
+}
 
+/**
+ * Processes object values recursively.
+ */
+function processObjectValues(node: any, tables: Set<string>): void {
   Object.values(node).forEach((value) => {
     if (value && typeof value === "object") {
       if (Array.isArray(value)) {
-        value.forEach((item: any) => {
-          if (item && typeof item === "object") {
-            extractTablesFromNode(item, tables);
-          }
-        });
+        processArrayValues(value, tables);
       } else {
         extractTablesFromNode(value, tables);
       }
@@ -370,6 +389,72 @@ function processRecursively(node: any, tables: Set<string>): void {
   });
 }
 
+function processRecursively(node: any, tables: Set<string>): void {
+  const isColumnRefAlias = node.type === "column_ref" && !node.table;
+  const hasName = Boolean(node.name);
+  const hasNoTable = !node.table;
+  const isNotTableType = node.type !== "table" && node.type !== "dual";
+  const isShortName = hasName && node.name.length <= 2;
+  const isShortNameAlias = hasName && hasNoTable && isNotTableType && isShortName;
+  const isLikelyAlias = isColumnRefAlias || isShortNameAlias;
+
+  if (isLikelyAlias || Array.isArray(node)) {
+    return;
+  }
+
+  processObjectValues(node, tables);
+}
+
+/**
+ * Processes DML statement types (UPDATE, INSERT, DELETE).
+ */
+function processDmlStatements(node: any, tables: Set<string>): void {
+  if (node.type === "update" && node.table) {
+    extractTablesFromNode(node.table, tables);
+  } else if (node.type === "insert" && node.table) {
+    extractTablesFromNode(node.table, tables);
+  } else if (node.type === "delete" && node.from) {
+    extractTablesFromNode(node.from, tables);
+  }
+}
+
+/**
+ * Processes SELECT-specific clauses (WHERE, HAVING, ORDER BY, GROUP BY).
+ */
+function processSelectClauses(node: any, tables: Set<string>): void {
+  if (node.where) {
+    extractTablesFromNode(node.where, tables);
+  }
+  if (node.having) {
+    extractTablesFromNode(node.having, tables);
+  }
+  processOrderByOrGroupBy(node.orderby || node.order_by, tables);
+  processOrderByOrGroupBy(node.groupby || node.group_by, tables);
+}
+
+/**
+ * Processes subquery and SELECT statement types.
+ */
+function processSubqueryAndSelect(node: any, tables: Set<string>): void {
+  if (node.type === "subquery" || node.type === "select") {
+    if (node.ast) {
+      extractTablesFromNode(node.ast, tables);
+    }
+    if (node.from) {
+      extractTablesFromNode(node.from, tables);
+    }
+  }
+}
+
+/**
+ * Processes set operations (UNION, INTERSECT, EXCEPT).
+ */
+function processSetOperations(node: any, tables: Set<string>): void {
+  processUnion(node, tables);
+  processUnionOperation(node, tables);
+  processNext(node, tables);
+}
+
 /**
  * Recursively extracts table names from SQL AST node.
  * Handles regular tables, CTEs, subqueries, and complex query structures.
@@ -391,58 +476,20 @@ function extractTablesFromNode(node: any, tables: Set<string>): void {
   // Extract tables from FROM and JOIN clauses
   processFromAndJoin(node, tables);
 
-  // Handle subqueries explicitly
-  if (node.type === "subquery" || node.type === "select") {
-    if (node.ast) {
-      extractTablesFromNode(node.ast, tables);
-    }
-    if (node.from) {
-      extractTablesFromNode(node.from, tables);
-    }
-  }
+  // Handle subqueries and SELECT statements
+  processSubqueryAndSelect(node, tables);
 
-  // Extract tables from WHERE clause (may contain subqueries)
-  if (node.where) {
-    extractTablesFromNode(node.where, tables);
-  }
+  // Process SELECT-specific clauses
+  processSelectClauses(node, tables);
 
   // Extract tables from SELECT columns (may contain subqueries)
   processSelectColumns(node, tables);
 
-  // Extract tables from HAVING clause (may contain subqueries)
-  if (node.having) {
-    extractTablesFromNode(node.having, tables);
-  }
-
-  // Extract tables from ORDER BY clause (may contain subqueries)
-  processOrderByOrGroupBy(node.orderby || node.order_by, tables);
-
-  // Extract tables from GROUP BY clause (may contain subqueries)
-  processOrderByOrGroupBy(node.groupby || node.group_by, tables);
+  // Process DML statements (UPDATE, INSERT, DELETE)
+  processDmlStatements(node, tables);
 
-  // Extract tables from UPDATE statement
-  if (node.type === "update" && node.table) {
-    extractTablesFromNode(node.table, tables);
-  }
-
-  // Extract tables from INSERT statement
-  if (node.type === "insert" && node.table) {
-    extractTablesFromNode(node.table, tables);
-  }
-
-  // Extract tables from DELETE statement
-  if (node.type === "delete" && node.from) {
-    extractTablesFromNode(node.from, tables);
-  }
-
-  // Extract tables from UNION operations
-  processUnion(node, tables);
-
-  // Handle node types for UNION/INTERSECT/EXCEPT operations
-  processUnionOperation(node, tables);
-
-  // Handle _next property (alternative UNION structure)
-  processNext(node, tables);
+  // Process set operations (UNION, INTERSECT, EXCEPT)
+  processSetOperations(node, tables);
 
   // Recursively process all object properties
   processRecursively(node, tables);
@@ -456,6 +503,33 @@ function extractTablesFromNode(node: any, tables: Set<string>): void {
  * @param options - ForgeSQL ORM options for logging
  * @returns Comma-separated string of unique table names in backticks
  */
+/**
+ * Formats table names as backticked comma-separated string.
+ */
+function formatBacktickedValues(tables: Set<string>): string {
+  return Array.from(tables)
+    .sort((a, b) => a.localeCompare(b, undefined, { sensitivity: "base", numeric: true }))
+    .map((table) => `\`${table}\``)
+    .join(",");
+}
+
+/**
+ * Extracts table names using regex fallback.
+ */
+function extractTablesWithRegex(sql: string): Set<string> {
+  const regex = /`([^`]+)`/g;
+  const matches = new Set<string>();
+  let match;
+
+  while ((match = regex.exec(sql.toLowerCase())) !== null) {
+    if (!match[1].startsWith("a_")) {
+      matches.add(match[1]);
+    }
+  }
+
+  return matches;
+}
+
 export function extractBacktickedValues(sql: string, options: ForgeSqlOrmOptions): string {
   // Try to use node-sql-parser first
   try {
@@ -471,11 +545,7 @@ export function extractBacktickedValues(sql: string, options: ForgeSqlOrmOptions
     });
 
     if (tables.size > 0) {
-      // Sort to ensure consistent order for the same input
-      const backtickedValues = Array.from(tables)
-        .sort((a, b) => a.localeCompare(b, undefined, { sensitivity: "base", numeric: true }))
-        .map((table) => `\`${table}\``)
-        .join(",");
+      const backtickedValues = formatBacktickedValues(tables);
       if (options.logCache) {
         // eslint-disable-next-line no-console
         console.warn(`Extracted backticked values: ${backtickedValues}`);
@@ -494,18 +564,6 @@ export function extractBacktickedValues(sql: string, options: ForgeSqlOrmOptions
   }
 
   // Fallback to regex-based extraction (original logic)
-  const regex = /`([^`]+)`/g;
-  const matches = new Set<string>();
-  let match;
-
-  while ((match = regex.exec(sql.toLowerCase())) !== null) {
-    if (!match[1].startsWith("a_")) {
-      matches.add(`\`${match[1]}\``);
-    }
-  }
-
-  // Sort to ensure consistent order for the same input
-  return Array.from(matches)
-    .sort((a, b) => a.localeCompare(b, undefined, { sensitivity: "base", numeric: true }))
-    .join(",");
+  const matches = extractTablesWithRegex(sql);
+  return formatBacktickedValues(matches);
 }

package/src/utils/cacheUtils.ts

@@ -48,6 +48,30 @@ function nowPlusSeconds(secondsToAdd: number): number {
   return Math.floor(dt.toSeconds());
 }
 
+/**
+ * Logs a message to console.debug when options.logCache is enabled.
+ *
+ * @param message - Message to log
+ * @param options - ForgeSQL ORM options (optional)
+ */
+function debugLog(message: string, options?: ForgeSqlOrmOptions): void {
+  if (options?.logCache) {
+    // eslint-disable-next-line no-console
+    console.debug(message);
+  }
+} /**
+ * Logs a message to console.debug when options.logCache is enabled.
+ *
+ * @param message - Message to log
+ * @param options - ForgeSQL ORM options (optional)
+ */
+function warnLog(message: string, options?: ForgeSqlOrmOptions): void {
+  if (options?.logCache) {
+    // eslint-disable-next-line no-console
+    console.warn(message);
+  }
+}
+
 /**
  * Generates a hash key for a query based on its SQL and parameters.
  *
@@ -66,19 +90,20 @@ export function hashKey(query: Query): string {
  *
  * @param results - Array of cache entries to delete
  * @param cacheEntityName - Name of the cache entity
+ * @param options - Forge SQL ORM properties
  * @returns Promise that resolves when all deletions are complete
  */
 async function deleteCacheEntriesInBatches(
   results: Array<{ key: string }>,
   cacheEntityName: string,
+  options?: ForgeSqlOrmOptions,
 ): Promise<void> {
   for (let i = 0; i < results.length; i += CACHE_CONSTANTS.BATCH_SIZE) {
     const batch = results.slice(i, i + CACHE_CONSTANTS.BATCH_SIZE);
-    let transactionBuilder = kvs.transact();
-    for (const result of batch) {
-      transactionBuilder = transactionBuilder.delete(result.key, { entityName: cacheEntityName });
-    }
-    await transactionBuilder.execute();
+    const batchResult = await kvs.batchDelete(
+      batch.map((result) => ({ key: result.key, entityName: cacheEntityName })),
+    );
+    batchResult.failedKeys.forEach((failedKey) => warnLog(JSON.stringify(failedKey), options));
   }
 }
 
@@ -124,12 +149,9 @@ async function clearCursorCache(
 
   const listResult = await entityQueryBuilder.limit(100).getMany();
 
-  if (options.logCache) {
-    // eslint-disable-next-line no-console
-    console.warn(`clear cache Records: ${JSON.stringify(listResult.results.map((r) => r.key))}`);
-  }
+  debugLog(`clear cache Records: ${JSON.stringify(listResult.results.map((r) => r.key))}`, options);
 
-  await deleteCacheEntriesInBatches(listResult.results, cacheEntityName);
+  await deleteCacheEntriesInBatches(listResult.results, cacheEntityName, options);
 
   if (listResult.nextCursor) {
     return (
@@ -172,10 +194,10 @@ async function clearExpirationCursorCache(
 
   const listResult = await entityQueryBuilder.limit(100).getMany();
 
-  if (options.logCache) {
-    // eslint-disable-next-line no-console
-    console.warn(`clear expired Records: ${JSON.stringify(listResult.results.map((r) => r.key))}`);
-  }
+  debugLog(
+    `clear expired Records: ${JSON.stringify(listResult.results.map((r) => r.key))}`,
+    options,
+  );
 
   await deleteCacheEntriesInBatches(listResult.results, cacheEntityName);
 
@@ -265,14 +287,12 @@ export async function clearTablesCache(
       "clearing cache",
     );
   } finally {
-    if (options.logCache) {
-      const duration = DateTime.now().toSeconds() - startTime.toSeconds();
-      // eslint-disable-next-line no-console
-      console.info(`Cleared ${totalRecords} cache records in ${duration} seconds`);
-    }
+    const duration = DateTime.now().toSeconds() - startTime.toSeconds();
+    debugLog(`Cleared ${totalRecords} cache records in ${duration} seconds`, options);
   }
 }
 /**
+ * since https://developer.atlassian.com/platform/forge/changelog/#CHANGE-3038
  * Clears expired cache entries with retry logic and performance logging.
  *
  * @param options - ForgeSQL ORM options
@@ -293,16 +313,17 @@ export async function clearExpiredCache(options: ForgeSqlOrmOptions): Promise<vo
     );
   } finally {
     const duration = DateTime.now().toSeconds() - startTime.toSeconds();
-    if (options?.logCache) {
-      // eslint-disable-next-line no-console
-      console.debug(`Cleared ${totalRecords} expired cache records in ${duration} seconds`);
-    }
+    debugLog(`Cleared ${totalRecords} expired cache records in ${duration} seconds`, options);
   }
 }
 
 /**
  * Retrieves data from cache if it exists and is not expired.
  *
+ * Note: Due to Forge KVS asynchronous deletion (up to 48 hours), expired entries may still
+ * be returned. This function checks the expiration timestamp to filter out expired entries.
+ * If cache growth impacts performance, use the Clear Cache Scheduler Trigger.
+ *
  * @param query - Query object with toSQL method
  * @param options - ForgeSQL ORM options
  * @returns Cached data if found and valid, undefined otherwise
@@ -325,27 +346,27 @@ export async function getFromCache<T>(
 
   // Skip cache if table is in cache context (will be cleared)
   if (await isTableContainsTableInCacheContext(sqlQuery.sql, options)) {
-    if (options.logCache) {
-      // eslint-disable-next-line no-console
-      console.warn(`Context contains value to clear. Skip getting from cache`);
-    }
+    debugLog("Context contains value to clear. Skip getting from cache", options);
     return undefined;
   }
 
   try {
     const cacheResult = await kvs.entity<CacheEntity>(options.cacheEntityName).get(key);
 
-    if (
-      cacheResult &&
-      (cacheResult[expirationName] as number) >= getCurrentTime() &&
-      extractBacktickedValues(sqlQuery.sql, options) === cacheResult[entityQueryName]
-    ) {
-      if (options.logCache) {
-        // eslint-disable-next-line no-console
-        console.warn(`Get value from cache, cacheKey: ${key}`);
+    if (cacheResult) {
+      if (
+        (cacheResult[expirationName] as number) >= getCurrentTime() &&
+        extractBacktickedValues(sqlQuery.sql, options) === cacheResult[entityQueryName]
+      ) {
+        debugLog(`Get value from cache, cacheKey: ${key}`, options);
+        const results = cacheResult[dataName];
+        return JSON.parse(results as string);
+      } else {
+        debugLog(
+          `Expired cache entry still exists (will be automatically removed within 48 hours per Forge KVS TTL documentation), cacheKey: ${key}`,
+          options,
+        );
       }
-      const results = cacheResult[dataName];
-      return JSON.parse(results as string);
     }
   } catch (error: any) {
     // eslint-disable-next-line no-console
@@ -358,11 +379,18 @@ export async function getFromCache<T>(
 /**
  * Stores query results in cache with specified TTL.
  *
+ * Uses Forge KVS TTL feature to set expiration. Note that expired data deletion is asynchronous:
+ * expired data is not removed immediately upon expiry. Deletion may take up to 48 hours.
+ * During this window, read operations may still return expired results. If your app requires
+ * strict expiry semantics, consider using the Clear Cache Scheduler Trigger to proactively
+ * clean up expired entries, especially if cache growth impacts INSERT/UPDATE performance.
+ *
  * @param query - Query object with toSQL method
  * @param options - ForgeSQL ORM options
  * @param results - Data to cache
- * @param cacheTtl - Time to live in seconds
+ * @param cacheTtl - Time to live in seconds (maximum TTL is 1 year from write time)
  * @returns Promise that resolves when data is stored in cache
+ * @see https://developer.atlassian.com/platform/forge/runtime-reference/storage-api-basic-api/#ttl
  */
 export async function setCacheResult(
   query: { toSQL: () => Query },
@@ -385,10 +413,7 @@ export async function setCacheResult(
 
     // Skip cache if table is in cache context (will be cleared)
     if (await isTableContainsTableInCacheContext(sqlQuery.sql, options)) {
-      if (options.logCache) {
-        // eslint-disable-next-line no-console
-        console.warn(`Context contains value to clear. Skip setting from cache`);
-      }
+      debugLog("Context contains value to clear. Skip setting from cache", options);
       return;
     }
 
@@ -400,17 +425,15 @@ export async function setCacheResult(
         key,
         {
           [entityQueryName]: extractBacktickedValues(sqlQuery.sql, options),
-          [expirationName]: nowPlusSeconds(cacheTtl),
+          [expirationName]: nowPlusSeconds(cacheTtl + 2),
           [dataName]: JSON.stringify(results),
         },
         { entityName: options.cacheEntityName },
+        { ttl: { value: cacheTtl, unit: "SECONDS" } },
       )
       .execute();
 
-    if (options.logCache) {
-      // eslint-disable-next-line no-console
-      console.warn(`Store value to cache, cacheKey: ${key}`);
-    }
+    debugLog(`Store value to cache, cacheKey: ${key}`, options);
   } catch (error: any) {
     // eslint-disable-next-line no-console
     console.error(`Error setting cache: ${error.message}`, error);