forge-sql-orm 2.1.24 → 2.1.26

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "forge-sql-orm",
-  "version": "2.1.24",
+  "version": "2.1.26",
   "description": "Drizzle ORM integration for Atlassian @forge/sql. Provides a custom driver, schema migration, two levels of caching (local and global via @forge/kvs), optimistic locking, and query analysis.",
   "main": "dist/index.js",
   "homepage": "https://github.com/forge-sql-orm/forge-sql-orm#readme",
@@ -27,23 +27,23 @@
   "devDependencies": {
     "@eslint/js": "^9.39.2",
     "@types/luxon": "^3.7.1",
-    "@types/node": "^25.2.0",
-    "@typescript-eslint/eslint-plugin": "^8.54.0",
-    "@typescript-eslint/parser": "^8.54.0",
-    "@vitest/coverage-v8": "^4.0.18",
-    "@vitest/ui": "^4.0.18",
+    "@types/node": "^25.5.2",
+    "@typescript-eslint/eslint-plugin": "^8.58.1",
+    "@typescript-eslint/parser": "^8.58.1",
+    "@vitest/coverage-v8": "^4.1.3",
+    "@vitest/ui": "^4.1.3",
     "eslint": "^9.39.2",
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-import": "^2.32.0",
     "eslint-plugin-vitest": "^0.5.4",
     "husky": "^9.1.7",
-    "knip": "^5.83.0",
+    "knip": "^6.3.1",
     "patch-package": "^8.0.1",
     "prettier": "^3.8.1",
     "ts-node": "^10.9.2",
     "typescript": "^5.9.3",
     "uuid": "^13.0.0",
-    "vitest": "^4.0.18"
+    "vitest": "^4.1.3"
   },
   "license": "MIT",
   "author": "Vasyl Zakharchenko",
@@ -71,15 +71,15 @@
     "README.md"
   ],
   "peerDependencies": {
-    "@forge/sql": "^3.0.17",
-    "drizzle-orm": "^0.45.1"
+    "@forge/sql": "^3.0.21",
+    "drizzle-orm": "^0.45.2"
   },
   "optionalDependencies": {
-    "@forge/kvs": "^1.2.5"
+    "@forge/kvs": "^1.4.0"
   },
   "dependencies": {
-    "@forge/api": "^7.0.1",
-    "@forge/events": "^2.0.17",
+    "@forge/api": "^7.1.2",
+    "@forge/events": "^2.1.2",
     "luxon": "^3.7.2",
     "node-sql-parser": "^5.4.0"
   },

package/src/core/ForgeSQLQueryBuilder.ts

@@ -1282,6 +1282,32 @@ export interface ForgeSqlOrmOptions {
 /**
  * Creates a custom type for date/time fields with specified format and timestamp validation.
  */
+function normalizeDateOnlyValue(value: unknown): unknown {
+  if (typeof value !== "string") {
+    return value;
+  }
+
+  const trimmedValue = value.trim();
+
+  const dotSeparatedMatch = /^(\d{2})\.(\d{2})\.(\d{4})$/.exec(trimmedValue);
+  if (dotSeparatedMatch) {
+    const [, day, month, year] = dotSeparatedMatch;
+    return `${year}-${month}-${day} 00:00:00.000`;
+  }
+
+  const slashSeparatedMatch = /^(\d{2})\/(\d{2})\/(\d{4})$/.exec(trimmedValue);
+  if (slashSeparatedMatch) {
+    const [, day, month, year] = slashSeparatedMatch;
+    return `${year}-${month}-${day} 00:00:00.000`;
+  }
+
+  if (trimmedValue.length === 10) {
+    return `${trimmedValue} 00:00:00.000`;
+  }
+
+  return value;
+}
+
 function createDateCustomType(dataType: string, format: string, isTimeStamp: boolean) {
   return customType<{
     data: Date;
@@ -1295,6 +1321,9 @@ function createDateCustomType(dataType: string, format: string, isTimeStamp: boo
       return formatDateTime(value, format, isTimeStamp);
     },
     fromDriver(value: unknown) {
+      if (value === null || value === undefined) {
+        return value as unknown as Date;
+      }
       return parseDateTime(value as string, format);
     },
   });
@@ -1306,11 +1335,25 @@ function createDateCustomType(dataType: string, format: string, isTimeStamp: boo
  *
  * @type {CustomType}
  */
-export const forgeDateTimeString = createDateCustomType(
-  "datetime",
-  "yyyy-MM-dd' 'HH:mm:ss.SSS",
-  false,
-);
+export const forgeDateTimeString = customType<{
+  data: Date;
+  driver: string;
+  config: { format?: string };
+}>({
+  dataType() {
+    return "datetime";
+  },
+  toDriver(value: Date) {
+    return formatDateTime(value, "yyyy-MM-dd' 'HH:mm:ss.SSS", false);
+  },
+  fromDriver(value: unknown) {
+    if (value === null || value === undefined) {
+      return value as unknown as Date;
+    }
+    const normalizedValue = normalizeDateOnlyValue(value);
+    return parseDateTime(normalizedValue as string, "yyyy-MM-dd' 'HH:mm:ss.SSS");
+  },
+});
 
 /**
  * Custom type for MySQL timestamp fields.

package/src/core/VectorTiDB.ts

@@ -0,0 +1,192 @@
+import { customType } from "drizzle-orm/mysql-core";
+import { sql, type SQL, type AnyColumn } from "drizzle-orm";
+
+type VectorConfig = {
+  dimension?: number;
+};
+
+function validateVectorValue(value: number[]): void {
+  if (!Array.isArray(value)) {
+    throw new Error("TiDB vector value must be an array of numbers");
+  }
+
+  for (const item of value) {
+    if (typeof item !== "number" || !Number.isFinite(item)) {
+      throw new Error("TiDB vector contains invalid number");
+    }
+  }
+}
+
+function parseVectorText(value: string): number[] {
+  const trimmed = value.trim();
+  if (!trimmed.startsWith("[") || !trimmed.endsWith("]")) {
+    throw new Error(`Invalid TiDB vector text: ${value}`);
+  }
+
+  // TiDB stores vectors as textual representation, e.g. "[0.3,0.5,-0.1]".
+  const parsed = JSON.parse(trimmed) as unknown;
+  if (!Array.isArray(parsed)) {
+    throw new Error(`Invalid TiDB vector text: ${value}`);
+  }
+
+  const result = parsed.map((item) => {
+    if (typeof item !== "number" || !Number.isFinite(item)) {
+      throw new Error(`Invalid TiDB vector element: ${String(item)}`);
+    }
+    return item;
+  });
+
+  return result;
+}
+
+function vectorToText(value: number[]): string {
+  validateVectorValue(value);
+  return `[${value.join(",")}]`;
+}
+
+/**
+ * Input accepted by TiDB vector SQL helpers.
+ *
+ * - `number[]`: converted to `VEC_FROM_TEXT('[...]')`
+ * - `string`: treated as raw SQL vector expression, for example:
+ *   `CAST('[0.3, 0.5, -0.1]' AS VECTOR)`
+ * - `SQL` / `AnyColumn`: passed through as-is
+ */
+type VectorInput = string | number[] | SQL | AnyColumn;
+
+function vectorExpr(value: VectorInput): SQL {
+  if (Array.isArray(value)) {
+    return sql.raw(`VEC_FROM_TEXT('${vectorToText(value)}')`);
+  }
+  if (typeof value === "string") {
+    return sql.raw(value);
+  }
+  return sql`${value}`;
+}
+
+/**
+ * TiDB `VECTOR` column type (same call shapes as other Drizzle MySQL builders).
+ *
+ * - `vectorTiDBType('embedding', { dimension: 1536 })` — SQL column name + fixed dimension (`VECTOR(1536)`).
+ * - `vectorTiDBType({ dimension: 1536 })` — config only; column name comes from the object key in `mysqlTable`.
+ * - `vectorTiDBType('embedding')` or `vectorTiDBType()` — `VECTOR` without a fixed size in DDL.
+ *
+ * Values are `number[]` in application code; the driver maps to TiDB’s text form.
+ */
+export const vectorTiDBType = customType<{
+  data: number[];
+  driverData: string;
+  config: VectorConfig;
+}>({
+  dataType(config) {
+    const dim = config?.dimension;
+    return dim ? `vector(${dim})` : "vector";
+  },
+  toDriver(value: number[]) {
+    validateVectorValue(value);
+    return `[${value.join(",")}]`;
+  },
+  fromDriver(value: unknown) {
+    if (value === null || value === undefined) {
+      return value as unknown as number[];
+    }
+    if (typeof value !== "string") {
+      throw new Error(`Invalid TiDB vector driver value type: ${typeof value}`);
+    }
+    return parseVectorText(value);
+  },
+});
+
+/**
+ * Converts a text representation of a vector into a TiDB VECTOR expression.
+ *
+ * TiDB function: `VEC_FROM_TEXT(string)`.
+ *
+ * @example
+ * vecFromText("[1, 2, 3]")
+ */
+export function vecFromText(text: string): SQL {
+  return sql`VEC_FROM_TEXT('${text}')`;
+}
+
+/**
+ * Converts a vector value/expression to its normalized string representation.
+ *
+ * TiDB function: `VEC_AS_TEXT(vector)`.
+ *
+ * @example
+ * vecAsText(table.embedding)
+ */
+export function vecAsText(vector: VectorInput): SQL<string> {
+  return sql<string>`VEC_AS_TEXT(${vectorExpr(vector)})`;
+}
+
+/**
+ * Returns the dimension (number of elements) of a vector.
+ *
+ * TiDB function: `VEC_DIMS(vector)`.
+ *
+ * Accepts vector column/expression, `sql.raw(...)`, raw SQL string expression,
+ * or `number[]`.
+ *
+ * @example
+ * vecDims(table.embedding)
+ * vecDims(sql.raw("CAST('[0.3, 0.5, -0.1]' AS VECTOR)"))
+ * vecDims("CAST('[0.3, 0.5, -0.1]' AS VECTOR)")
+ */
+export function vecDims(vector: VectorInput): SQL<number> {
+  return sql<number>`VEC_DIMS(${vectorExpr(vector)})`;
+}
+
+/**
+ * Calculates L2 norm (Euclidean norm) of a vector.
+ *
+ * TiDB function: `VEC_L2_NORM(vector)`.
+ */
+export function vecL2Norm(vector: VectorInput): SQL<number> {
+  return sql`VEC_L2_NORM(${vectorExpr(vector)})`;
+}
+
+/**
+ * Calculates L2 distance (Euclidean distance) between two vectors.
+ *
+ * TiDB function: `VEC_L2_DISTANCE(vector1, vector2)`.
+ *
+ * Both vectors must have the same dimensions.
+ */
+export function vecL2Distance(left: VectorInput, right: VectorInput): SQL<number> {
+  return sql<number>`VEC_L2_DISTANCE(${vectorExpr(left)}, ${vectorExpr(right)})`;
+}
+
+/**
+ * Calculates cosine distance between two vectors.
+ *
+ * TiDB function: `VEC_COSINE_DISTANCE(vector1, vector2)`.
+ *
+ * Both vectors must have the same dimensions.
+ */
+export function vecCosineDistance(left: VectorInput, right: VectorInput): SQL<number> {
+  return sql<number>`VEC_COSINE_DISTANCE(${vectorExpr(left)}, ${vectorExpr(right)})`;
+}
+
+/**
+ * Calculates negative inner product distance between two vectors.
+ *
+ * TiDB function: `VEC_NEGATIVE_INNER_PRODUCT(vector1, vector2)`.
+ *
+ * Both vectors must have the same dimensions.
+ */
+export function vecNegativeInnerProduct(left: VectorInput, right: VectorInput): SQL<number> {
+  return sql<number>`VEC_NEGATIVE_INNER_PRODUCT(${vectorExpr(left)}, ${vectorExpr(right)})`;
+}
+
+/**
+ * Calculates L1 distance (Manhattan distance) between two vectors.
+ *
+ * TiDB function: `VEC_L1_DISTANCE(vector1, vector2)`.
+ *
+ * Both vectors must have the same dimensions.
+ */
+export function vecL1Distance(left: VectorInput, right: VectorInput): SQL<number> {
+  return sql<number>`VEC_L1_DISTANCE(${vectorExpr(left)}, ${vectorExpr(right)})`;
+}

package/src/core/index.ts

@@ -2,3 +2,4 @@ export * from "./ForgeSQLQueryBuilder";
 export * from "./ForgeSQLCrudOperations";
 export * from "./ForgeSQLSelectOperations";
 export * from "./SystemTables";
+export * from "./VectorTiDB";

package/src/utils/cacheUtils.ts

@@ -48,6 +48,30 @@ function nowPlusSeconds(secondsToAdd: number): number {
   return Math.floor(dt.toSeconds());
 }
 
+/**
+ * Logs a message to console.debug when options.logCache is enabled.
+ *
+ * @param message - Message to log
+ * @param options - ForgeSQL ORM options (optional)
+ */
+function debugLog(message: string, options?: ForgeSqlOrmOptions): void {
+  if (options?.logCache) {
+    // eslint-disable-next-line no-console
+    console.debug(message);
+  }
+} /**
+ * Logs a message to console.debug when options.logCache is enabled.
+ *
+ * @param message - Message to log
+ * @param options - ForgeSQL ORM options (optional)
+ */
+function warnLog(message: string, options?: ForgeSqlOrmOptions): void {
+  if (options?.logCache) {
+    // eslint-disable-next-line no-console
+    console.warn(message);
+  }
+}
+
 /**
  * Generates a hash key for a query based on its SQL and parameters.
  *
@@ -66,19 +90,20 @@ export function hashKey(query: Query): string {
  *
  * @param results - Array of cache entries to delete
  * @param cacheEntityName - Name of the cache entity
+ * @param options - Forge SQL ORM properties
  * @returns Promise that resolves when all deletions are complete
  */
 async function deleteCacheEntriesInBatches(
   results: Array<{ key: string }>,
   cacheEntityName: string,
+  options?: ForgeSqlOrmOptions,
 ): Promise<void> {
   for (let i = 0; i < results.length; i += CACHE_CONSTANTS.BATCH_SIZE) {
     const batch = results.slice(i, i + CACHE_CONSTANTS.BATCH_SIZE);
-    let transactionBuilder = kvs.transact();
-    for (const result of batch) {
-      transactionBuilder = transactionBuilder.delete(result.key, { entityName: cacheEntityName });
-    }
-    await transactionBuilder.execute();
+    const batchResult = await kvs.batchDelete(
+      batch.map((result) => ({ key: result.key, entityName: cacheEntityName })),
+    );
+    batchResult.failedKeys.forEach((failedKey) => warnLog(JSON.stringify(failedKey), options));
   }
 }
 
@@ -124,12 +149,9 @@ async function clearCursorCache(
 
   const listResult = await entityQueryBuilder.limit(100).getMany();
 
-  if (options.logCache) {
-    // eslint-disable-next-line no-console
-    console.warn(`clear cache Records: ${JSON.stringify(listResult.results.map((r) => r.key))}`);
-  }
+  debugLog(`clear cache Records: ${JSON.stringify(listResult.results.map((r) => r.key))}`, options);
 
-  await deleteCacheEntriesInBatches(listResult.results, cacheEntityName);
+  await deleteCacheEntriesInBatches(listResult.results, cacheEntityName, options);
 
   if (listResult.nextCursor) {
     return (
@@ -172,10 +194,10 @@ async function clearExpirationCursorCache(
 
   const listResult = await entityQueryBuilder.limit(100).getMany();
 
-  if (options.logCache) {
-    // eslint-disable-next-line no-console
-    console.warn(`clear expired Records: ${JSON.stringify(listResult.results.map((r) => r.key))}`);
-  }
+  debugLog(
+    `clear expired Records: ${JSON.stringify(listResult.results.map((r) => r.key))}`,
+    options,
+  );
 
   await deleteCacheEntriesInBatches(listResult.results, cacheEntityName);
 
@@ -265,14 +287,12 @@ export async function clearTablesCache(
       "clearing cache",
     );
   } finally {
-    if (options.logCache) {
-      const duration = DateTime.now().toSeconds() - startTime.toSeconds();
-      // eslint-disable-next-line no-console
-      console.info(`Cleared ${totalRecords} cache records in ${duration} seconds`);
-    }
+    const duration = DateTime.now().toSeconds() - startTime.toSeconds();
+    debugLog(`Cleared ${totalRecords} cache records in ${duration} seconds`, options);
   }
 }
 /**
+ * since https://developer.atlassian.com/platform/forge/changelog/#CHANGE-3038
  * Clears expired cache entries with retry logic and performance logging.
  *
  * @param options - ForgeSQL ORM options
@@ -293,16 +313,17 @@ export async function clearExpiredCache(options: ForgeSqlOrmOptions): Promise<vo
     );
   } finally {
     const duration = DateTime.now().toSeconds() - startTime.toSeconds();
-    if (options?.logCache) {
-      // eslint-disable-next-line no-console
-      console.debug(`Cleared ${totalRecords} expired cache records in ${duration} seconds`);
-    }
+    debugLog(`Cleared ${totalRecords} expired cache records in ${duration} seconds`, options);
   }
 }
 
 /**
  * Retrieves data from cache if it exists and is not expired.
  *
+ * Note: Due to Forge KVS asynchronous deletion (up to 48 hours), expired entries may still
+ * be returned. This function checks the expiration timestamp to filter out expired entries.
+ * If cache growth impacts performance, use the Clear Cache Scheduler Trigger.
+ *
  * @param query - Query object with toSQL method
  * @param options - ForgeSQL ORM options
  * @returns Cached data if found and valid, undefined otherwise
@@ -325,27 +346,27 @@ export async function getFromCache<T>(
 
   // Skip cache if table is in cache context (will be cleared)
   if (await isTableContainsTableInCacheContext(sqlQuery.sql, options)) {
-    if (options.logCache) {
-      // eslint-disable-next-line no-console
-      console.warn(`Context contains value to clear. Skip getting from cache`);
-    }
+    debugLog("Context contains value to clear. Skip getting from cache", options);
     return undefined;
   }
 
   try {
     const cacheResult = await kvs.entity<CacheEntity>(options.cacheEntityName).get(key);
 
-    if (
-      cacheResult &&
-      (cacheResult[expirationName] as number) >= getCurrentTime() &&
-      extractBacktickedValues(sqlQuery.sql, options) === cacheResult[entityQueryName]
-    ) {
-      if (options.logCache) {
-        // eslint-disable-next-line no-console
-        console.warn(`Get value from cache, cacheKey: ${key}`);
+    if (cacheResult) {
+      if (
+        (cacheResult[expirationName] as number) >= getCurrentTime() &&
+        extractBacktickedValues(sqlQuery.sql, options) === cacheResult[entityQueryName]
+      ) {
+        debugLog(`Get value from cache, cacheKey: ${key}`, options);
+        const results = cacheResult[dataName];
+        return JSON.parse(results as string);
+      } else {
+        debugLog(
+          `Expired cache entry still exists (will be automatically removed within 48 hours per Forge KVS TTL documentation), cacheKey: ${key}`,
+          options,
+        );
       }
-      const results = cacheResult[dataName];
-      return JSON.parse(results as string);
     }
   } catch (error: any) {
     // eslint-disable-next-line no-console
@@ -358,11 +379,18 @@ export async function getFromCache<T>(
 /**
  * Stores query results in cache with specified TTL.
  *
+ * Uses Forge KVS TTL feature to set expiration. Note that expired data deletion is asynchronous:
+ * expired data is not removed immediately upon expiry. Deletion may take up to 48 hours.
+ * During this window, read operations may still return expired results. If your app requires
+ * strict expiry semantics, consider using the Clear Cache Scheduler Trigger to proactively
+ * clean up expired entries, especially if cache growth impacts INSERT/UPDATE performance.
+ *
  * @param query - Query object with toSQL method
  * @param options - ForgeSQL ORM options
  * @param results - Data to cache
- * @param cacheTtl - Time to live in seconds
+ * @param cacheTtl - Time to live in seconds (maximum TTL is 1 year from write time)
  * @returns Promise that resolves when data is stored in cache
+ * @see https://developer.atlassian.com/platform/forge/runtime-reference/storage-api-basic-api/#ttl
  */
 export async function setCacheResult(
   query: { toSQL: () => Query },
@@ -385,10 +413,7 @@ export async function setCacheResult(
 
     // Skip cache if table is in cache context (will be cleared)
     if (await isTableContainsTableInCacheContext(sqlQuery.sql, options)) {
-      if (options.logCache) {
-        // eslint-disable-next-line no-console
-        console.warn(`Context contains value to clear. Skip setting from cache`);
-      }
+      debugLog("Context contains value to clear. Skip setting from cache", options);
       return;
     }
 
@@ -400,17 +425,15 @@ export async function setCacheResult(
         key,
         {
           [entityQueryName]: extractBacktickedValues(sqlQuery.sql, options),
-          [expirationName]: nowPlusSeconds(cacheTtl),
+          [expirationName]: nowPlusSeconds(cacheTtl + 2),
           [dataName]: JSON.stringify(results),
         },
         { entityName: options.cacheEntityName },
+        { ttl: { value: cacheTtl, unit: "SECONDS" } },
       )
       .execute();
 
-    if (options.logCache) {
-      // eslint-disable-next-line no-console
-      console.warn(`Store value to cache, cacheKey: ${key}`);
-    }
+    debugLog(`Store value to cache, cacheKey: ${key}`, options);
   } catch (error: any) {
     // eslint-disable-next-line no-console
     console.error(`Error setting cache: ${error.message}`, error);

package/src/webtriggers/clearCacheSchedulerTrigger.ts

@@ -2,10 +2,28 @@ import { clearExpiredCache } from "../utils/cacheUtils";
 import { ForgeSqlOrmOptions } from "../core/ForgeSQLQueryBuilder";
 
 /**
- * Scheduler trigger for clearing expired cache entries.
+ * Scheduler trigger for proactively clearing expired cache entries.
  *
- * This trigger should be configured as a Forge scheduler to automatically
- * clean up expired cache entries based on their TTL (Time To Live).
+ * **Why this trigger is needed:**
+ *
+ * While forge-sql-orm uses Forge KVS TTL feature to mark entries as expired, **actual deletion
+ * is asynchronous and may take up to 48 hours**. During this window, expired entries remain in
+ * storage and can impact INSERT/UPDATE performance if the cache grows large.
+ *
+ * This scheduler trigger proactively cleans up expired entries by querying the expiration index
+ * and deleting entries where expiration < now, preventing cache growth from impacting data
+ * modification operations.
+ *
+ * **When to use:**
+ * - Your cache grows large over time
+ * - INSERT/UPDATE operations are slowing down due to cache size
+ * - You need strict expiry semantics (immediate cleanup)
+ * - You want to reduce storage costs proactively
+ *
+ * **When optional:**
+ * - Small cache footprint
+ * - No performance impact on data modifications
+ * - You can tolerate expired entries being returned for up to 48 hours
  *
  * @note This function is automatically disabled in production environments and will return a 500 error if called.
  *
@@ -28,7 +46,7 @@ import { ForgeSqlOrmOptions } from "../core/ForgeSQLQueryBuilder";
  *
  * @example
  * ```yaml
- * # In manifest.yml
+ * # In manifest.yml (optional - only if cache growth impacts INSERT/UPDATE performance)
  * scheduledTrigger:
  *   - key: clear-cache-trigger
  *     function: clearCache
@@ -38,6 +56,8 @@ import { ForgeSqlOrmOptions } from "../core/ForgeSQLQueryBuilder";
  *   - key: clearCache
  *     handler: index.clearCache
  * ```
+ *
+ * @see https://developer.atlassian.com/platform/forge/runtime-reference/storage-api-basic-api/#ttl
  */
 export const clearCacheSchedulerTrigger = async (options?: ForgeSqlOrmOptions) => {
   try {