forge-sql-orm 2.0.30 → 2.1.1

package/src/utils/cacheUtils.ts

@@ -0,0 +1,403 @@
+import { DateTime } from "luxon";
+import * as crypto from "crypto";
+import { Query } from "drizzle-orm";
+import { AnyMySqlTable } from "drizzle-orm/mysql-core";
+import { getTableName } from "drizzle-orm/table";
+import { Filter, FilterConditions, kvs, WhereConditions } from "@forge/kvs";
+import { ForgeSqlOrmOptions } from "../core/ForgeSQLQueryBuilder";
+import { cacheApplicationContext, isTableContainsTableInCacheContext } from "./cacheContextUtils";
+
+// Constants for better maintainability
+const CACHE_CONSTANTS = {
+  BATCH_SIZE: 25,
+  MAX_RETRY_ATTEMPTS: 3,
+  INITIAL_RETRY_DELAY: 1000,
+  RETRY_DELAY_MULTIPLIER: 2,
+  DEFAULT_ENTITY_QUERY_NAME: "sql",
+  DEFAULT_EXPIRATION_NAME: "expiration",
+  DEFAULT_DATA_NAME: "data",
+  HASH_LENGTH: 32,
+} as const;
+
+// Types for better type safety
+type CacheEntity = {
+  [key: string]: string | number;
+};
+
+/**
+ * Gets the current Unix timestamp in seconds.
+ *
+ * @returns Current timestamp as integer
+ */
+function getCurrentTime(): number {
+  const dt = DateTime.now();
+  return Math.floor(dt.toSeconds());
+}
+
+/**
+ * Calculates a future timestamp by adding seconds to the current time.
+ * Validates that the result is within 32-bit integer range.
+ *
+ * @param secondsToAdd - Number of seconds to add to current time
+ * @returns Future timestamp in seconds
+ * @throws Error if the result is out of 32-bit integer range
+ */
+function nowPlusSeconds(secondsToAdd: number): number {
+  const dt = DateTime.now().plus({ seconds: secondsToAdd });
+  return Math.floor(dt.toSeconds());
+}
+
+/**
+ * Generates a hash key for a query based on its SQL and parameters.
+ *
+ * @param query - The Drizzle query object
+ * @returns 32-character hexadecimal hash
+ */
+export function hashKey(query: Query): string {
+  const h = crypto.createHash("sha256");
+  h.update(query.sql.toLowerCase());
+  h.update(JSON.stringify(query.params));
+  return "CachedQuery_" + h.digest("hex").slice(0, CACHE_CONSTANTS.HASH_LENGTH);
+}
+
+/**
+ * Deletes cache entries in batches to respect Forge limits and timeouts.
+ *
+ * @param results - Array of cache entries to delete
+ * @param cacheEntityName - Name of the cache entity
+ * @returns Promise that resolves when all deletions are complete
+ */
+async function deleteCacheEntriesInBatches(
+  results: Array<{ key: string }>,
+  cacheEntityName: string,
+): Promise<void> {
+  for (let i = 0; i < results.length; i += CACHE_CONSTANTS.BATCH_SIZE) {
+    const batch = results.slice(i, i + CACHE_CONSTANTS.BATCH_SIZE);
+    let transactionBuilder = kvs.transact();
+    batch.forEach((result) => {
+      transactionBuilder = transactionBuilder.delete(result.key, { entityName: cacheEntityName });
+    });
+    await transactionBuilder.execute();
+  }
+}
+
+/**
+ * Clears cache entries for specific tables using cursor-based pagination.
+ *
+ * @param tables - Array of table names to clear cache for
+ * @param cursor - Pagination cursor for large result sets
+ * @param options - ForgeSQL ORM options
+ * @returns Total number of deleted cache entries
+ */
+async function clearCursorCache(
+  tables: string[],
+  cursor: string,
+  options: ForgeSqlOrmOptions,
+): Promise<number> {
+  const cacheEntityName = options.cacheEntityName;
+  if (!cacheEntityName) {
+    throw new Error("cacheEntityName is not configured");
+  }
+
+  const entityQueryName = options.cacheEntityQueryName ?? CACHE_CONSTANTS.DEFAULT_ENTITY_QUERY_NAME;
+  let filters = new Filter<{
+    [entityQueryName]: string;
+  }>();
+
+  for (const table of tables) {
+    const wrapIfNeeded = options.cacheWrapTable ? `\`${table}\`` : table;
+    filters.or(entityQueryName, FilterConditions.contains(wrapIfNeeded?.toLowerCase()));
+  }
+
+  let entityQueryBuilder = kvs
+    .entity<{
+      [entityQueryName]: string;
+    }>(cacheEntityName)
+    .query()
+    .index(entityQueryName)
+    .filters(filters);
+
+  if (cursor) {
+    entityQueryBuilder = entityQueryBuilder.cursor(cursor);
+  }
+
+  const listResult = await entityQueryBuilder.limit(100).getMany();
+
+  if (options.logRawSqlQuery) {
+    console.warn(`clear cache Records: ${JSON.stringify(listResult.results.map((r) => r.key))}`);
+  }
+
+  await deleteCacheEntriesInBatches(listResult.results, cacheEntityName);
+
+  if (listResult.nextCursor) {
+    return (
+      listResult.results.length + (await clearCursorCache(tables, listResult.nextCursor, options))
+    );
+  } else {
+    return listResult.results.length;
+  }
+}
+
+/**
+ * Clears expired cache entries using cursor-based pagination.
+ *
+ * @param cursor - Pagination cursor for large result sets
+ * @param options - ForgeSQL ORM options
+ * @returns Total number of deleted expired cache entries
+ */
+async function clearExpirationCursorCache(
+  cursor: string,
+  options: ForgeSqlOrmOptions,
+): Promise<number> {
+  const cacheEntityName = options.cacheEntityName;
+  if (!cacheEntityName) {
+    throw new Error("cacheEntityName is not configured");
+  }
+
+  const entityExpirationName =
+    options.cacheEntityExpirationName ?? CACHE_CONSTANTS.DEFAULT_EXPIRATION_NAME;
+  let entityQueryBuilder = kvs
+    .entity<{
+      [entityExpirationName]: number;
+    }>(cacheEntityName)
+    .query()
+    .index(entityExpirationName)
+    .where(WhereConditions.lessThan(Math.floor(DateTime.now().toSeconds())));
+
+  if (cursor) {
+    entityQueryBuilder = entityQueryBuilder.cursor(cursor);
+  }
+
+  const listResult = await entityQueryBuilder.limit(100).getMany();
+
+  if (options.logRawSqlQuery) {
+    console.warn(`clear expired Records: ${JSON.stringify(listResult.results.map((r) => r.key))}`);
+  }
+
+  await deleteCacheEntriesInBatches(listResult.results, cacheEntityName);
+
+  if (listResult.nextCursor) {
+    return (
+      listResult.results.length + (await clearExpirationCursorCache(listResult.nextCursor, options))
+    );
+  } else {
+    return listResult.results.length;
+  }
+}
+
+/**
+ * Executes a function with retry logic and exponential backoff.
+ *
+ * @param operation - Function to execute with retry
+ * @param operationName - Name of the operation for logging
+ * @param options - ForgeSQL ORM options for logging
+ * @returns Promise that resolves to the operation result
+ */
+async function executeWithRetry<T>(operation: () => Promise<T>, operationName: string): Promise<T> {
+  let attempt = 0;
+  let delay = CACHE_CONSTANTS.INITIAL_RETRY_DELAY;
+
+  while (attempt < CACHE_CONSTANTS.MAX_RETRY_ATTEMPTS) {
+    try {
+      return await operation();
+    } catch (err: any) {
+      console.warn(`Error during ${operationName}: ${err.message}, retry ${attempt}`, err);
+      attempt++;
+
+      if (attempt >= CACHE_CONSTANTS.MAX_RETRY_ATTEMPTS) {
+        console.error(`Error during ${operationName}: ${err.message}`, err);
+        throw err;
+      }
+
+      await new Promise((resolve) => setTimeout(resolve, delay));
+      delay *= CACHE_CONSTANTS.RETRY_DELAY_MULTIPLIER;
+    }
+  }
+
+  throw new Error(`Maximum retry attempts exceeded for ${operationName}`);
+}
+
+/**
+ * Clears cache for a specific table.
+ * Uses cache context if available, otherwise clears immediately.
+ *
+ * @param schema - The table schema to clear cache for
+ * @param options - ForgeSQL ORM options
+ */
+export async function clearCache<T extends AnyMySqlTable>(
+  schema: T,
+  options: ForgeSqlOrmOptions,
+): Promise<void> {
+  const tableName = getTableName(schema);
+  if (cacheApplicationContext.getStore()) {
+    cacheApplicationContext.getStore()?.tables.add(tableName);
+  } else {
+    await clearTablesCache([tableName], options);
+  }
+}
+
+/**
+ * Clears cache for multiple tables with retry logic and performance logging.
+ *
+ * @param tables - Array of table names to clear cache for
+ * @param options - ForgeSQL ORM options
+ * @returns Promise that resolves when cache clearing is complete
+ */
+export async function clearTablesCache(
+  tables: string[],
+  options: ForgeSqlOrmOptions,
+): Promise<void> {
+  if (!options.cacheEntityName) {
+    throw new Error("cacheEntityName is not configured");
+  }
+
+  const startTime = DateTime.now();
+  let totalRecords = 0;
+
+  try {
+    totalRecords = await executeWithRetry(
+      () => clearCursorCache(tables, "", options),
+      "clearing cache",
+    );
+  } finally {
+    if (options.logRawSqlQuery) {
+      const duration = DateTime.now().toSeconds() - startTime.toSeconds();
+      console.info(`Cleared ${totalRecords} cache records in ${duration} seconds`);
+    }
+  }
+}
+/**
+ * Clears expired cache entries with retry logic and performance logging.
+ *
+ * @param options - ForgeSQL ORM options
+ * @returns Promise that resolves when expired cache clearing is complete
+ */
+export async function clearExpiredCache(options: ForgeSqlOrmOptions): Promise<void> {
+  if (!options.cacheEntityName) {
+    throw new Error("cacheEntityName is not configured");
+  }
+
+  const startTime = DateTime.now();
+  let totalRecords = 0;
+
+  try {
+    totalRecords = await executeWithRetry(
+      () => clearExpirationCursorCache("", options),
+      "clearing expired cache",
+    );
+  } finally {
+    const duration = DateTime.now().toSeconds() - startTime.toSeconds();
+    console.info(`Cleared ${totalRecords} expired cache records in ${duration} seconds`);
+  }
+}
+
+/**
+ * Retrieves data from cache if it exists and is not expired.
+ *
+ * @param query - Query object with toSQL method
+ * @param options - ForgeSQL ORM options
+ * @returns Cached data if found and valid, undefined otherwise
+ */
+export async function getFromCache<T>(
+  query: { toSQL: () => Query },
+  options: ForgeSqlOrmOptions,
+): Promise<T | undefined> {
+  if (!options.cacheEntityName) {
+    throw new Error("cacheEntityName is not configured");
+  }
+
+  const entityQueryName = options.cacheEntityQueryName ?? CACHE_CONSTANTS.DEFAULT_ENTITY_QUERY_NAME;
+  const expirationName =
+    options.cacheEntityExpirationName ?? CACHE_CONSTANTS.DEFAULT_EXPIRATION_NAME;
+  const dataName = options.cacheEntityDataName ?? CACHE_CONSTANTS.DEFAULT_DATA_NAME;
+
+  const sqlQuery = query.toSQL();
+  const key = hashKey(sqlQuery);
+
+  // Skip cache if table is in cache context (will be cleared)
+  if (await isTableContainsTableInCacheContext(sqlQuery.sql, options)) {
+    if (options.logRawSqlQuery) {
+      console.warn(`Context contains value to clear. Skip getting from cache`);
+    }
+    return undefined;
+  }
+
+  try {
+    const cacheResult = await kvs.entity<CacheEntity>(options.cacheEntityName).get(key);
+
+    if (
+      cacheResult &&
+      (cacheResult[expirationName] as number) >= getCurrentTime() &&
+      sqlQuery.sql.toLowerCase() === cacheResult[entityQueryName]
+    ) {
+      if (options.logRawSqlQuery) {
+        console.warn(`Get value from cache, cacheKey: ${key}`);
+      }
+      const results = cacheResult[dataName];
+      return JSON.parse(results as string);
+    }
+  } catch (error: any) {
+    console.error(`Error getting from cache: ${error.message}`, error);
+  }
+
+  return undefined;
+}
+
+/**
+ * Stores query results in cache with specified TTL.
+ *
+ * @param query - Query object with toSQL method
+ * @param options - ForgeSQL ORM options
+ * @param results - Data to cache
+ * @param cacheTtl - Time to live in seconds
+ * @returns Promise that resolves when data is stored in cache
+ */
+export async function setCacheResult(
+  query: { toSQL: () => Query },
+  options: ForgeSqlOrmOptions,
+  results: unknown,
+  cacheTtl: number,
+): Promise<void> {
+  if (!options.cacheEntityName) {
+    throw new Error("cacheEntityName is not configured");
+  }
+
+  try {
+    const entityQueryName =
+      options.cacheEntityQueryName ?? CACHE_CONSTANTS.DEFAULT_ENTITY_QUERY_NAME;
+    const expirationName =
+      options.cacheEntityExpirationName ?? CACHE_CONSTANTS.DEFAULT_EXPIRATION_NAME;
+    const dataName = options.cacheEntityDataName ?? CACHE_CONSTANTS.DEFAULT_DATA_NAME;
+
+    const sqlQuery = query.toSQL();
+
+    // Skip cache if table is in cache context (will be cleared)
+    if (await isTableContainsTableInCacheContext(sqlQuery.sql, options)) {
+      if (options.logRawSqlQuery) {
+        console.warn(`Context contains value to clear. Skip setting from cache`);
+      }
+      return;
+    }
+
+    const key = hashKey(sqlQuery);
+
+    await kvs
+      .transact()
+      .set(
+        key,
+        {
+          [entityQueryName]: sqlQuery.sql.toLowerCase(),
+          [expirationName]: nowPlusSeconds(cacheTtl),
+          [dataName]: JSON.stringify(results),
+        },
+        { entityName: options.cacheEntityName },
+      )
+      .execute();
+
+    if (options.logRawSqlQuery) {
+      console.warn(`Store value to cache, cacheKey: ${key}`);
+    }
+  } catch (error: any) {
+    console.error(`Error setting cache: ${error.message}`, error);
+  }
+}

package/src/utils/sqlUtils.ts

@@ -74,6 +74,48 @@ export const parseDateTime = (value: string | Date, format: string): Date => {
   return result;
 };
 
+/**
+ * Helper function to validate and format a date-like value using Luxon DateTime.
+ * @param value - Date object, ISO/RFC2822/SQL/HTTP string, or timestamp (number|string).
+ * @param format - DateTime format string (Luxon format tokens).
+ * @returns Formatted date string.
+ * @throws Error if value cannot be parsed as a valid date.
+ */
+export function formatDateTime(value: Date | string | number, format: string): string {
+    let dt: DateTime|null = null;
+
+    if (value instanceof Date) {
+        dt = DateTime.fromJSDate(value);
+    } else if (typeof value === "string") {
+        // Перебор парсеров
+        for (const parser of [
+            DateTime.fromISO,
+            DateTime.fromRFC2822,
+            DateTime.fromSQL,
+            DateTime.fromHTTP,
+        ]) {
+            dt = parser(value);
+            if (dt.isValid) break;
+        }
+        if (!dt?.isValid) {
+            const parsed = Number(value);
+            if (!isNaN(parsed)) {
+                dt = DateTime.fromMillis(parsed);
+            }
+        }
+    } else if (typeof value === "number") {
+        dt = DateTime.fromMillis(value);
+    } else {
+        throw new Error("Unsupported type");
+    }
+
+    if (!dt?.isValid) {
+        throw new Error("Invalid Date");
+    }
+
+    return dt.toFormat(format);
+}
+
 /**
  * Gets primary keys from the schema.
  * @template T - The type of the table schema

package/src/webtriggers/clearCacheSchedulerTrigger.ts

@@ -0,0 +1,79 @@
+import { clearExpiredCache } from "../utils/cacheUtils";
+import { ForgeSqlOrmOptions } from "../core/ForgeSQLQueryBuilder";
+
+/**
+ * Scheduler trigger for clearing expired cache entries.
+ *
+ * This trigger should be configured as a Forge scheduler to automatically
+ * clean up expired cache entries based on their TTL (Time To Live).
+ *
+ * @param options - Optional ForgeSQL ORM configuration. If not provided,
+ *                  uses default cache settings with cacheEntityName: "cache"
+ * @returns Promise that resolves to HTTP response object
+ *
+ * @example
+ * ```typescript
+ * // In your index.ts
+ * import { clearCacheSchedulerTrigger } from "forge-sql-orm";
+ *
+ * export const clearCache = () => {
+ *   return clearCacheSchedulerTrigger({
+ *     cacheEntityName: "cache",
+ *     logRawSqlQuery: true
+ *   });
+ * };
+ * ```
+ *
+ * @example
+ * ```yaml
+ * # In manifest.yml
+ * scheduledTrigger:
+ *   - key: clear-cache-trigger
+ *     function: clearCache
+ *     interval: fiveMinute
+ *
+ * function:
+ *   - key: clearCache
+ *     handler: index.clearCache
+ * ```
+ */
+export const clearCacheSchedulerTrigger = async (options?: ForgeSqlOrmOptions) => {
+  try {
+    const newOptions: ForgeSqlOrmOptions = options ?? {
+      logRawSqlQuery: false,
+      disableOptimisticLocking: false,
+      cacheTTL: 120,
+      cacheEntityName: "cache",
+      cacheEntityQueryName: "sql",
+      cacheEntityExpirationName: "expiration",
+      cacheEntityDataName: "data",
+    };
+    if (!newOptions.cacheEntityName) {
+      throw new Error("cacheEntityName is not configured");
+    }
+    await clearExpiredCache(newOptions);
+
+    return {
+      headers: { "Content-Type": ["application/json"] },
+      statusCode: 200,
+      statusText: "OK",
+      body: JSON.stringify({
+        success: true,
+        message: "Cache cleanup completed successfully",
+        timestamp: new Date().toISOString(),
+      }),
+    };
+  } catch (error) {
+    console.error("Error during cache cleanup: ", JSON.stringify(error));
+    return {
+      headers: { "Content-Type": ["application/json"] },
+      statusCode: 500,
+      statusText: "Internal Server Error",
+      body: JSON.stringify({
+        success: false,
+        error: error instanceof Error ? error.message : "Unknown error during cache cleanup",
+        timestamp: new Date().toISOString(),
+      }),
+    };
+  }
+};

package/src/webtriggers/index.ts

@@ -2,6 +2,7 @@ export * from "./dropMigrationWebTrigger";
 export * from "./applyMigrationsWebTrigger";
 export * from "./fetchSchemaWebTrigger";
 export * from "./dropTablesMigrationWebTrigger";
+export * from "./clearCacheSchedulerTrigger";
 
 export interface TriggerResponse<BODY> {
   body?: BODY;

package/dist/lib/drizzle/extensions/selectAliased.d.ts

@@ -1,9 +0,0 @@
-import { MySqlRemoteDatabase } from "drizzle-orm/mysql-proxy";
-import type { SelectedFields } from "drizzle-orm/mysql-core/query-builders/select.types";
-import { MySqlSelectBuilder } from "drizzle-orm/mysql-core";
-import { MySqlRemotePreparedQueryHKT } from "drizzle-orm/mysql-proxy";
-export declare function patchDbWithSelectAliased(db: MySqlRemoteDatabase<any>): MySqlRemoteDatabase<any> & {
-    selectAliased: <TSelection extends SelectedFields>(fields: TSelection) => MySqlSelectBuilder<TSelection, MySqlRemotePreparedQueryHKT>;
-    selectAliasedDistinct: <TSelection extends SelectedFields>(fields: TSelection) => MySqlSelectBuilder<TSelection, MySqlRemotePreparedQueryHKT>;
-};
-//# sourceMappingURL=selectAliased.d.ts.map

package/dist/lib/drizzle/extensions/selectAliased.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"selectAliased.d.ts","sourceRoot":"","sources":["../../../../src/lib/drizzle/extensions/selectAliased.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oDAAoD,CAAC;AAEzF,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,2BAA2B,EAAE,MAAM,yBAAyB,CAAC;AAkDtE,wBAAgB,wBAAwB,CAAC,EAAE,EAAE,mBAAmB,CAAC,GAAG,CAAC,GAAG,mBAAmB,CAAC,GAAG,CAAC,GAAG;IACjG,aAAa,EAAE,CAAC,UAAU,SAAS,cAAc,EAC/C,MAAM,EAAE,UAAU,KACf,kBAAkB,CAAC,UAAU,EAAE,2BAA2B,CAAC,CAAC;IACjE,qBAAqB,EAAE,CAAC,UAAU,SAAS,cAAc,EACvD,MAAM,EAAE,UAAU,KACf,kBAAkB,CAAC,UAAU,EAAE,2BAA2B,CAAC,CAAC;CAClE,CAUA"}

package/src/lib/drizzle/extensions/selectAliased.ts

@@ -1,72 +0,0 @@
-import { MySqlRemoteDatabase } from "drizzle-orm/mysql-proxy";
-import type { SelectedFields } from "drizzle-orm/mysql-core/query-builders/select.types";
-import { applyFromDriverTransform, mapSelectFieldsWithAlias } from "../../..";
-import { MySqlSelectBuilder } from "drizzle-orm/mysql-core";
-import { MySqlRemotePreparedQueryHKT } from "drizzle-orm/mysql-proxy";
-
-function createAliasedSelectBuilder<TSelection extends SelectedFields>(
-  db: MySqlRemoteDatabase<any>,
-  fields: TSelection,
-  selectFn: (selections: any) => MySqlSelectBuilder<TSelection, MySqlRemotePreparedQueryHKT>,
-): MySqlSelectBuilder<TSelection, MySqlRemotePreparedQueryHKT> {
-  const { selections, aliasMap } = mapSelectFieldsWithAlias(fields);
-  const builder = selectFn(selections);
-
-  const wrapBuilder = (rawBuilder: any): any => {
-    return new Proxy(rawBuilder, {
-      get(target, prop, receiver) {
-        if (prop === "execute") {
-          return async (...args: any[]) => {
-            const rows = await target.execute(...args);
-            return applyFromDriverTransform(rows, selections, aliasMap);
-          };
-        }
-
-        if (prop === "then") {
-          return (onfulfilled: any, onrejected: any) =>
-            target.execute().then((rows: unknown[]) => {
-              const transformed = applyFromDriverTransform(rows, selections, aliasMap);
-              return onfulfilled?.(transformed);
-            }, onrejected);
-        }
-
-        const value = Reflect.get(target, prop, receiver);
-
-        if (typeof value === "function") {
-          return (...args: any[]) => {
-            const result = value.apply(target, args);
-
-            if (typeof result === "object" && result !== null && "execute" in result) {
-              return wrapBuilder(result);
-            }
-
-            return result;
-          };
-        }
-
-        return value;
-      },
-    });
-  };
-
-  return wrapBuilder(builder);
-}
-
-export function patchDbWithSelectAliased(db: MySqlRemoteDatabase<any>): MySqlRemoteDatabase<any> & {
-  selectAliased: <TSelection extends SelectedFields>(
-    fields: TSelection,
-  ) => MySqlSelectBuilder<TSelection, MySqlRemotePreparedQueryHKT>;
-  selectAliasedDistinct: <TSelection extends SelectedFields>(
-    fields: TSelection,
-  ) => MySqlSelectBuilder<TSelection, MySqlRemotePreparedQueryHKT>;
-} {
-  db.selectAliased = function <TSelection extends SelectedFields>(fields: TSelection) {
-    return createAliasedSelectBuilder(db, fields, (selections) => db.select(selections));
-  };
-
-  db.selectAliasedDistinct = function <TSelection extends SelectedFields>(fields: TSelection) {
-    return createAliasedSelectBuilder(db, fields, (selections) => db.selectDistinct(selections));
-  };
-
-  return db;
-}