forge-sql-orm 2.1.11 → 2.1.12

package/README.md

@@ -18,7 +18,7 @@
 - ✅ **Custom Drizzle Driver** for direct integration with @forge/sql
 - ✅ **Local Cache System (Level 1)** for in-memory query optimization within single resolver invocation scope
 - ✅ **Global Cache System (Level 2)** with cross-invocation caching, automatic cache invalidation and context-aware operations (using [@forge/kvs](https://developer.atlassian.com/platform/forge/storage-reference/storage-api-custom-entities/) )
-- ✅ **Performance Monitoring**: Query execution metrics and analysis capabilities with automatic error analysis for timeout and OOM errors
+- ✅ **Performance Monitoring**: Query execution metrics and analysis capabilities with automatic error analysis for timeout and OOM errors, plus scheduled slow query monitoring with execution plans
 - ✅ **Type-Safe Query Building**: Write SQL queries with full TypeScript support
 - ✅ **Supports complex SQL queries** with joins and filtering using Drizzle ORM
 - ✅ **Advanced Query Methods**: `selectFrom()`, `selectDistinctFrom()`, `selectCacheableFrom()`, `selectDistinctCacheableFrom()` for all-column queries with field aliasing
@@ -64,7 +64,8 @@
 ### 🔒 Advanced Features
 - [Optimistic Locking](#optimistic-locking)
 - [Query Analysis and Performance Optimization](#query-analysis-and-performance-optimization)
-- [Performance Monitoring](#performance-monitoring)
+- [Automatic Error Analysis](#automatic-error-analysis) - Automatic timeout and OOM error detection with execution plans
+- [Slow Query Monitoring](#slow-query-monitoring) - Scheduled monitoring of slow queries with execution plans
 - [Date and Time Types](#date-and-time-types)
 
 ### 🛠️ Development Tools
@@ -294,10 +295,8 @@ resolver.define("fetch", async (req: Request) => {
           console.warn(`[Performance Warning fetch] Resolver exceeded DB time: ${totalDbExecutionTime} ms`);
           await printQueriesWithPlan(); // Optionally log or capture diagnostics for further analysis
         } else if (totalDbExecutionTime > threshold) {
-          console.debug(`[Performance Debug] High DB time: ${totalDbExecutionTime} ms`);
+          console.debug(`[Performance Debug fetch] High DB time: ${totalDbExecutionTime} ms`);
         }
-        
-        console.log(`DB response size: ${totalResponseSize} bytes`);
       }
     );
   } catch (e) {
@@ -360,8 +359,9 @@ const rawUsers = await forgeSQL.execute(
 );
 
 // Raw SQL with caching
+// ⚠️ IMPORTANT: When using executeCacheable(), all table names must be wrapped with backticks (`)
 const cachedRawUsers = await forgeSQL.executeCacheable(
-  "SELECT * FROM users WHERE active = ?", 
+  "SELECT * FROM `users` WHERE active = ?", 
   [true], 
   300
 );
@@ -481,8 +481,9 @@ const rawUsers = await forgeSQL.execute(
 );
 
 // Raw SQL with caching
+// ⚠️ IMPORTANT: When using executeCacheable(), all table names must be wrapped with backticks (`)
 const cachedRawUsers = await forgeSQL.executeCacheable(
-  "SELECT * FROM users WHERE active = ?", 
+  "SELECT * FROM `users` WHERE active = ?", 
   [true], 
   300
 );
@@ -617,6 +618,9 @@ Please review the [official @forge/kvs quotas and limits](https://developer.atla
 - Monitor cache usage to stay within quotas
 - Use appropriate TTL values
 
+**⚠️ Important Cache Limitations:**
+- **Table names starting with `a_`**: Tables whose names start with `a_` (case-insensitive) are automatically ignored in cache operations. KVS Cache will not work with such tables, and they will be excluded from cache invalidation and cache key generation. This is by design to support special system tables or temporary tables.
+
 ### Step 1: Install Dependencies
 
 ```bash
@@ -1136,8 +1140,10 @@ const user = await forgeSQL
     .execute("SELECT * FROM users WHERE id = ?", [1]);
 
 // Using forgeSQL.executeCacheable() - Execute raw SQL with local and global caching
+// ⚠️ IMPORTANT: When using executeCacheable(), all table names in SQL queries must be wrapped with backticks (`)
+// Example: SELECT * FROM `users` WHERE id = ? (NOT: SELECT * FROM users WHERE id = ?)
 const user = await forgeSQL
-    .executeCacheable("SELECT * FROM users WHERE id = ?", [1], 300);
+    .executeCacheable("SELECT * FROM `users` WHERE id = ?", [1], 300);
 
 // Using forgeSQL.getDrizzleQueryBuilder()
 const user = await forgeSQL
@@ -1266,8 +1272,10 @@ const users = await forgeSQL
   .execute("SELECT * FROM users WHERE active = ?", [true]);
 
 // Using executeCacheable() for raw SQL with local and global caching
+// ⚠️ IMPORTANT: When using executeCacheable(), all table names in SQL queries must be wrapped with backticks (`)
+// Example: SELECT * FROM `users` WHERE active = ? (NOT: SELECT * FROM users WHERE active = ?)
 const users = await forgeSQL
-  .executeCacheable("SELECT * FROM users WHERE active = ?", [true], 300);
+  .executeCacheable("SELECT * FROM `users` WHERE active = ?", [true], 300);
 
 // Using executeWithMetadata() for capturing execution metrics and performance monitoring
 const usersWithMetadata = await forgeSQL.executeWithMetadata(
@@ -1760,8 +1768,9 @@ await forgeSQL.executeWithLocalContext(async () => {
     .where(eq(users.active, true));
   
   // Raw SQL with multi-level caching
+  // ⚠️ IMPORTANT: When using executeCacheable(), all table names must be wrapped with backticks (`)
   const rawUsers = await forgeSQL.executeCacheable(
-    "SELECT id, name FROM users WHERE active = ?", 
+    "SELECT id, name FROM `users` WHERE active = ?", 
     [true], 
     300 // TTL in seconds
   );
@@ -1810,8 +1819,9 @@ const usersDistinct = await forgeSQL.selectDistinctCacheableFrom(Users)
   .where(eq(Users.active, true));
 
 // Raw SQL with local and global caching
+// ⚠️ IMPORTANT: When using executeCacheable(), all table names must be wrapped with backticks (`)
 const rawUsers = await forgeSQL.executeCacheable(
-  "SELECT * FROM users WHERE active = ?",
+  "SELECT * FROM `users` WHERE active = ?",
   [true],
   300 // TTL in seconds
 );
@@ -2122,6 +2132,34 @@ Configure in `manifest.yml`:
 - `hour` - Every hour
 - `day` - Every day
 
+### 5. Slow Query Scheduler Trigger
+
+This scheduler trigger automatically monitors and analyzes slow queries on a scheduled basis. For detailed information, see the [Slow Query Monitoring](#slow-query-monitoring) section.
+
+**Quick Setup:**
+
+```typescript
+import ForgeSQL, { slowQuerySchedulerTrigger } from "forge-sql-orm";
+
+const forgeSQL = new ForgeSQL();
+
+export const slowQueryTrigger = () =>
+  slowQuerySchedulerTrigger(forgeSQL, { hours: 1, timeout: 3000 });
+```
+
+Configure in `manifest.yml`:
+```yaml
+  scheduledTrigger:
+    - key: slow-query-trigger
+      function: slowQueryTrigger
+      interval: hour
+  function:
+    - key: slowQueryTrigger
+      handler: index.slowQueryTrigger
+```
+
+> **💡 Note**: For complete documentation, examples, and configuration options, see the [Slow Query Monitoring](#slow-query-monitoring) section.
+
 ### Important Notes
 
 **Security Considerations**:
@@ -2210,6 +2248,112 @@ The error analysis mechanism:
 
 > **💡 Tip**: The automatic error analysis only triggers for timeout and OOM errors. Other errors are logged normally without plan analysis.
 
+### Slow Query Monitoring
+
+Forge-SQL-ORM provides a scheduler trigger (`slowQuerySchedulerTrigger`) that automatically monitors and analyzes slow queries on an hourly basis. This trigger queries TiDB's slow query log system table and provides detailed performance information including SQL query text, memory usage, execution time, and execution plans.
+
+#### Key Features
+
+- **Automatic Monitoring**: Runs on a scheduled interval (recommended: hourly)
+- **Detailed Performance Metrics**: Memory usage, execution time, and execution plans
+- **Console Logging**: Results are automatically logged to the Forge Developer Console
+- **Configurable Time Window**: Analyze queries from the last N hours (default: 1 hour)
+- **Automatic Plan Retrieval**: Execution plans are included for all slow queries
+
+#### Basic Setup
+
+**1. Create the trigger function:**
+
+```typescript
+import ForgeSQL, { slowQuerySchedulerTrigger } from "forge-sql-orm";
+
+const forgeSQL = new ForgeSQL();
+
+// Monitor slow queries from the last hour (recommended for hourly schedule)
+export const slowQueryTrigger = () =>
+  slowQuerySchedulerTrigger(forgeSQL, { hours: 1, timeout: 3000 });
+```
+
+**2. Configure in `manifest.yml`:**
+
+```yaml
+modules:
+  scheduledTrigger:
+    - key: slow-query-trigger
+      function: slowQueryTrigger
+      interval: hour  # Run every hour
+  
+  function:
+    - key: slowQueryTrigger
+      handler: index.slowQueryTrigger
+```
+
+#### Configuration Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `hours` | `number` | `1` | Number of hours to look back for slow queries |
+| `timeout` | `number` | `3000` | Timeout in milliseconds for the diagnostic query execution |
+
+#### Example Console Output
+
+When slow queries are detected, you'll see output like this in the Forge Developer Console:
+
+```
+Found SlowQuery SQL: SELECT * FROM users u INNER JOIN orders o ON u.id = o.user_id WHERE u.active = ? | Memory: 8.50 MB | Time: 2500.00 ms
+ Plan:
+id task estRows operator info actRows execution info memory disk
+Projection_7 root 1000.00 forge_38dd1c6156b94bb59c2c9a45582bbfc7.users.id, ... 1000 time:2.5s, loops:1 8.50 MB N/A
+└─IndexHashJoin_14 root 1000.00 inner join, ... 1000 time:2.2s, loops:1 7.98 MB N/A
+
+Found SlowQuery SQL: SELECT * FROM products WHERE category = ? ORDER BY created_at DESC | Memory: 6.25 MB | Time: 1800.00 ms
+ Plan:
+...
+```
+
+#### Advanced Configuration
+
+```typescript
+import ForgeSQL, { slowQuerySchedulerTrigger } from "forge-sql-orm";
+
+const forgeSQL = new ForgeSQL();
+
+// Monitor queries from the last 6 hours (for less frequent checks)
+export const sixHourSlowQueryTrigger = () =>
+  slowQuerySchedulerTrigger(forgeSQL, { hours: 6, timeout: 5000 });
+
+// Monitor queries from the last 24 hours (daily monitoring)
+export const dailySlowQueryTrigger = () =>
+  slowQuerySchedulerTrigger(forgeSQL, { hours: 24, timeout: 3000 });
+```
+
+#### How It Works
+
+1. **Scheduled Execution**: The trigger runs automatically on the configured interval (hourly recommended)
+2. **Query Analysis**: Queries TiDB's slow query log system table for queries executed within the specified time window
+3. **Performance Metrics**: Extracts and logs:
+   - SQL query text (sanitized for readability)
+   - Maximum memory usage (in MB)
+   - Query execution time (in ms)
+   - Detailed execution plan
+4. **Console Logging**: Results are logged to the Forge Developer Console via `console.warn()` for easy monitoring
+
+#### Best Practices
+
+- **Hourly Intervals**: Use `interval: hour` for timely detection of slow queries
+- **Default Time Window**: 1 hour is recommended for hourly schedules to avoid overlap
+- **Monitor Regularly**: Check console logs regularly to identify patterns in slow queries
+
+#### Benefits
+
+- **Proactive Monitoring**: Catch slow queries before they become critical issues
+- **Performance Trends**: Track query performance over time
+- **Optimization Insights**: Execution plans help identify optimization opportunities
+- **Zero Manual Intervention**: Fully automated monitoring with scheduled execution
+- **Production Safe**: Works silently in the background, only logs when slow queries are found
+
+> **💡 Tip**: The trigger queries up to 50 slow queries to prevent excessive logging. Transient timeouts are usually fine; repeated timeouts indicate the diagnostic query itself is slow and should be investigated.
+
 ### Available Analysis Tools
 
 ```typescript
@@ -2370,8 +2514,9 @@ const rawUsers = await forgeSQL.execute(
   [true]
 );
 
+// ⚠️ IMPORTANT: When using executeCacheable(), all table names must be wrapped with backticks (`)
 const cachedRawUsers = await forgeSQL.executeCacheable(
-  "SELECT * FROM users WHERE active = ?", 
+  "SELECT * FROM `users` WHERE active = ?", 
   [true], 
   300
 );

package/dist/ForgeSQLORM.js

@@ -823,7 +823,14 @@ async function printQueriesWithPlan(forgeSQLORM, timeDiffMs, timeout) {
         drizzleOrm.and(
           drizzleOrm.isNotNull(statementsTable.digest),
           drizzleOrm.not(drizzleOrm.ilike(statementsTable.digestText, "%information_schema%")),
-          drizzleOrm.notInArray(statementsTable.stmtType, ["Use", "Set", "Show", "Commit", "Rollback", "Begin"]),
+          drizzleOrm.notInArray(statementsTable.stmtType, [
+            "Use",
+            "Set",
+            "Show",
+            "Commit",
+            "Rollback",
+            "Begin"
+          ]),
           drizzleOrm.gte(
             statementsTable.lastSeen,
             drizzleOrm.sql`DATE_SUB
@@ -885,9 +892,7 @@ async function slowQueryPerHours(forgeSQLORM, hours, timeout) {
       const message = `Found SlowQuery SQL: ${result.query} | Memory: ${memMaxMB.toFixed(2)} MB | Time: ${result.queryTime} ms
  Plan:${result.plan}`;
       response.push(message);
-      console.warn(
-        message
-      );
+      console.warn(message);
     });
     return response;
   } catch (error) {
@@ -895,16 +900,16 @@ async function slowQueryPerHours(forgeSQLORM, hours, timeout) {
       `Error occurred while retrieving query execution plan: ${error instanceof Error ? error.message : "Unknown error"}. Try again after some time`,
       error
     );
-    return [`Error occurred while retrieving query execution plan: ${error instanceof Error ? error.message : "Unknown error"}`];
+    return [
+      `Error occurred while retrieving query execution plan: ${error instanceof Error ? error.message : "Unknown error"}`
+    ];
   }
 }
 async function withTimeout(promise, message, timeoutMs2) {
   let timeoutId;
   const timeoutPromise = new Promise((_, reject) => {
     timeoutId = setTimeout(() => {
-      reject(
-        new Error(message)
-      );
+      reject(new Error(message));
     }, timeoutMs2);
   });
   try {
@@ -936,6 +941,17 @@ function nowPlusSeconds(secondsToAdd) {
   const dt = luxon.DateTime.now().plus({ seconds: secondsToAdd });
   return Math.floor(dt.toSeconds());
 }
+function extractBacktickedValues(sql2) {
+  const regex = /`([^`]+)`/g;
+  const matches = /* @__PURE__ */ new Set();
+  let match;
+  while ((match = regex.exec(sql2.toLowerCase())) !== null) {
+    if (!match[1].startsWith("a_")) {
+      matches.add(`\`${match[1]}\``);
+    }
+  }
+  return Array.from(matches).sort().join(",");
+}
 function hashKey(query) {
   const h = crypto__namespace.createHash("sha256");
   h.update(query.sql.toLowerCase());
@@ -1079,7 +1095,7 @@ async function getFromCache(query, options) {
   }
   try {
     const cacheResult = await kvs.kvs.entity(options.cacheEntityName).get(key);
-    if (cacheResult && cacheResult[expirationName] >= getCurrentTime() && sqlQuery.sql.toLowerCase() === cacheResult[entityQueryName]) {
+    if (cacheResult && cacheResult[expirationName] >= getCurrentTime() && extractBacktickedValues(sqlQuery.sql) === cacheResult[entityQueryName]) {
       if (options.logCache) {
         console.warn(`Get value from cache, cacheKey: ${key}`);
       }
@@ -1110,7 +1126,7 @@ async function setCacheResult(query, options, results, cacheTtl) {
     await kvs.kvs.transact().set(
       key,
       {
-        [entityQueryName]: sqlQuery.sql.toLowerCase(),
+        [entityQueryName]: extractBacktickedValues(sqlQuery.sql),
         [expirationName]: nowPlusSeconds(cacheTtl),
         [dataName]: JSON.stringify(results)
       },
@@ -1613,10 +1629,7 @@ async function saveMetaDataToContext(metadata) {
       if (process.env.NODE_ENV !== "test") {
         await new Promise((r) => setTimeout(r, 200));
       }
-      await printQueriesWithPlan(
-        context.forgeSQLORM,
-        Date.now() - context.beginTime.getTime()
-      );
+      await printQueriesWithPlan(context.forgeSQLORM, Date.now() - context.beginTime.getTime());
     };
     if (metadata) {
       context.totalResponseSize += metadata.responseSize;
@@ -1683,7 +1696,11 @@ async function processAllMethod(query, params) {
   if (params) {
     await sqlStatement.bindParams(...params);
   }
-  const result = await withTimeout(sqlStatement.execute(), timeoutMessage, timeoutMs);
+  const result = await withTimeout(
+    sqlStatement.execute(),
+    timeoutMessage,
+    timeoutMs
+  );
   await saveMetaDataToContext(result.metadata);
   if (!result.rows) {
     return { rows: [] };
@@ -1694,7 +1711,11 @@ async function processAllMethod(query, params) {
 const forgeDriver = async (query, params, method) => {
   const operationType = await getOperationType();
   if (operationType === "DDL") {
-    const result = await withTimeout(sql.sql.executeDDL(inlineParams(query, params)), timeoutMessage, timeoutMs);
+    const result = await withTimeout(
+      sql.sql.executeDDL(inlineParams(query, params)),
+      timeoutMessage,
+      timeoutMs
+    );
     return await processDDLResult(method, result);
   }
   if (method === "execute") {
@@ -3833,7 +3854,12 @@ const clearCacheSchedulerTrigger = async (options) => {
 };
 async function slowQuerySchedulerTrigger(forgeSQLORM, options) {
   try {
-    return getHttpResponse(200, JSON.stringify(await slowQueryPerHours(forgeSQLORM, options?.hours ?? 1, options?.timeout ?? 3e3)));
+    return getHttpResponse(
+      200,
+      JSON.stringify(
+        await slowQueryPerHours(forgeSQLORM, options?.hours ?? 1, options?.timeout ?? 3e3)
+      )
+    );
   } catch (error) {
     const errorMessage = error?.debug?.sqlMessage ?? error?.debug?.message ?? error.message ?? "Unknown error occurred";
     console.error(errorMessage);