forge-sql-orm 2.1.10 → 2.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +202 -254
- package/dist/ForgeSQLORM.js +3238 -3227
- package/dist/ForgeSQLORM.js.map +1 -1
- package/dist/ForgeSQLORM.mjs +3236 -3225
- package/dist/ForgeSQLORM.mjs.map +1 -1
- package/dist/core/ForgeSQLORM.d.ts +65 -11
- package/dist/core/ForgeSQLORM.d.ts.map +1 -1
- package/dist/core/ForgeSQLQueryBuilder.d.ts +66 -11
- package/dist/core/ForgeSQLQueryBuilder.d.ts.map +1 -1
- package/dist/core/SystemTables.d.ts +82 -82
- package/dist/utils/cacheUtils.d.ts.map +1 -1
- package/dist/utils/forgeDriver.d.ts.map +1 -1
- package/dist/utils/forgeDriverProxy.d.ts +6 -2
- package/dist/utils/forgeDriverProxy.d.ts.map +1 -1
- package/dist/utils/metadataContextUtils.d.ts +5 -2
- package/dist/utils/metadataContextUtils.d.ts.map +1 -1
- package/dist/utils/sqlUtils.d.ts +72 -1
- package/dist/utils/sqlUtils.d.ts.map +1 -1
- package/dist/webtriggers/index.d.ts +1 -1
- package/dist/webtriggers/index.d.ts.map +1 -1
- package/dist/webtriggers/slowQuerySchedulerTrigger.d.ts +67 -0
- package/dist/webtriggers/slowQuerySchedulerTrigger.d.ts.map +1 -0
- package/package.json +10 -10
- package/src/core/ForgeSQLORM.ts +164 -33
- package/src/core/ForgeSQLQueryBuilder.ts +65 -11
- package/src/core/SystemTables.ts +1 -1
- package/src/utils/cacheUtils.ts +3 -1
- package/src/utils/forgeDriver.ts +7 -34
- package/src/utils/forgeDriverProxy.ts +58 -6
- package/src/utils/metadataContextUtils.ts +21 -6
- package/src/utils/sqlUtils.ts +229 -2
- package/src/webtriggers/index.ts +1 -1
- package/src/webtriggers/slowQuerySchedulerTrigger.ts +82 -0
- package/dist/webtriggers/topSlowestStatementLastHourTrigger.d.ts +0 -114
- package/dist/webtriggers/topSlowestStatementLastHourTrigger.d.ts.map +0 -1
- package/src/webtriggers/topSlowestStatementLastHourTrigger.ts +0 -563
|
@@ -29,7 +29,6 @@ import {
|
|
|
29
29
|
DeleteAndEvictCacheType,
|
|
30
30
|
ExecuteQuery,
|
|
31
31
|
ExecuteQueryCacheable,
|
|
32
|
-
ForgeSQLMetadata,
|
|
33
32
|
InsertAndEvictCacheType,
|
|
34
33
|
SelectAliasedCacheableType,
|
|
35
34
|
SelectAliasedDistinctCacheableType,
|
|
@@ -544,32 +543,87 @@ export interface QueryBuilderForgeSql {
|
|
|
544
543
|
executeWithLocalCacheContextAndReturnValue<T>(cacheContext: () => Promise<T>): Promise<T>;
|
|
545
544
|
|
|
546
545
|
/**
|
|
547
|
-
* Executes a query and provides access to execution metadata.
|
|
546
|
+
* Executes a query and provides access to execution metadata with performance monitoring.
|
|
548
547
|
* This method allows you to capture detailed information about query execution
|
|
549
|
-
* including database execution time, response size, and
|
|
548
|
+
* including database execution time, response size, and query analysis capabilities.
|
|
549
|
+
*
|
|
550
|
+
* The method aggregates metrics across all database operations within the query function,
|
|
551
|
+
* making it ideal for monitoring resolver performance and detecting performance issues.
|
|
550
552
|
*
|
|
551
553
|
* @template T - The return type of the query
|
|
552
|
-
* @param query - A function that returns a Promise with the query result
|
|
553
|
-
* @param onMetadata - Callback function that receives execution metadata
|
|
554
|
+
* @param query - A function that returns a Promise with the query result. Can contain multiple database operations.
|
|
555
|
+
* @param onMetadata - Callback function that receives aggregated execution metadata
|
|
556
|
+
* @param onMetadata.totalDbExecutionTime - Total database execution time across all operations in the query function (in milliseconds)
|
|
557
|
+
* @param onMetadata.totalResponseSize - Total response size across all operations (in bytes)
|
|
558
|
+
* @param onMetadata.printQueries - Function to analyze and print query execution plans from CLUSTER_STATEMENTS_SUMMARY
|
|
554
559
|
* @returns Promise with the query result
|
|
560
|
+
*
|
|
555
561
|
* @example
|
|
556
562
|
* ```typescript
|
|
563
|
+
* // Basic usage with performance monitoring
|
|
557
564
|
* const result = await forgeSQL.executeWithMetadata(
|
|
558
|
-
* async () =>
|
|
559
|
-
*
|
|
560
|
-
*
|
|
561
|
-
*
|
|
562
|
-
*
|
|
565
|
+
* async () => {
|
|
566
|
+
* const users = await forgeSQL.selectFrom(usersTable);
|
|
567
|
+
* const orders = await forgeSQL.selectFrom(ordersTable).where(eq(ordersTable.userId, usersTable.id));
|
|
568
|
+
* return { users, orders };
|
|
569
|
+
* },
|
|
570
|
+
* (totalDbExecutionTime, totalResponseSize, printQueries) => {
|
|
571
|
+
* const threshold = 500; // ms baseline for this resolver
|
|
572
|
+
*
|
|
573
|
+
* if (totalDbExecutionTime > threshold * 1.5) {
|
|
574
|
+
* console.warn(`[Performance Warning] Resolver exceeded DB time: ${totalDbExecutionTime} ms`);
|
|
575
|
+
* await printQueries(); // Analyze and print query execution plans
|
|
576
|
+
* } else if (totalDbExecutionTime > threshold) {
|
|
577
|
+
* console.debug(`[Performance Debug] High DB time: ${totalDbExecutionTime} ms`);
|
|
578
|
+
* }
|
|
579
|
+
*
|
|
580
|
+
* console.log(`DB response size: ${totalResponseSize} bytes`);
|
|
563
581
|
* }
|
|
564
582
|
* );
|
|
565
583
|
* ```
|
|
584
|
+
*
|
|
585
|
+
* @example
|
|
586
|
+
* ```typescript
|
|
587
|
+
* // Resolver with performance monitoring
|
|
588
|
+
* resolver.define("fetch", async (req: Request) => {
|
|
589
|
+
* try {
|
|
590
|
+
* return await forgeSQL.executeWithMetadata(
|
|
591
|
+
* async () => {
|
|
592
|
+
* // Resolver logic with multiple queries
|
|
593
|
+
* const users = await forgeSQL.selectFrom(demoUsers);
|
|
594
|
+
* const orders = await forgeSQL.selectFrom(demoOrders)
|
|
595
|
+
* .where(eq(demoOrders.userId, demoUsers.id));
|
|
596
|
+
* return { users, orders };
|
|
597
|
+
* },
|
|
598
|
+
* async (totalDbExecutionTime, totalResponseSize, printQueries) => {
|
|
599
|
+
* const threshold = 500; // ms baseline for this resolver
|
|
600
|
+
*
|
|
601
|
+
* if (totalDbExecutionTime > threshold * 1.5) {
|
|
602
|
+
* console.warn(`[Performance Warning fetch] Resolver exceeded DB time: ${totalDbExecutionTime} ms`);
|
|
603
|
+
* await printQueries(); // Optionally log or capture diagnostics for further analysis
|
|
604
|
+
* } else if (totalDbExecutionTime > threshold) {
|
|
605
|
+
* console.debug(`[Performance Debug] High DB time: ${totalDbExecutionTime} ms`);
|
|
606
|
+
* }
|
|
607
|
+
*
|
|
608
|
+
* console.log(`DB response size: ${totalResponseSize} bytes`);
|
|
609
|
+
* }
|
|
610
|
+
* );
|
|
611
|
+
* } catch (e) {
|
|
612
|
+
* const error = e?.cause?.debug?.sqlMessage ?? e?.cause;
|
|
613
|
+
* console.error(error, e);
|
|
614
|
+
* throw error;
|
|
615
|
+
* }
|
|
616
|
+
* });
|
|
617
|
+
* ```
|
|
618
|
+
*
|
|
619
|
+
* @note **Important**: When multiple resolvers are running concurrently, their query data may also appear in `printQueries()` analysis, as it queries the global `CLUSTER_STATEMENTS_SUMMARY` table.
|
|
566
620
|
*/
|
|
567
621
|
executeWithMetadata<T>(
|
|
568
622
|
query: () => Promise<T>,
|
|
569
623
|
onMetadata: (
|
|
570
624
|
totalDbExecutionTime: number,
|
|
571
625
|
totalResponseSize: number,
|
|
572
|
-
|
|
626
|
+
printQueriesWithPlan: () => Promise<void>,
|
|
573
627
|
) => Promise<void> | void,
|
|
574
628
|
): Promise<T>;
|
|
575
629
|
/**
|
package/src/core/SystemTables.ts
CHANGED
|
@@ -21,7 +21,7 @@ export const migrations = mysqlTable("__migrations", {
|
|
|
21
21
|
|
|
22
22
|
const informationSchema = mysqlSchema("information_schema");
|
|
23
23
|
|
|
24
|
-
export const slowQuery = informationSchema.table("
|
|
24
|
+
export const slowQuery = informationSchema.table("CLUSTER_SLOW_QUERY", {
|
|
25
25
|
time: timestamp("Time", { fsp: 6, mode: "string" }).notNull(), // Timestamp when the slow query was recorded
|
|
26
26
|
|
|
27
27
|
txnStartTs: bigint("Txn_start_ts", { mode: "bigint", unsigned: true }), // Transaction start timestamp (TSO)
|
package/src/utils/cacheUtils.ts
CHANGED
|
@@ -332,7 +332,9 @@ export async function getFromCache<T>(
|
|
|
332
332
|
}
|
|
333
333
|
|
|
334
334
|
try {
|
|
335
|
-
const cacheResult = await kvs.entity<CacheEntity>(options.cacheEntityName).get(key)
|
|
335
|
+
const cacheResult = (await kvs.entity<CacheEntity>(options.cacheEntityName).get(key)) as
|
|
336
|
+
| CacheEntity
|
|
337
|
+
| undefined;
|
|
336
338
|
|
|
337
339
|
if (
|
|
338
340
|
cacheResult &&
|
package/src/utils/forgeDriver.ts
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import { sql, UpdateQueryResponse } from "@forge/sql";
|
|
2
2
|
import { saveMetaDataToContext } from "./metadataContextUtils";
|
|
3
3
|
import { getOperationType } from "./requestTypeContextUtils";
|
|
4
|
+
import {withTimeout} from "./sqlUtils";
|
|
5
|
+
|
|
6
|
+
const timeoutMs =10000;
|
|
7
|
+
const timeoutMessage = `Atlassian @forge/sql did not return a response within ${timeoutMs}ms (${timeoutMs / 1000} seconds), so the request is blocked. Possible causes: slow query, network issues, or exceeding Forge SQL limits.`;
|
|
4
8
|
|
|
5
9
|
/**
|
|
6
10
|
* Metadata structure for Forge SQL query results.
|
|
@@ -62,36 +66,6 @@ export function isUpdateQueryResponse(obj: unknown): obj is UpdateQueryResponse
|
|
|
62
66
|
);
|
|
63
67
|
}
|
|
64
68
|
|
|
65
|
-
/**
|
|
66
|
-
* Executes a promise with a timeout.
|
|
67
|
-
*
|
|
68
|
-
* @param promise - The promise to execute
|
|
69
|
-
* @param timeoutMs - Timeout in milliseconds (default: 10000ms)
|
|
70
|
-
* @returns Promise that resolves with the result or rejects on timeout
|
|
71
|
-
* @throws {Error} When the operation times out
|
|
72
|
-
*/
|
|
73
|
-
async function withTimeout<T>(promise: Promise<T>, timeoutMs: number = 10000): Promise<T> {
|
|
74
|
-
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
|
75
|
-
|
|
76
|
-
const timeoutPromise = new Promise<never>((_, reject) => {
|
|
77
|
-
timeoutId = setTimeout(() => {
|
|
78
|
-
reject(
|
|
79
|
-
new Error(
|
|
80
|
-
`Atlassian @forge/sql did not return a response within ${timeoutMs}ms (${timeoutMs / 1000} seconds), so the request is blocked. Possible causes: slow query, network issues, or exceeding Forge SQL limits.`,
|
|
81
|
-
),
|
|
82
|
-
);
|
|
83
|
-
}, timeoutMs);
|
|
84
|
-
});
|
|
85
|
-
|
|
86
|
-
try {
|
|
87
|
-
return await Promise.race([promise, timeoutPromise]);
|
|
88
|
-
} finally {
|
|
89
|
-
if (timeoutId) {
|
|
90
|
-
clearTimeout(timeoutId);
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
|
|
95
69
|
function inlineParams(sql: string, params: unknown[]): string {
|
|
96
70
|
let i = 0;
|
|
97
71
|
return sql.replace(/\?/g, () => {
|
|
@@ -148,7 +122,7 @@ async function processExecuteMethod(query: string, params: unknown[]): Promise<F
|
|
|
148
122
|
sqlStatement.bindParams(...params);
|
|
149
123
|
}
|
|
150
124
|
|
|
151
|
-
const result = await withTimeout(sqlStatement.execute());
|
|
125
|
+
const result = await withTimeout(sqlStatement.execute(), timeoutMessage, timeoutMs);
|
|
152
126
|
await saveMetaDataToContext(result.metadata as ForgeSQLMetadata);
|
|
153
127
|
if (!result.rows) {
|
|
154
128
|
return { rows: [[]] };
|
|
@@ -171,7 +145,7 @@ async function processAllMethod(query: string, params: unknown[]): Promise<Forge
|
|
|
171
145
|
await sqlStatement.bindParams(...params);
|
|
172
146
|
}
|
|
173
147
|
|
|
174
|
-
const result = (await withTimeout(sqlStatement.execute())) as ForgeSQLResult;
|
|
148
|
+
const result = (await withTimeout(sqlStatement.execute(), timeoutMessage, timeoutMs)) as ForgeSQLResult;
|
|
175
149
|
await saveMetaDataToContext(result.metadata);
|
|
176
150
|
|
|
177
151
|
if (!result.rows) {
|
|
@@ -212,10 +186,9 @@ export const forgeDriver = async (
|
|
|
212
186
|
method: QueryMethod,
|
|
213
187
|
): Promise<ForgeDriverResult> => {
|
|
214
188
|
const operationType = await getOperationType();
|
|
215
|
-
|
|
216
189
|
// Handle DDL operations
|
|
217
190
|
if (operationType === "DDL") {
|
|
218
|
-
const result = await withTimeout(sql.executeDDL(inlineParams(query, params)));
|
|
191
|
+
const result = await withTimeout(sql.executeDDL(inlineParams(query, params)), timeoutMessage, timeoutMs);
|
|
219
192
|
return await processDDLResult(method, result);
|
|
220
193
|
}
|
|
221
194
|
|
|
@@ -1,11 +1,33 @@
|
|
|
1
1
|
import { forgeDriver } from "./forgeDriver";
|
|
2
2
|
import { injectSqlHints, SqlHints } from "./sqlHints";
|
|
3
|
+
import { ForgeSqlOperation } from "../core/ForgeSQLQueryBuilder";
|
|
4
|
+
import { printQueriesWithPlan } from "./sqlUtils";
|
|
3
5
|
|
|
4
6
|
/**
|
|
5
|
-
*
|
|
7
|
+
* Error codes and constants for query analysis
|
|
8
|
+
*/
|
|
9
|
+
const QUERY_ERROR_CODES = {
|
|
10
|
+
TIMEOUT: "SQL_QUERY_TIMEOUT",
|
|
11
|
+
OUT_OF_MEMORY_ERRNO: 8175,
|
|
12
|
+
} as const;
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Delay to wait for CLUSTER_STATEMENTS_SUMMARY to be populated
|
|
16
|
+
*/
|
|
17
|
+
const STATEMENTS_SUMMARY_DELAY_MS = 200;
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Creates a proxy for the forgeDriver that injects SQL hints and handles query analysis
|
|
21
|
+
* @param forgeSqlOperation - The ForgeSQL operation instance
|
|
22
|
+
* @param options - SQL hints to inject
|
|
23
|
+
* @param logRawSqlQuery - Whether to log raw SQL queries
|
|
6
24
|
* @returns A proxied version of the forgeDriver
|
|
7
25
|
*/
|
|
8
|
-
export function createForgeDriverProxy(
|
|
26
|
+
export function createForgeDriverProxy(
|
|
27
|
+
forgeSqlOperation: ForgeSqlOperation,
|
|
28
|
+
options?: SqlHints,
|
|
29
|
+
logRawSqlQuery?: boolean,
|
|
30
|
+
) {
|
|
9
31
|
return async (
|
|
10
32
|
query: string,
|
|
11
33
|
params: any[],
|
|
@@ -20,16 +42,46 @@ export function createForgeDriverProxy(options?: SqlHints, logRawSqlQuery?: bool
|
|
|
20
42
|
|
|
21
43
|
if (options && logRawSqlQuery && modifiedQuery !== query) {
|
|
22
44
|
// eslint-disable-next-line no-console
|
|
23
|
-
console.debug(
|
|
45
|
+
console.debug(`SQL Hints injected: ${modifiedQuery}`);
|
|
24
46
|
}
|
|
47
|
+
|
|
48
|
+
const queryStartTime = Date.now();
|
|
49
|
+
|
|
25
50
|
try {
|
|
26
|
-
//
|
|
51
|
+
// Execute the query with injected hints
|
|
27
52
|
return await forgeDriver(modifiedQuery, params, method);
|
|
28
|
-
} catch (error) {
|
|
53
|
+
} catch (error: any) {
|
|
54
|
+
// Check if this is a timeout or out-of-memory error that we want to analyze
|
|
55
|
+
const isTimeoutError = error.code === QUERY_ERROR_CODES.TIMEOUT;
|
|
56
|
+
const isOutOfMemoryError =
|
|
57
|
+
error?.context?.debug?.errno === QUERY_ERROR_CODES.OUT_OF_MEMORY_ERRNO;
|
|
58
|
+
|
|
59
|
+
if (isTimeoutError || isOutOfMemoryError) {
|
|
60
|
+
if (isTimeoutError) {
|
|
61
|
+
// eslint-disable-next-line no-console
|
|
62
|
+
console.error(` TIMEOUT detected - Query exceeded time limit`);
|
|
63
|
+
} else {
|
|
64
|
+
// eslint-disable-next-line no-console
|
|
65
|
+
console.error(`OUT OF MEMORY detected - Query exceeded memory limit`);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// Wait for CLUSTER_STATEMENTS_SUMMARY to be populated with our failed query data
|
|
69
|
+
await new Promise((resolve) => setTimeout(resolve, STATEMENTS_SUMMARY_DELAY_MS));
|
|
70
|
+
|
|
71
|
+
const queryEndTime = Date.now();
|
|
72
|
+
const queryDuration = queryEndTime - queryStartTime;
|
|
73
|
+
|
|
74
|
+
// Analyze the failed query using CLUSTER_STATEMENTS_SUMMARY
|
|
75
|
+
await printQueriesWithPlan(forgeSqlOperation, queryDuration);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// Log SQL error details if requested
|
|
29
79
|
if (logRawSqlQuery) {
|
|
30
80
|
// eslint-disable-next-line no-console
|
|
31
|
-
console.debug(
|
|
81
|
+
console.debug(`SQL Error Details:`, JSON.stringify(error, null, 2));
|
|
32
82
|
}
|
|
83
|
+
|
|
84
|
+
// Re-throw the original error
|
|
33
85
|
throw error;
|
|
34
86
|
}
|
|
35
87
|
};
|
|
@@ -1,19 +1,34 @@
|
|
|
1
1
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
2
|
import { ForgeSQLMetadata } from "./forgeDriver";
|
|
3
|
+
import { ForgeSqlOperation } from "../core/ForgeSQLQueryBuilder";
|
|
4
|
+
import { printQueriesWithPlan } from "./sqlUtils";
|
|
3
5
|
|
|
4
6
|
export type MetadataQueryContext = {
|
|
5
7
|
totalDbExecutionTime: number;
|
|
6
8
|
totalResponseSize: number;
|
|
7
|
-
|
|
9
|
+
beginTime: Date;
|
|
10
|
+
printQueriesWithPlan: () => Promise<void>;
|
|
11
|
+
forgeSQLORM: ForgeSqlOperation;
|
|
8
12
|
};
|
|
9
13
|
export const metadataQueryContext = new AsyncLocalStorage<MetadataQueryContext>();
|
|
10
14
|
|
|
11
|
-
export async function saveMetaDataToContext(metadata
|
|
15
|
+
export async function saveMetaDataToContext(metadata?: ForgeSQLMetadata): Promise<void> {
|
|
12
16
|
const context = metadataQueryContext.getStore();
|
|
13
|
-
if (context
|
|
14
|
-
context.
|
|
15
|
-
|
|
16
|
-
|
|
17
|
+
if (context) {
|
|
18
|
+
context.printQueriesWithPlan = async () => {
|
|
19
|
+
if (process.env.NODE_ENV !== "test") {
|
|
20
|
+
await new Promise((r) => setTimeout(r, 200));
|
|
21
|
+
}
|
|
22
|
+
await printQueriesWithPlan(
|
|
23
|
+
context.forgeSQLORM,
|
|
24
|
+
Date.now() - context.beginTime.getTime(),
|
|
25
|
+
);
|
|
26
|
+
};
|
|
27
|
+
if (metadata) {
|
|
28
|
+
context.totalResponseSize += metadata.responseSize;
|
|
29
|
+
context.totalDbExecutionTime += metadata.dbExecutionTime;
|
|
30
|
+
}
|
|
31
|
+
// Log the results to console
|
|
17
32
|
}
|
|
18
33
|
}
|
|
19
34
|
|
package/src/utils/sqlUtils.ts
CHANGED
|
@@ -1,4 +1,18 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
and,
|
|
3
|
+
AnyColumn,
|
|
4
|
+
Column,
|
|
5
|
+
gte,
|
|
6
|
+
ilike,
|
|
7
|
+
isNotNull,
|
|
8
|
+
isTable,
|
|
9
|
+
ne,
|
|
10
|
+
not,
|
|
11
|
+
notInArray,
|
|
12
|
+
SQL,
|
|
13
|
+
sql,
|
|
14
|
+
StringChunk,
|
|
15
|
+
} from "drizzle-orm";
|
|
2
16
|
import { AnyMySqlTable, MySqlCustomColumn } from "drizzle-orm/mysql-core/index";
|
|
3
17
|
import { DateTime } from "luxon";
|
|
4
18
|
import { PrimaryKeyBuilder } from "drizzle-orm/mysql-core/primary-keys";
|
|
@@ -6,9 +20,16 @@ import { AnyIndexBuilder } from "drizzle-orm/mysql-core/indexes";
|
|
|
6
20
|
import { CheckBuilder } from "drizzle-orm/mysql-core/checks";
|
|
7
21
|
import { ForeignKeyBuilder } from "drizzle-orm/mysql-core/foreign-keys";
|
|
8
22
|
import { UniqueConstraintBuilder } from "drizzle-orm/mysql-core/unique-constraint";
|
|
9
|
-
import
|
|
23
|
+
import {
|
|
24
|
+
SelectedFields
|
|
25
|
+
} from "drizzle-orm/mysql-core/query-builders/select.types";
|
|
10
26
|
import { MySqlTable } from "drizzle-orm/mysql-core";
|
|
11
27
|
import { isSQLWrapper } from "drizzle-orm/sql/sql";
|
|
28
|
+
import {clusterStatementsSummary, slowQuery} from "../core/SystemTables";
|
|
29
|
+
import { ForgeSqlOperation } from "../core/ForgeSQLQueryBuilder";
|
|
30
|
+
import { ColumnDataType} from "drizzle-orm/column-builder";
|
|
31
|
+
import {AnyMySqlColumn} from "drizzle-orm/mysql-core/columns/common";
|
|
32
|
+
import type {ColumnBaseConfig} from "drizzle-orm/column";
|
|
12
33
|
|
|
13
34
|
/**
|
|
14
35
|
* Interface representing table metadata information
|
|
@@ -536,3 +557,209 @@ export function formatLimitOffset(limitOrOffset: number): number {
|
|
|
536
557
|
export function nextVal(sequenceName: string): number {
|
|
537
558
|
return sql.raw(`NEXTVAL(${sequenceName})`) as unknown as number;
|
|
538
559
|
}
|
|
560
|
+
|
|
561
|
+
/**
|
|
562
|
+
* Analyzes and prints query performance data from CLUSTER_STATEMENTS_SUMMARY table.
|
|
563
|
+
*
|
|
564
|
+
* This function queries the CLUSTER_STATEMENTS_SUMMARY table to find queries that were executed
|
|
565
|
+
* within the specified time window and prints detailed performance information including:
|
|
566
|
+
* - SQL query text
|
|
567
|
+
* - Memory usage (average and max in MB)
|
|
568
|
+
* - Execution time (average in ms)
|
|
569
|
+
* - Number of executions
|
|
570
|
+
* - Execution plan
|
|
571
|
+
*
|
|
572
|
+
* @param forgeSQLORM - The ForgeSQL operation instance for database access
|
|
573
|
+
* @param timeDiffMs - Time window in milliseconds to look back for queries (e.g., 1500 for last 1.5 seconds)
|
|
574
|
+
* @param timeout - Optional timeout in milliseconds for the query execution (defaults to 1500ms)
|
|
575
|
+
*
|
|
576
|
+
* @example
|
|
577
|
+
* ```typescript
|
|
578
|
+
* // Analyze queries from the last 2 seconds
|
|
579
|
+
* await printQueriesWithPlan(forgeSQLORM, 2000);
|
|
580
|
+
*
|
|
581
|
+
* // Analyze queries with custom timeout
|
|
582
|
+
* await printQueriesWithPlan(forgeSQLORM, 1000, 3000);
|
|
583
|
+
* ```
|
|
584
|
+
*
|
|
585
|
+
* @throws Does not throw - errors are logged to console.debug instead
|
|
586
|
+
*/
|
|
587
|
+
export async function printQueriesWithPlan(
|
|
588
|
+
forgeSQLORM: ForgeSqlOperation,
|
|
589
|
+
timeDiffMs: number,
|
|
590
|
+
timeout?: number,
|
|
591
|
+
) {
|
|
592
|
+
try {
|
|
593
|
+
const statementsTable = clusterStatementsSummary;
|
|
594
|
+
const timeoutMs = timeout ?? 3000;
|
|
595
|
+
const results = await withTimeout(
|
|
596
|
+
forgeSQLORM
|
|
597
|
+
.getDrizzleQueryBuilder()
|
|
598
|
+
.select({
|
|
599
|
+
digestText: withTidbHint(statementsTable.digestText),
|
|
600
|
+
avgLatency: statementsTable.avgLatency,
|
|
601
|
+
avgMem: statementsTable.avgMem,
|
|
602
|
+
execCount: statementsTable.execCount,
|
|
603
|
+
plan: statementsTable.plan,
|
|
604
|
+
stmtType: statementsTable.stmtType,
|
|
605
|
+
})
|
|
606
|
+
.from(statementsTable)
|
|
607
|
+
.where(
|
|
608
|
+
and(
|
|
609
|
+
isNotNull(statementsTable.digest),
|
|
610
|
+
not(ilike(statementsTable.digestText, "%information_schema%")),
|
|
611
|
+
notInArray(statementsTable.stmtType, ["Use", "Set", "Show","Commit","Rollback", "Begin"]),
|
|
612
|
+
gte(
|
|
613
|
+
statementsTable.lastSeen,
|
|
614
|
+
sql`DATE_SUB
|
|
615
|
+
(NOW(), INTERVAL
|
|
616
|
+
${timeDiffMs * 1000}
|
|
617
|
+
MICROSECOND
|
|
618
|
+
)`,
|
|
619
|
+
),
|
|
620
|
+
),
|
|
621
|
+
),
|
|
622
|
+
`Timeout ${timeoutMs}ms in printQueriesWithPlan - transient timeouts are usually fine; repeated timeouts mean this diagnostic query is consistently slow and should be investigated`
|
|
623
|
+
,
|
|
624
|
+
timeoutMs+200,
|
|
625
|
+
);
|
|
626
|
+
|
|
627
|
+
results.forEach((result) => {
|
|
628
|
+
// Average execution time (convert from nanoseconds to milliseconds)
|
|
629
|
+
const avgTimeMs = Number(result.avgLatency) / 1_000_000;
|
|
630
|
+
const avgMemMB = Number(result.avgMem) / 1_000_000;
|
|
631
|
+
|
|
632
|
+
// 1. Query info: SQL, memory, time, executions
|
|
633
|
+
// eslint-disable-next-line no-console
|
|
634
|
+
console.warn(
|
|
635
|
+
`SQL: ${result.digestText} | Memory: ${avgMemMB.toFixed(2)} MB | Time: ${avgTimeMs.toFixed(2)} ms | stmtType: ${result.stmtType} | Executions: ${result.execCount}\n Plan:${result.plan}`,
|
|
636
|
+
);
|
|
637
|
+
});
|
|
638
|
+
} catch (error) {
|
|
639
|
+
// eslint-disable-next-line no-console
|
|
640
|
+
console.debug(
|
|
641
|
+
`Error occurred while retrieving query execution plan: ${error instanceof Error ? error.message : "Unknown error"}. Try again after some time`,
|
|
642
|
+
error,
|
|
643
|
+
);
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
const SESSION_ALIAS_NAME_ORM = 'orm';
|
|
648
|
+
|
|
649
|
+
/**
|
|
650
|
+
* Analyzes and logs slow queries from the last specified number of hours.
|
|
651
|
+
*
|
|
652
|
+
* This function queries the slow query system table to find queries that were executed
|
|
653
|
+
* within the specified time window and logs detailed performance information including:
|
|
654
|
+
* - SQL query text
|
|
655
|
+
* - Maximum memory usage (in MB)
|
|
656
|
+
* - Query execution time (in ms)
|
|
657
|
+
* - Execution count
|
|
658
|
+
* - Execution plan
|
|
659
|
+
*
|
|
660
|
+
* @param forgeSQLORM - The ForgeSQL operation instance for database access
|
|
661
|
+
* @param hours - Number of hours to look back for slow queries (e.g., 1 for last hour, 24 for last day)
|
|
662
|
+
* @param timeout - Optional timeout in milliseconds for the query execution (defaults to 1500ms)
|
|
663
|
+
*
|
|
664
|
+
* @example
|
|
665
|
+
* ```typescript
|
|
666
|
+
* // Analyze slow queries from the last hour
|
|
667
|
+
* await slowQueryPerHours(forgeSQLORM, 1);
|
|
668
|
+
*
|
|
669
|
+
* // Analyze slow queries from the last 24 hours with custom timeout
|
|
670
|
+
* await slowQueryPerHours(forgeSQLORM, 24, 3000);
|
|
671
|
+
*
|
|
672
|
+
* // Analyze slow queries from the last 6 hours
|
|
673
|
+
* await slowQueryPerHours(forgeSQLORM, 6);
|
|
674
|
+
* ```
|
|
675
|
+
*
|
|
676
|
+
* @throws Does not throw - errors are logged to console.debug instead
|
|
677
|
+
*/
|
|
678
|
+
export async function slowQueryPerHours(forgeSQLORM: ForgeSqlOperation, hours:number, timeout?: number) {
|
|
679
|
+
try {
|
|
680
|
+
const timeoutMs = timeout ?? 1500;
|
|
681
|
+
const results = await withTimeout(
|
|
682
|
+
forgeSQLORM
|
|
683
|
+
.getDrizzleQueryBuilder()
|
|
684
|
+
.select({
|
|
685
|
+
query: withTidbHint(slowQuery.query),
|
|
686
|
+
queryTime: slowQuery.queryTime,
|
|
687
|
+
memMax: slowQuery.memMax,
|
|
688
|
+
plan: slowQuery.plan,
|
|
689
|
+
})
|
|
690
|
+
.from(slowQuery)
|
|
691
|
+
.where(
|
|
692
|
+
and(
|
|
693
|
+
isNotNull(slowQuery.digest),
|
|
694
|
+
ne(slowQuery.sessionAlias, SESSION_ALIAS_NAME_ORM),
|
|
695
|
+
gte(
|
|
696
|
+
slowQuery.time,
|
|
697
|
+
sql`DATE_SUB
|
|
698
|
+
(NOW(), INTERVAL
|
|
699
|
+
${hours}
|
|
700
|
+
HOUR
|
|
701
|
+
)`,
|
|
702
|
+
),
|
|
703
|
+
),
|
|
704
|
+
),
|
|
705
|
+
`Timeout ${timeoutMs}ms in slowQueryPerHours - transient timeouts are usually fine; repeated timeouts mean this diagnostic query is consistently slow and should be investigated`,
|
|
706
|
+
timeoutMs,
|
|
707
|
+
);
|
|
708
|
+
const response:string[] =[]
|
|
709
|
+
results.forEach((result) => {
|
|
710
|
+
// Convert memory from bytes to MB and handle null values
|
|
711
|
+
const memMaxMB = result.memMax ? Number(result.memMax) / 1_000_000 : 0;
|
|
712
|
+
|
|
713
|
+
const message = `Found SlowQuery SQL: ${result.query} | Memory: ${memMaxMB.toFixed(2)} MB | Time: ${result.queryTime} ms\n Plan:${result.plan}`;
|
|
714
|
+
response.push(message);
|
|
715
|
+
// 1. Query info: SQL, memory, time, executions
|
|
716
|
+
// eslint-disable-next-line no-console
|
|
717
|
+
console.warn(
|
|
718
|
+
message,
|
|
719
|
+
);
|
|
720
|
+
});
|
|
721
|
+
return response;
|
|
722
|
+
} catch (error) {
|
|
723
|
+
// eslint-disable-next-line no-console
|
|
724
|
+
console.debug(
|
|
725
|
+
`Error occurred while retrieving query execution plan: ${error instanceof Error ? error.message : "Unknown error"}. Try again after some time`,
|
|
726
|
+
error,
|
|
727
|
+
);
|
|
728
|
+
return [`Error occurred while retrieving query execution plan: ${error instanceof Error ? error.message : "Unknown error"}`]
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
/**
|
|
733
|
+
* Executes a promise with a timeout.
|
|
734
|
+
*
|
|
735
|
+
* @param promise - The promise to execute
|
|
736
|
+
* @param timeoutMs - Timeout in milliseconds
|
|
737
|
+
* @returns Promise that resolves with the result or rejects on timeout
|
|
738
|
+
* @throws {Error} When the operation times out
|
|
739
|
+
*/
|
|
740
|
+
export async function withTimeout<T>(promise: Promise<T>, message: string, timeoutMs: number): Promise<T> {
|
|
741
|
+
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
|
742
|
+
|
|
743
|
+
const timeoutPromise = new Promise<never>((_, reject) => {
|
|
744
|
+
timeoutId = setTimeout(() => {
|
|
745
|
+
reject(
|
|
746
|
+
new Error(message),
|
|
747
|
+
);
|
|
748
|
+
}, timeoutMs);
|
|
749
|
+
});
|
|
750
|
+
|
|
751
|
+
try {
|
|
752
|
+
return await Promise.race([promise, timeoutPromise]);
|
|
753
|
+
} finally {
|
|
754
|
+
if (timeoutId) {
|
|
755
|
+
clearTimeout(timeoutId);
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
export function withTidbHint<TDataType extends ColumnDataType, TPartial extends Partial<ColumnBaseConfig<TDataType, string>>>(column: AnyMySqlColumn<TPartial>): AnyMySqlColumn<TPartial> {
|
|
761
|
+
// We lie a bit to TypeScript here: at runtime this is a new SQL fragment,
|
|
762
|
+
// but returning TExpr keeps the column type info in downstream inference.
|
|
763
|
+
return sql`/*+ SET_VAR(tidb_session_alias=${sql.raw(`${SESSION_ALIAS_NAME_ORM}`)}) */ ${column}` as unknown as AnyMySqlColumn<TPartial>;
|
|
764
|
+
}
|
|
765
|
+
|
package/src/webtriggers/index.ts
CHANGED
|
@@ -3,7 +3,7 @@ export * from "./applyMigrationsWebTrigger";
|
|
|
3
3
|
export * from "./fetchSchemaWebTrigger";
|
|
4
4
|
export * from "./dropTablesMigrationWebTrigger";
|
|
5
5
|
export * from "./clearCacheSchedulerTrigger";
|
|
6
|
-
export * from "./
|
|
6
|
+
export * from "./slowQuerySchedulerTrigger";
|
|
7
7
|
|
|
8
8
|
export interface TriggerResponse<BODY> {
|
|
9
9
|
body?: BODY;
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import {getHttpResponse, TriggerResponse} from "./index";
|
|
2
|
+
import {slowQueryPerHours} from "../utils/sqlUtils";
|
|
3
|
+
import {ForgeSqlOperation} from "../core/ForgeSQLQueryBuilder";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Scheduler trigger for analyzing slow queries from the last specified number of hours.
|
|
7
|
+
*
|
|
8
|
+
* This trigger analyzes slow queries from TiDB's slow query log system table and provides
|
|
9
|
+
* detailed performance information including SQL query text, memory usage, execution time,
|
|
10
|
+
* and execution plans. It's designed to be used as a scheduled trigger in Atlassian Forge
|
|
11
|
+
* to monitor query performance over time.
|
|
12
|
+
*
|
|
13
|
+
* The function queries the slow query system table to find queries executed within the
|
|
14
|
+
* specified time window and logs detailed performance information to the console. Results
|
|
15
|
+
* are limited to the top 50 slow queries to prevent excessive output.
|
|
16
|
+
*
|
|
17
|
+
* @param forgeSQLORM - The ForgeSQL operation instance for database access
|
|
18
|
+
* @param options - Configuration options for the slow query analysis
|
|
19
|
+
* @param options.hours - Number of hours to look back for slow queries (default: 1)
|
|
20
|
+
* @param options.timeout - Timeout in milliseconds for the query execution (default: 2000ms)
|
|
21
|
+
*
|
|
22
|
+
* @returns Promise<TriggerResponse<string>> - HTTP response with JSON stringified query results or error message
|
|
23
|
+
*
|
|
24
|
+
* @example
|
|
25
|
+
* ```typescript
|
|
26
|
+
* import ForgeSQL, { slowQuerySchedulerTrigger } from "forge-sql-orm";
|
|
27
|
+
*
|
|
28
|
+
* const forgeSQL = new ForgeSQL();
|
|
29
|
+
*
|
|
30
|
+
* // Basic usage with default options (1 hour, 2000ms timeout)
|
|
31
|
+
* export const slowQueryTrigger = () =>
|
|
32
|
+
* slowQuerySchedulerTrigger(forgeSQL, { hours: 1, timeout: 2000 });
|
|
33
|
+
*
|
|
34
|
+
* // Analyze slow queries from the last 6 hours with extended timeout
|
|
35
|
+
* export const sixHourSlowQueryTrigger = () =>
|
|
36
|
+
* slowQuerySchedulerTrigger(forgeSQL, { hours: 6, timeout: 5000 });
|
|
37
|
+
*
|
|
38
|
+
* // Analyze slow queries from the last 24 hours
|
|
39
|
+
* export const dailySlowQueryTrigger = () =>
|
|
40
|
+
* slowQuerySchedulerTrigger(forgeSQL, { hours: 24, timeout: 3000 });
|
|
41
|
+
* ```
|
|
42
|
+
*
|
|
43
|
+
* @example
|
|
44
|
+
* ```yaml
|
|
45
|
+
* # manifest.yml configuration
|
|
46
|
+
* scheduledTrigger:
|
|
47
|
+
* - key: slow-query-trigger
|
|
48
|
+
* function: slowQueryTrigger
|
|
49
|
+
* interval: hour
|
|
50
|
+
*
|
|
51
|
+
* function:
|
|
52
|
+
* - key: slowQueryTrigger
|
|
53
|
+
* handler: index.slowQueryTrigger
|
|
54
|
+
* ```
|
|
55
|
+
*
|
|
56
|
+
* @remarks
|
|
57
|
+
* - Results are automatically logged to the Forge Developer Console via `console.warn()`
|
|
58
|
+
* - The function returns up to 50 slow queries to prevent excessive logging
|
|
59
|
+
* - Transient timeouts are usually fine; repeated timeouts indicate the diagnostic query itself is slow
|
|
60
|
+
* - This trigger is best used with hourly intervals to catch slow queries in a timely manner
|
|
61
|
+
* - Error responses return HTTP 500 with error details
|
|
62
|
+
*
|
|
63
|
+
* @see {@link slowQueryPerHours} - The underlying function that performs the actual query analysis
|
|
64
|
+
*/
|
|
65
|
+
export async function slowQuerySchedulerTrigger(forgeSQLORM: ForgeSqlOperation, options: {
|
|
66
|
+
hours: number,
|
|
67
|
+
timeout: number
|
|
68
|
+
}): Promise<TriggerResponse<string>> {
|
|
69
|
+
try {
|
|
70
|
+
return getHttpResponse<string>(200, JSON.stringify(await slowQueryPerHours(forgeSQLORM, options?.hours ?? 1, options?.timeout ?? 3000)));
|
|
71
|
+
} catch (error: any) {
|
|
72
|
+
const errorMessage =
|
|
73
|
+
error?.debug?.sqlMessage ??
|
|
74
|
+
error?.debug?.message ??
|
|
75
|
+
error.message ??
|
|
76
|
+
"Unknown error occurred";
|
|
77
|
+
// eslint-disable-next-line no-console
|
|
78
|
+
console.error(errorMessage);
|
|
79
|
+
return getHttpResponse<string>(500, errorMessage);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|