forge-sql-orm 2.0.30 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1090 -81
- package/dist/ForgeSQLORM.js +1080 -60
- package/dist/ForgeSQLORM.js.map +1 -1
- package/dist/ForgeSQLORM.mjs +1063 -60
- package/dist/ForgeSQLORM.mjs.map +1 -1
- package/dist/core/ForgeSQLAnalyseOperations.d.ts +1 -1
- package/dist/core/ForgeSQLAnalyseOperations.d.ts.map +1 -1
- package/dist/core/ForgeSQLCacheOperations.d.ts +119 -0
- package/dist/core/ForgeSQLCacheOperations.d.ts.map +1 -0
- package/dist/core/ForgeSQLCrudOperations.d.ts +38 -22
- package/dist/core/ForgeSQLCrudOperations.d.ts.map +1 -1
- package/dist/core/ForgeSQLORM.d.ts +104 -13
- package/dist/core/ForgeSQLORM.d.ts.map +1 -1
- package/dist/core/ForgeSQLQueryBuilder.d.ts +243 -15
- package/dist/core/ForgeSQLQueryBuilder.d.ts.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/lib/drizzle/extensions/additionalActions.d.ts +42 -0
- package/dist/lib/drizzle/extensions/additionalActions.d.ts.map +1 -0
- package/dist/utils/cacheContextUtils.d.ts +123 -0
- package/dist/utils/cacheContextUtils.d.ts.map +1 -0
- package/dist/utils/cacheUtils.d.ts +56 -0
- package/dist/utils/cacheUtils.d.ts.map +1 -0
- package/dist/utils/sqlUtils.d.ts +8 -0
- package/dist/utils/sqlUtils.d.ts.map +1 -1
- package/dist/webtriggers/clearCacheSchedulerTrigger.d.ts +46 -0
- package/dist/webtriggers/clearCacheSchedulerTrigger.d.ts.map +1 -0
- package/dist/webtriggers/index.d.ts +1 -0
- package/dist/webtriggers/index.d.ts.map +1 -1
- package/package.json +15 -12
- package/src/core/ForgeSQLAnalyseOperations.ts +1 -1
- package/src/core/ForgeSQLCacheOperations.ts +195 -0
- package/src/core/ForgeSQLCrudOperations.ts +49 -40
- package/src/core/ForgeSQLORM.ts +443 -34
- package/src/core/ForgeSQLQueryBuilder.ts +291 -20
- package/src/index.ts +1 -1
- package/src/lib/drizzle/extensions/additionalActions.ts +548 -0
- package/src/lib/drizzle/extensions/types.d.ts +68 -10
- package/src/utils/cacheContextUtils.ts +210 -0
- package/src/utils/cacheUtils.ts +403 -0
- package/src/utils/sqlUtils.ts +16 -0
- package/src/webtriggers/clearCacheSchedulerTrigger.ts +79 -0
- package/src/webtriggers/index.ts +1 -0
- package/dist/lib/drizzle/extensions/selectAliased.d.ts +0 -9
- package/dist/lib/drizzle/extensions/selectAliased.d.ts.map +0 -1
- package/src/lib/drizzle/extensions/selectAliased.ts +0 -72
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
|
+
import { AnyMySqlSelectQueryBuilder, AnyMySqlTable } from "drizzle-orm/mysql-core";
|
|
3
|
+
import { getTableName } from "drizzle-orm/table";
|
|
4
|
+
import { ForgeSqlOrmOptions } from "../core/ForgeSQLQueryBuilder";
|
|
5
|
+
import { MySqlSelectDynamic } from "drizzle-orm/mysql-core/query-builders/select.types";
|
|
6
|
+
import { hashKey } from "./cacheUtils";
|
|
7
|
+
import { Query } from "drizzle-orm/sql/sql";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Interface representing the cache application context.
|
|
11
|
+
* Stores information about tables that are being processed within a cache context.
|
|
12
|
+
*/
|
|
13
|
+
export interface CacheApplicationContext {
|
|
14
|
+
/** Set of table names (in lowercase) that are being processed within the cache context */
|
|
15
|
+
tables: Set<string>;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Interface representing the local cache application context.
|
|
20
|
+
* Stores cached query results in memory for the duration of a local cache context.
|
|
21
|
+
*
|
|
22
|
+
* @interface LocalCacheApplicationContext
|
|
23
|
+
*/
|
|
24
|
+
export interface LocalCacheApplicationContext {
|
|
25
|
+
/**
|
|
26
|
+
* Cache object mapping query hash keys to cached results
|
|
27
|
+
* @property {Record<string, {sql: string, data: unknown[]}>} cache - Map of query keys to cached data
|
|
28
|
+
*/
|
|
29
|
+
cache: Record<
|
|
30
|
+
string,
|
|
31
|
+
{
|
|
32
|
+
sql: string;
|
|
33
|
+
data: unknown[];
|
|
34
|
+
}
|
|
35
|
+
>;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* AsyncLocalStorage instance for managing cache context across async operations.
|
|
40
|
+
* This allows tracking which tables are being processed within a cache context
|
|
41
|
+
* without explicitly passing context through function parameters.
|
|
42
|
+
*/
|
|
43
|
+
export const cacheApplicationContext = new AsyncLocalStorage<CacheApplicationContext>();
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* AsyncLocalStorage instance for managing local cache context across async operations.
|
|
47
|
+
* This allows storing and retrieving cached query results within a local cache context
|
|
48
|
+
* without explicitly passing context through function parameters.
|
|
49
|
+
*/
|
|
50
|
+
export const localCacheApplicationContext = new AsyncLocalStorage<LocalCacheApplicationContext>();
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Saves a table name to the current cache context if one exists.
|
|
54
|
+
* This function is used to track which tables are being processed within
|
|
55
|
+
* a cache context for proper cache invalidation.
|
|
56
|
+
*
|
|
57
|
+
* @param table - The Drizzle table schema to track
|
|
58
|
+
* @returns Promise that resolves when the table is saved to context
|
|
59
|
+
*
|
|
60
|
+
* @example
|
|
61
|
+
* ```typescript
|
|
62
|
+
* await saveTableIfInsideCacheContext(usersTable);
|
|
63
|
+
* ```
|
|
64
|
+
*/
|
|
65
|
+
export async function saveTableIfInsideCacheContext<T extends AnyMySqlTable>(
|
|
66
|
+
table: T,
|
|
67
|
+
): Promise<void> {
|
|
68
|
+
const context = cacheApplicationContext.getStore();
|
|
69
|
+
if (context) {
|
|
70
|
+
const tableName = getTableName(table).toLowerCase();
|
|
71
|
+
context.tables.add(tableName);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Saves a query result to the local cache context.
|
|
77
|
+
* This function stores query results in memory for the duration of the local cache context.
|
|
78
|
+
*
|
|
79
|
+
* @param query - The Drizzle query to cache
|
|
80
|
+
* @param rows - The query result data to cache
|
|
81
|
+
* @returns Promise that resolves when the data is saved to local cache
|
|
82
|
+
*
|
|
83
|
+
* @example
|
|
84
|
+
* ```typescript
|
|
85
|
+
* const query = db.select({ id: users.id, name: users.name }).from(users);
|
|
86
|
+
* const results = await query.execute();
|
|
87
|
+
* await saveQueryLocalCacheQuery(query, results);
|
|
88
|
+
* ```
|
|
89
|
+
*/
|
|
90
|
+
export async function saveQueryLocalCacheQuery<
|
|
91
|
+
T extends MySqlSelectDynamic<AnyMySqlSelectQueryBuilder>,
|
|
92
|
+
>(query: T, rows: unknown[]): Promise<void> {
|
|
93
|
+
const context = localCacheApplicationContext.getStore();
|
|
94
|
+
if (context) {
|
|
95
|
+
if (!context.cache) {
|
|
96
|
+
context.cache = {};
|
|
97
|
+
}
|
|
98
|
+
const sql = query as { toSQL: () => Query };
|
|
99
|
+
const key = hashKey(sql.toSQL());
|
|
100
|
+
context.cache[key] = {
|
|
101
|
+
sql: sql.toSQL().sql.toLowerCase(),
|
|
102
|
+
data: rows,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Retrieves a query result from the local cache context.
|
|
109
|
+
* This function checks if a query result is already cached in memory.
|
|
110
|
+
*
|
|
111
|
+
* @param query - The Drizzle query to check for cached results
|
|
112
|
+
* @returns Promise that resolves to cached data if found, undefined otherwise
|
|
113
|
+
*
|
|
114
|
+
* @example
|
|
115
|
+
* ```typescript
|
|
116
|
+
* const query = db.select({ id: users.id, name: users.name }).from(users);
|
|
117
|
+
* const cachedResult = await getQueryLocalCacheQuery(query);
|
|
118
|
+
* if (cachedResult) {
|
|
119
|
+
* return cachedResult; // Use cached data
|
|
120
|
+
* }
|
|
121
|
+
* // Execute query and cache result
|
|
122
|
+
* ```
|
|
123
|
+
*/
|
|
124
|
+
export async function getQueryLocalCacheQuery<
|
|
125
|
+
T extends MySqlSelectDynamic<AnyMySqlSelectQueryBuilder>,
|
|
126
|
+
>(query: T): Promise<unknown[] | undefined> {
|
|
127
|
+
const context = localCacheApplicationContext.getStore();
|
|
128
|
+
if (context) {
|
|
129
|
+
if (!context.cache) {
|
|
130
|
+
context.cache = {};
|
|
131
|
+
}
|
|
132
|
+
const sql = query as { toSQL: () => Query };
|
|
133
|
+
const key = hashKey(sql.toSQL());
|
|
134
|
+
if (context.cache[key] && context.cache[key].sql === sql.toSQL().sql.toLowerCase()) {
|
|
135
|
+
return context.cache[key].data;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
return undefined;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Evicts cached queries from the local cache context that involve the specified table.
|
|
143
|
+
* This function removes cached query results that contain the given table name.
|
|
144
|
+
*
|
|
145
|
+
* @param table - The Drizzle table schema to evict cache for
|
|
146
|
+
* @param options - ForgeSQL ORM options containing cache configuration
|
|
147
|
+
* @returns Promise that resolves when cache eviction is complete
|
|
148
|
+
*
|
|
149
|
+
* @example
|
|
150
|
+
* ```typescript
|
|
151
|
+
* // After inserting/updating/deleting from users table
|
|
152
|
+
* await evictLocalCacheQuery(usersTable, forgeSqlOptions);
|
|
153
|
+
* // All cached queries involving users table are now removed
|
|
154
|
+
* ```
|
|
155
|
+
*/
|
|
156
|
+
export async function evictLocalCacheQuery<T extends AnyMySqlTable>(
|
|
157
|
+
table: T,
|
|
158
|
+
options: ForgeSqlOrmOptions,
|
|
159
|
+
): Promise<void> {
|
|
160
|
+
const context = localCacheApplicationContext.getStore();
|
|
161
|
+
if (context) {
|
|
162
|
+
if (!context.cache) {
|
|
163
|
+
context.cache = {};
|
|
164
|
+
}
|
|
165
|
+
const tableName = getTableName(table);
|
|
166
|
+
const searchString = options.cacheWrapTable ? `\`${tableName}\`` : tableName;
|
|
167
|
+
const keyToEvicts: string[] = [];
|
|
168
|
+
Object.keys(context.cache).forEach((key) => {
|
|
169
|
+
if (context.cache[key].sql.includes(searchString)) {
|
|
170
|
+
keyToEvicts.push(key);
|
|
171
|
+
}
|
|
172
|
+
});
|
|
173
|
+
keyToEvicts.forEach((key) => delete context.cache[key]);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Checks if the given SQL query contains any tables that are currently being processed
|
|
179
|
+
* within a cache context. This is used to determine if cache should be bypassed
|
|
180
|
+
* for queries that affect tables being modified within the context.
|
|
181
|
+
*
|
|
182
|
+
* @param sql - The SQL query string to check
|
|
183
|
+
* @param options - ForgeSQL ORM options containing cache configuration
|
|
184
|
+
* @returns Promise that resolves to true if the SQL contains tables in cache context
|
|
185
|
+
*
|
|
186
|
+
* @example
|
|
187
|
+
* ```typescript
|
|
188
|
+
* const shouldBypassCache = await isTableContainsTableInCacheContext(
|
|
189
|
+
* "SELECT * FROM users WHERE id = 1",
|
|
190
|
+
* forgeSqlOptions
|
|
191
|
+
* );
|
|
192
|
+
* ```
|
|
193
|
+
*/
|
|
194
|
+
export async function isTableContainsTableInCacheContext(
|
|
195
|
+
sql: string,
|
|
196
|
+
options: ForgeSqlOrmOptions,
|
|
197
|
+
): Promise<boolean> {
|
|
198
|
+
const context = cacheApplicationContext.getStore();
|
|
199
|
+
if (!context) {
|
|
200
|
+
return false;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
const tables = Array.from(context.tables);
|
|
204
|
+
const lowerSql = sql.toLowerCase();
|
|
205
|
+
|
|
206
|
+
return tables.some((table) => {
|
|
207
|
+
const tablePattern = options.cacheWrapTable ? `\`${table}\`` : table;
|
|
208
|
+
return lowerSql.includes(tablePattern);
|
|
209
|
+
});
|
|
210
|
+
}
|
|
@@ -0,0 +1,403 @@
|
|
|
1
|
+
import { DateTime } from "luxon";
|
|
2
|
+
import * as crypto from "crypto";
|
|
3
|
+
import { Query } from "drizzle-orm";
|
|
4
|
+
import { AnyMySqlTable } from "drizzle-orm/mysql-core";
|
|
5
|
+
import { getTableName } from "drizzle-orm/table";
|
|
6
|
+
import { Filter, FilterConditions, kvs, WhereConditions } from "@forge/kvs";
|
|
7
|
+
import { ForgeSqlOrmOptions } from "../core/ForgeSQLQueryBuilder";
|
|
8
|
+
import { cacheApplicationContext, isTableContainsTableInCacheContext } from "./cacheContextUtils";
|
|
9
|
+
|
|
10
|
+
// Constants for better maintainability
|
|
11
|
+
const CACHE_CONSTANTS = {
|
|
12
|
+
BATCH_SIZE: 25,
|
|
13
|
+
MAX_RETRY_ATTEMPTS: 3,
|
|
14
|
+
INITIAL_RETRY_DELAY: 1000,
|
|
15
|
+
RETRY_DELAY_MULTIPLIER: 2,
|
|
16
|
+
DEFAULT_ENTITY_QUERY_NAME: "sql",
|
|
17
|
+
DEFAULT_EXPIRATION_NAME: "expiration",
|
|
18
|
+
DEFAULT_DATA_NAME: "data",
|
|
19
|
+
HASH_LENGTH: 32,
|
|
20
|
+
} as const;
|
|
21
|
+
|
|
22
|
+
// Types for better type safety
|
|
23
|
+
type CacheEntity = {
|
|
24
|
+
[key: string]: string | number;
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Gets the current Unix timestamp in seconds.
|
|
29
|
+
*
|
|
30
|
+
* @returns Current timestamp as integer
|
|
31
|
+
*/
|
|
32
|
+
function getCurrentTime(): number {
|
|
33
|
+
const dt = DateTime.now();
|
|
34
|
+
return Math.floor(dt.toSeconds());
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Calculates a future timestamp by adding seconds to the current time.
|
|
39
|
+
* Validates that the result is within 32-bit integer range.
|
|
40
|
+
*
|
|
41
|
+
* @param secondsToAdd - Number of seconds to add to current time
|
|
42
|
+
* @returns Future timestamp in seconds
|
|
43
|
+
* @throws Error if the result is out of 32-bit integer range
|
|
44
|
+
*/
|
|
45
|
+
function nowPlusSeconds(secondsToAdd: number): number {
|
|
46
|
+
const dt = DateTime.now().plus({ seconds: secondsToAdd });
|
|
47
|
+
return Math.floor(dt.toSeconds());
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Generates a hash key for a query based on its SQL and parameters.
|
|
52
|
+
*
|
|
53
|
+
* @param query - The Drizzle query object
|
|
54
|
+
* @returns 32-character hexadecimal hash
|
|
55
|
+
*/
|
|
56
|
+
export function hashKey(query: Query): string {
|
|
57
|
+
const h = crypto.createHash("sha256");
|
|
58
|
+
h.update(query.sql.toLowerCase());
|
|
59
|
+
h.update(JSON.stringify(query.params));
|
|
60
|
+
return "CachedQuery_" + h.digest("hex").slice(0, CACHE_CONSTANTS.HASH_LENGTH);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Deletes cache entries in batches to respect Forge limits and timeouts.
|
|
65
|
+
*
|
|
66
|
+
* @param results - Array of cache entries to delete
|
|
67
|
+
* @param cacheEntityName - Name of the cache entity
|
|
68
|
+
* @returns Promise that resolves when all deletions are complete
|
|
69
|
+
*/
|
|
70
|
+
async function deleteCacheEntriesInBatches(
|
|
71
|
+
results: Array<{ key: string }>,
|
|
72
|
+
cacheEntityName: string,
|
|
73
|
+
): Promise<void> {
|
|
74
|
+
for (let i = 0; i < results.length; i += CACHE_CONSTANTS.BATCH_SIZE) {
|
|
75
|
+
const batch = results.slice(i, i + CACHE_CONSTANTS.BATCH_SIZE);
|
|
76
|
+
let transactionBuilder = kvs.transact();
|
|
77
|
+
batch.forEach((result) => {
|
|
78
|
+
transactionBuilder = transactionBuilder.delete(result.key, { entityName: cacheEntityName });
|
|
79
|
+
});
|
|
80
|
+
await transactionBuilder.execute();
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Clears cache entries for specific tables using cursor-based pagination.
|
|
86
|
+
*
|
|
87
|
+
* @param tables - Array of table names to clear cache for
|
|
88
|
+
* @param cursor - Pagination cursor for large result sets
|
|
89
|
+
* @param options - ForgeSQL ORM options
|
|
90
|
+
* @returns Total number of deleted cache entries
|
|
91
|
+
*/
|
|
92
|
+
async function clearCursorCache(
|
|
93
|
+
tables: string[],
|
|
94
|
+
cursor: string,
|
|
95
|
+
options: ForgeSqlOrmOptions,
|
|
96
|
+
): Promise<number> {
|
|
97
|
+
const cacheEntityName = options.cacheEntityName;
|
|
98
|
+
if (!cacheEntityName) {
|
|
99
|
+
throw new Error("cacheEntityName is not configured");
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const entityQueryName = options.cacheEntityQueryName ?? CACHE_CONSTANTS.DEFAULT_ENTITY_QUERY_NAME;
|
|
103
|
+
let filters = new Filter<{
|
|
104
|
+
[entityQueryName]: string;
|
|
105
|
+
}>();
|
|
106
|
+
|
|
107
|
+
for (const table of tables) {
|
|
108
|
+
const wrapIfNeeded = options.cacheWrapTable ? `\`${table}\`` : table;
|
|
109
|
+
filters.or(entityQueryName, FilterConditions.contains(wrapIfNeeded?.toLowerCase()));
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
let entityQueryBuilder = kvs
|
|
113
|
+
.entity<{
|
|
114
|
+
[entityQueryName]: string;
|
|
115
|
+
}>(cacheEntityName)
|
|
116
|
+
.query()
|
|
117
|
+
.index(entityQueryName)
|
|
118
|
+
.filters(filters);
|
|
119
|
+
|
|
120
|
+
if (cursor) {
|
|
121
|
+
entityQueryBuilder = entityQueryBuilder.cursor(cursor);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
const listResult = await entityQueryBuilder.limit(100).getMany();
|
|
125
|
+
|
|
126
|
+
if (options.logRawSqlQuery) {
|
|
127
|
+
console.warn(`clear cache Records: ${JSON.stringify(listResult.results.map((r) => r.key))}`);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
await deleteCacheEntriesInBatches(listResult.results, cacheEntityName);
|
|
131
|
+
|
|
132
|
+
if (listResult.nextCursor) {
|
|
133
|
+
return (
|
|
134
|
+
listResult.results.length + (await clearCursorCache(tables, listResult.nextCursor, options))
|
|
135
|
+
);
|
|
136
|
+
} else {
|
|
137
|
+
return listResult.results.length;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Clears expired cache entries using cursor-based pagination.
|
|
143
|
+
*
|
|
144
|
+
* @param cursor - Pagination cursor for large result sets
|
|
145
|
+
* @param options - ForgeSQL ORM options
|
|
146
|
+
* @returns Total number of deleted expired cache entries
|
|
147
|
+
*/
|
|
148
|
+
async function clearExpirationCursorCache(
|
|
149
|
+
cursor: string,
|
|
150
|
+
options: ForgeSqlOrmOptions,
|
|
151
|
+
): Promise<number> {
|
|
152
|
+
const cacheEntityName = options.cacheEntityName;
|
|
153
|
+
if (!cacheEntityName) {
|
|
154
|
+
throw new Error("cacheEntityName is not configured");
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const entityExpirationName =
|
|
158
|
+
options.cacheEntityExpirationName ?? CACHE_CONSTANTS.DEFAULT_EXPIRATION_NAME;
|
|
159
|
+
let entityQueryBuilder = kvs
|
|
160
|
+
.entity<{
|
|
161
|
+
[entityExpirationName]: number;
|
|
162
|
+
}>(cacheEntityName)
|
|
163
|
+
.query()
|
|
164
|
+
.index(entityExpirationName)
|
|
165
|
+
.where(WhereConditions.lessThan(Math.floor(DateTime.now().toSeconds())));
|
|
166
|
+
|
|
167
|
+
if (cursor) {
|
|
168
|
+
entityQueryBuilder = entityQueryBuilder.cursor(cursor);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
const listResult = await entityQueryBuilder.limit(100).getMany();
|
|
172
|
+
|
|
173
|
+
if (options.logRawSqlQuery) {
|
|
174
|
+
console.warn(`clear expired Records: ${JSON.stringify(listResult.results.map((r) => r.key))}`);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
await deleteCacheEntriesInBatches(listResult.results, cacheEntityName);
|
|
178
|
+
|
|
179
|
+
if (listResult.nextCursor) {
|
|
180
|
+
return (
|
|
181
|
+
listResult.results.length + (await clearExpirationCursorCache(listResult.nextCursor, options))
|
|
182
|
+
);
|
|
183
|
+
} else {
|
|
184
|
+
return listResult.results.length;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* Executes a function with retry logic and exponential backoff.
|
|
190
|
+
*
|
|
191
|
+
* @param operation - Function to execute with retry
|
|
192
|
+
* @param operationName - Name of the operation for logging
|
|
193
|
+
* @param options - ForgeSQL ORM options for logging
|
|
194
|
+
* @returns Promise that resolves to the operation result
|
|
195
|
+
*/
|
|
196
|
+
async function executeWithRetry<T>(operation: () => Promise<T>, operationName: string): Promise<T> {
|
|
197
|
+
let attempt = 0;
|
|
198
|
+
let delay = CACHE_CONSTANTS.INITIAL_RETRY_DELAY;
|
|
199
|
+
|
|
200
|
+
while (attempt < CACHE_CONSTANTS.MAX_RETRY_ATTEMPTS) {
|
|
201
|
+
try {
|
|
202
|
+
return await operation();
|
|
203
|
+
} catch (err: any) {
|
|
204
|
+
console.warn(`Error during ${operationName}: ${err.message}, retry ${attempt}`, err);
|
|
205
|
+
attempt++;
|
|
206
|
+
|
|
207
|
+
if (attempt >= CACHE_CONSTANTS.MAX_RETRY_ATTEMPTS) {
|
|
208
|
+
console.error(`Error during ${operationName}: ${err.message}`, err);
|
|
209
|
+
throw err;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
213
|
+
delay *= CACHE_CONSTANTS.RETRY_DELAY_MULTIPLIER;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
throw new Error(`Maximum retry attempts exceeded for ${operationName}`);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Clears cache for a specific table.
|
|
222
|
+
* Uses cache context if available, otherwise clears immediately.
|
|
223
|
+
*
|
|
224
|
+
* @param schema - The table schema to clear cache for
|
|
225
|
+
* @param options - ForgeSQL ORM options
|
|
226
|
+
*/
|
|
227
|
+
export async function clearCache<T extends AnyMySqlTable>(
|
|
228
|
+
schema: T,
|
|
229
|
+
options: ForgeSqlOrmOptions,
|
|
230
|
+
): Promise<void> {
|
|
231
|
+
const tableName = getTableName(schema);
|
|
232
|
+
if (cacheApplicationContext.getStore()) {
|
|
233
|
+
cacheApplicationContext.getStore()?.tables.add(tableName);
|
|
234
|
+
} else {
|
|
235
|
+
await clearTablesCache([tableName], options);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* Clears cache for multiple tables with retry logic and performance logging.
|
|
241
|
+
*
|
|
242
|
+
* @param tables - Array of table names to clear cache for
|
|
243
|
+
* @param options - ForgeSQL ORM options
|
|
244
|
+
* @returns Promise that resolves when cache clearing is complete
|
|
245
|
+
*/
|
|
246
|
+
export async function clearTablesCache(
|
|
247
|
+
tables: string[],
|
|
248
|
+
options: ForgeSqlOrmOptions,
|
|
249
|
+
): Promise<void> {
|
|
250
|
+
if (!options.cacheEntityName) {
|
|
251
|
+
throw new Error("cacheEntityName is not configured");
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
const startTime = DateTime.now();
|
|
255
|
+
let totalRecords = 0;
|
|
256
|
+
|
|
257
|
+
try {
|
|
258
|
+
totalRecords = await executeWithRetry(
|
|
259
|
+
() => clearCursorCache(tables, "", options),
|
|
260
|
+
"clearing cache",
|
|
261
|
+
);
|
|
262
|
+
} finally {
|
|
263
|
+
if (options.logRawSqlQuery) {
|
|
264
|
+
const duration = DateTime.now().toSeconds() - startTime.toSeconds();
|
|
265
|
+
console.info(`Cleared ${totalRecords} cache records in ${duration} seconds`);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* Clears expired cache entries with retry logic and performance logging.
|
|
271
|
+
*
|
|
272
|
+
* @param options - ForgeSQL ORM options
|
|
273
|
+
* @returns Promise that resolves when expired cache clearing is complete
|
|
274
|
+
*/
|
|
275
|
+
export async function clearExpiredCache(options: ForgeSqlOrmOptions): Promise<void> {
|
|
276
|
+
if (!options.cacheEntityName) {
|
|
277
|
+
throw new Error("cacheEntityName is not configured");
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
const startTime = DateTime.now();
|
|
281
|
+
let totalRecords = 0;
|
|
282
|
+
|
|
283
|
+
try {
|
|
284
|
+
totalRecords = await executeWithRetry(
|
|
285
|
+
() => clearExpirationCursorCache("", options),
|
|
286
|
+
"clearing expired cache",
|
|
287
|
+
);
|
|
288
|
+
} finally {
|
|
289
|
+
const duration = DateTime.now().toSeconds() - startTime.toSeconds();
|
|
290
|
+
console.info(`Cleared ${totalRecords} expired cache records in ${duration} seconds`);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/**
|
|
295
|
+
* Retrieves data from cache if it exists and is not expired.
|
|
296
|
+
*
|
|
297
|
+
* @param query - Query object with toSQL method
|
|
298
|
+
* @param options - ForgeSQL ORM options
|
|
299
|
+
* @returns Cached data if found and valid, undefined otherwise
|
|
300
|
+
*/
|
|
301
|
+
export async function getFromCache<T>(
|
|
302
|
+
query: { toSQL: () => Query },
|
|
303
|
+
options: ForgeSqlOrmOptions,
|
|
304
|
+
): Promise<T | undefined> {
|
|
305
|
+
if (!options.cacheEntityName) {
|
|
306
|
+
throw new Error("cacheEntityName is not configured");
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
const entityQueryName = options.cacheEntityQueryName ?? CACHE_CONSTANTS.DEFAULT_ENTITY_QUERY_NAME;
|
|
310
|
+
const expirationName =
|
|
311
|
+
options.cacheEntityExpirationName ?? CACHE_CONSTANTS.DEFAULT_EXPIRATION_NAME;
|
|
312
|
+
const dataName = options.cacheEntityDataName ?? CACHE_CONSTANTS.DEFAULT_DATA_NAME;
|
|
313
|
+
|
|
314
|
+
const sqlQuery = query.toSQL();
|
|
315
|
+
const key = hashKey(sqlQuery);
|
|
316
|
+
|
|
317
|
+
// Skip cache if table is in cache context (will be cleared)
|
|
318
|
+
if (await isTableContainsTableInCacheContext(sqlQuery.sql, options)) {
|
|
319
|
+
if (options.logRawSqlQuery) {
|
|
320
|
+
console.warn(`Context contains value to clear. Skip getting from cache`);
|
|
321
|
+
}
|
|
322
|
+
return undefined;
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
try {
|
|
326
|
+
const cacheResult = await kvs.entity<CacheEntity>(options.cacheEntityName).get(key);
|
|
327
|
+
|
|
328
|
+
if (
|
|
329
|
+
cacheResult &&
|
|
330
|
+
(cacheResult[expirationName] as number) >= getCurrentTime() &&
|
|
331
|
+
sqlQuery.sql.toLowerCase() === cacheResult[entityQueryName]
|
|
332
|
+
) {
|
|
333
|
+
if (options.logRawSqlQuery) {
|
|
334
|
+
console.warn(`Get value from cache, cacheKey: ${key}`);
|
|
335
|
+
}
|
|
336
|
+
const results = cacheResult[dataName];
|
|
337
|
+
return JSON.parse(results as string);
|
|
338
|
+
}
|
|
339
|
+
} catch (error: any) {
|
|
340
|
+
console.error(`Error getting from cache: ${error.message}`, error);
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
return undefined;
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/**
|
|
347
|
+
* Stores query results in cache with specified TTL.
|
|
348
|
+
*
|
|
349
|
+
* @param query - Query object with toSQL method
|
|
350
|
+
* @param options - ForgeSQL ORM options
|
|
351
|
+
* @param results - Data to cache
|
|
352
|
+
* @param cacheTtl - Time to live in seconds
|
|
353
|
+
* @returns Promise that resolves when data is stored in cache
|
|
354
|
+
*/
|
|
355
|
+
export async function setCacheResult(
|
|
356
|
+
query: { toSQL: () => Query },
|
|
357
|
+
options: ForgeSqlOrmOptions,
|
|
358
|
+
results: unknown,
|
|
359
|
+
cacheTtl: number,
|
|
360
|
+
): Promise<void> {
|
|
361
|
+
if (!options.cacheEntityName) {
|
|
362
|
+
throw new Error("cacheEntityName is not configured");
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
try {
|
|
366
|
+
const entityQueryName =
|
|
367
|
+
options.cacheEntityQueryName ?? CACHE_CONSTANTS.DEFAULT_ENTITY_QUERY_NAME;
|
|
368
|
+
const expirationName =
|
|
369
|
+
options.cacheEntityExpirationName ?? CACHE_CONSTANTS.DEFAULT_EXPIRATION_NAME;
|
|
370
|
+
const dataName = options.cacheEntityDataName ?? CACHE_CONSTANTS.DEFAULT_DATA_NAME;
|
|
371
|
+
|
|
372
|
+
const sqlQuery = query.toSQL();
|
|
373
|
+
|
|
374
|
+
// Skip cache if table is in cache context (will be cleared)
|
|
375
|
+
if (await isTableContainsTableInCacheContext(sqlQuery.sql, options)) {
|
|
376
|
+
if (options.logRawSqlQuery) {
|
|
377
|
+
console.warn(`Context contains value to clear. Skip setting from cache`);
|
|
378
|
+
}
|
|
379
|
+
return;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
const key = hashKey(sqlQuery);
|
|
383
|
+
|
|
384
|
+
await kvs
|
|
385
|
+
.transact()
|
|
386
|
+
.set(
|
|
387
|
+
key,
|
|
388
|
+
{
|
|
389
|
+
[entityQueryName]: sqlQuery.sql.toLowerCase(),
|
|
390
|
+
[expirationName]: nowPlusSeconds(cacheTtl),
|
|
391
|
+
[dataName]: JSON.stringify(results),
|
|
392
|
+
},
|
|
393
|
+
{ entityName: options.cacheEntityName },
|
|
394
|
+
)
|
|
395
|
+
.execute();
|
|
396
|
+
|
|
397
|
+
if (options.logRawSqlQuery) {
|
|
398
|
+
console.warn(`Store value to cache, cacheKey: ${key}`);
|
|
399
|
+
}
|
|
400
|
+
} catch (error: any) {
|
|
401
|
+
console.error(`Error setting cache: ${error.message}`, error);
|
|
402
|
+
}
|
|
403
|
+
}
|
package/src/utils/sqlUtils.ts
CHANGED
|
@@ -74,6 +74,22 @@ export const parseDateTime = (value: string | Date, format: string): Date => {
|
|
|
74
74
|
return result;
|
|
75
75
|
};
|
|
76
76
|
|
|
77
|
+
/**
|
|
78
|
+
* Helper function to validate and format Date objects using DateTime
|
|
79
|
+
* @param value - Date object to validate and format
|
|
80
|
+
* @param format - DateTime format string
|
|
81
|
+
* @returns Formatted date string
|
|
82
|
+
* @throws Error if date is invalid
|
|
83
|
+
*/
|
|
84
|
+
export function formatDateTime(value: Date, format: string): string {
|
|
85
|
+
const fromJSDate = DateTime.fromJSDate(value);
|
|
86
|
+
if (fromJSDate.isValid) {
|
|
87
|
+
return fromJSDate.toFormat(format);
|
|
88
|
+
} else {
|
|
89
|
+
throw new Error("Invalid Date");
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
77
93
|
/**
|
|
78
94
|
* Gets primary keys from the schema.
|
|
79
95
|
* @template T - The type of the table schema
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { clearExpiredCache } from "../utils/cacheUtils";
|
|
2
|
+
import { ForgeSqlOrmOptions } from "../core/ForgeSQLQueryBuilder";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Scheduler trigger for clearing expired cache entries.
|
|
6
|
+
*
|
|
7
|
+
* This trigger should be configured as a Forge scheduler to automatically
|
|
8
|
+
* clean up expired cache entries based on their TTL (Time To Live).
|
|
9
|
+
*
|
|
10
|
+
* @param options - Optional ForgeSQL ORM configuration. If not provided,
|
|
11
|
+
* uses default cache settings with cacheEntityName: "cache"
|
|
12
|
+
* @returns Promise that resolves to HTTP response object
|
|
13
|
+
*
|
|
14
|
+
* @example
|
|
15
|
+
* ```typescript
|
|
16
|
+
* // In your index.ts
|
|
17
|
+
* import { clearCacheSchedulerTrigger } from "forge-sql-orm";
|
|
18
|
+
*
|
|
19
|
+
* export const clearCache = () => {
|
|
20
|
+
* return clearCacheSchedulerTrigger({
|
|
21
|
+
* cacheEntityName: "cache",
|
|
22
|
+
* logRawSqlQuery: true
|
|
23
|
+
* });
|
|
24
|
+
* };
|
|
25
|
+
* ```
|
|
26
|
+
*
|
|
27
|
+
* @example
|
|
28
|
+
* ```yaml
|
|
29
|
+
* # In manifest.yml
|
|
30
|
+
* scheduledTrigger:
|
|
31
|
+
* - key: clear-cache-trigger
|
|
32
|
+
* function: clearCache
|
|
33
|
+
* interval: fiveMinute
|
|
34
|
+
*
|
|
35
|
+
* function:
|
|
36
|
+
* - key: clearCache
|
|
37
|
+
* handler: index.clearCache
|
|
38
|
+
* ```
|
|
39
|
+
*/
|
|
40
|
+
export const clearCacheSchedulerTrigger = async (options?: ForgeSqlOrmOptions) => {
|
|
41
|
+
try {
|
|
42
|
+
const newOptions: ForgeSqlOrmOptions = options ?? {
|
|
43
|
+
logRawSqlQuery: false,
|
|
44
|
+
disableOptimisticLocking: false,
|
|
45
|
+
cacheTTL: 120,
|
|
46
|
+
cacheEntityName: "cache",
|
|
47
|
+
cacheEntityQueryName: "sql",
|
|
48
|
+
cacheEntityExpirationName: "expiration",
|
|
49
|
+
cacheEntityDataName: "data",
|
|
50
|
+
};
|
|
51
|
+
if (!newOptions.cacheEntityName) {
|
|
52
|
+
throw new Error("cacheEntityName is not configured");
|
|
53
|
+
}
|
|
54
|
+
await clearExpiredCache(newOptions);
|
|
55
|
+
|
|
56
|
+
return {
|
|
57
|
+
headers: { "Content-Type": ["application/json"] },
|
|
58
|
+
statusCode: 200,
|
|
59
|
+
statusText: "OK",
|
|
60
|
+
body: JSON.stringify({
|
|
61
|
+
success: true,
|
|
62
|
+
message: "Cache cleanup completed successfully",
|
|
63
|
+
timestamp: new Date().toISOString(),
|
|
64
|
+
}),
|
|
65
|
+
};
|
|
66
|
+
} catch (error) {
|
|
67
|
+
console.error("Error during cache cleanup: ", JSON.stringify(error));
|
|
68
|
+
return {
|
|
69
|
+
headers: { "Content-Type": ["application/json"] },
|
|
70
|
+
statusCode: 500,
|
|
71
|
+
statusText: "Internal Server Error",
|
|
72
|
+
body: JSON.stringify({
|
|
73
|
+
success: false,
|
|
74
|
+
error: error instanceof Error ? error.message : "Unknown error during cache cleanup",
|
|
75
|
+
timestamp: new Date().toISOString(),
|
|
76
|
+
}),
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
};
|