@databricks/appkit 0.1.5 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +57 -2
- package/CLAUDE.md +57 -2
- package/NOTICE.md +2 -0
- package/README.md +21 -15
- package/bin/appkit-lint.js +129 -0
- package/dist/analytics/analytics.d.ts.map +1 -1
- package/dist/analytics/analytics.js +33 -33
- package/dist/analytics/analytics.js.map +1 -1
- package/dist/analytics/query.js +8 -2
- package/dist/analytics/query.js.map +1 -1
- package/dist/app/index.d.ts +5 -1
- package/dist/app/index.d.ts.map +1 -1
- package/dist/app/index.js +41 -10
- package/dist/app/index.js.map +1 -1
- package/dist/appkit/package.js +1 -1
- package/dist/cache/index.d.ts.map +1 -1
- package/dist/cache/index.js +24 -3
- package/dist/cache/index.js.map +1 -1
- package/dist/cache/storage/persistent.js +12 -6
- package/dist/cache/storage/persistent.js.map +1 -1
- package/dist/connectors/lakebase/client.js +25 -14
- package/dist/connectors/lakebase/client.js.map +1 -1
- package/dist/connectors/sql-warehouse/client.js +68 -28
- package/dist/connectors/sql-warehouse/client.js.map +1 -1
- package/dist/context/service-context.js +13 -8
- package/dist/context/service-context.js.map +1 -1
- package/dist/errors/authentication.d.ts +38 -0
- package/dist/errors/authentication.d.ts.map +1 -0
- package/dist/errors/authentication.js +48 -0
- package/dist/errors/authentication.js.map +1 -0
- package/dist/errors/base.d.ts +58 -0
- package/dist/errors/base.d.ts.map +1 -0
- package/dist/errors/base.js +70 -0
- package/dist/errors/base.js.map +1 -0
- package/dist/errors/configuration.d.ts +38 -0
- package/dist/errors/configuration.d.ts.map +1 -0
- package/dist/errors/configuration.js +45 -0
- package/dist/errors/configuration.js.map +1 -0
- package/dist/errors/connection.d.ts +42 -0
- package/dist/errors/connection.d.ts.map +1 -0
- package/dist/errors/connection.js +54 -0
- package/dist/errors/connection.js.map +1 -0
- package/dist/errors/execution.d.ts +42 -0
- package/dist/errors/execution.d.ts.map +1 -0
- package/dist/errors/execution.js +51 -0
- package/dist/errors/execution.js.map +1 -0
- package/dist/errors/index.js +28 -0
- package/dist/errors/index.js.map +1 -0
- package/dist/errors/initialization.d.ts +34 -0
- package/dist/errors/initialization.d.ts.map +1 -0
- package/dist/errors/initialization.js +42 -0
- package/dist/errors/initialization.js.map +1 -0
- package/dist/errors/server.d.ts +38 -0
- package/dist/errors/server.d.ts.map +1 -0
- package/dist/errors/server.js +45 -0
- package/dist/errors/server.js.map +1 -0
- package/dist/errors/tunnel.d.ts +38 -0
- package/dist/errors/tunnel.d.ts.map +1 -0
- package/dist/errors/tunnel.js +51 -0
- package/dist/errors/tunnel.js.map +1 -0
- package/dist/errors/validation.d.ts +36 -0
- package/dist/errors/validation.d.ts.map +1 -0
- package/dist/errors/validation.js +45 -0
- package/dist/errors/validation.js.map +1 -0
- package/dist/index.d.ts +12 -3
- package/dist/index.js +18 -3
- package/dist/index.js.map +1 -0
- package/dist/logging/logger.js +179 -0
- package/dist/logging/logger.js.map +1 -0
- package/dist/logging/sampling.js +56 -0
- package/dist/logging/sampling.js.map +1 -0
- package/dist/logging/wide-event-emitter.js +108 -0
- package/dist/logging/wide-event-emitter.js.map +1 -0
- package/dist/logging/wide-event.js +167 -0
- package/dist/logging/wide-event.js.map +1 -0
- package/dist/plugin/dev-reader.d.ts.map +1 -1
- package/dist/plugin/dev-reader.js +8 -3
- package/dist/plugin/dev-reader.js.map +1 -1
- package/dist/plugin/interceptors/cache.js.map +1 -1
- package/dist/plugin/interceptors/retry.js +10 -2
- package/dist/plugin/interceptors/retry.js.map +1 -1
- package/dist/plugin/interceptors/telemetry.js +24 -9
- package/dist/plugin/interceptors/telemetry.js.map +1 -1
- package/dist/plugin/interceptors/timeout.js +4 -0
- package/dist/plugin/interceptors/timeout.js.map +1 -1
- package/dist/plugin/plugin.d.ts +1 -1
- package/dist/plugin/plugin.d.ts.map +1 -1
- package/dist/plugin/plugin.js +9 -4
- package/dist/plugin/plugin.js.map +1 -1
- package/dist/server/index.d.ts.map +1 -1
- package/dist/server/index.js +22 -17
- package/dist/server/index.js.map +1 -1
- package/dist/server/remote-tunnel/remote-tunnel-controller.js +4 -2
- package/dist/server/remote-tunnel/remote-tunnel-controller.js.map +1 -1
- package/dist/server/remote-tunnel/remote-tunnel-manager.js +10 -8
- package/dist/server/remote-tunnel/remote-tunnel-manager.js.map +1 -1
- package/dist/server/vite-dev-server.js +8 -3
- package/dist/server/vite-dev-server.js.map +1 -1
- package/dist/stream/arrow-stream-processor.js +13 -6
- package/dist/stream/arrow-stream-processor.js.map +1 -1
- package/dist/stream/buffers.js +5 -1
- package/dist/stream/buffers.js.map +1 -1
- package/dist/stream/stream-manager.d.ts.map +1 -1
- package/dist/stream/stream-manager.js +47 -36
- package/dist/stream/stream-manager.js.map +1 -1
- package/dist/stream/types.js.map +1 -1
- package/dist/telemetry/index.d.ts +2 -2
- package/dist/telemetry/index.js +2 -2
- package/dist/telemetry/instrumentations.js +14 -10
- package/dist/telemetry/instrumentations.js.map +1 -1
- package/dist/telemetry/telemetry-manager.js +8 -6
- package/dist/telemetry/telemetry-manager.js.map +1 -1
- package/dist/telemetry/trace-sampler.js +33 -0
- package/dist/telemetry/trace-sampler.js.map +1 -0
- package/dist/type-generator/index.js +4 -2
- package/dist/type-generator/index.js.map +1 -1
- package/dist/type-generator/query-registry.js +13 -3
- package/dist/type-generator/query-registry.js.map +1 -1
- package/dist/type-generator/vite-plugin.d.ts.map +1 -1
- package/dist/type-generator/vite-plugin.js +5 -3
- package/dist/type-generator/vite-plugin.js.map +1 -1
- package/dist/utils/env-validator.js +5 -1
- package/dist/utils/env-validator.js.map +1 -1
- package/dist/utils/path-exclusions.js +66 -0
- package/dist/utils/path-exclusions.js.map +1 -0
- package/llms.txt +57 -2
- package/package.json +4 -1
package/dist/app/index.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
import { createLogger } from "../logging/logger.js";
|
|
1
2
|
import fs from "node:fs/promises";
|
|
2
3
|
import path from "node:path";
|
|
3
4
|
|
|
4
5
|
//#region src/app/index.ts
|
|
6
|
+
const logger = createLogger("app");
|
|
5
7
|
var AppManager = class {
|
|
6
8
|
/**
|
|
7
9
|
* Retrieves a query file by key from the queries directory
|
|
@@ -14,31 +16,60 @@ var AppManager = class {
|
|
|
14
16
|
*/
|
|
15
17
|
async getAppQuery(queryKey, req, devFileReader) {
|
|
16
18
|
if (!queryKey || !/^[a-zA-Z0-9_-]+$/.test(queryKey)) {
|
|
17
|
-
|
|
19
|
+
logger.error("Invalid query key format: %s. Only alphanumeric characters, underscores, and hyphens are allowed.", queryKey);
|
|
18
20
|
return null;
|
|
19
21
|
}
|
|
20
|
-
const queryFilePath = path.join(process.cwd(), "config/queries", `${queryKey}.sql`);
|
|
21
|
-
const resolvedPath = path.resolve(queryFilePath);
|
|
22
22
|
const queriesDir = path.resolve(process.cwd(), "config/queries");
|
|
23
|
-
|
|
24
|
-
|
|
23
|
+
const oboFileName = `${queryKey}.obo.sql`;
|
|
24
|
+
const defaultFileName = `${queryKey}.sql`;
|
|
25
|
+
let queryFileName = null;
|
|
26
|
+
let isAsUser = false;
|
|
27
|
+
try {
|
|
28
|
+
const files = await fs.readdir(queriesDir);
|
|
29
|
+
if (files.includes(oboFileName)) {
|
|
30
|
+
queryFileName = oboFileName;
|
|
31
|
+
isAsUser = true;
|
|
32
|
+
if (files.includes(defaultFileName)) logger.warn(`Both ${oboFileName} and ${defaultFileName} found for query ${queryKey}. Using ${oboFileName}.`);
|
|
33
|
+
} else if (files.includes(defaultFileName)) {
|
|
34
|
+
queryFileName = defaultFileName;
|
|
35
|
+
isAsUser = false;
|
|
36
|
+
}
|
|
37
|
+
} catch (error) {
|
|
38
|
+
logger.error(`Failed to read queries directory: ${error.message}`);
|
|
39
|
+
return null;
|
|
40
|
+
}
|
|
41
|
+
if (!queryFileName) {
|
|
42
|
+
logger.error(`Query file not found: ${queryKey}`);
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
45
|
+
const queryFilePath = path.join(queriesDir, queryFileName);
|
|
46
|
+
const resolvedPath = path.resolve(queryFilePath);
|
|
47
|
+
const resolvedQueriesDir = path.resolve(queriesDir);
|
|
48
|
+
if (!resolvedPath.startsWith(resolvedQueriesDir)) {
|
|
49
|
+
logger.error(`Invalid query path: path traversal detected`);
|
|
25
50
|
return null;
|
|
26
51
|
}
|
|
27
52
|
if (req?.query?.dev !== void 0 && devFileReader && req) try {
|
|
28
53
|
const relativePath = path.relative(process.cwd(), resolvedPath);
|
|
29
|
-
return
|
|
54
|
+
return {
|
|
55
|
+
query: await devFileReader.readFile(relativePath, req),
|
|
56
|
+
isAsUser
|
|
57
|
+
};
|
|
30
58
|
} catch (error) {
|
|
31
|
-
|
|
59
|
+
logger.error(`Failed to read query from dev tunnel: ${error.message}`);
|
|
32
60
|
return null;
|
|
33
61
|
}
|
|
34
62
|
try {
|
|
35
|
-
return
|
|
63
|
+
return {
|
|
64
|
+
query: await fs.readFile(resolvedPath, "utf8"),
|
|
65
|
+
isAsUser
|
|
66
|
+
};
|
|
36
67
|
} catch (error) {
|
|
37
68
|
if (error.code === "ENOENT") {
|
|
38
|
-
|
|
69
|
+
logger.error(`Failed to read query from server filesystem: ${error.message}`);
|
|
39
70
|
return null;
|
|
40
71
|
}
|
|
41
|
-
|
|
72
|
+
logger.error(`Failed to read query from server filesystem: ${error.message}`);
|
|
42
73
|
return null;
|
|
43
74
|
}
|
|
44
75
|
}
|
package/dist/app/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","names":[],"sources":["../../src/app/index.ts"],"sourcesContent":["import fs from \"node:fs/promises\";\nimport path from \"node:path\";\n\ninterface RequestLike {\n query?: Record<string, any>;\n headers: Record<string, string | string[] | undefined>;\n}\n\ninterface DevFileReader {\n readFile(filePath: string, req: RequestLike): Promise<string>;\n}\n\nexport class AppManager {\n /**\n * Retrieves a query file by key from the queries directory\n * In dev mode with a request context, reads from local filesystem via WebSocket\n * @param queryKey - The query file name (without extension)\n * @param req - Optional request object to detect dev mode\n * @param devFileReader - Optional DevFileReader instance to read files from local filesystem\n * @returns The query content as a string\n * @throws Error if query key is invalid or file not found\n */\n async getAppQuery(\n queryKey: string,\n req?: RequestLike,\n devFileReader?: DevFileReader,\n ): Promise<
|
|
1
|
+
{"version":3,"file":"index.js","names":[],"sources":["../../src/app/index.ts"],"sourcesContent":["import fs from \"node:fs/promises\";\nimport path from \"node:path\";\nimport { createLogger } from \"../logging/logger\";\n\nconst logger = createLogger(\"app\");\n\ninterface RequestLike {\n query?: Record<string, any>;\n headers: Record<string, string | string[] | undefined>;\n}\n\ninterface DevFileReader {\n readFile(filePath: string, req: RequestLike): Promise<string>;\n}\n\ninterface QueryResult {\n query: string;\n isAsUser: boolean;\n}\n\nexport class AppManager {\n /**\n * Retrieves a query file by key from the queries directory\n * In dev mode with a request context, reads from local filesystem via WebSocket\n * @param queryKey - The query file name (without extension)\n * @param req - Optional request object to detect dev mode\n * @param devFileReader - Optional DevFileReader instance to read files from local filesystem\n * @returns The query content as a string\n * @throws Error if query key is invalid or file not found\n */\n async getAppQuery(\n queryKey: string,\n req?: RequestLike,\n devFileReader?: DevFileReader,\n ): Promise<QueryResult | null> {\n // Security: Sanitize query key to prevent path traversal\n if (!queryKey || !/^[a-zA-Z0-9_-]+$/.test(queryKey)) {\n logger.error(\n \"Invalid query key format: %s. Only alphanumeric characters, underscores, and hyphens are allowed.\",\n queryKey,\n );\n return null;\n }\n\n const queriesDir = path.resolve(process.cwd(), \"config/queries\");\n\n // priority order: .obo.sql first (asUser), then .sql (default)\n const oboFileName = `${queryKey}.obo.sql`;\n const defaultFileName = `${queryKey}.sql`;\n\n let queryFileName: string | null = null;\n let isAsUser: boolean = false;\n\n try {\n const files = await fs.readdir(queriesDir);\n\n // check for OBO query first\n if (files.includes(oboFileName)) {\n queryFileName = oboFileName;\n isAsUser = true;\n\n // check for both files and warn if both are present\n if (files.includes(defaultFileName)) {\n logger.warn(\n `Both ${oboFileName} and ${defaultFileName} found for query ${queryKey}. Using ${oboFileName}.`,\n );\n }\n // check for default query if OBO query is not present\n } else if (files.includes(defaultFileName)) {\n queryFileName = defaultFileName;\n isAsUser = false;\n }\n } catch (error) {\n logger.error(\n `Failed to read queries directory: ${(error as Error).message}`,\n );\n return null;\n }\n\n if (!queryFileName) {\n logger.error(`Query file not found: ${queryKey}`);\n return null;\n }\n\n const queryFilePath = path.join(queriesDir, queryFileName);\n\n // security: validate resolved path is within queries directory\n const resolvedPath = path.resolve(queryFilePath);\n const resolvedQueriesDir = path.resolve(queriesDir);\n\n if (!resolvedPath.startsWith(resolvedQueriesDir)) {\n logger.error(`Invalid query path: path traversal detected`);\n return null;\n }\n\n // check if we're in dev mode and should use WebSocket\n const isDevMode = req?.query?.dev !== undefined;\n if (isDevMode && devFileReader && req) {\n try {\n const relativePath = path.relative(process.cwd(), resolvedPath);\n return {\n query: await devFileReader.readFile(relativePath, req),\n isAsUser,\n };\n } catch (error) {\n logger.error(\n `Failed to read query from dev tunnel: ${(error as Error).message}`,\n );\n return null;\n }\n }\n\n // production mode: read from server filesystem\n try {\n const query = await fs.readFile(resolvedPath, \"utf8\");\n return { query, isAsUser };\n } catch (error) {\n if ((error as NodeJS.ErrnoException).code === \"ENOENT\") {\n logger.error(\n `Failed to read query from server filesystem: ${(error as Error).message}`,\n );\n return null;\n }\n\n logger.error(\n `Failed to read query from server filesystem: ${(error as Error).message}`,\n );\n return null;\n }\n }\n}\n\nexport type { DevFileReader, QueryResult, RequestLike };\n"],"mappings":";;;;;AAIA,MAAM,SAAS,aAAa,MAAM;AAgBlC,IAAa,aAAb,MAAwB;;;;;;;;;;CAUtB,MAAM,YACJ,UACA,KACA,eAC6B;AAE7B,MAAI,CAAC,YAAY,CAAC,mBAAmB,KAAK,SAAS,EAAE;AACnD,UAAO,MACL,qGACA,SACD;AACD,UAAO;;EAGT,MAAM,aAAa,KAAK,QAAQ,QAAQ,KAAK,EAAE,iBAAiB;EAGhE,MAAM,cAAc,GAAG,SAAS;EAChC,MAAM,kBAAkB,GAAG,SAAS;EAEpC,IAAI,gBAA+B;EACnC,IAAI,WAAoB;AAExB,MAAI;GACF,MAAM,QAAQ,MAAM,GAAG,QAAQ,WAAW;AAG1C,OAAI,MAAM,SAAS,YAAY,EAAE;AAC/B,oBAAgB;AAChB,eAAW;AAGX,QAAI,MAAM,SAAS,gBAAgB,CACjC,QAAO,KACL,QAAQ,YAAY,OAAO,gBAAgB,mBAAmB,SAAS,UAAU,YAAY,GAC9F;cAGM,MAAM,SAAS,gBAAgB,EAAE;AAC1C,oBAAgB;AAChB,eAAW;;WAEN,OAAO;AACd,UAAO,MACL,qCAAsC,MAAgB,UACvD;AACD,UAAO;;AAGT,MAAI,CAAC,eAAe;AAClB,UAAO,MAAM,yBAAyB,WAAW;AACjD,UAAO;;EAGT,MAAM,gBAAgB,KAAK,KAAK,YAAY,cAAc;EAG1D,MAAM,eAAe,KAAK,QAAQ,cAAc;EAChD,MAAM,qBAAqB,KAAK,QAAQ,WAAW;AAEnD,MAAI,CAAC,aAAa,WAAW,mBAAmB,EAAE;AAChD,UAAO,MAAM,8CAA8C;AAC3D,UAAO;;AAKT,MADkB,KAAK,OAAO,QAAQ,UACrB,iBAAiB,IAChC,KAAI;GACF,MAAM,eAAe,KAAK,SAAS,QAAQ,KAAK,EAAE,aAAa;AAC/D,UAAO;IACL,OAAO,MAAM,cAAc,SAAS,cAAc,IAAI;IACtD;IACD;WACM,OAAO;AACd,UAAO,MACL,yCAA0C,MAAgB,UAC3D;AACD,UAAO;;AAKX,MAAI;AAEF,UAAO;IAAE,OADK,MAAM,GAAG,SAAS,cAAc,OAAO;IACrC;IAAU;WACnB,OAAO;AACd,OAAK,MAAgC,SAAS,UAAU;AACtD,WAAO,MACL,gDAAiD,MAAgB,UAClE;AACD,WAAO;;AAGT,UAAO,MACL,gDAAiD,MAAgB,UAClE;AACD,UAAO"}
|
package/dist/appkit/package.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/cache/index.ts"],"sourcesContent":[],"mappings":";;;;;;
|
|
1
|
+
{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/cache/index.ts"],"sourcesContent":[],"mappings":";;;;;;AA4BA;;;;;;;;;;;;AA+Q6B,cA/QhB,YAAA,CA+QgB;0BAkDlB,uBAAA;mBAEN,IAAA;iBAawB,QAAA;iBAMZ,WAAA;UAUS,OAAA;UA0BT,MAAA;UAQW,gBAAA;EAAO,QAAA,iBAAA;;;;;;;;;;;4BAnVP;;;;;;;;kCAmBX,QAAQ,eACpB,QAAQ;;;;;;;;;;;;;;;;;;;;;;+DAuFC,QAAQ;;MAGjB,QAAQ;;;;;;uBAkHgB,QAAQ;;;;;;;;;;6BAkD1B;;MAEN;;;;;;uBAawB;;WAMZ;;;;;;oBAUS;;;;;;;;;WA0BT;;;;;sBAQW"}
|
package/dist/cache/index.js
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
|
+
import { createLogger } from "../logging/logger.js";
|
|
1
2
|
import { TelemetryManager } from "../telemetry/telemetry-manager.js";
|
|
2
3
|
import { SpanStatusCode } from "../telemetry/index.js";
|
|
4
|
+
import { AppKitError } from "../errors/base.js";
|
|
5
|
+
import { ExecutionError } from "../errors/execution.js";
|
|
6
|
+
import { InitializationError } from "../errors/initialization.js";
|
|
7
|
+
import { init_errors } from "../errors/index.js";
|
|
3
8
|
import { deepMerge } from "../utils/merge.js";
|
|
4
9
|
import { LakebaseConnector } from "../connectors/lakebase/client.js";
|
|
5
10
|
import "../connectors/index.js";
|
|
@@ -11,6 +16,8 @@ import { createHash } from "node:crypto";
|
|
|
11
16
|
import { WorkspaceClient } from "@databricks/sdk-experimental";
|
|
12
17
|
|
|
13
18
|
//#region src/cache/index.ts
|
|
19
|
+
init_errors();
|
|
20
|
+
const logger = createLogger("cache");
|
|
14
21
|
/**
|
|
15
22
|
* Cache manager class to handle cache operations.
|
|
16
23
|
* Can be used with in-memory storage or persistent storage (Lakebase).
|
|
@@ -61,7 +68,7 @@ var CacheManager = class CacheManager {
|
|
|
61
68
|
* @returns CacheManager instance
|
|
62
69
|
*/
|
|
63
70
|
static getInstanceSync() {
|
|
64
|
-
if (!CacheManager.instance) throw
|
|
71
|
+
if (!CacheManager.instance) throw InitializationError.notInitialized("CacheManager", "Ensure AppKit.create() has completed before accessing the cache");
|
|
65
72
|
return CacheManager.instance;
|
|
66
73
|
}
|
|
67
74
|
/**
|
|
@@ -143,6 +150,10 @@ var CacheManager = class CacheManager {
|
|
|
143
150
|
span.setAttribute("cache.hit", true);
|
|
144
151
|
span.setStatus({ code: SpanStatusCode.OK });
|
|
145
152
|
this.telemetryMetrics.cacheHitCount.add(1, { "cache.key": cacheKey });
|
|
153
|
+
logger.event()?.setExecution({
|
|
154
|
+
cache_hit: true,
|
|
155
|
+
cache_key: cacheKey
|
|
156
|
+
});
|
|
146
157
|
return cached.value;
|
|
147
158
|
}
|
|
148
159
|
const inFlight = this.inFlightRequests.get(cacheKey);
|
|
@@ -155,12 +166,21 @@ var CacheManager = class CacheManager {
|
|
|
155
166
|
"cache.key": cacheKey,
|
|
156
167
|
"cache.deduplication": "true"
|
|
157
168
|
});
|
|
169
|
+
logger.event()?.setExecution({
|
|
170
|
+
cache_hit: true,
|
|
171
|
+
cache_key: cacheKey,
|
|
172
|
+
cache_deduplication: true
|
|
173
|
+
});
|
|
158
174
|
span.end();
|
|
159
175
|
return inFlight;
|
|
160
176
|
}
|
|
161
177
|
span.setAttribute("cache.hit", false);
|
|
162
178
|
span.addEvent("cache.miss", { "cache.key": cacheKey });
|
|
163
179
|
this.telemetryMetrics.cacheMissCount.add(1, { "cache.key": cacheKey });
|
|
180
|
+
logger.event()?.setExecution({
|
|
181
|
+
cache_hit: false,
|
|
182
|
+
cache_key: cacheKey
|
|
183
|
+
});
|
|
164
184
|
const promise = fn().then(async (result$1) => {
|
|
165
185
|
await this.set(cacheKey, result$1, options);
|
|
166
186
|
span.addEvent("cache.value_stored", {
|
|
@@ -171,7 +191,8 @@ var CacheManager = class CacheManager {
|
|
|
171
191
|
}).catch((error) => {
|
|
172
192
|
span.recordException(error);
|
|
173
193
|
span.setStatus({ code: SpanStatusCode.ERROR });
|
|
174
|
-
throw error;
|
|
194
|
+
if (error instanceof AppKitError) throw error;
|
|
195
|
+
throw ExecutionError.statementFailed(error instanceof Error ? error.message : String(error));
|
|
175
196
|
}).finally(() => {
|
|
176
197
|
this.inFlightRequests.delete(cacheKey);
|
|
177
198
|
});
|
|
@@ -218,7 +239,7 @@ var CacheManager = class CacheManager {
|
|
|
218
239
|
this.lastCleanupAttempt = now;
|
|
219
240
|
this.cleanupInProgress = true;
|
|
220
241
|
this.storage.cleanupExpired().catch((error) => {
|
|
221
|
-
|
|
242
|
+
logger.debug("Error cleaning up expired entries: %O", error);
|
|
222
243
|
}).finally(() => {
|
|
223
244
|
this.cleanupInProgress = false;
|
|
224
245
|
});
|
package/dist/cache/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","names":["result"],"sources":["../../src/cache/index.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport { WorkspaceClient } from \"@databricks/sdk-experimental\";\nimport type { CacheConfig, CacheStorage } from \"shared\";\nimport { LakebaseConnector } from \"@/connectors\";\nimport type { Counter, TelemetryProvider } from \"../telemetry\";\nimport { SpanStatusCode, TelemetryManager } from \"../telemetry\";\nimport { deepMerge } from \"../utils\";\nimport { cacheDefaults } from \"./defaults\";\nimport { InMemoryStorage, PersistentStorage } from \"./storage\";\n\n/**\n * Cache manager class to handle cache operations.\n * Can be used with in-memory storage or persistent storage (Lakebase).\n *\n * The cache is automatically initialized by AppKit. Use `getInstanceSync()` to access\n * the singleton instance after initialization.\n *\n * @internal\n * @example\n * ```typescript\n * const cache = CacheManager.getInstanceSync();\n * const result = await cache.getOrExecute([\"users\", userId], () => fetchUser(userId), userKey);\n * ```\n */\nexport class CacheManager {\n private static readonly MIN_CLEANUP_INTERVAL_MS = 60_000;\n private readonly name: string = \"cache-manager\";\n private static instance: CacheManager | null = null;\n private static initPromise: Promise<CacheManager> | null = null;\n\n private storage: CacheStorage;\n private config: CacheConfig;\n private inFlightRequests: Map<string, Promise<unknown>>;\n private cleanupInProgress: boolean;\n private lastCleanupAttempt: number;\n\n // Telemetry\n private telemetry: TelemetryProvider;\n private telemetryMetrics: {\n cacheHitCount: Counter;\n cacheMissCount: Counter;\n };\n\n private constructor(storage: CacheStorage, config: CacheConfig) {\n this.storage = storage;\n this.config = config;\n this.inFlightRequests = new Map();\n this.cleanupInProgress = false;\n this.lastCleanupAttempt = 0;\n\n this.telemetry = TelemetryManager.getProvider(\n this.name,\n this.config.telemetry,\n );\n this.telemetryMetrics = {\n cacheHitCount: this.telemetry.getMeter().createCounter(\"cache.hit\", {\n description: \"Total number of cache hits\",\n unit: \"1\",\n }),\n cacheMissCount: this.telemetry.getMeter().createCounter(\"cache.miss\", {\n description: \"Total number of cache misses\",\n unit: \"1\",\n }),\n };\n }\n\n /**\n * Get the singleton instance of the cache manager (sync version).\n *\n * Throws if not initialized - ensure AppKit.create() has completed first.\n * @returns CacheManager instance\n */\n static getInstanceSync(): CacheManager {\n if (!CacheManager.instance) {\n throw new Error(\n \"CacheManager not initialized. Ensure AppKit.create() has completed before accessing the cache.\",\n );\n }\n\n return CacheManager.instance;\n }\n\n /**\n * Initialize and get the singleton instance of the cache manager.\n * Called internally by AppKit - prefer `getInstanceSync()` for plugin access.\n * @param userConfig - User configuration for the cache manager\n * @returns CacheManager instance\n * @internal\n */\n static async getInstance(\n userConfig?: Partial<CacheConfig>,\n ): Promise<CacheManager> {\n if (CacheManager.instance) {\n return CacheManager.instance;\n }\n\n if (!CacheManager.initPromise) {\n CacheManager.initPromise = CacheManager.create(userConfig).then(\n (instance) => {\n CacheManager.instance = instance;\n return instance;\n },\n );\n }\n\n return CacheManager.initPromise;\n }\n\n /**\n * Create a new cache manager instance\n *\n * Storage selection logic:\n * 1. If `storage` provided and healthy → use provided storage\n * 2. If `storage` provided but unhealthy → fallback to InMemory (or disable if strictPersistence)\n * 3. If no `storage` provided and Lakebase available → use Lakebase\n * 4. If no `storage` provided and Lakebase unavailable → fallback to InMemory (or disable if strictPersistence)\n *\n * @param userConfig - User configuration for the cache manager\n * @returns CacheManager instance\n */\n private static async create(\n userConfig?: Partial<CacheConfig>,\n ): Promise<CacheManager> {\n const config = deepMerge(cacheDefaults, userConfig);\n\n if (config.storage) {\n const isHealthy = await config.storage.healthCheck();\n if (isHealthy) {\n return new CacheManager(config.storage, config);\n }\n\n if (config.strictPersistence) {\n const disabledConfig = { ...config, enabled: false };\n return new CacheManager(\n new InMemoryStorage(disabledConfig),\n disabledConfig,\n );\n }\n\n return new CacheManager(new InMemoryStorage(config), config);\n }\n\n // try to use lakebase storage\n try {\n const workspaceClient = new WorkspaceClient({});\n const connector = new LakebaseConnector({ workspaceClient });\n const isHealthy = await connector.healthCheck();\n\n if (isHealthy) {\n const persistentStorage = new PersistentStorage(config, connector);\n await persistentStorage.initialize();\n return new CacheManager(persistentStorage, config);\n }\n } catch {\n // lakebase unavailable, continue with in-memory storage\n }\n\n if (config.strictPersistence) {\n const disabledConfig = { ...config, enabled: false };\n return new CacheManager(\n new InMemoryStorage(disabledConfig),\n disabledConfig,\n );\n }\n\n return new CacheManager(new InMemoryStorage(config), config);\n }\n\n /**\n * Get or execute a function and cache the result\n * @param key - Cache key\n * @param fn - Function to execute\n * @param userKey - User key\n * @param options - Options for the cache\n * @returns Promise of the result\n */\n async getOrExecute<T>(\n key: (string | number | object)[],\n fn: () => Promise<T>,\n userKey: string,\n options?: { ttl?: number },\n ): Promise<T> {\n if (!this.config.enabled) return fn();\n\n const cacheKey = this.generateKey(key, userKey);\n\n return this.telemetry.startActiveSpan(\n \"cache.getOrExecute\",\n {\n attributes: {\n \"cache.key\": cacheKey,\n \"cache.enabled\": this.config.enabled,\n \"cache.persistent\": this.storage.isPersistent(),\n },\n },\n async (span) => {\n try {\n // check if the value is in the cache\n const cached = await this.storage.get<T>(cacheKey);\n if (cached !== null) {\n span.setAttribute(\"cache.hit\", true);\n span.setStatus({ code: SpanStatusCode.OK });\n this.telemetryMetrics.cacheHitCount.add(1, {\n \"cache.key\": cacheKey,\n });\n return cached.value as T;\n }\n\n // check if the value is being processed by another request\n const inFlight = this.inFlightRequests.get(cacheKey);\n if (inFlight) {\n span.setAttribute(\"cache.hit\", true);\n span.setAttribute(\"cache.deduplication\", true);\n span.addEvent(\"cache.deduplication_used\", {\n \"cache.key\": cacheKey,\n });\n span.setStatus({ code: SpanStatusCode.OK });\n this.telemetryMetrics.cacheHitCount.add(1, {\n \"cache.key\": cacheKey,\n \"cache.deduplication\": \"true\",\n });\n span.end();\n return inFlight as Promise<T>;\n }\n\n // cache miss - execute function\n span.setAttribute(\"cache.hit\", false);\n span.addEvent(\"cache.miss\", { \"cache.key\": cacheKey });\n this.telemetryMetrics.cacheMissCount.add(1, {\n \"cache.key\": cacheKey,\n });\n\n const promise = fn()\n .then(async (result) => {\n await this.set(cacheKey, result, options);\n span.addEvent(\"cache.value_stored\", {\n \"cache.key\": cacheKey,\n \"cache.ttl\": options?.ttl ?? this.config.ttl ?? 3600,\n });\n return result;\n })\n .catch((error) => {\n span.recordException(error);\n span.setStatus({ code: SpanStatusCode.ERROR });\n throw error;\n })\n .finally(() => {\n this.inFlightRequests.delete(cacheKey);\n });\n\n this.inFlightRequests.set(cacheKey, promise);\n\n const result = await promise;\n span.setStatus({ code: SpanStatusCode.OK });\n return result;\n } catch (error) {\n span.recordException(error as Error);\n span.setStatus({ code: SpanStatusCode.ERROR });\n throw error;\n } finally {\n span.end();\n }\n },\n { name: this.name, includePrefix: true },\n );\n }\n\n /**\n * Get a cached value\n * @param key - Cache key\n * @returns Promise of the value or null if not found or expired\n */\n async get<T>(key: string): Promise<T | null> {\n if (!this.config.enabled) return null;\n\n // probabilistic cleanup trigger\n this.maybeCleanup();\n\n const entry = await this.storage.get<T>(key);\n if (!entry) return null;\n\n if (Date.now() > entry.expiry) {\n await this.storage.delete(key);\n return null;\n }\n return entry.value as T;\n }\n\n /** Probabilistically trigger cleanup of expired entries (fire-and-forget) */\n private maybeCleanup(): void {\n if (this.cleanupInProgress) return;\n if (!this.storage.isPersistent()) return;\n const now = Date.now();\n if (now - this.lastCleanupAttempt < CacheManager.MIN_CLEANUP_INTERVAL_MS)\n return;\n\n const probability = this.config.cleanupProbability ?? 0.01;\n\n if (Math.random() > probability) return;\n\n this.lastCleanupAttempt = now;\n\n this.cleanupInProgress = true;\n (this.storage as PersistentStorage)\n .cleanupExpired()\n .catch((error) => {\n console.debug(\"Error cleaning up expired entries:\", error);\n })\n .finally(() => {\n this.cleanupInProgress = false;\n });\n }\n\n /**\n * Set a value in the cache\n * @param key - Cache key\n * @param value - Value to set\n * @param options - Options for the cache\n * @returns Promise of the result\n */\n async set<T>(\n key: string,\n value: T,\n options?: { ttl?: number },\n ): Promise<void> {\n if (!this.config.enabled) return;\n\n const ttl = options?.ttl ?? this.config.ttl ?? 3600;\n const expiryTime = Date.now() + ttl * 1000;\n await this.storage.set(key, { value, expiry: expiryTime });\n }\n\n /**\n * Delete a value from the cache\n * @param key - Cache key\n * @returns Promise of the result\n */\n async delete(key: string): Promise<void> {\n if (!this.config.enabled) return;\n await this.storage.delete(key);\n }\n\n /** Clear the cache */\n async clear(): Promise<void> {\n await this.storage.clear();\n this.inFlightRequests.clear();\n }\n\n /**\n * Check if a value exists in the cache\n * @param key - Cache key\n * @returns Promise of true if the value exists, false otherwise\n */\n async has(key: string): Promise<boolean> {\n if (!this.config.enabled) return false;\n\n const entry = await this.storage.get(key);\n if (!entry) return false;\n\n if (Date.now() > entry.expiry) {\n await this.storage.delete(key);\n return false;\n }\n return true;\n }\n\n /**\n * Generate a cache key\n * @param parts - Parts of the key\n * @param userKey - User key\n * @returns Cache key\n */\n generateKey(parts: (string | number | object)[], userKey: string): string {\n const allParts = [userKey, ...parts];\n const serialized = JSON.stringify(allParts);\n return createHash(\"sha256\").update(serialized).digest(\"hex\");\n }\n\n /** Close the cache */\n async close(): Promise<void> {\n await this.storage.close();\n }\n\n /**\n * Check if the storage is healthy\n * @returns Promise of true if the storage is healthy, false otherwise\n */\n async isStorageHealthy(): Promise<boolean> {\n return this.storage.healthCheck();\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAwBA,IAAa,eAAb,MAAa,aAAa;;iCAC0B;;;kBAEH;;;qBACY;;CAe3D,AAAQ,YAAY,SAAuB,QAAqB;cAjBhC;AAkB9B,OAAK,UAAU;AACf,OAAK,SAAS;AACd,OAAK,mCAAmB,IAAI,KAAK;AACjC,OAAK,oBAAoB;AACzB,OAAK,qBAAqB;AAE1B,OAAK,YAAY,iBAAiB,YAChC,KAAK,MACL,KAAK,OAAO,UACb;AACD,OAAK,mBAAmB;GACtB,eAAe,KAAK,UAAU,UAAU,CAAC,cAAc,aAAa;IAClE,aAAa;IACb,MAAM;IACP,CAAC;GACF,gBAAgB,KAAK,UAAU,UAAU,CAAC,cAAc,cAAc;IACpE,aAAa;IACb,MAAM;IACP,CAAC;GACH;;;;;;;;CASH,OAAO,kBAAgC;AACrC,MAAI,CAAC,aAAa,SAChB,OAAM,IAAI,MACR,iGACD;AAGH,SAAO,aAAa;;;;;;;;;CAUtB,aAAa,YACX,YACuB;AACvB,MAAI,aAAa,SACf,QAAO,aAAa;AAGtB,MAAI,CAAC,aAAa,YAChB,cAAa,cAAc,aAAa,OAAO,WAAW,CAAC,MACxD,aAAa;AACZ,gBAAa,WAAW;AACxB,UAAO;IAEV;AAGH,SAAO,aAAa;;;;;;;;;;;;;;CAetB,aAAqB,OACnB,YACuB;EACvB,MAAM,SAAS,UAAU,eAAe,WAAW;AAEnD,MAAI,OAAO,SAAS;AAElB,OADkB,MAAM,OAAO,QAAQ,aAAa,CAElD,QAAO,IAAI,aAAa,OAAO,SAAS,OAAO;AAGjD,OAAI,OAAO,mBAAmB;IAC5B,MAAM,iBAAiB;KAAE,GAAG;KAAQ,SAAS;KAAO;AACpD,WAAO,IAAI,aACT,IAAI,gBAAgB,eAAe,EACnC,eACD;;AAGH,UAAO,IAAI,aAAa,IAAI,gBAAgB,OAAO,EAAE,OAAO;;AAI9D,MAAI;GAEF,MAAM,YAAY,IAAI,kBAAkB,EAAE,iBADlB,IAAI,gBAAgB,EAAE,CAAC,EACY,CAAC;AAG5D,OAFkB,MAAM,UAAU,aAAa,EAEhC;IACb,MAAM,oBAAoB,IAAI,kBAAkB,QAAQ,UAAU;AAClE,UAAM,kBAAkB,YAAY;AACpC,WAAO,IAAI,aAAa,mBAAmB,OAAO;;UAE9C;AAIR,MAAI,OAAO,mBAAmB;GAC5B,MAAM,iBAAiB;IAAE,GAAG;IAAQ,SAAS;IAAO;AACpD,UAAO,IAAI,aACT,IAAI,gBAAgB,eAAe,EACnC,eACD;;AAGH,SAAO,IAAI,aAAa,IAAI,gBAAgB,OAAO,EAAE,OAAO;;;;;;;;;;CAW9D,MAAM,aACJ,KACA,IACA,SACA,SACY;AACZ,MAAI,CAAC,KAAK,OAAO,QAAS,QAAO,IAAI;EAErC,MAAM,WAAW,KAAK,YAAY,KAAK,QAAQ;AAE/C,SAAO,KAAK,UAAU,gBACpB,sBACA,EACE,YAAY;GACV,aAAa;GACb,iBAAiB,KAAK,OAAO;GAC7B,oBAAoB,KAAK,QAAQ,cAAc;GAChD,EACF,EACD,OAAO,SAAS;AACd,OAAI;IAEF,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAO,SAAS;AAClD,QAAI,WAAW,MAAM;AACnB,UAAK,aAAa,aAAa,KAAK;AACpC,UAAK,UAAU,EAAE,MAAM,eAAe,IAAI,CAAC;AAC3C,UAAK,iBAAiB,cAAc,IAAI,GAAG,EACzC,aAAa,UACd,CAAC;AACF,YAAO,OAAO;;IAIhB,MAAM,WAAW,KAAK,iBAAiB,IAAI,SAAS;AACpD,QAAI,UAAU;AACZ,UAAK,aAAa,aAAa,KAAK;AACpC,UAAK,aAAa,uBAAuB,KAAK;AAC9C,UAAK,SAAS,4BAA4B,EACxC,aAAa,UACd,CAAC;AACF,UAAK,UAAU,EAAE,MAAM,eAAe,IAAI,CAAC;AAC3C,UAAK,iBAAiB,cAAc,IAAI,GAAG;MACzC,aAAa;MACb,uBAAuB;MACxB,CAAC;AACF,UAAK,KAAK;AACV,YAAO;;AAIT,SAAK,aAAa,aAAa,MAAM;AACrC,SAAK,SAAS,cAAc,EAAE,aAAa,UAAU,CAAC;AACtD,SAAK,iBAAiB,eAAe,IAAI,GAAG,EAC1C,aAAa,UACd,CAAC;IAEF,MAAM,UAAU,IAAI,CACjB,KAAK,OAAO,aAAW;AACtB,WAAM,KAAK,IAAI,UAAUA,UAAQ,QAAQ;AACzC,UAAK,SAAS,sBAAsB;MAClC,aAAa;MACb,aAAa,SAAS,OAAO,KAAK,OAAO,OAAO;MACjD,CAAC;AACF,YAAOA;MACP,CACD,OAAO,UAAU;AAChB,UAAK,gBAAgB,MAAM;AAC3B,UAAK,UAAU,EAAE,MAAM,eAAe,OAAO,CAAC;AAC9C,WAAM;MACN,CACD,cAAc;AACb,UAAK,iBAAiB,OAAO,SAAS;MACtC;AAEJ,SAAK,iBAAiB,IAAI,UAAU,QAAQ;IAE5C,MAAM,SAAS,MAAM;AACrB,SAAK,UAAU,EAAE,MAAM,eAAe,IAAI,CAAC;AAC3C,WAAO;YACA,OAAO;AACd,SAAK,gBAAgB,MAAe;AACpC,SAAK,UAAU,EAAE,MAAM,eAAe,OAAO,CAAC;AAC9C,UAAM;aACE;AACR,SAAK,KAAK;;KAGd;GAAE,MAAM,KAAK;GAAM,eAAe;GAAM,CACzC;;;;;;;CAQH,MAAM,IAAO,KAAgC;AAC3C,MAAI,CAAC,KAAK,OAAO,QAAS,QAAO;AAGjC,OAAK,cAAc;EAEnB,MAAM,QAAQ,MAAM,KAAK,QAAQ,IAAO,IAAI;AAC5C,MAAI,CAAC,MAAO,QAAO;AAEnB,MAAI,KAAK,KAAK,GAAG,MAAM,QAAQ;AAC7B,SAAM,KAAK,QAAQ,OAAO,IAAI;AAC9B,UAAO;;AAET,SAAO,MAAM;;;CAIf,AAAQ,eAAqB;AAC3B,MAAI,KAAK,kBAAmB;AAC5B,MAAI,CAAC,KAAK,QAAQ,cAAc,CAAE;EAClC,MAAM,MAAM,KAAK,KAAK;AACtB,MAAI,MAAM,KAAK,qBAAqB,aAAa,wBAC/C;EAEF,MAAM,cAAc,KAAK,OAAO,sBAAsB;AAEtD,MAAI,KAAK,QAAQ,GAAG,YAAa;AAEjC,OAAK,qBAAqB;AAE1B,OAAK,oBAAoB;AACzB,EAAC,KAAK,QACH,gBAAgB,CAChB,OAAO,UAAU;AAChB,WAAQ,MAAM,sCAAsC,MAAM;IAC1D,CACD,cAAc;AACb,QAAK,oBAAoB;IACzB;;;;;;;;;CAUN,MAAM,IACJ,KACA,OACA,SACe;AACf,MAAI,CAAC,KAAK,OAAO,QAAS;EAE1B,MAAM,MAAM,SAAS,OAAO,KAAK,OAAO,OAAO;EAC/C,MAAM,aAAa,KAAK,KAAK,GAAG,MAAM;AACtC,QAAM,KAAK,QAAQ,IAAI,KAAK;GAAE;GAAO,QAAQ;GAAY,CAAC;;;;;;;CAQ5D,MAAM,OAAO,KAA4B;AACvC,MAAI,CAAC,KAAK,OAAO,QAAS;AAC1B,QAAM,KAAK,QAAQ,OAAO,IAAI;;;CAIhC,MAAM,QAAuB;AAC3B,QAAM,KAAK,QAAQ,OAAO;AAC1B,OAAK,iBAAiB,OAAO;;;;;;;CAQ/B,MAAM,IAAI,KAA+B;AACvC,MAAI,CAAC,KAAK,OAAO,QAAS,QAAO;EAEjC,MAAM,QAAQ,MAAM,KAAK,QAAQ,IAAI,IAAI;AACzC,MAAI,CAAC,MAAO,QAAO;AAEnB,MAAI,KAAK,KAAK,GAAG,MAAM,QAAQ;AAC7B,SAAM,KAAK,QAAQ,OAAO,IAAI;AAC9B,UAAO;;AAET,SAAO;;;;;;;;CAST,YAAY,OAAqC,SAAyB;EACxE,MAAM,WAAW,CAAC,SAAS,GAAG,MAAM;EACpC,MAAM,aAAa,KAAK,UAAU,SAAS;AAC3C,SAAO,WAAW,SAAS,CAAC,OAAO,WAAW,CAAC,OAAO,MAAM;;;CAI9D,MAAM,QAAuB;AAC3B,QAAM,KAAK,QAAQ,OAAO;;;;;;CAO5B,MAAM,mBAAqC;AACzC,SAAO,KAAK,QAAQ,aAAa"}
|
|
1
|
+
{"version":3,"file":"index.js","names":["result"],"sources":["../../src/cache/index.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport { WorkspaceClient } from \"@databricks/sdk-experimental\";\nimport type { CacheConfig, CacheStorage } from \"shared\";\nimport { LakebaseConnector } from \"@/connectors\";\nimport { AppKitError, ExecutionError, InitializationError } from \"../errors\";\nimport { createLogger } from \"../logging/logger\";\nimport type { Counter, TelemetryProvider } from \"../telemetry\";\nimport { SpanStatusCode, TelemetryManager } from \"../telemetry\";\nimport { deepMerge } from \"../utils\";\nimport { cacheDefaults } from \"./defaults\";\nimport { InMemoryStorage, PersistentStorage } from \"./storage\";\n\nconst logger = createLogger(\"cache\");\n\n/**\n * Cache manager class to handle cache operations.\n * Can be used with in-memory storage or persistent storage (Lakebase).\n *\n * The cache is automatically initialized by AppKit. Use `getInstanceSync()` to access\n * the singleton instance after initialization.\n *\n * @internal\n * @example\n * ```typescript\n * const cache = CacheManager.getInstanceSync();\n * const result = await cache.getOrExecute([\"users\", userId], () => fetchUser(userId), userKey);\n * ```\n */\nexport class CacheManager {\n private static readonly MIN_CLEANUP_INTERVAL_MS = 60_000;\n private readonly name: string = \"cache-manager\";\n private static instance: CacheManager | null = null;\n private static initPromise: Promise<CacheManager> | null = null;\n\n private storage: CacheStorage;\n private config: CacheConfig;\n private inFlightRequests: Map<string, Promise<unknown>>;\n private cleanupInProgress: boolean;\n private lastCleanupAttempt: number;\n\n private telemetry: TelemetryProvider;\n private telemetryMetrics: {\n cacheHitCount: Counter;\n cacheMissCount: Counter;\n };\n\n private constructor(storage: CacheStorage, config: CacheConfig) {\n this.storage = storage;\n this.config = config;\n this.inFlightRequests = new Map();\n this.cleanupInProgress = false;\n this.lastCleanupAttempt = 0;\n\n this.telemetry = TelemetryManager.getProvider(\n this.name,\n this.config.telemetry,\n );\n this.telemetryMetrics = {\n cacheHitCount: this.telemetry.getMeter().createCounter(\"cache.hit\", {\n description: \"Total number of cache hits\",\n unit: \"1\",\n }),\n cacheMissCount: this.telemetry.getMeter().createCounter(\"cache.miss\", {\n description: \"Total number of cache misses\",\n unit: \"1\",\n }),\n };\n }\n\n /**\n * Get the singleton instance of the cache manager (sync version).\n *\n * Throws if not initialized - ensure AppKit.create() has completed first.\n * @returns CacheManager instance\n */\n static getInstanceSync(): CacheManager {\n if (!CacheManager.instance) {\n throw InitializationError.notInitialized(\n \"CacheManager\",\n \"Ensure AppKit.create() has completed before accessing the cache\",\n );\n }\n\n return CacheManager.instance;\n }\n\n /**\n * Initialize and get the singleton instance of the cache manager.\n * Called internally by AppKit - prefer `getInstanceSync()` for plugin access.\n * @param userConfig - User configuration for the cache manager\n * @returns CacheManager instance\n * @internal\n */\n static async getInstance(\n userConfig?: Partial<CacheConfig>,\n ): Promise<CacheManager> {\n if (CacheManager.instance) {\n return CacheManager.instance;\n }\n\n if (!CacheManager.initPromise) {\n CacheManager.initPromise = CacheManager.create(userConfig).then(\n (instance) => {\n CacheManager.instance = instance;\n return instance;\n },\n );\n }\n\n return CacheManager.initPromise;\n }\n\n /**\n * Create a new cache manager instance\n *\n * Storage selection logic:\n * 1. If `storage` provided and healthy → use provided storage\n * 2. If `storage` provided but unhealthy → fallback to InMemory (or disable if strictPersistence)\n * 3. If no `storage` provided and Lakebase available → use Lakebase\n * 4. If no `storage` provided and Lakebase unavailable → fallback to InMemory (or disable if strictPersistence)\n *\n * @param userConfig - User configuration for the cache manager\n * @returns CacheManager instance\n */\n private static async create(\n userConfig?: Partial<CacheConfig>,\n ): Promise<CacheManager> {\n const config = deepMerge(cacheDefaults, userConfig);\n\n if (config.storage) {\n const isHealthy = await config.storage.healthCheck();\n if (isHealthy) {\n return new CacheManager(config.storage, config);\n }\n\n if (config.strictPersistence) {\n const disabledConfig = { ...config, enabled: false };\n return new CacheManager(\n new InMemoryStorage(disabledConfig),\n disabledConfig,\n );\n }\n\n return new CacheManager(new InMemoryStorage(config), config);\n }\n\n // try to use lakebase storage\n try {\n const workspaceClient = new WorkspaceClient({});\n const connector = new LakebaseConnector({ workspaceClient });\n const isHealthy = await connector.healthCheck();\n\n if (isHealthy) {\n const persistentStorage = new PersistentStorage(config, connector);\n await persistentStorage.initialize();\n return new CacheManager(persistentStorage, config);\n }\n } catch {\n // lakebase unavailable, continue with in-memory storage\n }\n\n if (config.strictPersistence) {\n const disabledConfig = { ...config, enabled: false };\n return new CacheManager(\n new InMemoryStorage(disabledConfig),\n disabledConfig,\n );\n }\n\n return new CacheManager(new InMemoryStorage(config), config);\n }\n\n /**\n * Get or execute a function and cache the result\n * @param key - Cache key\n * @param fn - Function to execute\n * @param userKey - User key\n * @param options - Options for the cache\n * @returns Promise of the result\n */\n async getOrExecute<T>(\n key: (string | number | object)[],\n fn: () => Promise<T>,\n userKey: string,\n options?: { ttl?: number },\n ): Promise<T> {\n if (!this.config.enabled) return fn();\n\n const cacheKey = this.generateKey(key, userKey);\n\n return this.telemetry.startActiveSpan(\n \"cache.getOrExecute\",\n {\n attributes: {\n \"cache.key\": cacheKey,\n \"cache.enabled\": this.config.enabled,\n \"cache.persistent\": this.storage.isPersistent(),\n },\n },\n async (span) => {\n try {\n // check if the value is in the cache\n const cached = await this.storage.get<T>(cacheKey);\n if (cached !== null) {\n span.setAttribute(\"cache.hit\", true);\n span.setStatus({ code: SpanStatusCode.OK });\n this.telemetryMetrics.cacheHitCount.add(1, {\n \"cache.key\": cacheKey,\n });\n\n logger.event()?.setExecution({\n cache_hit: true,\n cache_key: cacheKey,\n });\n\n return cached.value as T;\n }\n\n // check if the value is being processed by another request\n const inFlight = this.inFlightRequests.get(cacheKey);\n if (inFlight) {\n span.setAttribute(\"cache.hit\", true);\n span.setAttribute(\"cache.deduplication\", true);\n span.addEvent(\"cache.deduplication_used\", {\n \"cache.key\": cacheKey,\n });\n span.setStatus({ code: SpanStatusCode.OK });\n this.telemetryMetrics.cacheHitCount.add(1, {\n \"cache.key\": cacheKey,\n \"cache.deduplication\": \"true\",\n });\n\n logger.event()?.setExecution({\n cache_hit: true,\n cache_key: cacheKey,\n cache_deduplication: true,\n });\n\n span.end();\n return inFlight as Promise<T>;\n }\n\n // cache miss - execute function\n span.setAttribute(\"cache.hit\", false);\n span.addEvent(\"cache.miss\", { \"cache.key\": cacheKey });\n this.telemetryMetrics.cacheMissCount.add(1, {\n \"cache.key\": cacheKey,\n });\n\n logger.event()?.setExecution({\n cache_hit: false,\n cache_key: cacheKey,\n });\n\n const promise = fn()\n .then(async (result) => {\n await this.set(cacheKey, result, options);\n span.addEvent(\"cache.value_stored\", {\n \"cache.key\": cacheKey,\n \"cache.ttl\": options?.ttl ?? this.config.ttl ?? 3600,\n });\n return result;\n })\n .catch((error) => {\n span.recordException(error);\n span.setStatus({ code: SpanStatusCode.ERROR });\n if (error instanceof AppKitError) {\n throw error;\n }\n throw ExecutionError.statementFailed(\n error instanceof Error ? error.message : String(error),\n );\n })\n .finally(() => {\n this.inFlightRequests.delete(cacheKey);\n });\n\n this.inFlightRequests.set(cacheKey, promise);\n\n const result = await promise;\n span.setStatus({ code: SpanStatusCode.OK });\n return result;\n } catch (error) {\n span.recordException(error as Error);\n span.setStatus({ code: SpanStatusCode.ERROR });\n throw error;\n } finally {\n span.end();\n }\n },\n { name: this.name, includePrefix: true },\n );\n }\n\n /**\n * Get a cached value\n * @param key - Cache key\n * @returns Promise of the value or null if not found or expired\n */\n async get<T>(key: string): Promise<T | null> {\n if (!this.config.enabled) return null;\n\n // probabilistic cleanup trigger\n this.maybeCleanup();\n\n const entry = await this.storage.get<T>(key);\n if (!entry) return null;\n\n if (Date.now() > entry.expiry) {\n await this.storage.delete(key);\n return null;\n }\n return entry.value as T;\n }\n\n /** Probabilistically trigger cleanup of expired entries (fire-and-forget) */\n private maybeCleanup(): void {\n if (this.cleanupInProgress) return;\n if (!this.storage.isPersistent()) return;\n const now = Date.now();\n if (now - this.lastCleanupAttempt < CacheManager.MIN_CLEANUP_INTERVAL_MS)\n return;\n\n const probability = this.config.cleanupProbability ?? 0.01;\n\n if (Math.random() > probability) return;\n\n this.lastCleanupAttempt = now;\n\n this.cleanupInProgress = true;\n (this.storage as PersistentStorage)\n .cleanupExpired()\n .catch((error) => {\n logger.debug(\"Error cleaning up expired entries: %O\", error);\n })\n .finally(() => {\n this.cleanupInProgress = false;\n });\n }\n\n /**\n * Set a value in the cache\n * @param key - Cache key\n * @param value - Value to set\n * @param options - Options for the cache\n * @returns Promise of the result\n */\n async set<T>(\n key: string,\n value: T,\n options?: { ttl?: number },\n ): Promise<void> {\n if (!this.config.enabled) return;\n\n const ttl = options?.ttl ?? this.config.ttl ?? 3600;\n const expiryTime = Date.now() + ttl * 1000;\n await this.storage.set(key, { value, expiry: expiryTime });\n }\n\n /**\n * Delete a value from the cache\n * @param key - Cache key\n * @returns Promise of the result\n */\n async delete(key: string): Promise<void> {\n if (!this.config.enabled) return;\n await this.storage.delete(key);\n }\n\n /** Clear the cache */\n async clear(): Promise<void> {\n await this.storage.clear();\n this.inFlightRequests.clear();\n }\n\n /**\n * Check if a value exists in the cache\n * @param key - Cache key\n * @returns Promise of true if the value exists, false otherwise\n */\n async has(key: string): Promise<boolean> {\n if (!this.config.enabled) return false;\n\n const entry = await this.storage.get(key);\n if (!entry) return false;\n\n if (Date.now() > entry.expiry) {\n await this.storage.delete(key);\n return false;\n }\n return true;\n }\n\n /**\n * Generate a cache key\n * @param parts - Parts of the key\n * @param userKey - User key\n * @returns Cache key\n */\n generateKey(parts: (string | number | object)[], userKey: string): string {\n const allParts = [userKey, ...parts];\n const serialized = JSON.stringify(allParts);\n return createHash(\"sha256\").update(serialized).digest(\"hex\");\n }\n\n /** Close the cache */\n async close(): Promise<void> {\n await this.storage.close();\n }\n\n /**\n * Check if the storage is healthy\n * @returns Promise of true if the storage is healthy, false otherwise\n */\n async isStorageHealthy(): Promise<boolean> {\n return this.storage.healthCheck();\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;aAI6E;AAQ7E,MAAM,SAAS,aAAa,QAAQ;;;;;;;;;;;;;;;AAgBpC,IAAa,eAAb,MAAa,aAAa;;iCAC0B;;;kBAEH;;;qBACY;;CAc3D,AAAQ,YAAY,SAAuB,QAAqB;cAhBhC;AAiB9B,OAAK,UAAU;AACf,OAAK,SAAS;AACd,OAAK,mCAAmB,IAAI,KAAK;AACjC,OAAK,oBAAoB;AACzB,OAAK,qBAAqB;AAE1B,OAAK,YAAY,iBAAiB,YAChC,KAAK,MACL,KAAK,OAAO,UACb;AACD,OAAK,mBAAmB;GACtB,eAAe,KAAK,UAAU,UAAU,CAAC,cAAc,aAAa;IAClE,aAAa;IACb,MAAM;IACP,CAAC;GACF,gBAAgB,KAAK,UAAU,UAAU,CAAC,cAAc,cAAc;IACpE,aAAa;IACb,MAAM;IACP,CAAC;GACH;;;;;;;;CASH,OAAO,kBAAgC;AACrC,MAAI,CAAC,aAAa,SAChB,OAAM,oBAAoB,eACxB,gBACA,kEACD;AAGH,SAAO,aAAa;;;;;;;;;CAUtB,aAAa,YACX,YACuB;AACvB,MAAI,aAAa,SACf,QAAO,aAAa;AAGtB,MAAI,CAAC,aAAa,YAChB,cAAa,cAAc,aAAa,OAAO,WAAW,CAAC,MACxD,aAAa;AACZ,gBAAa,WAAW;AACxB,UAAO;IAEV;AAGH,SAAO,aAAa;;;;;;;;;;;;;;CAetB,aAAqB,OACnB,YACuB;EACvB,MAAM,SAAS,UAAU,eAAe,WAAW;AAEnD,MAAI,OAAO,SAAS;AAElB,OADkB,MAAM,OAAO,QAAQ,aAAa,CAElD,QAAO,IAAI,aAAa,OAAO,SAAS,OAAO;AAGjD,OAAI,OAAO,mBAAmB;IAC5B,MAAM,iBAAiB;KAAE,GAAG;KAAQ,SAAS;KAAO;AACpD,WAAO,IAAI,aACT,IAAI,gBAAgB,eAAe,EACnC,eACD;;AAGH,UAAO,IAAI,aAAa,IAAI,gBAAgB,OAAO,EAAE,OAAO;;AAI9D,MAAI;GAEF,MAAM,YAAY,IAAI,kBAAkB,EAAE,iBADlB,IAAI,gBAAgB,EAAE,CAAC,EACY,CAAC;AAG5D,OAFkB,MAAM,UAAU,aAAa,EAEhC;IACb,MAAM,oBAAoB,IAAI,kBAAkB,QAAQ,UAAU;AAClE,UAAM,kBAAkB,YAAY;AACpC,WAAO,IAAI,aAAa,mBAAmB,OAAO;;UAE9C;AAIR,MAAI,OAAO,mBAAmB;GAC5B,MAAM,iBAAiB;IAAE,GAAG;IAAQ,SAAS;IAAO;AACpD,UAAO,IAAI,aACT,IAAI,gBAAgB,eAAe,EACnC,eACD;;AAGH,SAAO,IAAI,aAAa,IAAI,gBAAgB,OAAO,EAAE,OAAO;;;;;;;;;;CAW9D,MAAM,aACJ,KACA,IACA,SACA,SACY;AACZ,MAAI,CAAC,KAAK,OAAO,QAAS,QAAO,IAAI;EAErC,MAAM,WAAW,KAAK,YAAY,KAAK,QAAQ;AAE/C,SAAO,KAAK,UAAU,gBACpB,sBACA,EACE,YAAY;GACV,aAAa;GACb,iBAAiB,KAAK,OAAO;GAC7B,oBAAoB,KAAK,QAAQ,cAAc;GAChD,EACF,EACD,OAAO,SAAS;AACd,OAAI;IAEF,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAO,SAAS;AAClD,QAAI,WAAW,MAAM;AACnB,UAAK,aAAa,aAAa,KAAK;AACpC,UAAK,UAAU,EAAE,MAAM,eAAe,IAAI,CAAC;AAC3C,UAAK,iBAAiB,cAAc,IAAI,GAAG,EACzC,aAAa,UACd,CAAC;AAEF,YAAO,OAAO,EAAE,aAAa;MAC3B,WAAW;MACX,WAAW;MACZ,CAAC;AAEF,YAAO,OAAO;;IAIhB,MAAM,WAAW,KAAK,iBAAiB,IAAI,SAAS;AACpD,QAAI,UAAU;AACZ,UAAK,aAAa,aAAa,KAAK;AACpC,UAAK,aAAa,uBAAuB,KAAK;AAC9C,UAAK,SAAS,4BAA4B,EACxC,aAAa,UACd,CAAC;AACF,UAAK,UAAU,EAAE,MAAM,eAAe,IAAI,CAAC;AAC3C,UAAK,iBAAiB,cAAc,IAAI,GAAG;MACzC,aAAa;MACb,uBAAuB;MACxB,CAAC;AAEF,YAAO,OAAO,EAAE,aAAa;MAC3B,WAAW;MACX,WAAW;MACX,qBAAqB;MACtB,CAAC;AAEF,UAAK,KAAK;AACV,YAAO;;AAIT,SAAK,aAAa,aAAa,MAAM;AACrC,SAAK,SAAS,cAAc,EAAE,aAAa,UAAU,CAAC;AACtD,SAAK,iBAAiB,eAAe,IAAI,GAAG,EAC1C,aAAa,UACd,CAAC;AAEF,WAAO,OAAO,EAAE,aAAa;KAC3B,WAAW;KACX,WAAW;KACZ,CAAC;IAEF,MAAM,UAAU,IAAI,CACjB,KAAK,OAAO,aAAW;AACtB,WAAM,KAAK,IAAI,UAAUA,UAAQ,QAAQ;AACzC,UAAK,SAAS,sBAAsB;MAClC,aAAa;MACb,aAAa,SAAS,OAAO,KAAK,OAAO,OAAO;MACjD,CAAC;AACF,YAAOA;MACP,CACD,OAAO,UAAU;AAChB,UAAK,gBAAgB,MAAM;AAC3B,UAAK,UAAU,EAAE,MAAM,eAAe,OAAO,CAAC;AAC9C,SAAI,iBAAiB,YACnB,OAAM;AAER,WAAM,eAAe,gBACnB,iBAAiB,QAAQ,MAAM,UAAU,OAAO,MAAM,CACvD;MACD,CACD,cAAc;AACb,UAAK,iBAAiB,OAAO,SAAS;MACtC;AAEJ,SAAK,iBAAiB,IAAI,UAAU,QAAQ;IAE5C,MAAM,SAAS,MAAM;AACrB,SAAK,UAAU,EAAE,MAAM,eAAe,IAAI,CAAC;AAC3C,WAAO;YACA,OAAO;AACd,SAAK,gBAAgB,MAAe;AACpC,SAAK,UAAU,EAAE,MAAM,eAAe,OAAO,CAAC;AAC9C,UAAM;aACE;AACR,SAAK,KAAK;;KAGd;GAAE,MAAM,KAAK;GAAM,eAAe;GAAM,CACzC;;;;;;;CAQH,MAAM,IAAO,KAAgC;AAC3C,MAAI,CAAC,KAAK,OAAO,QAAS,QAAO;AAGjC,OAAK,cAAc;EAEnB,MAAM,QAAQ,MAAM,KAAK,QAAQ,IAAO,IAAI;AAC5C,MAAI,CAAC,MAAO,QAAO;AAEnB,MAAI,KAAK,KAAK,GAAG,MAAM,QAAQ;AAC7B,SAAM,KAAK,QAAQ,OAAO,IAAI;AAC9B,UAAO;;AAET,SAAO,MAAM;;;CAIf,AAAQ,eAAqB;AAC3B,MAAI,KAAK,kBAAmB;AAC5B,MAAI,CAAC,KAAK,QAAQ,cAAc,CAAE;EAClC,MAAM,MAAM,KAAK,KAAK;AACtB,MAAI,MAAM,KAAK,qBAAqB,aAAa,wBAC/C;EAEF,MAAM,cAAc,KAAK,OAAO,sBAAsB;AAEtD,MAAI,KAAK,QAAQ,GAAG,YAAa;AAEjC,OAAK,qBAAqB;AAE1B,OAAK,oBAAoB;AACzB,EAAC,KAAK,QACH,gBAAgB,CAChB,OAAO,UAAU;AAChB,UAAO,MAAM,yCAAyC,MAAM;IAC5D,CACD,cAAc;AACb,QAAK,oBAAoB;IACzB;;;;;;;;;CAUN,MAAM,IACJ,KACA,OACA,SACe;AACf,MAAI,CAAC,KAAK,OAAO,QAAS;EAE1B,MAAM,MAAM,SAAS,OAAO,KAAK,OAAO,OAAO;EAC/C,MAAM,aAAa,KAAK,KAAK,GAAG,MAAM;AACtC,QAAM,KAAK,QAAQ,IAAI,KAAK;GAAE;GAAO,QAAQ;GAAY,CAAC;;;;;;;CAQ5D,MAAM,OAAO,KAA4B;AACvC,MAAI,CAAC,KAAK,OAAO,QAAS;AAC1B,QAAM,KAAK,QAAQ,OAAO,IAAI;;;CAIhC,MAAM,QAAuB;AAC3B,QAAM,KAAK,QAAQ,OAAO;AAC1B,OAAK,iBAAiB,OAAO;;;;;;;CAQ/B,MAAM,IAAI,KAA+B;AACvC,MAAI,CAAC,KAAK,OAAO,QAAS,QAAO;EAEjC,MAAM,QAAQ,MAAM,KAAK,QAAQ,IAAI,IAAI;AACzC,MAAI,CAAC,MAAO,QAAO;AAEnB,MAAI,KAAK,KAAK,GAAG,MAAM,QAAQ;AAC7B,SAAM,KAAK,QAAQ,OAAO,IAAI;AAC9B,UAAO;;AAET,SAAO;;;;;;;;CAST,YAAY,OAAqC,SAAyB;EACxE,MAAM,WAAW,CAAC,SAAS,GAAG,MAAM;EACpC,MAAM,aAAa,KAAK,UAAU,SAAS;AAC3C,SAAO,WAAW,SAAS,CAAC,OAAO,WAAW,CAAC,OAAO,MAAM;;;CAI9D,MAAM,QAAuB;AAC3B,QAAM,KAAK,QAAQ,OAAO;;;;;;CAO5B,MAAM,mBAAqC;AACzC,SAAO,KAAK,QAAQ,aAAa"}
|
|
@@ -1,7 +1,13 @@
|
|
|
1
|
+
import { createLogger } from "../../logging/logger.js";
|
|
2
|
+
import { InitializationError } from "../../errors/initialization.js";
|
|
3
|
+
import { ValidationError } from "../../errors/validation.js";
|
|
4
|
+
import { init_errors } from "../../errors/index.js";
|
|
1
5
|
import { lakebaseStorageDefaults } from "./defaults.js";
|
|
2
6
|
import { createHash } from "node:crypto";
|
|
3
7
|
|
|
4
8
|
//#region src/cache/storage/persistent.ts
|
|
9
|
+
init_errors();
|
|
10
|
+
const logger = createLogger("cache:persistent");
|
|
5
11
|
/**
|
|
6
12
|
* Persistent cache storage implementation. Uses a least recently used (LRU) eviction policy
|
|
7
13
|
* to manage memory usage and ensure efficient cache operations.
|
|
@@ -33,7 +39,7 @@ var PersistentStorage = class {
|
|
|
33
39
|
await this.runMigrations();
|
|
34
40
|
this.initialized = true;
|
|
35
41
|
} catch (error) {
|
|
36
|
-
|
|
42
|
+
logger.error("Error in persistent storage initialization: %O", error);
|
|
37
43
|
throw error;
|
|
38
44
|
}
|
|
39
45
|
}
|
|
@@ -49,7 +55,7 @@ var PersistentStorage = class {
|
|
|
49
55
|
if (result.rows.length === 0) return null;
|
|
50
56
|
const entry = result.rows[0];
|
|
51
57
|
this.connector.query(`UPDATE ${this.tableName} SET last_accessed = NOW() WHERE key_hash = $1`, [keyHash]).catch(() => {
|
|
52
|
-
|
|
58
|
+
logger.debug("Error updating last_accessed time for key: %s", key);
|
|
53
59
|
});
|
|
54
60
|
return {
|
|
55
61
|
value: this.deserializeValue(entry.value),
|
|
@@ -68,7 +74,7 @@ var PersistentStorage = class {
|
|
|
68
74
|
const keyBytes = Buffer.from(key, "utf-8");
|
|
69
75
|
const valueBytes = this.serializeValue(entry.value);
|
|
70
76
|
const byteSize = keyBytes.length + valueBytes.length;
|
|
71
|
-
if (byteSize > this.maxEntryBytes) throw
|
|
77
|
+
if (byteSize > this.maxEntryBytes) throw ValidationError.invalidValue("cache entry size", byteSize, `maximum ${this.maxEntryBytes} bytes`);
|
|
72
78
|
if (Math.random() < this.evictionCheckProbability) {
|
|
73
79
|
if (await this.totalBytes() + byteSize > this.maxBytes) await this.evictBySize(byteSize);
|
|
74
80
|
}
|
|
@@ -169,7 +175,7 @@ var PersistentStorage = class {
|
|
|
169
175
|
}
|
|
170
176
|
/** Generate a 64-bit hash for the cache key using SHA256 */
|
|
171
177
|
hashKey(key) {
|
|
172
|
-
if (!key) throw
|
|
178
|
+
if (!key) throw ValidationError.missingField("key");
|
|
173
179
|
return createHash("sha256").update(key).digest().readBigInt64BE(0);
|
|
174
180
|
}
|
|
175
181
|
/** Serialize a value to a buffer */
|
|
@@ -200,8 +206,8 @@ var PersistentStorage = class {
|
|
|
200
206
|
await this.connector.query(`CREATE INDEX IF NOT EXISTS idx_${this.tableName}_last_accessed ON ${this.tableName} (last_accessed); `);
|
|
201
207
|
await this.connector.query(`CREATE INDEX IF NOT EXISTS idx_${this.tableName}_byte_size ON ${this.tableName} (byte_size); `);
|
|
202
208
|
} catch (error) {
|
|
203
|
-
|
|
204
|
-
throw error;
|
|
209
|
+
logger.error("Error in running migrations for persistent storage: %O", error);
|
|
210
|
+
throw InitializationError.migrationFailed(error);
|
|
205
211
|
}
|
|
206
212
|
}
|
|
207
213
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"persistent.js","names":[],"sources":["../../../src/cache/storage/persistent.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport type { CacheConfig, CacheEntry, CacheStorage } from \"shared\";\nimport type { LakebaseConnector } from \"../../connectors\";\nimport { lakebaseStorageDefaults } from \"./defaults\";\n\n/**\n * Persistent cache storage implementation. Uses a least recently used (LRU) eviction policy\n * to manage memory usage and ensure efficient cache operations.\n *\n * @example\n * const persistentStorage = new PersistentStorage(config, connector);\n * await persistentStorage.initialize();\n * await persistentStorage.get(\"my-key\");\n * await persistentStorage.set(\"my-key\", \"my-value\");\n * await persistentStorage.delete(\"my-key\");\n * await persistentStorage.clear();\n * await persistentStorage.has(\"my-key\");\n *\n */\nexport class PersistentStorage implements CacheStorage {\n private readonly connector: LakebaseConnector;\n private readonly tableName: string;\n private readonly maxBytes: number;\n private readonly maxEntryBytes: number;\n private readonly evictionBatchSize: number;\n private readonly evictionCheckProbability: number;\n private initialized: boolean;\n\n constructor(config: CacheConfig, connector: LakebaseConnector) {\n this.connector = connector;\n this.maxBytes = config.maxBytes ?? lakebaseStorageDefaults.maxBytes;\n this.maxEntryBytes =\n config.maxEntryBytes ?? lakebaseStorageDefaults.maxEntryBytes;\n this.evictionBatchSize = lakebaseStorageDefaults.evictionBatchSize;\n this.evictionCheckProbability =\n config.evictionCheckProbability ??\n lakebaseStorageDefaults.evictionCheckProbability;\n this.tableName = lakebaseStorageDefaults.tableName; // hardcoded, safe for now\n this.initialized = false;\n }\n\n /** Initialize the persistent storage and run migrations if necessary */\n async initialize(): Promise<void> {\n if (this.initialized) return;\n\n try {\n await this.runMigrations();\n this.initialized = true;\n } catch (error) {\n console.error(\"Error in persistent storage initialization:\", error);\n throw error;\n }\n }\n\n /**\n * Get a cached value from the persistent storage\n * @param key - Cache key\n * @returns Promise of the cached value or null if not found\n */\n async get<T>(key: string): Promise<CacheEntry<T> | null> {\n await this.ensureInitialized();\n\n const keyHash = this.hashKey(key);\n\n const result = await this.connector.query<{\n value: Buffer;\n expiry: string;\n }>(`SELECT value, expiry FROM ${this.tableName} WHERE key_hash = $1`, [\n keyHash,\n ]);\n\n if (result.rows.length === 0) return null;\n\n const entry = result.rows[0];\n\n // fire-and-forget update\n this.connector\n .query(\n `UPDATE ${this.tableName} SET last_accessed = NOW() WHERE key_hash = $1`,\n [keyHash],\n )\n .catch(() => {\n console.debug(\"Error updating last_accessed time for key:\", key);\n });\n\n return {\n value: this.deserializeValue<T>(entry.value),\n expiry: Number(entry.expiry),\n };\n }\n\n /**\n * Set a value in the persistent storage\n * @param key - Cache key\n * @param entry - Cache entry\n * @returns Promise of the result\n */\n async set<T>(key: string, entry: CacheEntry<T>): Promise<void> {\n await this.ensureInitialized();\n\n const keyHash = this.hashKey(key);\n const keyBytes = Buffer.from(key, \"utf-8\");\n const valueBytes = this.serializeValue(entry.value);\n const byteSize = keyBytes.length + valueBytes.length;\n\n if (byteSize > this.maxEntryBytes) {\n throw new Error(\n `Cache entry too large: ${byteSize} bytes exceeds maximum of ${this.maxEntryBytes} bytes`,\n );\n }\n\n // probabilistic eviction check\n if (Math.random() < this.evictionCheckProbability) {\n const totalBytes = await this.totalBytes();\n if (totalBytes + byteSize > this.maxBytes) {\n await this.evictBySize(byteSize);\n }\n }\n\n await this.connector.query(\n `INSERT INTO ${this.tableName} (key_hash, key, value, byte_size, expiry, created_at, last_accessed)\n VALUES ($1, $2, $3, $4, $5, NOW(), NOW())\n ON CONFLICT (key_hash)\n DO UPDATE SET value = $3, byte_size = $4, expiry = $5, last_accessed = NOW()\n `,\n [keyHash, keyBytes, valueBytes, byteSize, entry.expiry],\n );\n }\n\n /**\n * Delete a value from the persistent storage\n * @param key - Cache key\n * @returns Promise of the result\n */\n async delete(key: string): Promise<void> {\n await this.ensureInitialized();\n const keyHash = this.hashKey(key);\n await this.connector.query(\n `DELETE FROM ${this.tableName} WHERE key_hash = $1`,\n [keyHash],\n );\n }\n\n /** Clear the persistent storage */\n async clear(): Promise<void> {\n await this.ensureInitialized();\n await this.connector.query(`TRUNCATE TABLE ${this.tableName}`);\n }\n\n /**\n * Check if a value exists in the persistent storage\n * @param key - Cache key\n * @returns Promise of true if the value exists, false otherwise\n */\n async has(key: string): Promise<boolean> {\n await this.ensureInitialized();\n const keyHash = this.hashKey(key);\n\n const result = await this.connector.query<{ exists: boolean }>(\n `SELECT EXISTS(SELECT 1 FROM ${this.tableName} WHERE key_hash = $1) as exists`,\n [keyHash],\n );\n\n return result.rows[0]?.exists ?? false;\n }\n\n /**\n * Get the size of the persistent storage\n * @returns Promise of the size of the storage\n */\n async size(): Promise<number> {\n await this.ensureInitialized();\n\n const result = await this.connector.query<{ count: string }>(\n `SELECT COUNT(*) as count FROM ${this.tableName}`,\n );\n return parseInt(result.rows[0]?.count ?? \"0\", 10);\n }\n\n /** Get the total number of bytes in the persistent storage */\n async totalBytes(): Promise<number> {\n await this.ensureInitialized();\n\n const result = await this.connector.query<{ total: string }>(\n `SELECT COALESCE(SUM(byte_size), 0) as total FROM ${this.tableName}`,\n );\n return parseInt(result.rows[0]?.total ?? \"0\", 10);\n }\n\n /**\n * Check if the persistent storage is persistent\n * @returns true if the storage is persistent, false otherwise\n */\n isPersistent(): boolean {\n return true;\n }\n\n /**\n * Check if the persistent storage is healthy\n * @returns Promise of true if the storage is healthy, false otherwise\n */\n async healthCheck(): Promise<boolean> {\n try {\n return await this.connector.healthCheck();\n } catch {\n return false;\n }\n }\n\n /** Close the persistent storage */\n async close(): Promise<void> {\n await this.connector.close();\n }\n\n /**\n * Cleanup expired entries from the persistent storage\n * @returns Promise of the number of expired entries\n */\n async cleanupExpired(): Promise<number> {\n await this.ensureInitialized();\n const result = await this.connector.query<{ count: string }>(\n `WITH deleted as (DELETE FROM ${this.tableName} WHERE expiry < $1 RETURNING *) SELECT COUNT(*) as count FROM deleted`,\n [Date.now()],\n );\n return parseInt(result.rows[0]?.count ?? \"0\", 10);\n }\n\n /** Evict entries from the persistent storage by size */\n private async evictBySize(requiredBytes: number): Promise<void> {\n const freedByExpiry = await this.cleanupExpired();\n if (freedByExpiry > 0) {\n const currentBytes = await this.totalBytes();\n if (currentBytes + requiredBytes <= this.maxBytes) {\n return;\n }\n }\n\n await this.connector.query(\n `DELETE FROM ${this.tableName} WHERE key_hash IN\n (SELECT key_hash FROM ${this.tableName} ORDER BY last_accessed ASC LIMIT $1)`,\n [this.evictionBatchSize],\n );\n }\n\n /** Ensure the persistent storage is initialized */\n private async ensureInitialized(): Promise<void> {\n if (!this.initialized) {\n await this.initialize();\n }\n }\n\n /** Generate a 64-bit hash for the cache key using SHA256 */\n private hashKey(key: string): bigint {\n if (!key) throw new Error(\"Cache key cannot be empty\");\n const hash = createHash(\"sha256\").update(key).digest();\n return hash.readBigInt64BE(0);\n }\n\n /** Serialize a value to a buffer */\n private serializeValue<T>(value: T): Buffer {\n return Buffer.from(JSON.stringify(value), \"utf-8\");\n }\n\n /** Deserialize a value from a buffer */\n private deserializeValue<T>(buffer: Buffer): T {\n return JSON.parse(buffer.toString(\"utf-8\")) as T;\n }\n\n /** Run migrations for the persistent storage */\n private async runMigrations(): Promise<void> {\n try {\n await this.connector.query(`\n CREATE TABLE IF NOT EXISTS ${this.tableName} (\n id BIGSERIAL PRIMARY KEY,\n key_hash BIGINT NOT NULL,\n key BYTEA NOT NULL,\n value BYTEA NOT NULL,\n byte_size INTEGER NOT NULL,\n expiry BIGINT NOT NULL,\n created_at TIMESTAMP NOT NULL DEFAULT NOW(),\n last_accessed TIMESTAMP NOT NULL DEFAULT NOW()\n )\n `);\n\n // unique index on key_hash for fast lookups\n await this.connector.query(\n `CREATE UNIQUE INDEX IF NOT EXISTS idx_${this.tableName}_key_hash ON ${this.tableName} (key_hash);`,\n );\n\n // index on expiry for cleanup queries\n await this.connector.query(\n `CREATE INDEX IF NOT EXISTS idx_${this.tableName}_expiry ON ${this.tableName} (expiry); `,\n );\n\n // index on last_accessed for LRU eviction\n await this.connector.query(\n `CREATE INDEX IF NOT EXISTS idx_${this.tableName}_last_accessed ON ${this.tableName} (last_accessed); `,\n );\n\n // index on byte_size for monitoring\n await this.connector.query(\n `CREATE INDEX IF NOT EXISTS idx_${this.tableName}_byte_size ON ${this.tableName} (byte_size); `,\n );\n } catch (error) {\n console.error(\n \"Error in running migrations for persistent storage:\",\n error,\n );\n throw error;\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAmBA,IAAa,oBAAb,MAAuD;CASrD,YAAY,QAAqB,WAA8B;AAC7D,OAAK,YAAY;AACjB,OAAK,WAAW,OAAO,YAAY,wBAAwB;AAC3D,OAAK,gBACH,OAAO,iBAAiB,wBAAwB;AAClD,OAAK,oBAAoB,wBAAwB;AACjD,OAAK,2BACH,OAAO,4BACP,wBAAwB;AAC1B,OAAK,YAAY,wBAAwB;AACzC,OAAK,cAAc;;;CAIrB,MAAM,aAA4B;AAChC,MAAI,KAAK,YAAa;AAEtB,MAAI;AACF,SAAM,KAAK,eAAe;AAC1B,QAAK,cAAc;WACZ,OAAO;AACd,WAAQ,MAAM,+CAA+C,MAAM;AACnE,SAAM;;;;;;;;CASV,MAAM,IAAO,KAA4C;AACvD,QAAM,KAAK,mBAAmB;EAE9B,MAAM,UAAU,KAAK,QAAQ,IAAI;EAEjC,MAAM,SAAS,MAAM,KAAK,UAAU,MAGjC,6BAA6B,KAAK,UAAU,uBAAuB,CACpE,QACD,CAAC;AAEF,MAAI,OAAO,KAAK,WAAW,EAAG,QAAO;EAErC,MAAM,QAAQ,OAAO,KAAK;AAG1B,OAAK,UACF,MACC,UAAU,KAAK,UAAU,iDACzB,CAAC,QAAQ,CACV,CACA,YAAY;AACX,WAAQ,MAAM,8CAA8C,IAAI;IAChE;AAEJ,SAAO;GACL,OAAO,KAAK,iBAAoB,MAAM,MAAM;GAC5C,QAAQ,OAAO,MAAM,OAAO;GAC7B;;;;;;;;CASH,MAAM,IAAO,KAAa,OAAqC;AAC7D,QAAM,KAAK,mBAAmB;EAE9B,MAAM,UAAU,KAAK,QAAQ,IAAI;EACjC,MAAM,WAAW,OAAO,KAAK,KAAK,QAAQ;EAC1C,MAAM,aAAa,KAAK,eAAe,MAAM,MAAM;EACnD,MAAM,WAAW,SAAS,SAAS,WAAW;AAE9C,MAAI,WAAW,KAAK,cAClB,OAAM,IAAI,MACR,0BAA0B,SAAS,4BAA4B,KAAK,cAAc,QACnF;AAIH,MAAI,KAAK,QAAQ,GAAG,KAAK,0BAEvB;OADmB,MAAM,KAAK,YAAY,GACzB,WAAW,KAAK,SAC/B,OAAM,KAAK,YAAY,SAAS;;AAIpC,QAAM,KAAK,UAAU,MACnB,eAAe,KAAK,UAAU;;;;SAK9B;GAAC;GAAS;GAAU;GAAY;GAAU,MAAM;GAAO,CACxD;;;;;;;CAQH,MAAM,OAAO,KAA4B;AACvC,QAAM,KAAK,mBAAmB;EAC9B,MAAM,UAAU,KAAK,QAAQ,IAAI;AACjC,QAAM,KAAK,UAAU,MACnB,eAAe,KAAK,UAAU,uBAC9B,CAAC,QAAQ,CACV;;;CAIH,MAAM,QAAuB;AAC3B,QAAM,KAAK,mBAAmB;AAC9B,QAAM,KAAK,UAAU,MAAM,kBAAkB,KAAK,YAAY;;;;;;;CAQhE,MAAM,IAAI,KAA+B;AACvC,QAAM,KAAK,mBAAmB;EAC9B,MAAM,UAAU,KAAK,QAAQ,IAAI;AAOjC,UALe,MAAM,KAAK,UAAU,MAClC,+BAA+B,KAAK,UAAU,kCAC9C,CAAC,QAAQ,CACV,EAEa,KAAK,IAAI,UAAU;;;;;;CAOnC,MAAM,OAAwB;AAC5B,QAAM,KAAK,mBAAmB;EAE9B,MAAM,SAAS,MAAM,KAAK,UAAU,MAClC,iCAAiC,KAAK,YACvC;AACD,SAAO,SAAS,OAAO,KAAK,IAAI,SAAS,KAAK,GAAG;;;CAInD,MAAM,aAA8B;AAClC,QAAM,KAAK,mBAAmB;EAE9B,MAAM,SAAS,MAAM,KAAK,UAAU,MAClC,oDAAoD,KAAK,YAC1D;AACD,SAAO,SAAS,OAAO,KAAK,IAAI,SAAS,KAAK,GAAG;;;;;;CAOnD,eAAwB;AACtB,SAAO;;;;;;CAOT,MAAM,cAAgC;AACpC,MAAI;AACF,UAAO,MAAM,KAAK,UAAU,aAAa;UACnC;AACN,UAAO;;;;CAKX,MAAM,QAAuB;AAC3B,QAAM,KAAK,UAAU,OAAO;;;;;;CAO9B,MAAM,iBAAkC;AACtC,QAAM,KAAK,mBAAmB;EAC9B,MAAM,SAAS,MAAM,KAAK,UAAU,MAClC,gCAAgC,KAAK,UAAU,wEAC/C,CAAC,KAAK,KAAK,CAAC,CACb;AACD,SAAO,SAAS,OAAO,KAAK,IAAI,SAAS,KAAK,GAAG;;;CAInD,MAAc,YAAY,eAAsC;AAE9D,MADsB,MAAM,KAAK,gBAAgB,GAC7B,GAElB;OADqB,MAAM,KAAK,YAAY,GACzB,iBAAiB,KAAK,SACvC;;AAIJ,QAAM,KAAK,UAAU,MACnB,eAAe,KAAK,UAAU;8BACN,KAAK,UAAU,wCACvC,CAAC,KAAK,kBAAkB,CACzB;;;CAIH,MAAc,oBAAmC;AAC/C,MAAI,CAAC,KAAK,YACR,OAAM,KAAK,YAAY;;;CAK3B,AAAQ,QAAQ,KAAqB;AACnC,MAAI,CAAC,IAAK,OAAM,IAAI,MAAM,4BAA4B;AAEtD,SADa,WAAW,SAAS,CAAC,OAAO,IAAI,CAAC,QAAQ,CAC1C,eAAe,EAAE;;;CAI/B,AAAQ,eAAkB,OAAkB;AAC1C,SAAO,OAAO,KAAK,KAAK,UAAU,MAAM,EAAE,QAAQ;;;CAIpD,AAAQ,iBAAoB,QAAmB;AAC7C,SAAO,KAAK,MAAM,OAAO,SAAS,QAAQ,CAAC;;;CAI7C,MAAc,gBAA+B;AAC3C,MAAI;AACF,SAAM,KAAK,UAAU,MAAM;yCACQ,KAAK,UAAU;;;;;;;;;;cAU1C;AAGR,SAAM,KAAK,UAAU,MACnB,yCAAyC,KAAK,UAAU,eAAe,KAAK,UAAU,cACvF;AAGD,SAAM,KAAK,UAAU,MACnB,kCAAkC,KAAK,UAAU,aAAa,KAAK,UAAU,aAC9E;AAGD,SAAM,KAAK,UAAU,MACnB,kCAAkC,KAAK,UAAU,oBAAoB,KAAK,UAAU,oBACrF;AAGD,SAAM,KAAK,UAAU,MACnB,kCAAkC,KAAK,UAAU,gBAAgB,KAAK,UAAU,gBACjF;WACM,OAAO;AACd,WAAQ,MACN,uDACA,MACD;AACD,SAAM"}
|
|
1
|
+
{"version":3,"file":"persistent.js","names":[],"sources":["../../../src/cache/storage/persistent.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport type { CacheConfig, CacheEntry, CacheStorage } from \"shared\";\nimport type { LakebaseConnector } from \"../../connectors\";\nimport { InitializationError, ValidationError } from \"../../errors\";\nimport { createLogger } from \"../../logging/logger\";\nimport { lakebaseStorageDefaults } from \"./defaults\";\n\nconst logger = createLogger(\"cache:persistent\");\n\n/**\n * Persistent cache storage implementation. Uses a least recently used (LRU) eviction policy\n * to manage memory usage and ensure efficient cache operations.\n *\n * @example\n * const persistentStorage = new PersistentStorage(config, connector);\n * await persistentStorage.initialize();\n * await persistentStorage.get(\"my-key\");\n * await persistentStorage.set(\"my-key\", \"my-value\");\n * await persistentStorage.delete(\"my-key\");\n * await persistentStorage.clear();\n * await persistentStorage.has(\"my-key\");\n *\n */\nexport class PersistentStorage implements CacheStorage {\n private readonly connector: LakebaseConnector;\n private readonly tableName: string;\n private readonly maxBytes: number;\n private readonly maxEntryBytes: number;\n private readonly evictionBatchSize: number;\n private readonly evictionCheckProbability: number;\n private initialized: boolean;\n\n constructor(config: CacheConfig, connector: LakebaseConnector) {\n this.connector = connector;\n this.maxBytes = config.maxBytes ?? lakebaseStorageDefaults.maxBytes;\n this.maxEntryBytes =\n config.maxEntryBytes ?? lakebaseStorageDefaults.maxEntryBytes;\n this.evictionBatchSize = lakebaseStorageDefaults.evictionBatchSize;\n this.evictionCheckProbability =\n config.evictionCheckProbability ??\n lakebaseStorageDefaults.evictionCheckProbability;\n this.tableName = lakebaseStorageDefaults.tableName; // hardcoded, safe for now\n this.initialized = false;\n }\n\n /** Initialize the persistent storage and run migrations if necessary */\n async initialize(): Promise<void> {\n if (this.initialized) return;\n\n try {\n await this.runMigrations();\n this.initialized = true;\n } catch (error) {\n logger.error(\"Error in persistent storage initialization: %O\", error);\n throw error;\n }\n }\n\n /**\n * Get a cached value from the persistent storage\n * @param key - Cache key\n * @returns Promise of the cached value or null if not found\n */\n async get<T>(key: string): Promise<CacheEntry<T> | null> {\n await this.ensureInitialized();\n\n const keyHash = this.hashKey(key);\n\n const result = await this.connector.query<{\n value: Buffer;\n expiry: string;\n }>(`SELECT value, expiry FROM ${this.tableName} WHERE key_hash = $1`, [\n keyHash,\n ]);\n\n if (result.rows.length === 0) return null;\n\n const entry = result.rows[0];\n\n // fire-and-forget update\n this.connector\n .query(\n `UPDATE ${this.tableName} SET last_accessed = NOW() WHERE key_hash = $1`,\n [keyHash],\n )\n .catch(() => {\n logger.debug(\"Error updating last_accessed time for key: %s\", key);\n });\n\n return {\n value: this.deserializeValue<T>(entry.value),\n expiry: Number(entry.expiry),\n };\n }\n\n /**\n * Set a value in the persistent storage\n * @param key - Cache key\n * @param entry - Cache entry\n * @returns Promise of the result\n */\n async set<T>(key: string, entry: CacheEntry<T>): Promise<void> {\n await this.ensureInitialized();\n\n const keyHash = this.hashKey(key);\n const keyBytes = Buffer.from(key, \"utf-8\");\n const valueBytes = this.serializeValue(entry.value);\n const byteSize = keyBytes.length + valueBytes.length;\n\n if (byteSize > this.maxEntryBytes) {\n throw ValidationError.invalidValue(\n \"cache entry size\",\n byteSize,\n `maximum ${this.maxEntryBytes} bytes`,\n );\n }\n\n // probabilistic eviction check\n if (Math.random() < this.evictionCheckProbability) {\n const totalBytes = await this.totalBytes();\n if (totalBytes + byteSize > this.maxBytes) {\n await this.evictBySize(byteSize);\n }\n }\n\n await this.connector.query(\n `INSERT INTO ${this.tableName} (key_hash, key, value, byte_size, expiry, created_at, last_accessed)\n VALUES ($1, $2, $3, $4, $5, NOW(), NOW())\n ON CONFLICT (key_hash)\n DO UPDATE SET value = $3, byte_size = $4, expiry = $5, last_accessed = NOW()\n `,\n [keyHash, keyBytes, valueBytes, byteSize, entry.expiry],\n );\n }\n\n /**\n * Delete a value from the persistent storage\n * @param key - Cache key\n * @returns Promise of the result\n */\n async delete(key: string): Promise<void> {\n await this.ensureInitialized();\n const keyHash = this.hashKey(key);\n await this.connector.query(\n `DELETE FROM ${this.tableName} WHERE key_hash = $1`,\n [keyHash],\n );\n }\n\n /** Clear the persistent storage */\n async clear(): Promise<void> {\n await this.ensureInitialized();\n await this.connector.query(`TRUNCATE TABLE ${this.tableName}`);\n }\n\n /**\n * Check if a value exists in the persistent storage\n * @param key - Cache key\n * @returns Promise of true if the value exists, false otherwise\n */\n async has(key: string): Promise<boolean> {\n await this.ensureInitialized();\n const keyHash = this.hashKey(key);\n\n const result = await this.connector.query<{ exists: boolean }>(\n `SELECT EXISTS(SELECT 1 FROM ${this.tableName} WHERE key_hash = $1) as exists`,\n [keyHash],\n );\n\n return result.rows[0]?.exists ?? false;\n }\n\n /**\n * Get the size of the persistent storage\n * @returns Promise of the size of the storage\n */\n async size(): Promise<number> {\n await this.ensureInitialized();\n\n const result = await this.connector.query<{ count: string }>(\n `SELECT COUNT(*) as count FROM ${this.tableName}`,\n );\n return parseInt(result.rows[0]?.count ?? \"0\", 10);\n }\n\n /** Get the total number of bytes in the persistent storage */\n async totalBytes(): Promise<number> {\n await this.ensureInitialized();\n\n const result = await this.connector.query<{ total: string }>(\n `SELECT COALESCE(SUM(byte_size), 0) as total FROM ${this.tableName}`,\n );\n return parseInt(result.rows[0]?.total ?? \"0\", 10);\n }\n\n /**\n * Check if the persistent storage is persistent\n * @returns true if the storage is persistent, false otherwise\n */\n isPersistent(): boolean {\n return true;\n }\n\n /**\n * Check if the persistent storage is healthy\n * @returns Promise of true if the storage is healthy, false otherwise\n */\n async healthCheck(): Promise<boolean> {\n try {\n return await this.connector.healthCheck();\n } catch {\n return false;\n }\n }\n\n /** Close the persistent storage */\n async close(): Promise<void> {\n await this.connector.close();\n }\n\n /**\n * Cleanup expired entries from the persistent storage\n * @returns Promise of the number of expired entries\n */\n async cleanupExpired(): Promise<number> {\n await this.ensureInitialized();\n const result = await this.connector.query<{ count: string }>(\n `WITH deleted as (DELETE FROM ${this.tableName} WHERE expiry < $1 RETURNING *) SELECT COUNT(*) as count FROM deleted`,\n [Date.now()],\n );\n return parseInt(result.rows[0]?.count ?? \"0\", 10);\n }\n\n /** Evict entries from the persistent storage by size */\n private async evictBySize(requiredBytes: number): Promise<void> {\n const freedByExpiry = await this.cleanupExpired();\n if (freedByExpiry > 0) {\n const currentBytes = await this.totalBytes();\n if (currentBytes + requiredBytes <= this.maxBytes) {\n return;\n }\n }\n\n await this.connector.query(\n `DELETE FROM ${this.tableName} WHERE key_hash IN\n (SELECT key_hash FROM ${this.tableName} ORDER BY last_accessed ASC LIMIT $1)`,\n [this.evictionBatchSize],\n );\n }\n\n /** Ensure the persistent storage is initialized */\n private async ensureInitialized(): Promise<void> {\n if (!this.initialized) {\n await this.initialize();\n }\n }\n\n /** Generate a 64-bit hash for the cache key using SHA256 */\n private hashKey(key: string): bigint {\n if (!key) throw ValidationError.missingField(\"key\");\n const hash = createHash(\"sha256\").update(key).digest();\n return hash.readBigInt64BE(0);\n }\n\n /** Serialize a value to a buffer */\n private serializeValue<T>(value: T): Buffer {\n return Buffer.from(JSON.stringify(value), \"utf-8\");\n }\n\n /** Deserialize a value from a buffer */\n private deserializeValue<T>(buffer: Buffer): T {\n return JSON.parse(buffer.toString(\"utf-8\")) as T;\n }\n\n /** Run migrations for the persistent storage */\n private async runMigrations(): Promise<void> {\n try {\n await this.connector.query(`\n CREATE TABLE IF NOT EXISTS ${this.tableName} (\n id BIGSERIAL PRIMARY KEY,\n key_hash BIGINT NOT NULL,\n key BYTEA NOT NULL,\n value BYTEA NOT NULL,\n byte_size INTEGER NOT NULL,\n expiry BIGINT NOT NULL,\n created_at TIMESTAMP NOT NULL DEFAULT NOW(),\n last_accessed TIMESTAMP NOT NULL DEFAULT NOW()\n )\n `);\n\n // unique index on key_hash for fast lookups\n await this.connector.query(\n `CREATE UNIQUE INDEX IF NOT EXISTS idx_${this.tableName}_key_hash ON ${this.tableName} (key_hash);`,\n );\n\n // index on expiry for cleanup queries\n await this.connector.query(\n `CREATE INDEX IF NOT EXISTS idx_${this.tableName}_expiry ON ${this.tableName} (expiry); `,\n );\n\n // index on last_accessed for LRU eviction\n await this.connector.query(\n `CREATE INDEX IF NOT EXISTS idx_${this.tableName}_last_accessed ON ${this.tableName} (last_accessed); `,\n );\n\n // index on byte_size for monitoring\n await this.connector.query(\n `CREATE INDEX IF NOT EXISTS idx_${this.tableName}_byte_size ON ${this.tableName} (byte_size); `,\n );\n } catch (error) {\n logger.error(\n \"Error in running migrations for persistent storage: %O\",\n error,\n );\n throw InitializationError.migrationFailed(error as Error);\n }\n }\n}\n"],"mappings":";;;;;;;;aAGoE;AAIpE,MAAM,SAAS,aAAa,mBAAmB;;;;;;;;;;;;;;;AAgB/C,IAAa,oBAAb,MAAuD;CASrD,YAAY,QAAqB,WAA8B;AAC7D,OAAK,YAAY;AACjB,OAAK,WAAW,OAAO,YAAY,wBAAwB;AAC3D,OAAK,gBACH,OAAO,iBAAiB,wBAAwB;AAClD,OAAK,oBAAoB,wBAAwB;AACjD,OAAK,2BACH,OAAO,4BACP,wBAAwB;AAC1B,OAAK,YAAY,wBAAwB;AACzC,OAAK,cAAc;;;CAIrB,MAAM,aAA4B;AAChC,MAAI,KAAK,YAAa;AAEtB,MAAI;AACF,SAAM,KAAK,eAAe;AAC1B,QAAK,cAAc;WACZ,OAAO;AACd,UAAO,MAAM,kDAAkD,MAAM;AACrE,SAAM;;;;;;;;CASV,MAAM,IAAO,KAA4C;AACvD,QAAM,KAAK,mBAAmB;EAE9B,MAAM,UAAU,KAAK,QAAQ,IAAI;EAEjC,MAAM,SAAS,MAAM,KAAK,UAAU,MAGjC,6BAA6B,KAAK,UAAU,uBAAuB,CACpE,QACD,CAAC;AAEF,MAAI,OAAO,KAAK,WAAW,EAAG,QAAO;EAErC,MAAM,QAAQ,OAAO,KAAK;AAG1B,OAAK,UACF,MACC,UAAU,KAAK,UAAU,iDACzB,CAAC,QAAQ,CACV,CACA,YAAY;AACX,UAAO,MAAM,iDAAiD,IAAI;IAClE;AAEJ,SAAO;GACL,OAAO,KAAK,iBAAoB,MAAM,MAAM;GAC5C,QAAQ,OAAO,MAAM,OAAO;GAC7B;;;;;;;;CASH,MAAM,IAAO,KAAa,OAAqC;AAC7D,QAAM,KAAK,mBAAmB;EAE9B,MAAM,UAAU,KAAK,QAAQ,IAAI;EACjC,MAAM,WAAW,OAAO,KAAK,KAAK,QAAQ;EAC1C,MAAM,aAAa,KAAK,eAAe,MAAM,MAAM;EACnD,MAAM,WAAW,SAAS,SAAS,WAAW;AAE9C,MAAI,WAAW,KAAK,cAClB,OAAM,gBAAgB,aACpB,oBACA,UACA,WAAW,KAAK,cAAc,QAC/B;AAIH,MAAI,KAAK,QAAQ,GAAG,KAAK,0BAEvB;OADmB,MAAM,KAAK,YAAY,GACzB,WAAW,KAAK,SAC/B,OAAM,KAAK,YAAY,SAAS;;AAIpC,QAAM,KAAK,UAAU,MACnB,eAAe,KAAK,UAAU;;;;SAK9B;GAAC;GAAS;GAAU;GAAY;GAAU,MAAM;GAAO,CACxD;;;;;;;CAQH,MAAM,OAAO,KAA4B;AACvC,QAAM,KAAK,mBAAmB;EAC9B,MAAM,UAAU,KAAK,QAAQ,IAAI;AACjC,QAAM,KAAK,UAAU,MACnB,eAAe,KAAK,UAAU,uBAC9B,CAAC,QAAQ,CACV;;;CAIH,MAAM,QAAuB;AAC3B,QAAM,KAAK,mBAAmB;AAC9B,QAAM,KAAK,UAAU,MAAM,kBAAkB,KAAK,YAAY;;;;;;;CAQhE,MAAM,IAAI,KAA+B;AACvC,QAAM,KAAK,mBAAmB;EAC9B,MAAM,UAAU,KAAK,QAAQ,IAAI;AAOjC,UALe,MAAM,KAAK,UAAU,MAClC,+BAA+B,KAAK,UAAU,kCAC9C,CAAC,QAAQ,CACV,EAEa,KAAK,IAAI,UAAU;;;;;;CAOnC,MAAM,OAAwB;AAC5B,QAAM,KAAK,mBAAmB;EAE9B,MAAM,SAAS,MAAM,KAAK,UAAU,MAClC,iCAAiC,KAAK,YACvC;AACD,SAAO,SAAS,OAAO,KAAK,IAAI,SAAS,KAAK,GAAG;;;CAInD,MAAM,aAA8B;AAClC,QAAM,KAAK,mBAAmB;EAE9B,MAAM,SAAS,MAAM,KAAK,UAAU,MAClC,oDAAoD,KAAK,YAC1D;AACD,SAAO,SAAS,OAAO,KAAK,IAAI,SAAS,KAAK,GAAG;;;;;;CAOnD,eAAwB;AACtB,SAAO;;;;;;CAOT,MAAM,cAAgC;AACpC,MAAI;AACF,UAAO,MAAM,KAAK,UAAU,aAAa;UACnC;AACN,UAAO;;;;CAKX,MAAM,QAAuB;AAC3B,QAAM,KAAK,UAAU,OAAO;;;;;;CAO9B,MAAM,iBAAkC;AACtC,QAAM,KAAK,mBAAmB;EAC9B,MAAM,SAAS,MAAM,KAAK,UAAU,MAClC,gCAAgC,KAAK,UAAU,wEAC/C,CAAC,KAAK,KAAK,CAAC,CACb;AACD,SAAO,SAAS,OAAO,KAAK,IAAI,SAAS,KAAK,GAAG;;;CAInD,MAAc,YAAY,eAAsC;AAE9D,MADsB,MAAM,KAAK,gBAAgB,GAC7B,GAElB;OADqB,MAAM,KAAK,YAAY,GACzB,iBAAiB,KAAK,SACvC;;AAIJ,QAAM,KAAK,UAAU,MACnB,eAAe,KAAK,UAAU;8BACN,KAAK,UAAU,wCACvC,CAAC,KAAK,kBAAkB,CACzB;;;CAIH,MAAc,oBAAmC;AAC/C,MAAI,CAAC,KAAK,YACR,OAAM,KAAK,YAAY;;;CAK3B,AAAQ,QAAQ,KAAqB;AACnC,MAAI,CAAC,IAAK,OAAM,gBAAgB,aAAa,MAAM;AAEnD,SADa,WAAW,SAAS,CAAC,OAAO,IAAI,CAAC,QAAQ,CAC1C,eAAe,EAAE;;;CAI/B,AAAQ,eAAkB,OAAkB;AAC1C,SAAO,OAAO,KAAK,KAAK,UAAU,MAAM,EAAE,QAAQ;;;CAIpD,AAAQ,iBAAoB,QAAmB;AAC7C,SAAO,KAAK,MAAM,OAAO,SAAS,QAAQ,CAAC;;;CAI7C,MAAc,gBAA+B;AAC3C,MAAI;AACF,SAAM,KAAK,UAAU,MAAM;yCACQ,KAAK,UAAU;;;;;;;;;;cAU1C;AAGR,SAAM,KAAK,UAAU,MACnB,yCAAyC,KAAK,UAAU,eAAe,KAAK,UAAU,cACvF;AAGD,SAAM,KAAK,UAAU,MACnB,kCAAkC,KAAK,UAAU,aAAa,KAAK,UAAU,aAC9E;AAGD,SAAM,KAAK,UAAU,MACnB,kCAAkC,KAAK,UAAU,oBAAoB,KAAK,UAAU,oBACrF;AAGD,SAAM,KAAK,UAAU,MACnB,kCAAkC,KAAK,UAAU,gBAAgB,KAAK,UAAU,gBACjF;WACM,OAAO;AACd,UAAO,MACL,0DACA,MACD;AACD,SAAM,oBAAoB,gBAAgB,MAAe"}
|
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
import { __toCommonJS } from "../../_virtual/rolldown_runtime.js";
|
|
2
|
+
import { createLogger } from "../../logging/logger.js";
|
|
2
3
|
import { TelemetryManager } from "../../telemetry/telemetry-manager.js";
|
|
3
4
|
import { SpanStatusCode } from "../../telemetry/index.js";
|
|
5
|
+
import { AppKitError } from "../../errors/base.js";
|
|
6
|
+
import { AuthenticationError } from "../../errors/authentication.js";
|
|
7
|
+
import { ConfigurationError } from "../../errors/configuration.js";
|
|
8
|
+
import { ConnectionError } from "../../errors/connection.js";
|
|
9
|
+
import { ValidationError } from "../../errors/validation.js";
|
|
10
|
+
import { init_errors } from "../../errors/index.js";
|
|
4
11
|
import { deepMerge } from "../../utils/merge.js";
|
|
5
12
|
import { lakebaseDefaults } from "./defaults.js";
|
|
6
13
|
import { context_exports, init_context } from "../../context/index.js";
|
|
@@ -9,6 +16,8 @@ import { ApiClient, Config } from "@databricks/sdk-experimental";
|
|
|
9
16
|
import pg from "pg";
|
|
10
17
|
|
|
11
18
|
//#region src/connectors/lakebase/client.ts
|
|
19
|
+
init_errors();
|
|
20
|
+
const logger = createLogger("connectors:lakebase");
|
|
12
21
|
/**
|
|
13
22
|
* Enterprise-grade connector for Databricks Lakebase
|
|
14
23
|
* @example Simplest - everything from env/context
|
|
@@ -43,7 +52,7 @@ var LakebaseConnector = class {
|
|
|
43
52
|
unit: "ms"
|
|
44
53
|
})
|
|
45
54
|
};
|
|
46
|
-
if (this.config.maxPoolSize < 1) throw
|
|
55
|
+
if (this.config.maxPoolSize < 1) throw ValidationError.invalidValue("maxPoolSize", this.config.maxPoolSize, "at least 1");
|
|
47
56
|
}
|
|
48
57
|
/**
|
|
49
58
|
* Execute a SQL query
|
|
@@ -82,7 +91,8 @@ var LakebaseConnector = class {
|
|
|
82
91
|
}
|
|
83
92
|
span.recordException(error);
|
|
84
93
|
span.setStatus({ code: SpanStatusCode.ERROR });
|
|
85
|
-
throw error;
|
|
94
|
+
if (error instanceof AppKitError) throw error;
|
|
95
|
+
throw ConnectionError.queryFailed(error);
|
|
86
96
|
} finally {
|
|
87
97
|
const duration = Date.now() - startTime;
|
|
88
98
|
this.telemetryMetrics.queryCount.add(1);
|
|
@@ -150,7 +160,8 @@ var LakebaseConnector = class {
|
|
|
150
160
|
}
|
|
151
161
|
span.recordException(error);
|
|
152
162
|
span.setStatus({ code: SpanStatusCode.ERROR });
|
|
153
|
-
throw error;
|
|
163
|
+
if (error instanceof AppKitError) throw error;
|
|
164
|
+
throw ConnectionError.transactionFailed(error);
|
|
154
165
|
} finally {
|
|
155
166
|
client.release();
|
|
156
167
|
const duration = Date.now() - startTime;
|
|
@@ -181,7 +192,7 @@ var LakebaseConnector = class {
|
|
|
181
192
|
async close() {
|
|
182
193
|
if (this.pool) {
|
|
183
194
|
await this.pool.end().catch((error) => {
|
|
184
|
-
|
|
195
|
+
logger.error("Error closing connection pool: %O", error);
|
|
185
196
|
});
|
|
186
197
|
this.pool = null;
|
|
187
198
|
}
|
|
@@ -202,12 +213,12 @@ var LakebaseConnector = class {
|
|
|
202
213
|
this.config.workspaceClient = client;
|
|
203
214
|
return client;
|
|
204
215
|
} catch (_error) {
|
|
205
|
-
throw
|
|
216
|
+
throw ConnectionError.clientUnavailable("Databricks workspace client", "Either pass it in config or ensure ServiceContext is initialized");
|
|
206
217
|
}
|
|
207
218
|
}
|
|
208
219
|
/** Get or create connection pool */
|
|
209
220
|
async getPool() {
|
|
210
|
-
if (!this.connectionConfig) throw
|
|
221
|
+
if (!this.connectionConfig) throw ConfigurationError.invalidConnection("Lakebase", "Set PGHOST, PGDATABASE, PGAPPNAME env vars, provide a connectionString, or pass explicit config");
|
|
211
222
|
if (!this.pool) {
|
|
212
223
|
const creds = await this.getCredentials();
|
|
213
224
|
this.pool = this.createPool(creds);
|
|
@@ -229,7 +240,7 @@ var LakebaseConnector = class {
|
|
|
229
240
|
ssl: sslMode === "require" ? { rejectUnauthorized: true } : false
|
|
230
241
|
});
|
|
231
242
|
pool.on("error", (error) => {
|
|
232
|
-
|
|
243
|
+
logger.error("Connection pool error: %s (code: %s)", error.message, error.code);
|
|
233
244
|
});
|
|
234
245
|
return pool;
|
|
235
246
|
}
|
|
@@ -256,20 +267,20 @@ var LakebaseConnector = class {
|
|
|
256
267
|
const oldPool = this.pool;
|
|
257
268
|
this.pool = null;
|
|
258
269
|
oldPool.end().catch((error) => {
|
|
259
|
-
|
|
270
|
+
logger.error("Error closing old connection pool during rotation: %O", error);
|
|
260
271
|
});
|
|
261
272
|
}
|
|
262
273
|
}
|
|
263
274
|
/** Fetch username from Databricks */
|
|
264
275
|
async fetchUsername() {
|
|
265
276
|
const user = await this.getWorkspaceClient().currentUser.me();
|
|
266
|
-
if (!user.userName) throw
|
|
277
|
+
if (!user.userName) throw AuthenticationError.userLookupFailed();
|
|
267
278
|
return user.userName;
|
|
268
279
|
}
|
|
269
280
|
/** Fetch password (OAuth token) from Databricks */
|
|
270
281
|
async fetchPassword() {
|
|
271
282
|
const apiClient = new ApiClient(new Config({ host: this.getWorkspaceClient().config.host }));
|
|
272
|
-
if (!this.connectionConfig.appName) throw
|
|
283
|
+
if (!this.connectionConfig.appName) throw ConfigurationError.resourceNotFound("Database app name");
|
|
273
284
|
const credentials = await apiClient.request({
|
|
274
285
|
path: `/api/2.0/database/credentials`,
|
|
275
286
|
method: "POST",
|
|
@@ -280,7 +291,7 @@ var LakebaseConnector = class {
|
|
|
280
291
|
request_id: randomUUID()
|
|
281
292
|
}
|
|
282
293
|
});
|
|
283
|
-
if (!this.validateCredentials(credentials)) throw
|
|
294
|
+
if (!this.validateCredentials(credentials)) throw AuthenticationError.credentialsFailed(this.connectionConfig.appName);
|
|
284
295
|
const expiresAt = new Date(credentials.expiration_time).getTime();
|
|
285
296
|
return {
|
|
286
297
|
token: credentials.token,
|
|
@@ -316,10 +327,10 @@ var LakebaseConnector = class {
|
|
|
316
327
|
const pgHost = process.env.PGHOST;
|
|
317
328
|
const pgDatabase = process.env.PGDATABASE;
|
|
318
329
|
const pgAppName = process.env.PGAPPNAME;
|
|
319
|
-
if (!pgHost || !pgDatabase || !pgAppName) throw
|
|
330
|
+
if (!pgHost || !pgDatabase || !pgAppName) throw ConfigurationError.invalidConnection("Lakebase", "Required env vars: PGHOST, PGDATABASE, PGAPPNAME. Optional: PGPORT (default: 5432), PGSSLMODE (default: require)");
|
|
320
331
|
const pgPort = process.env.PGPORT;
|
|
321
332
|
const port = pgPort ? parseInt(pgPort, 10) : 5432;
|
|
322
|
-
if (Number.isNaN(port)) throw
|
|
333
|
+
if (Number.isNaN(port)) throw ValidationError.invalidValue("port", pgPort, "a number");
|
|
323
334
|
return {
|
|
324
335
|
host: pgHost,
|
|
325
336
|
database: pgDatabase,
|
|
@@ -331,7 +342,7 @@ var LakebaseConnector = class {
|
|
|
331
342
|
parseConnectionString(connectionString) {
|
|
332
343
|
const url = new URL(connectionString);
|
|
333
344
|
const appName = url.searchParams.get("appName");
|
|
334
|
-
if (!appName) throw
|
|
345
|
+
if (!appName) throw ConfigurationError.missingConnectionParam("appName");
|
|
335
346
|
return {
|
|
336
347
|
host: url.hostname,
|
|
337
348
|
database: url.pathname.slice(1),
|