@databricks/appkit 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +89 -12
- package/CLAUDE.md +89 -12
- package/NOTICE.md +4 -0
- package/README.md +21 -15
- package/bin/appkit-lint.js +129 -0
- package/dist/analytics/analytics.d.ts +33 -8
- package/dist/analytics/analytics.d.ts.map +1 -1
- package/dist/analytics/analytics.js +67 -27
- package/dist/analytics/analytics.js.map +1 -1
- package/dist/analytics/defaults.js.map +1 -1
- package/dist/analytics/query.js +12 -6
- package/dist/analytics/query.js.map +1 -1
- package/dist/app/index.d.ts.map +1 -1
- package/dist/app/index.js +7 -5
- package/dist/app/index.js.map +1 -1
- package/dist/appkit/package.js +1 -1
- package/dist/cache/defaults.js.map +1 -1
- package/dist/cache/index.d.ts +1 -0
- package/dist/cache/index.d.ts.map +1 -1
- package/dist/cache/index.js +25 -5
- package/dist/cache/index.js.map +1 -1
- package/dist/cache/storage/memory.js.map +1 -1
- package/dist/cache/storage/persistent.js +12 -6
- package/dist/cache/storage/persistent.js.map +1 -1
- package/dist/connectors/lakebase/client.js +31 -21
- package/dist/connectors/lakebase/client.js.map +1 -1
- package/dist/connectors/lakebase/defaults.js.map +1 -1
- package/dist/connectors/sql-warehouse/client.js +68 -28
- package/dist/connectors/sql-warehouse/client.js.map +1 -1
- package/dist/connectors/sql-warehouse/defaults.js.map +1 -1
- package/dist/context/execution-context.js +75 -0
- package/dist/context/execution-context.js.map +1 -0
- package/dist/context/index.js +27 -0
- package/dist/context/index.js.map +1 -0
- package/dist/context/service-context.js +154 -0
- package/dist/context/service-context.js.map +1 -0
- package/dist/context/user-context.js +15 -0
- package/dist/context/user-context.js.map +1 -0
- package/dist/core/appkit.d.ts +3 -0
- package/dist/core/appkit.d.ts.map +1 -1
- package/dist/core/appkit.js +7 -0
- package/dist/core/appkit.js.map +1 -1
- package/dist/errors/authentication.d.ts +38 -0
- package/dist/errors/authentication.d.ts.map +1 -0
- package/dist/errors/authentication.js +48 -0
- package/dist/errors/authentication.js.map +1 -0
- package/dist/errors/base.d.ts +58 -0
- package/dist/errors/base.d.ts.map +1 -0
- package/dist/errors/base.js +70 -0
- package/dist/errors/base.js.map +1 -0
- package/dist/errors/configuration.d.ts +38 -0
- package/dist/errors/configuration.d.ts.map +1 -0
- package/dist/errors/configuration.js +45 -0
- package/dist/errors/configuration.js.map +1 -0
- package/dist/errors/connection.d.ts +42 -0
- package/dist/errors/connection.d.ts.map +1 -0
- package/dist/errors/connection.js +54 -0
- package/dist/errors/connection.js.map +1 -0
- package/dist/errors/execution.d.ts +42 -0
- package/dist/errors/execution.d.ts.map +1 -0
- package/dist/errors/execution.js +51 -0
- package/dist/errors/execution.js.map +1 -0
- package/dist/errors/index.js +28 -0
- package/dist/errors/index.js.map +1 -0
- package/dist/errors/initialization.d.ts +34 -0
- package/dist/errors/initialization.d.ts.map +1 -0
- package/dist/errors/initialization.js +42 -0
- package/dist/errors/initialization.js.map +1 -0
- package/dist/errors/server.d.ts +38 -0
- package/dist/errors/server.d.ts.map +1 -0
- package/dist/errors/server.js +45 -0
- package/dist/errors/server.js.map +1 -0
- package/dist/errors/tunnel.d.ts +38 -0
- package/dist/errors/tunnel.d.ts.map +1 -0
- package/dist/errors/tunnel.js +51 -0
- package/dist/errors/tunnel.js.map +1 -0
- package/dist/errors/validation.d.ts +36 -0
- package/dist/errors/validation.d.ts.map +1 -0
- package/dist/errors/validation.js +45 -0
- package/dist/errors/validation.js.map +1 -0
- package/dist/index.d.ts +12 -4
- package/dist/index.js +12 -4
- package/dist/index.js.map +1 -1
- package/dist/logging/logger.js +179 -0
- package/dist/logging/logger.js.map +1 -0
- package/dist/logging/sampling.js +56 -0
- package/dist/logging/sampling.js.map +1 -0
- package/dist/logging/wide-event-emitter.js +108 -0
- package/dist/logging/wide-event-emitter.js.map +1 -0
- package/dist/logging/wide-event.js +167 -0
- package/dist/logging/wide-event.js.map +1 -0
- package/dist/plugin/dev-reader.d.ts.map +1 -1
- package/dist/plugin/dev-reader.js +8 -3
- package/dist/plugin/dev-reader.js.map +1 -1
- package/dist/plugin/interceptors/cache.js.map +1 -1
- package/dist/plugin/interceptors/retry.js +10 -2
- package/dist/plugin/interceptors/retry.js.map +1 -1
- package/dist/plugin/interceptors/telemetry.js +24 -9
- package/dist/plugin/interceptors/telemetry.js.map +1 -1
- package/dist/plugin/interceptors/timeout.js +4 -0
- package/dist/plugin/interceptors/timeout.js.map +1 -1
- package/dist/plugin/plugin.d.ts +38 -4
- package/dist/plugin/plugin.d.ts.map +1 -1
- package/dist/plugin/plugin.js +86 -5
- package/dist/plugin/plugin.js.map +1 -1
- package/dist/plugin/to-plugin.d.ts +4 -0
- package/dist/plugin/to-plugin.d.ts.map +1 -1
- package/dist/plugin/to-plugin.js +3 -0
- package/dist/plugin/to-plugin.js.map +1 -1
- package/dist/server/index.d.ts +3 -0
- package/dist/server/index.d.ts.map +1 -1
- package/dist/server/index.js +25 -21
- package/dist/server/index.js.map +1 -1
- package/dist/server/remote-tunnel/remote-tunnel-controller.js +4 -2
- package/dist/server/remote-tunnel/remote-tunnel-controller.js.map +1 -1
- package/dist/server/remote-tunnel/remote-tunnel-manager.js +10 -8
- package/dist/server/remote-tunnel/remote-tunnel-manager.js.map +1 -1
- package/dist/server/utils.js.map +1 -1
- package/dist/server/vite-dev-server.js +8 -5
- package/dist/server/vite-dev-server.js.map +1 -1
- package/dist/shared/src/sql/helpers.js.map +1 -1
- package/dist/stream/arrow-stream-processor.js +13 -6
- package/dist/stream/arrow-stream-processor.js.map +1 -1
- package/dist/stream/buffers.js +5 -1
- package/dist/stream/buffers.js.map +1 -1
- package/dist/stream/sse-writer.js.map +1 -1
- package/dist/stream/stream-manager.d.ts.map +1 -1
- package/dist/stream/stream-manager.js +47 -36
- package/dist/stream/stream-manager.js.map +1 -1
- package/dist/stream/stream-registry.js.map +1 -1
- package/dist/stream/types.js.map +1 -1
- package/dist/telemetry/index.d.ts +2 -2
- package/dist/telemetry/index.js +2 -2
- package/dist/telemetry/instrumentations.js +14 -10
- package/dist/telemetry/instrumentations.js.map +1 -1
- package/dist/telemetry/telemetry-manager.js +8 -6
- package/dist/telemetry/telemetry-manager.js.map +1 -1
- package/dist/telemetry/trace-sampler.js +33 -0
- package/dist/telemetry/trace-sampler.js.map +1 -0
- package/dist/type-generator/index.js +4 -2
- package/dist/type-generator/index.js.map +1 -1
- package/dist/type-generator/query-registry.js +4 -2
- package/dist/type-generator/query-registry.js.map +1 -1
- package/dist/type-generator/types.js.map +1 -1
- package/dist/type-generator/vite-plugin.d.ts.map +1 -1
- package/dist/type-generator/vite-plugin.js +5 -3
- package/dist/type-generator/vite-plugin.js.map +1 -1
- package/dist/utils/env-validator.js +5 -5
- package/dist/utils/env-validator.js.map +1 -1
- package/dist/utils/merge.js +1 -5
- package/dist/utils/merge.js.map +1 -1
- package/dist/utils/path-exclusions.js +66 -0
- package/dist/utils/path-exclusions.js.map +1 -0
- package/dist/utils/vite-config-merge.js +1 -5
- package/dist/utils/vite-config-merge.js.map +1 -1
- package/llms.txt +89 -12
- package/package.json +6 -1
- package/dist/utils/databricks-client-middleware.d.ts +0 -17
- package/dist/utils/databricks-client-middleware.d.ts.map +0 -1
- package/dist/utils/databricks-client-middleware.js +0 -117
- package/dist/utils/databricks-client-middleware.js.map +0 -1
- package/dist/utils/index.js +0 -26
- package/dist/utils/index.js.map +0 -1
|
@@ -1,7 +1,13 @@
|
|
|
1
|
+
import { createLogger } from "../../logging/logger.js";
|
|
2
|
+
import { InitializationError } from "../../errors/initialization.js";
|
|
3
|
+
import { ValidationError } from "../../errors/validation.js";
|
|
4
|
+
import { init_errors } from "../../errors/index.js";
|
|
1
5
|
import { lakebaseStorageDefaults } from "./defaults.js";
|
|
2
6
|
import { createHash } from "node:crypto";
|
|
3
7
|
|
|
4
8
|
//#region src/cache/storage/persistent.ts
|
|
9
|
+
init_errors();
|
|
10
|
+
const logger = createLogger("cache:persistent");
|
|
5
11
|
/**
|
|
6
12
|
* Persistent cache storage implementation. Uses a least recently used (LRU) eviction policy
|
|
7
13
|
* to manage memory usage and ensure efficient cache operations.
|
|
@@ -33,7 +39,7 @@ var PersistentStorage = class {
|
|
|
33
39
|
await this.runMigrations();
|
|
34
40
|
this.initialized = true;
|
|
35
41
|
} catch (error) {
|
|
36
|
-
|
|
42
|
+
logger.error("Error in persistent storage initialization: %O", error);
|
|
37
43
|
throw error;
|
|
38
44
|
}
|
|
39
45
|
}
|
|
@@ -49,7 +55,7 @@ var PersistentStorage = class {
|
|
|
49
55
|
if (result.rows.length === 0) return null;
|
|
50
56
|
const entry = result.rows[0];
|
|
51
57
|
this.connector.query(`UPDATE ${this.tableName} SET last_accessed = NOW() WHERE key_hash = $1`, [keyHash]).catch(() => {
|
|
52
|
-
|
|
58
|
+
logger.debug("Error updating last_accessed time for key: %s", key);
|
|
53
59
|
});
|
|
54
60
|
return {
|
|
55
61
|
value: this.deserializeValue(entry.value),
|
|
@@ -68,7 +74,7 @@ var PersistentStorage = class {
|
|
|
68
74
|
const keyBytes = Buffer.from(key, "utf-8");
|
|
69
75
|
const valueBytes = this.serializeValue(entry.value);
|
|
70
76
|
const byteSize = keyBytes.length + valueBytes.length;
|
|
71
|
-
if (byteSize > this.maxEntryBytes) throw
|
|
77
|
+
if (byteSize > this.maxEntryBytes) throw ValidationError.invalidValue("cache entry size", byteSize, `maximum ${this.maxEntryBytes} bytes`);
|
|
72
78
|
if (Math.random() < this.evictionCheckProbability) {
|
|
73
79
|
if (await this.totalBytes() + byteSize > this.maxBytes) await this.evictBySize(byteSize);
|
|
74
80
|
}
|
|
@@ -169,7 +175,7 @@ var PersistentStorage = class {
|
|
|
169
175
|
}
|
|
170
176
|
/** Generate a 64-bit hash for the cache key using SHA256 */
|
|
171
177
|
hashKey(key) {
|
|
172
|
-
if (!key) throw
|
|
178
|
+
if (!key) throw ValidationError.missingField("key");
|
|
173
179
|
return createHash("sha256").update(key).digest().readBigInt64BE(0);
|
|
174
180
|
}
|
|
175
181
|
/** Serialize a value to a buffer */
|
|
@@ -200,8 +206,8 @@ var PersistentStorage = class {
|
|
|
200
206
|
await this.connector.query(`CREATE INDEX IF NOT EXISTS idx_${this.tableName}_last_accessed ON ${this.tableName} (last_accessed); `);
|
|
201
207
|
await this.connector.query(`CREATE INDEX IF NOT EXISTS idx_${this.tableName}_byte_size ON ${this.tableName} (byte_size); `);
|
|
202
208
|
} catch (error) {
|
|
203
|
-
|
|
204
|
-
throw error;
|
|
209
|
+
logger.error("Error in running migrations for persistent storage: %O", error);
|
|
210
|
+
throw InitializationError.migrationFailed(error);
|
|
205
211
|
}
|
|
206
212
|
}
|
|
207
213
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"persistent.js","names":[],"sources":["../../../src/cache/storage/persistent.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport type { CacheConfig, CacheEntry, CacheStorage } from \"shared\";\nimport type { LakebaseConnector } from \"../../connectors\";\nimport { lakebaseStorageDefaults } from \"./defaults\";\n\n/**\n * Persistent cache storage implementation. Uses a least recently used (LRU) eviction policy\n * to manage memory usage and ensure efficient cache operations.\n *\n * @example\n * const persistentStorage = new PersistentStorage(config, connector);\n * await persistentStorage.initialize();\n * await persistentStorage.get(\"my-key\");\n * await persistentStorage.set(\"my-key\", \"my-value\");\n * await persistentStorage.delete(\"my-key\");\n * await persistentStorage.clear();\n * await persistentStorage.has(\"my-key\");\n *\n */\nexport class PersistentStorage implements CacheStorage {\n private readonly connector: LakebaseConnector;\n private readonly tableName: string;\n private readonly maxBytes: number;\n private readonly maxEntryBytes: number;\n private readonly evictionBatchSize: number;\n private readonly evictionCheckProbability: number;\n private initialized: boolean;\n\n constructor(config: CacheConfig, connector: LakebaseConnector) {\n this.connector = connector;\n this.maxBytes = config.maxBytes ?? lakebaseStorageDefaults.maxBytes;\n this.maxEntryBytes =\n config.maxEntryBytes ?? lakebaseStorageDefaults.maxEntryBytes;\n this.evictionBatchSize = lakebaseStorageDefaults.evictionBatchSize;\n this.evictionCheckProbability =\n config.evictionCheckProbability ??\n lakebaseStorageDefaults.evictionCheckProbability;\n this.tableName = lakebaseStorageDefaults.tableName; // hardcoded, safe for now\n this.initialized = false;\n }\n\n /** Initialize the persistent storage and run migrations if necessary */\n async initialize(): Promise<void> {\n if (this.initialized) return;\n\n try {\n await this.runMigrations();\n this.initialized = true;\n } catch (error) {\n console.error(\"Error in persistent storage initialization:\", error);\n throw error;\n }\n }\n\n /**\n * Get a cached value from the persistent storage\n * @param key - Cache key\n * @returns Promise of the cached value or null if not found\n */\n async get<T>(key: string): Promise<CacheEntry<T> | null> {\n await this.ensureInitialized();\n\n const keyHash = this.hashKey(key);\n\n const result = await this.connector.query<{\n value: Buffer;\n expiry: string;\n }>(`SELECT value, expiry FROM ${this.tableName} WHERE key_hash = $1`, [\n keyHash,\n ]);\n\n if (result.rows.length === 0) return null;\n\n const entry = result.rows[0];\n\n // fire-and-forget update\n this.connector\n .query(\n `UPDATE ${this.tableName} SET last_accessed = NOW() WHERE key_hash = $1`,\n [keyHash],\n )\n .catch(() => {\n console.debug(\"Error updating last_accessed time for key:\", key);\n });\n\n return {\n value: this.deserializeValue<T>(entry.value),\n expiry: Number(entry.expiry),\n };\n }\n\n /**\n * Set a value in the persistent storage\n * @param key - Cache key\n * @param entry - Cache entry\n * @returns Promise of the result\n */\n async set<T>(key: string, entry: CacheEntry<T>): Promise<void> {\n await this.ensureInitialized();\n\n const keyHash = this.hashKey(key);\n const keyBytes = Buffer.from(key, \"utf-8\");\n const valueBytes = this.serializeValue(entry.value);\n const byteSize = keyBytes.length + valueBytes.length;\n\n if (byteSize > this.maxEntryBytes) {\n throw new Error(\n `Cache entry too large: ${byteSize} bytes exceeds maximum of ${this.maxEntryBytes} bytes`,\n );\n }\n\n // probabilistic eviction check\n if (Math.random() < this.evictionCheckProbability) {\n const totalBytes = await this.totalBytes();\n if (totalBytes + byteSize > this.maxBytes) {\n await this.evictBySize(byteSize);\n }\n }\n\n await this.connector.query(\n `INSERT INTO ${this.tableName} (key_hash, key, value, byte_size, expiry, created_at, last_accessed)\n VALUES ($1, $2, $3, $4, $5, NOW(), NOW())\n ON CONFLICT (key_hash)\n DO UPDATE SET value = $3, byte_size = $4, expiry = $5, last_accessed = NOW()\n `,\n [keyHash, keyBytes, valueBytes, byteSize, entry.expiry],\n );\n }\n\n /**\n * Delete a value from the persistent storage\n * @param key - Cache key\n * @returns Promise of the result\n */\n async delete(key: string): Promise<void> {\n await this.ensureInitialized();\n const keyHash = this.hashKey(key);\n await this.connector.query(\n `DELETE FROM ${this.tableName} WHERE key_hash = $1`,\n [keyHash],\n );\n }\n\n /** Clear the persistent storage */\n async clear(): Promise<void> {\n await this.ensureInitialized();\n await this.connector.query(`TRUNCATE TABLE ${this.tableName}`);\n }\n\n /**\n * Check if a value exists in the persistent storage\n * @param key - Cache key\n * @returns Promise of true if the value exists, false otherwise\n */\n async has(key: string): Promise<boolean> {\n await this.ensureInitialized();\n const keyHash = this.hashKey(key);\n\n const result = await this.connector.query<{ exists: boolean }>(\n `SELECT EXISTS(SELECT 1 FROM ${this.tableName} WHERE key_hash = $1) as exists`,\n [keyHash],\n );\n\n return result.rows[0]?.exists ?? false;\n }\n\n /**\n * Get the size of the persistent storage\n * @returns Promise of the size of the storage\n */\n async size(): Promise<number> {\n await this.ensureInitialized();\n\n const result = await this.connector.query<{ count: string }>(\n `SELECT COUNT(*) as count FROM ${this.tableName}`,\n );\n return parseInt(result.rows[0]?.count ?? \"0\", 10);\n }\n\n /** Get the total number of bytes in the persistent storage */\n async totalBytes(): Promise<number> {\n await this.ensureInitialized();\n\n const result = await this.connector.query<{ total: string }>(\n `SELECT COALESCE(SUM(byte_size), 0) as total FROM ${this.tableName}`,\n );\n return parseInt(result.rows[0]?.total ?? \"0\", 10);\n }\n\n /**\n * Check if the persistent storage is persistent\n * @returns true if the storage is persistent, false otherwise\n */\n isPersistent(): boolean {\n return true;\n }\n\n /**\n * Check if the persistent storage is healthy\n * @returns Promise of true if the storage is healthy, false otherwise\n */\n async healthCheck(): Promise<boolean> {\n try {\n return await this.connector.healthCheck();\n } catch {\n return false;\n }\n }\n\n /** Close the persistent storage */\n async close(): Promise<void> {\n await this.connector.close();\n }\n\n /**\n * Cleanup expired entries from the persistent storage\n * @returns Promise of the number of expired entries\n */\n async cleanupExpired(): Promise<number> {\n await this.ensureInitialized();\n const result = await this.connector.query<{ count: string }>(\n `WITH deleted as (DELETE FROM ${this.tableName} WHERE expiry < $1 RETURNING *) SELECT COUNT(*) as count FROM deleted`,\n [Date.now()],\n );\n return parseInt(result.rows[0]?.count ?? \"0\", 10);\n }\n\n /** Evict entries from the persistent storage by size */\n private async evictBySize(requiredBytes: number): Promise<void> {\n const freedByExpiry = await this.cleanupExpired();\n if (freedByExpiry > 0) {\n const currentBytes = await this.totalBytes();\n if (currentBytes + requiredBytes <= this.maxBytes) {\n return;\n }\n }\n\n await this.connector.query(\n `DELETE FROM ${this.tableName} WHERE key_hash IN\n (SELECT key_hash FROM ${this.tableName} ORDER BY last_accessed ASC LIMIT $1)`,\n [this.evictionBatchSize],\n );\n }\n\n /** Ensure the persistent storage is initialized */\n private async ensureInitialized(): Promise<void> {\n if (!this.initialized) {\n await this.initialize();\n }\n }\n\n /** Generate a 64-bit hash for the cache key using SHA256 */\n private hashKey(key: string): bigint {\n if (!key) throw new Error(\"Cache key cannot be empty\");\n const hash = createHash(\"sha256\").update(key).digest();\n return hash.readBigInt64BE(0);\n }\n\n /** Serialize a value to a buffer */\n private serializeValue<T>(value: T): Buffer {\n return Buffer.from(JSON.stringify(value), \"utf-8\");\n }\n\n /** Deserialize a value from a buffer */\n private deserializeValue<T>(buffer: Buffer): T {\n return JSON.parse(buffer.toString(\"utf-8\")) as T;\n }\n\n /** Run migrations for the persistent storage */\n private async runMigrations(): Promise<void> {\n try {\n await this.connector.query(`\n CREATE TABLE IF NOT EXISTS ${this.tableName} (\n id BIGSERIAL PRIMARY KEY,\n key_hash BIGINT NOT NULL,\n key BYTEA NOT NULL,\n value BYTEA NOT NULL,\n byte_size INTEGER NOT NULL,\n expiry BIGINT NOT NULL,\n created_at TIMESTAMP NOT NULL DEFAULT NOW(),\n last_accessed TIMESTAMP NOT NULL DEFAULT NOW()\n )\n `);\n\n // unique index on key_hash for fast lookups\n await this.connector.query(\n `CREATE UNIQUE INDEX IF NOT EXISTS idx_${this.tableName}_key_hash ON ${this.tableName} (key_hash);`,\n );\n\n // index on expiry for cleanup queries\n await this.connector.query(\n `CREATE INDEX IF NOT EXISTS idx_${this.tableName}_expiry ON ${this.tableName} (expiry); `,\n );\n\n // index on last_accessed for LRU eviction\n await this.connector.query(\n `CREATE INDEX IF NOT EXISTS idx_${this.tableName}_last_accessed ON ${this.tableName} (last_accessed); `,\n );\n\n // index on byte_size for monitoring\n await this.connector.query(\n `CREATE INDEX IF NOT EXISTS idx_${this.tableName}_byte_size ON ${this.tableName} (byte_size); `,\n );\n } catch (error) {\n console.error(\n \"Error in running migrations for persistent storage:\",\n error,\n );\n throw error;\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAmBA,IAAa,oBAAb,MAAuD;CASrD,YAAY,QAAqB,WAA8B;AAC7D,OAAK,YAAY;AACjB,OAAK,WAAW,OAAO,YAAY,wBAAwB;AAC3D,OAAK,gBACH,OAAO,iBAAiB,wBAAwB;AAClD,OAAK,oBAAoB,wBAAwB;AACjD,OAAK,2BACH,OAAO,4BACP,wBAAwB;AAC1B,OAAK,YAAY,wBAAwB;AACzC,OAAK,cAAc;;;CAIrB,MAAM,aAA4B;AAChC,MAAI,KAAK,YAAa;AAEtB,MAAI;AACF,SAAM,KAAK,eAAe;AAC1B,QAAK,cAAc;WACZ,OAAO;AACd,WAAQ,MAAM,+CAA+C,MAAM;AACnE,SAAM;;;;;;;;CASV,MAAM,IAAO,KAA4C;AACvD,QAAM,KAAK,mBAAmB;EAE9B,MAAM,UAAU,KAAK,QAAQ,IAAI;EAEjC,MAAM,SAAS,MAAM,KAAK,UAAU,MAGjC,6BAA6B,KAAK,UAAU,uBAAuB,CACpE,QACD,CAAC;AAEF,MAAI,OAAO,KAAK,WAAW,EAAG,QAAO;EAErC,MAAM,QAAQ,OAAO,KAAK;AAG1B,OAAK,UACF,MACC,UAAU,KAAK,UAAU,iDACzB,CAAC,QAAQ,CACV,CACA,YAAY;AACX,WAAQ,MAAM,8CAA8C,IAAI;IAChE;AAEJ,SAAO;GACL,OAAO,KAAK,iBAAoB,MAAM,MAAM;GAC5C,QAAQ,OAAO,MAAM,OAAO;GAC7B;;;;;;;;CASH,MAAM,IAAO,KAAa,OAAqC;AAC7D,QAAM,KAAK,mBAAmB;EAE9B,MAAM,UAAU,KAAK,QAAQ,IAAI;EACjC,MAAM,WAAW,OAAO,KAAK,KAAK,QAAQ;EAC1C,MAAM,aAAa,KAAK,eAAe,MAAM,MAAM;EACnD,MAAM,WAAW,SAAS,SAAS,WAAW;AAE9C,MAAI,WAAW,KAAK,cAClB,OAAM,IAAI,MACR,0BAA0B,SAAS,4BAA4B,KAAK,cAAc,QACnF;AAIH,MAAI,KAAK,QAAQ,GAAG,KAAK,0BAEvB;OADmB,MAAM,KAAK,YAAY,GACzB,WAAW,KAAK,SAC/B,OAAM,KAAK,YAAY,SAAS;;AAIpC,QAAM,KAAK,UAAU,MACnB,eAAe,KAAK,UAAU;;;;SAK9B;GAAC;GAAS;GAAU;GAAY;GAAU,MAAM;GAAO,CACxD;;;;;;;CAQH,MAAM,OAAO,KAA4B;AACvC,QAAM,KAAK,mBAAmB;EAC9B,MAAM,UAAU,KAAK,QAAQ,IAAI;AACjC,QAAM,KAAK,UAAU,MACnB,eAAe,KAAK,UAAU,uBAC9B,CAAC,QAAQ,CACV;;;CAIH,MAAM,QAAuB;AAC3B,QAAM,KAAK,mBAAmB;AAC9B,QAAM,KAAK,UAAU,MAAM,kBAAkB,KAAK,YAAY;;;;;;;CAQhE,MAAM,IAAI,KAA+B;AACvC,QAAM,KAAK,mBAAmB;EAC9B,MAAM,UAAU,KAAK,QAAQ,IAAI;AAOjC,UALe,MAAM,KAAK,UAAU,MAClC,+BAA+B,KAAK,UAAU,kCAC9C,CAAC,QAAQ,CACV,EAEa,KAAK,IAAI,UAAU;;;;;;CAOnC,MAAM,OAAwB;AAC5B,QAAM,KAAK,mBAAmB;EAE9B,MAAM,SAAS,MAAM,KAAK,UAAU,MAClC,iCAAiC,KAAK,YACvC;AACD,SAAO,SAAS,OAAO,KAAK,IAAI,SAAS,KAAK,GAAG;;;CAInD,MAAM,aAA8B;AAClC,QAAM,KAAK,mBAAmB;EAE9B,MAAM,SAAS,MAAM,KAAK,UAAU,MAClC,oDAAoD,KAAK,YAC1D;AACD,SAAO,SAAS,OAAO,KAAK,IAAI,SAAS,KAAK,GAAG;;;;;;CAOnD,eAAwB;AACtB,SAAO;;;;;;CAOT,MAAM,cAAgC;AACpC,MAAI;AACF,UAAO,MAAM,KAAK,UAAU,aAAa;UACnC;AACN,UAAO;;;;CAKX,MAAM,QAAuB;AAC3B,QAAM,KAAK,UAAU,OAAO;;;;;;CAO9B,MAAM,iBAAkC;AACtC,QAAM,KAAK,mBAAmB;EAC9B,MAAM,SAAS,MAAM,KAAK,UAAU,MAClC,gCAAgC,KAAK,UAAU,wEAC/C,CAAC,KAAK,KAAK,CAAC,CACb;AACD,SAAO,SAAS,OAAO,KAAK,IAAI,SAAS,KAAK,GAAG;;;CAInD,MAAc,YAAY,eAAsC;AAE9D,MADsB,MAAM,KAAK,gBAAgB,GAC7B,GAElB;OADqB,MAAM,KAAK,YAAY,GACzB,iBAAiB,KAAK,SACvC;;AAIJ,QAAM,KAAK,UAAU,MACnB,eAAe,KAAK,UAAU;8BACN,KAAK,UAAU,wCACvC,CAAC,KAAK,kBAAkB,CACzB;;;CAIH,MAAc,oBAAmC;AAC/C,MAAI,CAAC,KAAK,YACR,OAAM,KAAK,YAAY;;;CAK3B,AAAQ,QAAQ,KAAqB;AACnC,MAAI,CAAC,IAAK,OAAM,IAAI,MAAM,4BAA4B;AAEtD,SADa,WAAW,SAAS,CAAC,OAAO,IAAI,CAAC,QAAQ,CAC1C,eAAe,EAAE;;;CAI/B,AAAQ,eAAkB,OAAkB;AAC1C,SAAO,OAAO,KAAK,KAAK,UAAU,MAAM,EAAE,QAAQ;;;CAIpD,AAAQ,iBAAoB,QAAmB;AAC7C,SAAO,KAAK,MAAM,OAAO,SAAS,QAAQ,CAAC;;;CAI7C,MAAc,gBAA+B;AAC3C,MAAI;AACF,SAAM,KAAK,UAAU,MAAM;yCACQ,KAAK,UAAU;;;;;;;;;;cAU1C;AAGR,SAAM,KAAK,UAAU,MACnB,yCAAyC,KAAK,UAAU,eAAe,KAAK,UAAU,cACvF;AAGD,SAAM,KAAK,UAAU,MACnB,kCAAkC,KAAK,UAAU,aAAa,KAAK,UAAU,aAC9E;AAGD,SAAM,KAAK,UAAU,MACnB,kCAAkC,KAAK,UAAU,oBAAoB,KAAK,UAAU,oBACrF;AAGD,SAAM,KAAK,UAAU,MACnB,kCAAkC,KAAK,UAAU,gBAAgB,KAAK,UAAU,gBACjF;WACM,OAAO;AACd,WAAQ,MACN,uDACA,MACD;AACD,SAAM"}
|
|
1
|
+
{"version":3,"file":"persistent.js","names":[],"sources":["../../../src/cache/storage/persistent.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport type { CacheConfig, CacheEntry, CacheStorage } from \"shared\";\nimport type { LakebaseConnector } from \"../../connectors\";\nimport { InitializationError, ValidationError } from \"../../errors\";\nimport { createLogger } from \"../../logging/logger\";\nimport { lakebaseStorageDefaults } from \"./defaults\";\n\nconst logger = createLogger(\"cache:persistent\");\n\n/**\n * Persistent cache storage implementation. Uses a least recently used (LRU) eviction policy\n * to manage memory usage and ensure efficient cache operations.\n *\n * @example\n * const persistentStorage = new PersistentStorage(config, connector);\n * await persistentStorage.initialize();\n * await persistentStorage.get(\"my-key\");\n * await persistentStorage.set(\"my-key\", \"my-value\");\n * await persistentStorage.delete(\"my-key\");\n * await persistentStorage.clear();\n * await persistentStorage.has(\"my-key\");\n *\n */\nexport class PersistentStorage implements CacheStorage {\n private readonly connector: LakebaseConnector;\n private readonly tableName: string;\n private readonly maxBytes: number;\n private readonly maxEntryBytes: number;\n private readonly evictionBatchSize: number;\n private readonly evictionCheckProbability: number;\n private initialized: boolean;\n\n constructor(config: CacheConfig, connector: LakebaseConnector) {\n this.connector = connector;\n this.maxBytes = config.maxBytes ?? lakebaseStorageDefaults.maxBytes;\n this.maxEntryBytes =\n config.maxEntryBytes ?? lakebaseStorageDefaults.maxEntryBytes;\n this.evictionBatchSize = lakebaseStorageDefaults.evictionBatchSize;\n this.evictionCheckProbability =\n config.evictionCheckProbability ??\n lakebaseStorageDefaults.evictionCheckProbability;\n this.tableName = lakebaseStorageDefaults.tableName; // hardcoded, safe for now\n this.initialized = false;\n }\n\n /** Initialize the persistent storage and run migrations if necessary */\n async initialize(): Promise<void> {\n if (this.initialized) return;\n\n try {\n await this.runMigrations();\n this.initialized = true;\n } catch (error) {\n logger.error(\"Error in persistent storage initialization: %O\", error);\n throw error;\n }\n }\n\n /**\n * Get a cached value from the persistent storage\n * @param key - Cache key\n * @returns Promise of the cached value or null if not found\n */\n async get<T>(key: string): Promise<CacheEntry<T> | null> {\n await this.ensureInitialized();\n\n const keyHash = this.hashKey(key);\n\n const result = await this.connector.query<{\n value: Buffer;\n expiry: string;\n }>(`SELECT value, expiry FROM ${this.tableName} WHERE key_hash = $1`, [\n keyHash,\n ]);\n\n if (result.rows.length === 0) return null;\n\n const entry = result.rows[0];\n\n // fire-and-forget update\n this.connector\n .query(\n `UPDATE ${this.tableName} SET last_accessed = NOW() WHERE key_hash = $1`,\n [keyHash],\n )\n .catch(() => {\n logger.debug(\"Error updating last_accessed time for key: %s\", key);\n });\n\n return {\n value: this.deserializeValue<T>(entry.value),\n expiry: Number(entry.expiry),\n };\n }\n\n /**\n * Set a value in the persistent storage\n * @param key - Cache key\n * @param entry - Cache entry\n * @returns Promise of the result\n */\n async set<T>(key: string, entry: CacheEntry<T>): Promise<void> {\n await this.ensureInitialized();\n\n const keyHash = this.hashKey(key);\n const keyBytes = Buffer.from(key, \"utf-8\");\n const valueBytes = this.serializeValue(entry.value);\n const byteSize = keyBytes.length + valueBytes.length;\n\n if (byteSize > this.maxEntryBytes) {\n throw ValidationError.invalidValue(\n \"cache entry size\",\n byteSize,\n `maximum ${this.maxEntryBytes} bytes`,\n );\n }\n\n // probabilistic eviction check\n if (Math.random() < this.evictionCheckProbability) {\n const totalBytes = await this.totalBytes();\n if (totalBytes + byteSize > this.maxBytes) {\n await this.evictBySize(byteSize);\n }\n }\n\n await this.connector.query(\n `INSERT INTO ${this.tableName} (key_hash, key, value, byte_size, expiry, created_at, last_accessed)\n VALUES ($1, $2, $3, $4, $5, NOW(), NOW())\n ON CONFLICT (key_hash)\n DO UPDATE SET value = $3, byte_size = $4, expiry = $5, last_accessed = NOW()\n `,\n [keyHash, keyBytes, valueBytes, byteSize, entry.expiry],\n );\n }\n\n /**\n * Delete a value from the persistent storage\n * @param key - Cache key\n * @returns Promise of the result\n */\n async delete(key: string): Promise<void> {\n await this.ensureInitialized();\n const keyHash = this.hashKey(key);\n await this.connector.query(\n `DELETE FROM ${this.tableName} WHERE key_hash = $1`,\n [keyHash],\n );\n }\n\n /** Clear the persistent storage */\n async clear(): Promise<void> {\n await this.ensureInitialized();\n await this.connector.query(`TRUNCATE TABLE ${this.tableName}`);\n }\n\n /**\n * Check if a value exists in the persistent storage\n * @param key - Cache key\n * @returns Promise of true if the value exists, false otherwise\n */\n async has(key: string): Promise<boolean> {\n await this.ensureInitialized();\n const keyHash = this.hashKey(key);\n\n const result = await this.connector.query<{ exists: boolean }>(\n `SELECT EXISTS(SELECT 1 FROM ${this.tableName} WHERE key_hash = $1) as exists`,\n [keyHash],\n );\n\n return result.rows[0]?.exists ?? false;\n }\n\n /**\n * Get the size of the persistent storage\n * @returns Promise of the size of the storage\n */\n async size(): Promise<number> {\n await this.ensureInitialized();\n\n const result = await this.connector.query<{ count: string }>(\n `SELECT COUNT(*) as count FROM ${this.tableName}`,\n );\n return parseInt(result.rows[0]?.count ?? \"0\", 10);\n }\n\n /** Get the total number of bytes in the persistent storage */\n async totalBytes(): Promise<number> {\n await this.ensureInitialized();\n\n const result = await this.connector.query<{ total: string }>(\n `SELECT COALESCE(SUM(byte_size), 0) as total FROM ${this.tableName}`,\n );\n return parseInt(result.rows[0]?.total ?? \"0\", 10);\n }\n\n /**\n * Check if the persistent storage is persistent\n * @returns true if the storage is persistent, false otherwise\n */\n isPersistent(): boolean {\n return true;\n }\n\n /**\n * Check if the persistent storage is healthy\n * @returns Promise of true if the storage is healthy, false otherwise\n */\n async healthCheck(): Promise<boolean> {\n try {\n return await this.connector.healthCheck();\n } catch {\n return false;\n }\n }\n\n /** Close the persistent storage */\n async close(): Promise<void> {\n await this.connector.close();\n }\n\n /**\n * Cleanup expired entries from the persistent storage\n * @returns Promise of the number of expired entries\n */\n async cleanupExpired(): Promise<number> {\n await this.ensureInitialized();\n const result = await this.connector.query<{ count: string }>(\n `WITH deleted as (DELETE FROM ${this.tableName} WHERE expiry < $1 RETURNING *) SELECT COUNT(*) as count FROM deleted`,\n [Date.now()],\n );\n return parseInt(result.rows[0]?.count ?? \"0\", 10);\n }\n\n /** Evict entries from the persistent storage by size */\n private async evictBySize(requiredBytes: number): Promise<void> {\n const freedByExpiry = await this.cleanupExpired();\n if (freedByExpiry > 0) {\n const currentBytes = await this.totalBytes();\n if (currentBytes + requiredBytes <= this.maxBytes) {\n return;\n }\n }\n\n await this.connector.query(\n `DELETE FROM ${this.tableName} WHERE key_hash IN\n (SELECT key_hash FROM ${this.tableName} ORDER BY last_accessed ASC LIMIT $1)`,\n [this.evictionBatchSize],\n );\n }\n\n /** Ensure the persistent storage is initialized */\n private async ensureInitialized(): Promise<void> {\n if (!this.initialized) {\n await this.initialize();\n }\n }\n\n /** Generate a 64-bit hash for the cache key using SHA256 */\n private hashKey(key: string): bigint {\n if (!key) throw ValidationError.missingField(\"key\");\n const hash = createHash(\"sha256\").update(key).digest();\n return hash.readBigInt64BE(0);\n }\n\n /** Serialize a value to a buffer */\n private serializeValue<T>(value: T): Buffer {\n return Buffer.from(JSON.stringify(value), \"utf-8\");\n }\n\n /** Deserialize a value from a buffer */\n private deserializeValue<T>(buffer: Buffer): T {\n return JSON.parse(buffer.toString(\"utf-8\")) as T;\n }\n\n /** Run migrations for the persistent storage */\n private async runMigrations(): Promise<void> {\n try {\n await this.connector.query(`\n CREATE TABLE IF NOT EXISTS ${this.tableName} (\n id BIGSERIAL PRIMARY KEY,\n key_hash BIGINT NOT NULL,\n key BYTEA NOT NULL,\n value BYTEA NOT NULL,\n byte_size INTEGER NOT NULL,\n expiry BIGINT NOT NULL,\n created_at TIMESTAMP NOT NULL DEFAULT NOW(),\n last_accessed TIMESTAMP NOT NULL DEFAULT NOW()\n )\n `);\n\n // unique index on key_hash for fast lookups\n await this.connector.query(\n `CREATE UNIQUE INDEX IF NOT EXISTS idx_${this.tableName}_key_hash ON ${this.tableName} (key_hash);`,\n );\n\n // index on expiry for cleanup queries\n await this.connector.query(\n `CREATE INDEX IF NOT EXISTS idx_${this.tableName}_expiry ON ${this.tableName} (expiry); `,\n );\n\n // index on last_accessed for LRU eviction\n await this.connector.query(\n `CREATE INDEX IF NOT EXISTS idx_${this.tableName}_last_accessed ON ${this.tableName} (last_accessed); `,\n );\n\n // index on byte_size for monitoring\n await this.connector.query(\n `CREATE INDEX IF NOT EXISTS idx_${this.tableName}_byte_size ON ${this.tableName} (byte_size); `,\n );\n } catch (error) {\n logger.error(\n \"Error in running migrations for persistent storage: %O\",\n error,\n );\n throw InitializationError.migrationFailed(error as Error);\n }\n }\n}\n"],"mappings":";;;;;;;;aAGoE;AAIpE,MAAM,SAAS,aAAa,mBAAmB;;;;;;;;;;;;;;;AAgB/C,IAAa,oBAAb,MAAuD;CASrD,YAAY,QAAqB,WAA8B;AAC7D,OAAK,YAAY;AACjB,OAAK,WAAW,OAAO,YAAY,wBAAwB;AAC3D,OAAK,gBACH,OAAO,iBAAiB,wBAAwB;AAClD,OAAK,oBAAoB,wBAAwB;AACjD,OAAK,2BACH,OAAO,4BACP,wBAAwB;AAC1B,OAAK,YAAY,wBAAwB;AACzC,OAAK,cAAc;;;CAIrB,MAAM,aAA4B;AAChC,MAAI,KAAK,YAAa;AAEtB,MAAI;AACF,SAAM,KAAK,eAAe;AAC1B,QAAK,cAAc;WACZ,OAAO;AACd,UAAO,MAAM,kDAAkD,MAAM;AACrE,SAAM;;;;;;;;CASV,MAAM,IAAO,KAA4C;AACvD,QAAM,KAAK,mBAAmB;EAE9B,MAAM,UAAU,KAAK,QAAQ,IAAI;EAEjC,MAAM,SAAS,MAAM,KAAK,UAAU,MAGjC,6BAA6B,KAAK,UAAU,uBAAuB,CACpE,QACD,CAAC;AAEF,MAAI,OAAO,KAAK,WAAW,EAAG,QAAO;EAErC,MAAM,QAAQ,OAAO,KAAK;AAG1B,OAAK,UACF,MACC,UAAU,KAAK,UAAU,iDACzB,CAAC,QAAQ,CACV,CACA,YAAY;AACX,UAAO,MAAM,iDAAiD,IAAI;IAClE;AAEJ,SAAO;GACL,OAAO,KAAK,iBAAoB,MAAM,MAAM;GAC5C,QAAQ,OAAO,MAAM,OAAO;GAC7B;;;;;;;;CASH,MAAM,IAAO,KAAa,OAAqC;AAC7D,QAAM,KAAK,mBAAmB;EAE9B,MAAM,UAAU,KAAK,QAAQ,IAAI;EACjC,MAAM,WAAW,OAAO,KAAK,KAAK,QAAQ;EAC1C,MAAM,aAAa,KAAK,eAAe,MAAM,MAAM;EACnD,MAAM,WAAW,SAAS,SAAS,WAAW;AAE9C,MAAI,WAAW,KAAK,cAClB,OAAM,gBAAgB,aACpB,oBACA,UACA,WAAW,KAAK,cAAc,QAC/B;AAIH,MAAI,KAAK,QAAQ,GAAG,KAAK,0BAEvB;OADmB,MAAM,KAAK,YAAY,GACzB,WAAW,KAAK,SAC/B,OAAM,KAAK,YAAY,SAAS;;AAIpC,QAAM,KAAK,UAAU,MACnB,eAAe,KAAK,UAAU;;;;SAK9B;GAAC;GAAS;GAAU;GAAY;GAAU,MAAM;GAAO,CACxD;;;;;;;CAQH,MAAM,OAAO,KAA4B;AACvC,QAAM,KAAK,mBAAmB;EAC9B,MAAM,UAAU,KAAK,QAAQ,IAAI;AACjC,QAAM,KAAK,UAAU,MACnB,eAAe,KAAK,UAAU,uBAC9B,CAAC,QAAQ,CACV;;;CAIH,MAAM,QAAuB;AAC3B,QAAM,KAAK,mBAAmB;AAC9B,QAAM,KAAK,UAAU,MAAM,kBAAkB,KAAK,YAAY;;;;;;;CAQhE,MAAM,IAAI,KAA+B;AACvC,QAAM,KAAK,mBAAmB;EAC9B,MAAM,UAAU,KAAK,QAAQ,IAAI;AAOjC,UALe,MAAM,KAAK,UAAU,MAClC,+BAA+B,KAAK,UAAU,kCAC9C,CAAC,QAAQ,CACV,EAEa,KAAK,IAAI,UAAU;;;;;;CAOnC,MAAM,OAAwB;AAC5B,QAAM,KAAK,mBAAmB;EAE9B,MAAM,SAAS,MAAM,KAAK,UAAU,MAClC,iCAAiC,KAAK,YACvC;AACD,SAAO,SAAS,OAAO,KAAK,IAAI,SAAS,KAAK,GAAG;;;CAInD,MAAM,aAA8B;AAClC,QAAM,KAAK,mBAAmB;EAE9B,MAAM,SAAS,MAAM,KAAK,UAAU,MAClC,oDAAoD,KAAK,YAC1D;AACD,SAAO,SAAS,OAAO,KAAK,IAAI,SAAS,KAAK,GAAG;;;;;;CAOnD,eAAwB;AACtB,SAAO;;;;;;CAOT,MAAM,cAAgC;AACpC,MAAI;AACF,UAAO,MAAM,KAAK,UAAU,aAAa;UACnC;AACN,UAAO;;;;CAKX,MAAM,QAAuB;AAC3B,QAAM,KAAK,UAAU,OAAO;;;;;;CAO9B,MAAM,iBAAkC;AACtC,QAAM,KAAK,mBAAmB;EAC9B,MAAM,SAAS,MAAM,KAAK,UAAU,MAClC,gCAAgC,KAAK,UAAU,wEAC/C,CAAC,KAAK,KAAK,CAAC,CACb;AACD,SAAO,SAAS,OAAO,KAAK,IAAI,SAAS,KAAK,GAAG;;;CAInD,MAAc,YAAY,eAAsC;AAE9D,MADsB,MAAM,KAAK,gBAAgB,GAC7B,GAElB;OADqB,MAAM,KAAK,YAAY,GACzB,iBAAiB,KAAK,SACvC;;AAIJ,QAAM,KAAK,UAAU,MACnB,eAAe,KAAK,UAAU;8BACN,KAAK,UAAU,wCACvC,CAAC,KAAK,kBAAkB,CACzB;;;CAIH,MAAc,oBAAmC;AAC/C,MAAI,CAAC,KAAK,YACR,OAAM,KAAK,YAAY;;;CAK3B,AAAQ,QAAQ,KAAqB;AACnC,MAAI,CAAC,IAAK,OAAM,gBAAgB,aAAa,MAAM;AAEnD,SADa,WAAW,SAAS,CAAC,OAAO,IAAI,CAAC,QAAQ,CAC1C,eAAe,EAAE;;;CAI/B,AAAQ,eAAkB,OAAkB;AAC1C,SAAO,OAAO,KAAK,KAAK,UAAU,MAAM,EAAE,QAAQ;;;CAIpD,AAAQ,iBAAoB,QAAmB;AAC7C,SAAO,KAAK,MAAM,OAAO,SAAS,QAAQ,CAAC;;;CAI7C,MAAc,gBAA+B;AAC3C,MAAI;AACF,SAAM,KAAK,UAAU,MAAM;yCACQ,KAAK,UAAU;;;;;;;;;;cAU1C;AAGR,SAAM,KAAK,UAAU,MACnB,yCAAyC,KAAK,UAAU,eAAe,KAAK,UAAU,cACvF;AAGD,SAAM,KAAK,UAAU,MACnB,kCAAkC,KAAK,UAAU,aAAa,KAAK,UAAU,aAC9E;AAGD,SAAM,KAAK,UAAU,MACnB,kCAAkC,KAAK,UAAU,oBAAoB,KAAK,UAAU,oBACrF;AAGD,SAAM,KAAK,UAAU,MACnB,kCAAkC,KAAK,UAAU,gBAAgB,KAAK,UAAU,gBACjF;WACM,OAAO;AACd,UAAO,MACL,0DACA,MACD;AACD,SAAM,oBAAoB,gBAAgB,MAAe"}
|
|
@@ -1,15 +1,23 @@
|
|
|
1
1
|
import { __toCommonJS } from "../../_virtual/rolldown_runtime.js";
|
|
2
|
+
import { createLogger } from "../../logging/logger.js";
|
|
2
3
|
import { TelemetryManager } from "../../telemetry/telemetry-manager.js";
|
|
3
4
|
import { SpanStatusCode } from "../../telemetry/index.js";
|
|
5
|
+
import { AppKitError } from "../../errors/base.js";
|
|
6
|
+
import { AuthenticationError } from "../../errors/authentication.js";
|
|
7
|
+
import { ConfigurationError } from "../../errors/configuration.js";
|
|
8
|
+
import { ConnectionError } from "../../errors/connection.js";
|
|
9
|
+
import { ValidationError } from "../../errors/validation.js";
|
|
10
|
+
import { init_errors } from "../../errors/index.js";
|
|
4
11
|
import { deepMerge } from "../../utils/merge.js";
|
|
5
|
-
import { init_utils, utils_exports } from "../../utils/index.js";
|
|
6
12
|
import { lakebaseDefaults } from "./defaults.js";
|
|
13
|
+
import { context_exports, init_context } from "../../context/index.js";
|
|
7
14
|
import { randomUUID } from "node:crypto";
|
|
8
15
|
import { ApiClient, Config } from "@databricks/sdk-experimental";
|
|
9
16
|
import pg from "pg";
|
|
10
17
|
|
|
11
18
|
//#region src/connectors/lakebase/client.ts
|
|
12
|
-
|
|
19
|
+
init_errors();
|
|
20
|
+
const logger = createLogger("connectors:lakebase");
|
|
13
21
|
/**
|
|
14
22
|
* Enterprise-grade connector for Databricks Lakebase
|
|
15
23
|
* @example Simplest - everything from env/context
|
|
@@ -44,7 +52,7 @@ var LakebaseConnector = class {
|
|
|
44
52
|
unit: "ms"
|
|
45
53
|
})
|
|
46
54
|
};
|
|
47
|
-
if (this.config.maxPoolSize < 1) throw
|
|
55
|
+
if (this.config.maxPoolSize < 1) throw ValidationError.invalidValue("maxPoolSize", this.config.maxPoolSize, "at least 1");
|
|
48
56
|
}
|
|
49
57
|
/**
|
|
50
58
|
* Execute a SQL query
|
|
@@ -83,7 +91,8 @@ var LakebaseConnector = class {
|
|
|
83
91
|
}
|
|
84
92
|
span.recordException(error);
|
|
85
93
|
span.setStatus({ code: SpanStatusCode.ERROR });
|
|
86
|
-
throw error;
|
|
94
|
+
if (error instanceof AppKitError) throw error;
|
|
95
|
+
throw ConnectionError.queryFailed(error);
|
|
87
96
|
} finally {
|
|
88
97
|
const duration = Date.now() - startTime;
|
|
89
98
|
this.telemetryMetrics.queryCount.add(1);
|
|
@@ -151,7 +160,8 @@ var LakebaseConnector = class {
|
|
|
151
160
|
}
|
|
152
161
|
span.recordException(error);
|
|
153
162
|
span.setStatus({ code: SpanStatusCode.ERROR });
|
|
154
|
-
throw error;
|
|
163
|
+
if (error instanceof AppKitError) throw error;
|
|
164
|
+
throw ConnectionError.transactionFailed(error);
|
|
155
165
|
} finally {
|
|
156
166
|
client.release();
|
|
157
167
|
const duration = Date.now() - startTime;
|
|
@@ -182,7 +192,7 @@ var LakebaseConnector = class {
|
|
|
182
192
|
async close() {
|
|
183
193
|
if (this.pool) {
|
|
184
194
|
await this.pool.end().catch((error) => {
|
|
185
|
-
|
|
195
|
+
logger.error("Error closing connection pool: %O", error);
|
|
186
196
|
});
|
|
187
197
|
this.pool = null;
|
|
188
198
|
}
|
|
@@ -194,21 +204,21 @@ var LakebaseConnector = class {
|
|
|
194
204
|
process.on("SIGINT", () => this.close());
|
|
195
205
|
this.close();
|
|
196
206
|
}
|
|
197
|
-
/** Get Databricks workspace client - from config or
|
|
207
|
+
/** Get Databricks workspace client - from config or execution context */
|
|
198
208
|
getWorkspaceClient() {
|
|
199
209
|
if (this.config.workspaceClient) return this.config.workspaceClient;
|
|
200
210
|
try {
|
|
201
|
-
const {
|
|
202
|
-
const
|
|
203
|
-
this.config.workspaceClient =
|
|
204
|
-
return
|
|
211
|
+
const { getWorkspaceClient: getClient } = (init_context(), __toCommonJS(context_exports));
|
|
212
|
+
const client = getClient();
|
|
213
|
+
this.config.workspaceClient = client;
|
|
214
|
+
return client;
|
|
205
215
|
} catch (_error) {
|
|
206
|
-
throw
|
|
216
|
+
throw ConnectionError.clientUnavailable("Databricks workspace client", "Either pass it in config or ensure ServiceContext is initialized");
|
|
207
217
|
}
|
|
208
218
|
}
|
|
209
219
|
/** Get or create connection pool */
|
|
210
220
|
async getPool() {
|
|
211
|
-
if (!this.connectionConfig) throw
|
|
221
|
+
if (!this.connectionConfig) throw ConfigurationError.invalidConnection("Lakebase", "Set PGHOST, PGDATABASE, PGAPPNAME env vars, provide a connectionString, or pass explicit config");
|
|
212
222
|
if (!this.pool) {
|
|
213
223
|
const creds = await this.getCredentials();
|
|
214
224
|
this.pool = this.createPool(creds);
|
|
@@ -230,7 +240,7 @@ var LakebaseConnector = class {
|
|
|
230
240
|
ssl: sslMode === "require" ? { rejectUnauthorized: true } : false
|
|
231
241
|
});
|
|
232
242
|
pool.on("error", (error) => {
|
|
233
|
-
|
|
243
|
+
logger.error("Connection pool error: %s (code: %s)", error.message, error.code);
|
|
234
244
|
});
|
|
235
245
|
return pool;
|
|
236
246
|
}
|
|
@@ -257,20 +267,20 @@ var LakebaseConnector = class {
|
|
|
257
267
|
const oldPool = this.pool;
|
|
258
268
|
this.pool = null;
|
|
259
269
|
oldPool.end().catch((error) => {
|
|
260
|
-
|
|
270
|
+
logger.error("Error closing old connection pool during rotation: %O", error);
|
|
261
271
|
});
|
|
262
272
|
}
|
|
263
273
|
}
|
|
264
274
|
/** Fetch username from Databricks */
|
|
265
275
|
async fetchUsername() {
|
|
266
276
|
const user = await this.getWorkspaceClient().currentUser.me();
|
|
267
|
-
if (!user.userName) throw
|
|
277
|
+
if (!user.userName) throw AuthenticationError.userLookupFailed();
|
|
268
278
|
return user.userName;
|
|
269
279
|
}
|
|
270
280
|
/** Fetch password (OAuth token) from Databricks */
|
|
271
281
|
async fetchPassword() {
|
|
272
282
|
const apiClient = new ApiClient(new Config({ host: this.getWorkspaceClient().config.host }));
|
|
273
|
-
if (!this.connectionConfig.appName) throw
|
|
283
|
+
if (!this.connectionConfig.appName) throw ConfigurationError.resourceNotFound("Database app name");
|
|
274
284
|
const credentials = await apiClient.request({
|
|
275
285
|
path: `/api/2.0/database/credentials`,
|
|
276
286
|
method: "POST",
|
|
@@ -281,7 +291,7 @@ var LakebaseConnector = class {
|
|
|
281
291
|
request_id: randomUUID()
|
|
282
292
|
}
|
|
283
293
|
});
|
|
284
|
-
if (!this.validateCredentials(credentials)) throw
|
|
294
|
+
if (!this.validateCredentials(credentials)) throw AuthenticationError.credentialsFailed(this.connectionConfig.appName);
|
|
285
295
|
const expiresAt = new Date(credentials.expiration_time).getTime();
|
|
286
296
|
return {
|
|
287
297
|
token: credentials.token,
|
|
@@ -317,10 +327,10 @@ var LakebaseConnector = class {
|
|
|
317
327
|
const pgHost = process.env.PGHOST;
|
|
318
328
|
const pgDatabase = process.env.PGDATABASE;
|
|
319
329
|
const pgAppName = process.env.PGAPPNAME;
|
|
320
|
-
if (!pgHost || !pgDatabase || !pgAppName) throw
|
|
330
|
+
if (!pgHost || !pgDatabase || !pgAppName) throw ConfigurationError.invalidConnection("Lakebase", "Required env vars: PGHOST, PGDATABASE, PGAPPNAME. Optional: PGPORT (default: 5432), PGSSLMODE (default: require)");
|
|
321
331
|
const pgPort = process.env.PGPORT;
|
|
322
332
|
const port = pgPort ? parseInt(pgPort, 10) : 5432;
|
|
323
|
-
if (Number.isNaN(port)) throw
|
|
333
|
+
if (Number.isNaN(port)) throw ValidationError.invalidValue("port", pgPort, "a number");
|
|
324
334
|
return {
|
|
325
335
|
host: pgHost,
|
|
326
336
|
database: pgDatabase,
|
|
@@ -332,7 +342,7 @@ var LakebaseConnector = class {
|
|
|
332
342
|
parseConnectionString(connectionString) {
|
|
333
343
|
const url = new URL(connectionString);
|
|
334
344
|
const appName = url.searchParams.get("appName");
|
|
335
|
-
if (!appName) throw
|
|
345
|
+
if (!appName) throw ConfigurationError.missingConnectionParam("appName");
|
|
336
346
|
return {
|
|
337
347
|
host: url.hostname,
|
|
338
348
|
database: url.pathname.slice(1),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.js","names":[],"sources":["../../../src/connectors/lakebase/client.ts"],"sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport type { WorkspaceClient } from \"@databricks/sdk-experimental\";\nimport { ApiClient, Config } from \"@databricks/sdk-experimental\";\nimport pg from \"pg\";\nimport {\n type Counter,\n type Histogram,\n SpanStatusCode,\n TelemetryManager,\n type TelemetryProvider,\n} from \"@/telemetry\";\nimport { deepMerge } from \"../../utils\";\nimport { lakebaseDefaults } from \"./defaults\";\nimport type {\n LakebaseConfig,\n LakebaseConnectionConfig,\n LakebaseCredentials,\n} from \"./types\";\n\n/**\n * Enterprise-grade connector for Databricks Lakebase\n * @example Simplest - everything from env/context\n * ```typescript\n * const connector = new LakebaseConnector();\n * await connector.query('SELECT * FROM users');\n * ```\n *\n * @example With explicit connection string\n * ```typescript\n * const connector = new LakebaseConnector({\n * connectionString: 'postgresql://...'\n * });\n * ```\n */\nexport class LakebaseConnector {\n private readonly name: string = \"lakebase\";\n private readonly CACHE_BUFFER_MS = 2 * 60 * 1000;\n private readonly config: LakebaseConfig;\n private readonly connectionConfig: LakebaseConnectionConfig;\n private pool: pg.Pool | null = null;\n private credentials: LakebaseCredentials | null = null;\n\n // telemetry\n private readonly telemetry: TelemetryProvider;\n private readonly telemetryMetrics: {\n queryCount: Counter;\n queryDuration: Histogram;\n };\n\n constructor(userConfig?: Partial<LakebaseConfig>) {\n this.config = deepMerge(lakebaseDefaults, userConfig);\n this.connectionConfig = this.parseConnectionConfig();\n\n this.telemetry = TelemetryManager.getProvider(\n this.name,\n this.config.telemetry,\n );\n this.telemetryMetrics = {\n queryCount: this.telemetry\n .getMeter()\n .createCounter(\"lakebase.query.count\", {\n description: \"Total number of queries executed\",\n unit: \"1\",\n }),\n queryDuration: this.telemetry\n .getMeter()\n .createHistogram(\"lakebase.query.duration\", {\n description: \"Duration of queries executed\",\n unit: \"ms\",\n }),\n };\n\n // validate configuration\n if (this.config.maxPoolSize < 1) {\n throw new Error(\"maxPoolSize must be at least 1\");\n }\n }\n\n /**\n * Execute a SQL query\n *\n * @example\n * ```typescript\n * const users = await connector.query('SELECT * FROM users');\n * const user = await connector.query('SELECT * FROM users WHERE id = $1', [123]);\n * ```\n */\n async query<T extends pg.QueryResultRow>(\n sql: string,\n params?: any[],\n retryCount: number = 0,\n ): Promise<pg.QueryResult<T>> {\n const startTime = Date.now();\n\n return this.telemetry.startActiveSpan(\n \"lakebase.query\",\n {\n attributes: {\n \"db.system\": \"lakebase\",\n \"db.statement\": sql.substring(0, 500),\n \"db.retry_count\": retryCount,\n },\n },\n async (span) => {\n try {\n const pool = await this.getPool();\n const result = await pool.query<T>(sql, params);\n span.setAttribute(\"db.rows_affected\", result.rowCount ?? 0);\n span.setStatus({ code: SpanStatusCode.OK });\n return result;\n } catch (error) {\n // retry on auth failure\n if (this.isAuthError(error)) {\n span.addEvent(\"auth_error_retry\");\n await this.rotateCredentials();\n const newPool = await this.getPool();\n const result = await newPool.query<T>(sql, params);\n span.setAttribute(\"db.rows_affected\", result.rowCount ?? 0);\n span.setStatus({ code: SpanStatusCode.OK });\n return result;\n }\n\n // retry on transient errors, but only once\n if (this.isTransientError(error) && retryCount < 1) {\n span.addEvent(\"transient_error_retry\");\n await new Promise((resolve) => setTimeout(resolve, 100));\n return await this.query<T>(sql, params, retryCount + 1);\n }\n\n span.recordException(error as Error);\n span.setStatus({ code: SpanStatusCode.ERROR });\n\n throw error;\n } finally {\n const duration = Date.now() - startTime;\n this.telemetryMetrics.queryCount.add(1);\n this.telemetryMetrics.queryDuration.record(duration);\n span.end();\n }\n },\n );\n }\n\n /**\n * Execute a transaction\n *\n * COMMIT and ROLLBACK are automatically managed by the transaction function.\n *\n * @param callback - Callback function to execute within the transaction context\n * @example\n * ```typescript\n * await connector.transaction(async (client) => {\n * await client.query('INSERT INTO accounts (name) VALUES ($1)', ['Alice']);\n * await client.query('INSERT INTO logs (action) VALUES ($1)', ['Created Alice']);\n * });\n * ```\n */\n async transaction<T>(\n callback: (client: pg.PoolClient) => Promise<T>,\n retryCount: number = 0,\n ): Promise<T> {\n const startTime = Date.now();\n return this.telemetry.startActiveSpan(\n \"lakebase.transaction\",\n {\n attributes: {\n \"db.system\": \"lakebase\",\n \"db.retry_count\": retryCount,\n },\n },\n async (span) => {\n const pool = await this.getPool();\n const client = await pool.connect();\n try {\n await client.query(\"BEGIN\");\n const result = await callback(client);\n await client.query(\"COMMIT\");\n span.setStatus({ code: SpanStatusCode.OK });\n return result;\n } catch (error) {\n try {\n await client.query(\"ROLLBACK\");\n } catch {}\n // retry on auth failure\n if (this.isAuthError(error)) {\n span.addEvent(\"auth_error_retry\");\n client.release();\n await this.rotateCredentials();\n const newPool = await this.getPool();\n const retryClient = await newPool.connect();\n try {\n await client.query(\"BEGIN\");\n const result = await callback(retryClient);\n await client.query(\"COMMIT\");\n span.setStatus({ code: SpanStatusCode.OK });\n return result;\n } catch (retryError) {\n try {\n await retryClient.query(\"ROLLBACK\");\n } catch {}\n throw retryError;\n } finally {\n retryClient.release();\n }\n }\n\n // retry on transient errors, but only once\n if (this.isTransientError(error) && retryCount < 1) {\n span.addEvent(\"transaction_error_retry\");\n client.release();\n await new Promise((resolve) => setTimeout(resolve, 100));\n return await this.transaction<T>(callback, retryCount + 1);\n }\n span.recordException(error as Error);\n span.setStatus({ code: SpanStatusCode.ERROR });\n throw error;\n } finally {\n client.release();\n const duration = Date.now() - startTime;\n this.telemetryMetrics.queryCount.add(1);\n this.telemetryMetrics.queryDuration.record(duration);\n span.end();\n }\n },\n );\n }\n\n /** Check if database connection is healthy */\n async healthCheck(): Promise<boolean> {\n return this.telemetry.startActiveSpan(\n \"lakebase.healthCheck\",\n {},\n async (span) => {\n try {\n const result = await this.query<{ result: number }>(\n \"SELECT 1 as result\",\n );\n const healthy = result.rows[0]?.result === 1;\n span.setAttribute(\"db.healthy\", healthy);\n span.setStatus({ code: SpanStatusCode.OK });\n return healthy;\n } catch {\n span.setAttribute(\"db.healthy\", false);\n span.setStatus({ code: SpanStatusCode.ERROR });\n return false;\n } finally {\n span.end();\n }\n },\n );\n }\n\n /** Close connection pool (call on shutdown) */\n async close(): Promise<void> {\n if (this.pool) {\n await this.pool.end().catch((error: unknown) => {\n console.error(\"Error closing connection pool:\", error);\n });\n this.pool = null;\n }\n this.credentials = null;\n }\n\n /** Setup graceful shutdown to close connection pools */\n shutdown(): void {\n process.on(\"SIGTERM\", () => this.close());\n process.on(\"SIGINT\", () => this.close());\n this.close();\n }\n\n /** Get Databricks workspace client - from config or request context */\n private getWorkspaceClient(): WorkspaceClient {\n if (this.config.workspaceClient) {\n return this.config.workspaceClient;\n }\n\n try {\n const { getRequestContext } = require(\"../../utils\");\n const { serviceDatabricksClient } = getRequestContext();\n\n // cache it for subsequent calls\n this.config.workspaceClient = serviceDatabricksClient;\n return serviceDatabricksClient;\n } catch (_error) {\n throw new Error(\n \"Databricks workspace client not available. Either pass it in config or use within AppKit request context.\",\n );\n }\n }\n\n /** Get or create connection pool */\n private async getPool(): Promise<pg.Pool> {\n if (!this.connectionConfig) {\n throw new Error(\n \"Lakebase connection not configured. \" +\n \"Set PGHOST, PGDATABASE, PGAPPNAME env vars, provide a connectionString, or pass explicit config.\",\n );\n }\n\n if (!this.pool) {\n const creds = await this.getCredentials();\n this.pool = this.createPool(creds);\n }\n return this.pool;\n }\n\n /** Create PostgreSQL pool */\n private createPool(credentials: {\n username: string;\n password: string;\n }): pg.Pool {\n const { host, database, port, sslMode } = this.connectionConfig;\n\n const pool = new pg.Pool({\n host,\n port,\n database,\n user: credentials.username,\n password: credentials.password,\n max: this.config.maxPoolSize,\n idleTimeoutMillis: this.config.idleTimeoutMs,\n connectionTimeoutMillis: this.config.connectionTimeoutMs,\n ssl: sslMode === \"require\" ? { rejectUnauthorized: true } : false,\n });\n\n pool.on(\"error\", (error: Error & { code?: string }) => {\n console.error(\"Connection pool error:\", error.message, {\n code: error.code,\n });\n });\n\n return pool;\n }\n\n /** Get or fetch credentials with caching */\n private async getCredentials(): Promise<{\n username: string;\n password: string;\n }> {\n const now = Date.now();\n\n // return cached if still valid\n if (\n this.credentials &&\n now < this.credentials.expiresAt - this.CACHE_BUFFER_MS\n ) {\n return this.credentials;\n }\n\n // fetch new credentials\n const username = await this.fetchUsername();\n const { token, expiresAt } = await this.fetchPassword();\n\n this.credentials = {\n username,\n password: token,\n expiresAt,\n };\n\n return { username, password: token };\n }\n\n /** Rotate credentials and recreate pool */\n private async rotateCredentials(): Promise<void> {\n // clear cached credentials\n this.credentials = null;\n\n if (this.pool) {\n const oldPool = this.pool;\n this.pool = null;\n oldPool.end().catch((error: unknown) => {\n console.error(\n \"Error closing old connection pool during rotation:\",\n error,\n );\n });\n }\n }\n\n /** Fetch username from Databricks */\n private async fetchUsername(): Promise<string> {\n const workspaceClient = this.getWorkspaceClient();\n const user = await workspaceClient.currentUser.me();\n if (!user.userName) {\n throw new Error(\"Failed to get current user from Databricks workspace\");\n }\n return user.userName;\n }\n\n /** Fetch password (OAuth token) from Databricks */\n private async fetchPassword(): Promise<{ token: string; expiresAt: number }> {\n const workspaceClient = this.getWorkspaceClient();\n const config = new Config({ host: workspaceClient.config.host });\n const apiClient = new ApiClient(config);\n\n if (!this.connectionConfig.appName) {\n throw new Error(`Database app name not found in connection config`);\n }\n\n const credentials = await apiClient.request({\n path: `/api/2.0/database/credentials`,\n method: \"POST\",\n headers: new Headers(),\n raw: false,\n payload: {\n instance_names: [this.connectionConfig.appName],\n request_id: randomUUID(),\n },\n });\n\n if (!this.validateCredentials(credentials)) {\n throw new Error(\n `Failed to generate credentials for instance: ${this.connectionConfig.appName}`,\n );\n }\n\n const expiresAt = new Date(credentials.expiration_time).getTime();\n\n return { token: credentials.token, expiresAt };\n }\n\n /** Check if error is auth failure */\n private isAuthError(error: unknown): boolean {\n return (\n typeof error === \"object\" &&\n error !== null &&\n \"code\" in error &&\n (error as any).code === \"28P01\"\n );\n }\n\n /** Check if error is transient */\n private isTransientError(error: unknown): boolean {\n if (typeof error !== \"object\" || error === null || !(\"code\" in error)) {\n return false;\n }\n\n const code = (error as any).code;\n return (\n code === \"ECONNRESET\" ||\n code === \"ECONNREFUSED\" ||\n code === \"ETIMEDOUT\" ||\n code === \"57P01\" || // admin_shutdown\n code === \"57P03\" || // cannot_connect_now\n code === \"08006\" || // connection_failure\n code === \"08003\" || // connection_does_not_exist\n code === \"08000\" // connection_exception\n );\n }\n\n /** Type guard for credentials */\n private validateCredentials(\n value: unknown,\n ): value is { token: string; expiration_time: string } {\n if (typeof value !== \"object\" || value === null) {\n return false;\n }\n\n const credentials = value as { token: string; expiration_time: string };\n return (\n \"token\" in credentials &&\n typeof credentials.token === \"string\" &&\n \"expiration_time\" in credentials &&\n typeof credentials.expiration_time === \"string\" &&\n new Date(credentials.expiration_time).getTime() > Date.now()\n );\n }\n\n /** Parse connection configuration from config or environment */\n private parseConnectionConfig(): LakebaseConnectionConfig {\n if (this.config.connectionString) {\n return this.parseConnectionString(this.config.connectionString);\n }\n\n // get connection from config\n if (this.config.host && this.config.database && this.config.appName) {\n return {\n host: this.config.host,\n database: this.config.database,\n port: this.config.port ?? 5432,\n sslMode: this.config.sslMode ?? \"require\",\n appName: this.config.appName,\n };\n }\n\n // get connection from environment variables\n const pgHost = process.env.PGHOST;\n const pgDatabase = process.env.PGDATABASE;\n const pgAppName = process.env.PGAPPNAME;\n if (!pgHost || !pgDatabase || !pgAppName) {\n throw new Error(\n \"Lakebase connection not configured. Required env vars: PGHOST, PGDATABASE, PGAPPNAME. \" +\n \"Optional: PGPORT (default: 5432), PGSSLMODE (default: require).\",\n );\n }\n const pgPort = process.env.PGPORT;\n const port = pgPort ? parseInt(pgPort, 10) : 5432;\n\n if (Number.isNaN(port)) {\n throw new Error(`Invalid port: ${pgPort}. Must be a number.`);\n }\n\n const pgSSLMode = process.env.PGSSLMODE;\n const sslMode =\n (pgSSLMode as \"require\" | \"disable\" | \"prefer\") || \"require\";\n\n return {\n host: pgHost,\n database: pgDatabase,\n port,\n sslMode,\n appName: pgAppName,\n };\n }\n\n private parseConnectionString(\n connectionString: string,\n ): LakebaseConnectionConfig {\n const url = new URL(connectionString);\n const appName = url.searchParams.get(\"appName\");\n if (!appName) {\n throw new Error(\"Connection string must include appName parameter\");\n }\n\n return {\n host: url.hostname,\n database: url.pathname.slice(1), // remove leading slash\n port: url.port ? parseInt(url.port, 10) : 5432,\n sslMode:\n (url.searchParams.get(\"sslmode\") as \"require\" | \"disable\" | \"prefer\") ??\n \"require\",\n appName: appName,\n };\n }\n}\n"],"mappings":";;;;;;;;;;;YAWwC;;;;;;;;;;;;;;;;AAuBxC,IAAa,oBAAb,MAA+B;CAe7B,YAAY,YAAsC;cAdlB;yBACG,MAAS;cAGb;qBACmB;AAUhD,OAAK,SAAS,UAAU,kBAAkB,WAAW;AACrD,OAAK,mBAAmB,KAAK,uBAAuB;AAEpD,OAAK,YAAY,iBAAiB,YAChC,KAAK,MACL,KAAK,OAAO,UACb;AACD,OAAK,mBAAmB;GACtB,YAAY,KAAK,UACd,UAAU,CACV,cAAc,wBAAwB;IACrC,aAAa;IACb,MAAM;IACP,CAAC;GACJ,eAAe,KAAK,UACjB,UAAU,CACV,gBAAgB,2BAA2B;IAC1C,aAAa;IACb,MAAM;IACP,CAAC;GACL;AAGD,MAAI,KAAK,OAAO,cAAc,EAC5B,OAAM,IAAI,MAAM,iCAAiC;;;;;;;;;;;CAarD,MAAM,MACJ,KACA,QACA,aAAqB,GACO;EAC5B,MAAM,YAAY,KAAK,KAAK;AAE5B,SAAO,KAAK,UAAU,gBACpB,kBACA,EACE,YAAY;GACV,aAAa;GACb,gBAAgB,IAAI,UAAU,GAAG,IAAI;GACrC,kBAAkB;GACnB,EACF,EACD,OAAO,SAAS;AACd,OAAI;IAEF,MAAM,SAAS,OADF,MAAM,KAAK,SAAS,EACP,MAAS,KAAK,OAAO;AAC/C,SAAK,aAAa,oBAAoB,OAAO,YAAY,EAAE;AAC3D,SAAK,UAAU,EAAE,MAAM,eAAe,IAAI,CAAC;AAC3C,WAAO;YACA,OAAO;AAEd,QAAI,KAAK,YAAY,MAAM,EAAE;AAC3B,UAAK,SAAS,mBAAmB;AACjC,WAAM,KAAK,mBAAmB;KAE9B,MAAM,SAAS,OADC,MAAM,KAAK,SAAS,EACP,MAAS,KAAK,OAAO;AAClD,UAAK,aAAa,oBAAoB,OAAO,YAAY,EAAE;AAC3D,UAAK,UAAU,EAAE,MAAM,eAAe,IAAI,CAAC;AAC3C,YAAO;;AAIT,QAAI,KAAK,iBAAiB,MAAM,IAAI,aAAa,GAAG;AAClD,UAAK,SAAS,wBAAwB;AACtC,WAAM,IAAI,SAAS,YAAY,WAAW,SAAS,IAAI,CAAC;AACxD,YAAO,MAAM,KAAK,MAAS,KAAK,QAAQ,aAAa,EAAE;;AAGzD,SAAK,gBAAgB,MAAe;AACpC,SAAK,UAAU,EAAE,MAAM,eAAe,OAAO,CAAC;AAE9C,UAAM;aACE;IACR,MAAM,WAAW,KAAK,KAAK,GAAG;AAC9B,SAAK,iBAAiB,WAAW,IAAI,EAAE;AACvC,SAAK,iBAAiB,cAAc,OAAO,SAAS;AACpD,SAAK,KAAK;;IAGf;;;;;;;;;;;;;;;;CAiBH,MAAM,YACJ,UACA,aAAqB,GACT;EACZ,MAAM,YAAY,KAAK,KAAK;AAC5B,SAAO,KAAK,UAAU,gBACpB,wBACA,EACE,YAAY;GACV,aAAa;GACb,kBAAkB;GACnB,EACF,EACD,OAAO,SAAS;GAEd,MAAM,SAAS,OADF,MAAM,KAAK,SAAS,EACP,SAAS;AACnC,OAAI;AACF,UAAM,OAAO,MAAM,QAAQ;IAC3B,MAAM,SAAS,MAAM,SAAS,OAAO;AACrC,UAAM,OAAO,MAAM,SAAS;AAC5B,SAAK,UAAU,EAAE,MAAM,eAAe,IAAI,CAAC;AAC3C,WAAO;YACA,OAAO;AACd,QAAI;AACF,WAAM,OAAO,MAAM,WAAW;YACxB;AAER,QAAI,KAAK,YAAY,MAAM,EAAE;AAC3B,UAAK,SAAS,mBAAmB;AACjC,YAAO,SAAS;AAChB,WAAM,KAAK,mBAAmB;KAE9B,MAAM,cAAc,OADJ,MAAM,KAAK,SAAS,EACF,SAAS;AAC3C,SAAI;AACF,YAAM,OAAO,MAAM,QAAQ;MAC3B,MAAM,SAAS,MAAM,SAAS,YAAY;AAC1C,YAAM,OAAO,MAAM,SAAS;AAC5B,WAAK,UAAU,EAAE,MAAM,eAAe,IAAI,CAAC;AAC3C,aAAO;cACA,YAAY;AACnB,UAAI;AACF,aAAM,YAAY,MAAM,WAAW;cAC7B;AACR,YAAM;eACE;AACR,kBAAY,SAAS;;;AAKzB,QAAI,KAAK,iBAAiB,MAAM,IAAI,aAAa,GAAG;AAClD,UAAK,SAAS,0BAA0B;AACxC,YAAO,SAAS;AAChB,WAAM,IAAI,SAAS,YAAY,WAAW,SAAS,IAAI,CAAC;AACxD,YAAO,MAAM,KAAK,YAAe,UAAU,aAAa,EAAE;;AAE5D,SAAK,gBAAgB,MAAe;AACpC,SAAK,UAAU,EAAE,MAAM,eAAe,OAAO,CAAC;AAC9C,UAAM;aACE;AACR,WAAO,SAAS;IAChB,MAAM,WAAW,KAAK,KAAK,GAAG;AAC9B,SAAK,iBAAiB,WAAW,IAAI,EAAE;AACvC,SAAK,iBAAiB,cAAc,OAAO,SAAS;AACpD,SAAK,KAAK;;IAGf;;;CAIH,MAAM,cAAgC;AACpC,SAAO,KAAK,UAAU,gBACpB,wBACA,EAAE,EACF,OAAO,SAAS;AACd,OAAI;IAIF,MAAM,WAHS,MAAM,KAAK,MACxB,qBACD,EACsB,KAAK,IAAI,WAAW;AAC3C,SAAK,aAAa,cAAc,QAAQ;AACxC,SAAK,UAAU,EAAE,MAAM,eAAe,IAAI,CAAC;AAC3C,WAAO;WACD;AACN,SAAK,aAAa,cAAc,MAAM;AACtC,SAAK,UAAU,EAAE,MAAM,eAAe,OAAO,CAAC;AAC9C,WAAO;aACC;AACR,SAAK,KAAK;;IAGf;;;CAIH,MAAM,QAAuB;AAC3B,MAAI,KAAK,MAAM;AACb,SAAM,KAAK,KAAK,KAAK,CAAC,OAAO,UAAmB;AAC9C,YAAQ,MAAM,kCAAkC,MAAM;KACtD;AACF,QAAK,OAAO;;AAEd,OAAK,cAAc;;;CAIrB,WAAiB;AACf,UAAQ,GAAG,iBAAiB,KAAK,OAAO,CAAC;AACzC,UAAQ,GAAG,gBAAgB,KAAK,OAAO,CAAC;AACxC,OAAK,OAAO;;;CAId,AAAQ,qBAAsC;AAC5C,MAAI,KAAK,OAAO,gBACd,QAAO,KAAK,OAAO;AAGrB,MAAI;GACF,MAAM,EAAE;GACR,MAAM,EAAE,4BAA4B,mBAAmB;AAGvD,QAAK,OAAO,kBAAkB;AAC9B,UAAO;WACA,QAAQ;AACf,SAAM,IAAI,MACR,4GACD;;;;CAKL,MAAc,UAA4B;AACxC,MAAI,CAAC,KAAK,iBACR,OAAM,IAAI,MACR,uIAED;AAGH,MAAI,CAAC,KAAK,MAAM;GACd,MAAM,QAAQ,MAAM,KAAK,gBAAgB;AACzC,QAAK,OAAO,KAAK,WAAW,MAAM;;AAEpC,SAAO,KAAK;;;CAId,AAAQ,WAAW,aAGP;EACV,MAAM,EAAE,MAAM,UAAU,MAAM,YAAY,KAAK;EAE/C,MAAM,OAAO,IAAI,GAAG,KAAK;GACvB;GACA;GACA;GACA,MAAM,YAAY;GAClB,UAAU,YAAY;GACtB,KAAK,KAAK,OAAO;GACjB,mBAAmB,KAAK,OAAO;GAC/B,yBAAyB,KAAK,OAAO;GACrC,KAAK,YAAY,YAAY,EAAE,oBAAoB,MAAM,GAAG;GAC7D,CAAC;AAEF,OAAK,GAAG,UAAU,UAAqC;AACrD,WAAQ,MAAM,0BAA0B,MAAM,SAAS,EACrD,MAAM,MAAM,MACb,CAAC;IACF;AAEF,SAAO;;;CAIT,MAAc,iBAGX;EACD,MAAM,MAAM,KAAK,KAAK;AAGtB,MACE,KAAK,eACL,MAAM,KAAK,YAAY,YAAY,KAAK,gBAExC,QAAO,KAAK;EAId,MAAM,WAAW,MAAM,KAAK,eAAe;EAC3C,MAAM,EAAE,OAAO,cAAc,MAAM,KAAK,eAAe;AAEvD,OAAK,cAAc;GACjB;GACA,UAAU;GACV;GACD;AAED,SAAO;GAAE;GAAU,UAAU;GAAO;;;CAItC,MAAc,oBAAmC;AAE/C,OAAK,cAAc;AAEnB,MAAI,KAAK,MAAM;GACb,MAAM,UAAU,KAAK;AACrB,QAAK,OAAO;AACZ,WAAQ,KAAK,CAAC,OAAO,UAAmB;AACtC,YAAQ,MACN,sDACA,MACD;KACD;;;;CAKN,MAAc,gBAAiC;EAE7C,MAAM,OAAO,MADW,KAAK,oBAAoB,CACd,YAAY,IAAI;AACnD,MAAI,CAAC,KAAK,SACR,OAAM,IAAI,MAAM,uDAAuD;AAEzE,SAAO,KAAK;;;CAId,MAAc,gBAA+D;EAG3E,MAAM,YAAY,IAAI,UADP,IAAI,OAAO,EAAE,MADJ,KAAK,oBAAoB,CACC,OAAO,MAAM,CAAC,CACzB;AAEvC,MAAI,CAAC,KAAK,iBAAiB,QACzB,OAAM,IAAI,MAAM,mDAAmD;EAGrE,MAAM,cAAc,MAAM,UAAU,QAAQ;GAC1C,MAAM;GACN,QAAQ;GACR,SAAS,IAAI,SAAS;GACtB,KAAK;GACL,SAAS;IACP,gBAAgB,CAAC,KAAK,iBAAiB,QAAQ;IAC/C,YAAY,YAAY;IACzB;GACF,CAAC;AAEF,MAAI,CAAC,KAAK,oBAAoB,YAAY,CACxC,OAAM,IAAI,MACR,gDAAgD,KAAK,iBAAiB,UACvE;EAGH,MAAM,YAAY,IAAI,KAAK,YAAY,gBAAgB,CAAC,SAAS;AAEjE,SAAO;GAAE,OAAO,YAAY;GAAO;GAAW;;;CAIhD,AAAQ,YAAY,OAAyB;AAC3C,SACE,OAAO,UAAU,YACjB,UAAU,QACV,UAAU,SACT,MAAc,SAAS;;;CAK5B,AAAQ,iBAAiB,OAAyB;AAChD,MAAI,OAAO,UAAU,YAAY,UAAU,QAAQ,EAAE,UAAU,OAC7D,QAAO;EAGT,MAAM,OAAQ,MAAc;AAC5B,SACE,SAAS,gBACT,SAAS,kBACT,SAAS,eACT,SAAS,WACT,SAAS,WACT,SAAS,WACT,SAAS,WACT,SAAS;;;CAKb,AAAQ,oBACN,OACqD;AACrD,MAAI,OAAO,UAAU,YAAY,UAAU,KACzC,QAAO;EAGT,MAAM,cAAc;AACpB,SACE,WAAW,eACX,OAAO,YAAY,UAAU,YAC7B,qBAAqB,eACrB,OAAO,YAAY,oBAAoB,YACvC,IAAI,KAAK,YAAY,gBAAgB,CAAC,SAAS,GAAG,KAAK,KAAK;;;CAKhE,AAAQ,wBAAkD;AACxD,MAAI,KAAK,OAAO,iBACd,QAAO,KAAK,sBAAsB,KAAK,OAAO,iBAAiB;AAIjE,MAAI,KAAK,OAAO,QAAQ,KAAK,OAAO,YAAY,KAAK,OAAO,QAC1D,QAAO;GACL,MAAM,KAAK,OAAO;GAClB,UAAU,KAAK,OAAO;GACtB,MAAM,KAAK,OAAO,QAAQ;GAC1B,SAAS,KAAK,OAAO,WAAW;GAChC,SAAS,KAAK,OAAO;GACtB;EAIH,MAAM,SAAS,QAAQ,IAAI;EAC3B,MAAM,aAAa,QAAQ,IAAI;EAC/B,MAAM,YAAY,QAAQ,IAAI;AAC9B,MAAI,CAAC,UAAU,CAAC,cAAc,CAAC,UAC7B,OAAM,IAAI,MACR,wJAED;EAEH,MAAM,SAAS,QAAQ,IAAI;EAC3B,MAAM,OAAO,SAAS,SAAS,QAAQ,GAAG,GAAG;AAE7C,MAAI,OAAO,MAAM,KAAK,CACpB,OAAM,IAAI,MAAM,iBAAiB,OAAO,qBAAqB;AAO/D,SAAO;GACL,MAAM;GACN,UAAU;GACV;GACA,SARgB,QAAQ,IAAI,aAEuB;GAOnD,SAAS;GACV;;CAGH,AAAQ,sBACN,kBAC0B;EAC1B,MAAM,MAAM,IAAI,IAAI,iBAAiB;EACrC,MAAM,UAAU,IAAI,aAAa,IAAI,UAAU;AAC/C,MAAI,CAAC,QACH,OAAM,IAAI,MAAM,mDAAmD;AAGrE,SAAO;GACL,MAAM,IAAI;GACV,UAAU,IAAI,SAAS,MAAM,EAAE;GAC/B,MAAM,IAAI,OAAO,SAAS,IAAI,MAAM,GAAG,GAAG;GAC1C,SACG,IAAI,aAAa,IAAI,UAAU,IAChC;GACO;GACV"}
|
|
1
|
+
{"version":3,"file":"client.js","names":[],"sources":["../../../src/connectors/lakebase/client.ts"],"sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport type { WorkspaceClient } from \"@databricks/sdk-experimental\";\nimport { ApiClient, Config } from \"@databricks/sdk-experimental\";\nimport pg from \"pg\";\nimport {\n type Counter,\n type Histogram,\n SpanStatusCode,\n TelemetryManager,\n type TelemetryProvider,\n} from \"@/telemetry\";\nimport {\n AppKitError,\n AuthenticationError,\n ConfigurationError,\n ConnectionError,\n ValidationError,\n} from \"../../errors\";\nimport { createLogger } from \"../../logging/logger\";\nimport { deepMerge } from \"../../utils\";\nimport { lakebaseDefaults } from \"./defaults\";\nimport type {\n LakebaseConfig,\n LakebaseConnectionConfig,\n LakebaseCredentials,\n} from \"./types\";\n\nconst logger = createLogger(\"connectors:lakebase\");\n\n/**\n * Enterprise-grade connector for Databricks Lakebase\n * @example Simplest - everything from env/context\n * ```typescript\n * const connector = new LakebaseConnector();\n * await connector.query('SELECT * FROM users');\n * ```\n *\n * @example With explicit connection string\n * ```typescript\n * const connector = new LakebaseConnector({\n * connectionString: 'postgresql://...'\n * });\n * ```\n */\nexport class LakebaseConnector {\n private readonly name: string = \"lakebase\";\n private readonly CACHE_BUFFER_MS = 2 * 60 * 1000;\n private readonly config: LakebaseConfig;\n private readonly connectionConfig: LakebaseConnectionConfig;\n private pool: pg.Pool | null = null;\n private credentials: LakebaseCredentials | null = null;\n\n // telemetry\n private readonly telemetry: TelemetryProvider;\n private readonly telemetryMetrics: {\n queryCount: Counter;\n queryDuration: Histogram;\n };\n\n constructor(userConfig?: Partial<LakebaseConfig>) {\n this.config = deepMerge(lakebaseDefaults, userConfig);\n this.connectionConfig = this.parseConnectionConfig();\n\n this.telemetry = TelemetryManager.getProvider(\n this.name,\n this.config.telemetry,\n );\n this.telemetryMetrics = {\n queryCount: this.telemetry\n .getMeter()\n .createCounter(\"lakebase.query.count\", {\n description: \"Total number of queries executed\",\n unit: \"1\",\n }),\n queryDuration: this.telemetry\n .getMeter()\n .createHistogram(\"lakebase.query.duration\", {\n description: \"Duration of queries executed\",\n unit: \"ms\",\n }),\n };\n\n // validate configuration\n if (this.config.maxPoolSize < 1) {\n throw ValidationError.invalidValue(\n \"maxPoolSize\",\n this.config.maxPoolSize,\n \"at least 1\",\n );\n }\n }\n\n /**\n * Execute a SQL query\n *\n * @example\n * ```typescript\n * const users = await connector.query('SELECT * FROM users');\n * const user = await connector.query('SELECT * FROM users WHERE id = $1', [123]);\n * ```\n */\n async query<T extends pg.QueryResultRow>(\n sql: string,\n params?: any[],\n retryCount: number = 0,\n ): Promise<pg.QueryResult<T>> {\n const startTime = Date.now();\n\n return this.telemetry.startActiveSpan(\n \"lakebase.query\",\n {\n attributes: {\n \"db.system\": \"lakebase\",\n \"db.statement\": sql.substring(0, 500),\n \"db.retry_count\": retryCount,\n },\n },\n async (span) => {\n try {\n const pool = await this.getPool();\n const result = await pool.query<T>(sql, params);\n span.setAttribute(\"db.rows_affected\", result.rowCount ?? 0);\n span.setStatus({ code: SpanStatusCode.OK });\n return result;\n } catch (error) {\n // retry on auth failure\n if (this.isAuthError(error)) {\n span.addEvent(\"auth_error_retry\");\n await this.rotateCredentials();\n const newPool = await this.getPool();\n const result = await newPool.query<T>(sql, params);\n span.setAttribute(\"db.rows_affected\", result.rowCount ?? 0);\n span.setStatus({ code: SpanStatusCode.OK });\n return result;\n }\n\n // retry on transient errors, but only once\n if (this.isTransientError(error) && retryCount < 1) {\n span.addEvent(\"transient_error_retry\");\n await new Promise((resolve) => setTimeout(resolve, 100));\n return await this.query<T>(sql, params, retryCount + 1);\n }\n\n span.recordException(error as Error);\n span.setStatus({ code: SpanStatusCode.ERROR });\n\n if (error instanceof AppKitError) {\n throw error;\n }\n throw ConnectionError.queryFailed(error as Error);\n } finally {\n const duration = Date.now() - startTime;\n this.telemetryMetrics.queryCount.add(1);\n this.telemetryMetrics.queryDuration.record(duration);\n span.end();\n }\n },\n );\n }\n\n /**\n * Execute a transaction\n *\n * COMMIT and ROLLBACK are automatically managed by the transaction function.\n *\n * @param callback - Callback function to execute within the transaction context\n * @example\n * ```typescript\n * await connector.transaction(async (client) => {\n * await client.query('INSERT INTO accounts (name) VALUES ($1)', ['Alice']);\n * await client.query('INSERT INTO logs (action) VALUES ($1)', ['Created Alice']);\n * });\n * ```\n */\n async transaction<T>(\n callback: (client: pg.PoolClient) => Promise<T>,\n retryCount: number = 0,\n ): Promise<T> {\n const startTime = Date.now();\n return this.telemetry.startActiveSpan(\n \"lakebase.transaction\",\n {\n attributes: {\n \"db.system\": \"lakebase\",\n \"db.retry_count\": retryCount,\n },\n },\n async (span) => {\n const pool = await this.getPool();\n const client = await pool.connect();\n try {\n await client.query(\"BEGIN\");\n const result = await callback(client);\n await client.query(\"COMMIT\");\n span.setStatus({ code: SpanStatusCode.OK });\n return result;\n } catch (error) {\n try {\n await client.query(\"ROLLBACK\");\n } catch {}\n // retry on auth failure\n if (this.isAuthError(error)) {\n span.addEvent(\"auth_error_retry\");\n client.release();\n await this.rotateCredentials();\n const newPool = await this.getPool();\n const retryClient = await newPool.connect();\n try {\n await client.query(\"BEGIN\");\n const result = await callback(retryClient);\n await client.query(\"COMMIT\");\n span.setStatus({ code: SpanStatusCode.OK });\n return result;\n } catch (retryError) {\n try {\n await retryClient.query(\"ROLLBACK\");\n } catch {}\n throw retryError;\n } finally {\n retryClient.release();\n }\n }\n\n // retry on transient errors, but only once\n if (this.isTransientError(error) && retryCount < 1) {\n span.addEvent(\"transaction_error_retry\");\n client.release();\n await new Promise((resolve) => setTimeout(resolve, 100));\n return await this.transaction<T>(callback, retryCount + 1);\n }\n span.recordException(error as Error);\n span.setStatus({ code: SpanStatusCode.ERROR });\n\n if (error instanceof AppKitError) {\n throw error;\n }\n throw ConnectionError.transactionFailed(error as Error);\n } finally {\n client.release();\n const duration = Date.now() - startTime;\n this.telemetryMetrics.queryCount.add(1);\n this.telemetryMetrics.queryDuration.record(duration);\n span.end();\n }\n },\n );\n }\n\n /** Check if database connection is healthy */\n async healthCheck(): Promise<boolean> {\n return this.telemetry.startActiveSpan(\n \"lakebase.healthCheck\",\n {},\n async (span) => {\n try {\n const result = await this.query<{ result: number }>(\n \"SELECT 1 as result\",\n );\n const healthy = result.rows[0]?.result === 1;\n span.setAttribute(\"db.healthy\", healthy);\n span.setStatus({ code: SpanStatusCode.OK });\n return healthy;\n } catch {\n span.setAttribute(\"db.healthy\", false);\n span.setStatus({ code: SpanStatusCode.ERROR });\n return false;\n } finally {\n span.end();\n }\n },\n );\n }\n\n /** Close connection pool (call on shutdown) */\n async close(): Promise<void> {\n if (this.pool) {\n await this.pool.end().catch((error: unknown) => {\n logger.error(\"Error closing connection pool: %O\", error);\n });\n this.pool = null;\n }\n this.credentials = null;\n }\n\n /** Setup graceful shutdown to close connection pools */\n shutdown(): void {\n process.on(\"SIGTERM\", () => this.close());\n process.on(\"SIGINT\", () => this.close());\n this.close();\n }\n\n /** Get Databricks workspace client - from config or execution context */\n private getWorkspaceClient(): WorkspaceClient {\n if (this.config.workspaceClient) {\n return this.config.workspaceClient;\n }\n\n try {\n const { getWorkspaceClient: getClient } = require(\"../../context\");\n const client = getClient();\n\n // cache it for subsequent calls\n this.config.workspaceClient = client;\n return client;\n } catch (_error) {\n throw ConnectionError.clientUnavailable(\n \"Databricks workspace client\",\n \"Either pass it in config or ensure ServiceContext is initialized\",\n );\n }\n }\n\n /** Get or create connection pool */\n private async getPool(): Promise<pg.Pool> {\n if (!this.connectionConfig) {\n throw ConfigurationError.invalidConnection(\n \"Lakebase\",\n \"Set PGHOST, PGDATABASE, PGAPPNAME env vars, provide a connectionString, or pass explicit config\",\n );\n }\n\n if (!this.pool) {\n const creds = await this.getCredentials();\n this.pool = this.createPool(creds);\n }\n return this.pool;\n }\n\n /** Create PostgreSQL pool */\n private createPool(credentials: {\n username: string;\n password: string;\n }): pg.Pool {\n const { host, database, port, sslMode } = this.connectionConfig;\n\n const pool = new pg.Pool({\n host,\n port,\n database,\n user: credentials.username,\n password: credentials.password,\n max: this.config.maxPoolSize,\n idleTimeoutMillis: this.config.idleTimeoutMs,\n connectionTimeoutMillis: this.config.connectionTimeoutMs,\n ssl: sslMode === \"require\" ? { rejectUnauthorized: true } : false,\n });\n\n pool.on(\"error\", (error: Error & { code?: string }) => {\n logger.error(\n \"Connection pool error: %s (code: %s)\",\n error.message,\n error.code,\n );\n });\n\n return pool;\n }\n\n /** Get or fetch credentials with caching */\n private async getCredentials(): Promise<{\n username: string;\n password: string;\n }> {\n const now = Date.now();\n\n // return cached if still valid\n if (\n this.credentials &&\n now < this.credentials.expiresAt - this.CACHE_BUFFER_MS\n ) {\n return this.credentials;\n }\n\n // fetch new credentials\n const username = await this.fetchUsername();\n const { token, expiresAt } = await this.fetchPassword();\n\n this.credentials = {\n username,\n password: token,\n expiresAt,\n };\n\n return { username, password: token };\n }\n\n /** Rotate credentials and recreate pool */\n private async rotateCredentials(): Promise<void> {\n // clear cached credentials\n this.credentials = null;\n\n if (this.pool) {\n const oldPool = this.pool;\n this.pool = null;\n oldPool.end().catch((error: unknown) => {\n logger.error(\n \"Error closing old connection pool during rotation: %O\",\n error,\n );\n });\n }\n }\n\n /** Fetch username from Databricks */\n private async fetchUsername(): Promise<string> {\n const workspaceClient = this.getWorkspaceClient();\n const user = await workspaceClient.currentUser.me();\n if (!user.userName) {\n throw AuthenticationError.userLookupFailed();\n }\n return user.userName;\n }\n\n /** Fetch password (OAuth token) from Databricks */\n private async fetchPassword(): Promise<{ token: string; expiresAt: number }> {\n const workspaceClient = this.getWorkspaceClient();\n const config = new Config({ host: workspaceClient.config.host });\n const apiClient = new ApiClient(config);\n\n if (!this.connectionConfig.appName) {\n throw ConfigurationError.resourceNotFound(\"Database app name\");\n }\n\n const credentials = await apiClient.request({\n path: `/api/2.0/database/credentials`,\n method: \"POST\",\n headers: new Headers(),\n raw: false,\n payload: {\n instance_names: [this.connectionConfig.appName],\n request_id: randomUUID(),\n },\n });\n\n if (!this.validateCredentials(credentials)) {\n throw AuthenticationError.credentialsFailed(\n this.connectionConfig.appName,\n );\n }\n\n const expiresAt = new Date(credentials.expiration_time).getTime();\n\n return { token: credentials.token, expiresAt };\n }\n\n /** Check if error is auth failure */\n private isAuthError(error: unknown): boolean {\n return (\n typeof error === \"object\" &&\n error !== null &&\n \"code\" in error &&\n (error as any).code === \"28P01\"\n );\n }\n\n /** Check if error is transient */\n private isTransientError(error: unknown): boolean {\n if (typeof error !== \"object\" || error === null || !(\"code\" in error)) {\n return false;\n }\n\n const code = (error as any).code;\n return (\n code === \"ECONNRESET\" ||\n code === \"ECONNREFUSED\" ||\n code === \"ETIMEDOUT\" ||\n code === \"57P01\" || // admin_shutdown\n code === \"57P03\" || // cannot_connect_now\n code === \"08006\" || // connection_failure\n code === \"08003\" || // connection_does_not_exist\n code === \"08000\" // connection_exception\n );\n }\n\n /** Type guard for credentials */\n private validateCredentials(\n value: unknown,\n ): value is { token: string; expiration_time: string } {\n if (typeof value !== \"object\" || value === null) {\n return false;\n }\n\n const credentials = value as { token: string; expiration_time: string };\n return (\n \"token\" in credentials &&\n typeof credentials.token === \"string\" &&\n \"expiration_time\" in credentials &&\n typeof credentials.expiration_time === \"string\" &&\n new Date(credentials.expiration_time).getTime() > Date.now()\n );\n }\n\n /** Parse connection configuration from config or environment */\n private parseConnectionConfig(): LakebaseConnectionConfig {\n if (this.config.connectionString) {\n return this.parseConnectionString(this.config.connectionString);\n }\n\n // get connection from config\n if (this.config.host && this.config.database && this.config.appName) {\n return {\n host: this.config.host,\n database: this.config.database,\n port: this.config.port ?? 5432,\n sslMode: this.config.sslMode ?? \"require\",\n appName: this.config.appName,\n };\n }\n\n // get connection from environment variables\n const pgHost = process.env.PGHOST;\n const pgDatabase = process.env.PGDATABASE;\n const pgAppName = process.env.PGAPPNAME;\n if (!pgHost || !pgDatabase || !pgAppName) {\n throw ConfigurationError.invalidConnection(\n \"Lakebase\",\n \"Required env vars: PGHOST, PGDATABASE, PGAPPNAME. Optional: PGPORT (default: 5432), PGSSLMODE (default: require)\",\n );\n }\n const pgPort = process.env.PGPORT;\n const port = pgPort ? parseInt(pgPort, 10) : 5432;\n\n if (Number.isNaN(port)) {\n throw ValidationError.invalidValue(\"port\", pgPort, \"a number\");\n }\n\n const pgSSLMode = process.env.PGSSLMODE;\n const sslMode =\n (pgSSLMode as \"require\" | \"disable\" | \"prefer\") || \"require\";\n\n return {\n host: pgHost,\n database: pgDatabase,\n port,\n sslMode,\n appName: pgAppName,\n };\n }\n\n private parseConnectionString(\n connectionString: string,\n ): LakebaseConnectionConfig {\n const url = new URL(connectionString);\n const appName = url.searchParams.get(\"appName\");\n if (!appName) {\n throw ConfigurationError.missingConnectionParam(\"appName\");\n }\n\n return {\n host: url.hostname,\n database: url.pathname.slice(1), // remove leading slash\n port: url.port ? parseInt(url.port, 10) : 5432,\n sslMode:\n (url.searchParams.get(\"sslmode\") as \"require\" | \"disable\" | \"prefer\") ??\n \"require\",\n appName: appName,\n };\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;aAiBsB;AAUtB,MAAM,SAAS,aAAa,sBAAsB;;;;;;;;;;;;;;;;AAiBlD,IAAa,oBAAb,MAA+B;CAe7B,YAAY,YAAsC;cAdlB;yBACG,MAAS;cAGb;qBACmB;AAUhD,OAAK,SAAS,UAAU,kBAAkB,WAAW;AACrD,OAAK,mBAAmB,KAAK,uBAAuB;AAEpD,OAAK,YAAY,iBAAiB,YAChC,KAAK,MACL,KAAK,OAAO,UACb;AACD,OAAK,mBAAmB;GACtB,YAAY,KAAK,UACd,UAAU,CACV,cAAc,wBAAwB;IACrC,aAAa;IACb,MAAM;IACP,CAAC;GACJ,eAAe,KAAK,UACjB,UAAU,CACV,gBAAgB,2BAA2B;IAC1C,aAAa;IACb,MAAM;IACP,CAAC;GACL;AAGD,MAAI,KAAK,OAAO,cAAc,EAC5B,OAAM,gBAAgB,aACpB,eACA,KAAK,OAAO,aACZ,aACD;;;;;;;;;;;CAaL,MAAM,MACJ,KACA,QACA,aAAqB,GACO;EAC5B,MAAM,YAAY,KAAK,KAAK;AAE5B,SAAO,KAAK,UAAU,gBACpB,kBACA,EACE,YAAY;GACV,aAAa;GACb,gBAAgB,IAAI,UAAU,GAAG,IAAI;GACrC,kBAAkB;GACnB,EACF,EACD,OAAO,SAAS;AACd,OAAI;IAEF,MAAM,SAAS,OADF,MAAM,KAAK,SAAS,EACP,MAAS,KAAK,OAAO;AAC/C,SAAK,aAAa,oBAAoB,OAAO,YAAY,EAAE;AAC3D,SAAK,UAAU,EAAE,MAAM,eAAe,IAAI,CAAC;AAC3C,WAAO;YACA,OAAO;AAEd,QAAI,KAAK,YAAY,MAAM,EAAE;AAC3B,UAAK,SAAS,mBAAmB;AACjC,WAAM,KAAK,mBAAmB;KAE9B,MAAM,SAAS,OADC,MAAM,KAAK,SAAS,EACP,MAAS,KAAK,OAAO;AAClD,UAAK,aAAa,oBAAoB,OAAO,YAAY,EAAE;AAC3D,UAAK,UAAU,EAAE,MAAM,eAAe,IAAI,CAAC;AAC3C,YAAO;;AAIT,QAAI,KAAK,iBAAiB,MAAM,IAAI,aAAa,GAAG;AAClD,UAAK,SAAS,wBAAwB;AACtC,WAAM,IAAI,SAAS,YAAY,WAAW,SAAS,IAAI,CAAC;AACxD,YAAO,MAAM,KAAK,MAAS,KAAK,QAAQ,aAAa,EAAE;;AAGzD,SAAK,gBAAgB,MAAe;AACpC,SAAK,UAAU,EAAE,MAAM,eAAe,OAAO,CAAC;AAE9C,QAAI,iBAAiB,YACnB,OAAM;AAER,UAAM,gBAAgB,YAAY,MAAe;aACzC;IACR,MAAM,WAAW,KAAK,KAAK,GAAG;AAC9B,SAAK,iBAAiB,WAAW,IAAI,EAAE;AACvC,SAAK,iBAAiB,cAAc,OAAO,SAAS;AACpD,SAAK,KAAK;;IAGf;;;;;;;;;;;;;;;;CAiBH,MAAM,YACJ,UACA,aAAqB,GACT;EACZ,MAAM,YAAY,KAAK,KAAK;AAC5B,SAAO,KAAK,UAAU,gBACpB,wBACA,EACE,YAAY;GACV,aAAa;GACb,kBAAkB;GACnB,EACF,EACD,OAAO,SAAS;GAEd,MAAM,SAAS,OADF,MAAM,KAAK,SAAS,EACP,SAAS;AACnC,OAAI;AACF,UAAM,OAAO,MAAM,QAAQ;IAC3B,MAAM,SAAS,MAAM,SAAS,OAAO;AACrC,UAAM,OAAO,MAAM,SAAS;AAC5B,SAAK,UAAU,EAAE,MAAM,eAAe,IAAI,CAAC;AAC3C,WAAO;YACA,OAAO;AACd,QAAI;AACF,WAAM,OAAO,MAAM,WAAW;YACxB;AAER,QAAI,KAAK,YAAY,MAAM,EAAE;AAC3B,UAAK,SAAS,mBAAmB;AACjC,YAAO,SAAS;AAChB,WAAM,KAAK,mBAAmB;KAE9B,MAAM,cAAc,OADJ,MAAM,KAAK,SAAS,EACF,SAAS;AAC3C,SAAI;AACF,YAAM,OAAO,MAAM,QAAQ;MAC3B,MAAM,SAAS,MAAM,SAAS,YAAY;AAC1C,YAAM,OAAO,MAAM,SAAS;AAC5B,WAAK,UAAU,EAAE,MAAM,eAAe,IAAI,CAAC;AAC3C,aAAO;cACA,YAAY;AACnB,UAAI;AACF,aAAM,YAAY,MAAM,WAAW;cAC7B;AACR,YAAM;eACE;AACR,kBAAY,SAAS;;;AAKzB,QAAI,KAAK,iBAAiB,MAAM,IAAI,aAAa,GAAG;AAClD,UAAK,SAAS,0BAA0B;AACxC,YAAO,SAAS;AAChB,WAAM,IAAI,SAAS,YAAY,WAAW,SAAS,IAAI,CAAC;AACxD,YAAO,MAAM,KAAK,YAAe,UAAU,aAAa,EAAE;;AAE5D,SAAK,gBAAgB,MAAe;AACpC,SAAK,UAAU,EAAE,MAAM,eAAe,OAAO,CAAC;AAE9C,QAAI,iBAAiB,YACnB,OAAM;AAER,UAAM,gBAAgB,kBAAkB,MAAe;aAC/C;AACR,WAAO,SAAS;IAChB,MAAM,WAAW,KAAK,KAAK,GAAG;AAC9B,SAAK,iBAAiB,WAAW,IAAI,EAAE;AACvC,SAAK,iBAAiB,cAAc,OAAO,SAAS;AACpD,SAAK,KAAK;;IAGf;;;CAIH,MAAM,cAAgC;AACpC,SAAO,KAAK,UAAU,gBACpB,wBACA,EAAE,EACF,OAAO,SAAS;AACd,OAAI;IAIF,MAAM,WAHS,MAAM,KAAK,MACxB,qBACD,EACsB,KAAK,IAAI,WAAW;AAC3C,SAAK,aAAa,cAAc,QAAQ;AACxC,SAAK,UAAU,EAAE,MAAM,eAAe,IAAI,CAAC;AAC3C,WAAO;WACD;AACN,SAAK,aAAa,cAAc,MAAM;AACtC,SAAK,UAAU,EAAE,MAAM,eAAe,OAAO,CAAC;AAC9C,WAAO;aACC;AACR,SAAK,KAAK;;IAGf;;;CAIH,MAAM,QAAuB;AAC3B,MAAI,KAAK,MAAM;AACb,SAAM,KAAK,KAAK,KAAK,CAAC,OAAO,UAAmB;AAC9C,WAAO,MAAM,qCAAqC,MAAM;KACxD;AACF,QAAK,OAAO;;AAEd,OAAK,cAAc;;;CAIrB,WAAiB;AACf,UAAQ,GAAG,iBAAiB,KAAK,OAAO,CAAC;AACzC,UAAQ,GAAG,gBAAgB,KAAK,OAAO,CAAC;AACxC,OAAK,OAAO;;;CAId,AAAQ,qBAAsC;AAC5C,MAAI,KAAK,OAAO,gBACd,QAAO,KAAK,OAAO;AAGrB,MAAI;GACF,MAAM,EAAE,oBAAoB;GAC5B,MAAM,SAAS,WAAW;AAG1B,QAAK,OAAO,kBAAkB;AAC9B,UAAO;WACA,QAAQ;AACf,SAAM,gBAAgB,kBACpB,+BACA,mEACD;;;;CAKL,MAAc,UAA4B;AACxC,MAAI,CAAC,KAAK,iBACR,OAAM,mBAAmB,kBACvB,YACA,kGACD;AAGH,MAAI,CAAC,KAAK,MAAM;GACd,MAAM,QAAQ,MAAM,KAAK,gBAAgB;AACzC,QAAK,OAAO,KAAK,WAAW,MAAM;;AAEpC,SAAO,KAAK;;;CAId,AAAQ,WAAW,aAGP;EACV,MAAM,EAAE,MAAM,UAAU,MAAM,YAAY,KAAK;EAE/C,MAAM,OAAO,IAAI,GAAG,KAAK;GACvB;GACA;GACA;GACA,MAAM,YAAY;GAClB,UAAU,YAAY;GACtB,KAAK,KAAK,OAAO;GACjB,mBAAmB,KAAK,OAAO;GAC/B,yBAAyB,KAAK,OAAO;GACrC,KAAK,YAAY,YAAY,EAAE,oBAAoB,MAAM,GAAG;GAC7D,CAAC;AAEF,OAAK,GAAG,UAAU,UAAqC;AACrD,UAAO,MACL,wCACA,MAAM,SACN,MAAM,KACP;IACD;AAEF,SAAO;;;CAIT,MAAc,iBAGX;EACD,MAAM,MAAM,KAAK,KAAK;AAGtB,MACE,KAAK,eACL,MAAM,KAAK,YAAY,YAAY,KAAK,gBAExC,QAAO,KAAK;EAId,MAAM,WAAW,MAAM,KAAK,eAAe;EAC3C,MAAM,EAAE,OAAO,cAAc,MAAM,KAAK,eAAe;AAEvD,OAAK,cAAc;GACjB;GACA,UAAU;GACV;GACD;AAED,SAAO;GAAE;GAAU,UAAU;GAAO;;;CAItC,MAAc,oBAAmC;AAE/C,OAAK,cAAc;AAEnB,MAAI,KAAK,MAAM;GACb,MAAM,UAAU,KAAK;AACrB,QAAK,OAAO;AACZ,WAAQ,KAAK,CAAC,OAAO,UAAmB;AACtC,WAAO,MACL,yDACA,MACD;KACD;;;;CAKN,MAAc,gBAAiC;EAE7C,MAAM,OAAO,MADW,KAAK,oBAAoB,CACd,YAAY,IAAI;AACnD,MAAI,CAAC,KAAK,SACR,OAAM,oBAAoB,kBAAkB;AAE9C,SAAO,KAAK;;;CAId,MAAc,gBAA+D;EAG3E,MAAM,YAAY,IAAI,UADP,IAAI,OAAO,EAAE,MADJ,KAAK,oBAAoB,CACC,OAAO,MAAM,CAAC,CACzB;AAEvC,MAAI,CAAC,KAAK,iBAAiB,QACzB,OAAM,mBAAmB,iBAAiB,oBAAoB;EAGhE,MAAM,cAAc,MAAM,UAAU,QAAQ;GAC1C,MAAM;GACN,QAAQ;GACR,SAAS,IAAI,SAAS;GACtB,KAAK;GACL,SAAS;IACP,gBAAgB,CAAC,KAAK,iBAAiB,QAAQ;IAC/C,YAAY,YAAY;IACzB;GACF,CAAC;AAEF,MAAI,CAAC,KAAK,oBAAoB,YAAY,CACxC,OAAM,oBAAoB,kBACxB,KAAK,iBAAiB,QACvB;EAGH,MAAM,YAAY,IAAI,KAAK,YAAY,gBAAgB,CAAC,SAAS;AAEjE,SAAO;GAAE,OAAO,YAAY;GAAO;GAAW;;;CAIhD,AAAQ,YAAY,OAAyB;AAC3C,SACE,OAAO,UAAU,YACjB,UAAU,QACV,UAAU,SACT,MAAc,SAAS;;;CAK5B,AAAQ,iBAAiB,OAAyB;AAChD,MAAI,OAAO,UAAU,YAAY,UAAU,QAAQ,EAAE,UAAU,OAC7D,QAAO;EAGT,MAAM,OAAQ,MAAc;AAC5B,SACE,SAAS,gBACT,SAAS,kBACT,SAAS,eACT,SAAS,WACT,SAAS,WACT,SAAS,WACT,SAAS,WACT,SAAS;;;CAKb,AAAQ,oBACN,OACqD;AACrD,MAAI,OAAO,UAAU,YAAY,UAAU,KACzC,QAAO;EAGT,MAAM,cAAc;AACpB,SACE,WAAW,eACX,OAAO,YAAY,UAAU,YAC7B,qBAAqB,eACrB,OAAO,YAAY,oBAAoB,YACvC,IAAI,KAAK,YAAY,gBAAgB,CAAC,SAAS,GAAG,KAAK,KAAK;;;CAKhE,AAAQ,wBAAkD;AACxD,MAAI,KAAK,OAAO,iBACd,QAAO,KAAK,sBAAsB,KAAK,OAAO,iBAAiB;AAIjE,MAAI,KAAK,OAAO,QAAQ,KAAK,OAAO,YAAY,KAAK,OAAO,QAC1D,QAAO;GACL,MAAM,KAAK,OAAO;GAClB,UAAU,KAAK,OAAO;GACtB,MAAM,KAAK,OAAO,QAAQ;GAC1B,SAAS,KAAK,OAAO,WAAW;GAChC,SAAS,KAAK,OAAO;GACtB;EAIH,MAAM,SAAS,QAAQ,IAAI;EAC3B,MAAM,aAAa,QAAQ,IAAI;EAC/B,MAAM,YAAY,QAAQ,IAAI;AAC9B,MAAI,CAAC,UAAU,CAAC,cAAc,CAAC,UAC7B,OAAM,mBAAmB,kBACvB,YACA,mHACD;EAEH,MAAM,SAAS,QAAQ,IAAI;EAC3B,MAAM,OAAO,SAAS,SAAS,QAAQ,GAAG,GAAG;AAE7C,MAAI,OAAO,MAAM,KAAK,CACpB,OAAM,gBAAgB,aAAa,QAAQ,QAAQ,WAAW;AAOhE,SAAO;GACL,MAAM;GACN,UAAU;GACV;GACA,SARgB,QAAQ,IAAI,aAEuB;GAOnD,SAAS;GACV;;CAGH,AAAQ,sBACN,kBAC0B;EAC1B,MAAM,MAAM,IAAI,IAAI,iBAAiB;EACrC,MAAM,UAAU,IAAI,aAAa,IAAI,UAAU;AAC/C,MAAI,CAAC,QACH,OAAM,mBAAmB,uBAAuB,UAAU;AAG5D,SAAO;GACL,MAAM,IAAI;GACV,UAAU,IAAI,SAAS,MAAM,EAAE;GAC/B,MAAM,IAAI,OAAO,SAAS,IAAI,MAAM,GAAG,GAAG;GAC1C,SACG,IAAI,aAAa,IAAI,UAAU,IAChC;GACO;GACV"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"defaults.js","names":[
|
|
1
|
+
{"version":3,"file":"defaults.js","names":[],"sources":["../../../src/connectors/lakebase/defaults.ts"],"sourcesContent":["import type { LakebaseConfig } from \"./types\";\n\n/** Default configuration for Lakebase connector */\nexport const lakebaseDefaults: LakebaseConfig = {\n port: 5432,\n sslMode: \"require\",\n maxPoolSize: 10,\n idleTimeoutMs: 30_000,\n connectionTimeoutMs: 10_000,\n};\n"],"mappings":";;AAGA,MAAa,mBAAmC;CAC9C,MAAM;CACN,SAAS;CACT,aAAa;CACb,eAAe;CACf,qBAAqB;CACtB"}
|
|
@@ -1,10 +1,18 @@
|
|
|
1
|
+
import { createLogger } from "../../logging/logger.js";
|
|
1
2
|
import { TelemetryManager } from "../../telemetry/telemetry-manager.js";
|
|
2
3
|
import { SpanKind, SpanStatusCode } from "../../telemetry/index.js";
|
|
4
|
+
import { AppKitError } from "../../errors/base.js";
|
|
5
|
+
import { ConnectionError } from "../../errors/connection.js";
|
|
6
|
+
import { ExecutionError } from "../../errors/execution.js";
|
|
7
|
+
import { ValidationError } from "../../errors/validation.js";
|
|
8
|
+
import { init_errors } from "../../errors/index.js";
|
|
3
9
|
import { ArrowStreamProcessor } from "../../stream/arrow-stream-processor.js";
|
|
4
10
|
import { executeStatementDefaults } from "./defaults.js";
|
|
5
11
|
import { Context } from "@databricks/sdk-experimental";
|
|
6
12
|
|
|
7
13
|
//#region src/connectors/sql-warehouse/client.ts
|
|
14
|
+
init_errors();
|
|
15
|
+
const logger = createLogger("connectors:sql-warehouse");
|
|
8
16
|
var SQLWarehouseConnector = class {
|
|
9
17
|
constructor(config) {
|
|
10
18
|
this.name = "sql-warehouse";
|
|
@@ -37,6 +45,7 @@ var SQLWarehouseConnector = class {
|
|
|
37
45
|
async executeStatement(workspaceClient, input, signal) {
|
|
38
46
|
const startTime = Date.now();
|
|
39
47
|
let success = false;
|
|
48
|
+
if (signal?.aborted) throw ExecutionError.canceled();
|
|
40
49
|
return this.telemetry.startActiveSpan("sql.query", {
|
|
41
50
|
kind: SpanKind.CLIENT,
|
|
42
51
|
attributes: {
|
|
@@ -48,9 +57,24 @@ var SQLWarehouseConnector = class {
|
|
|
48
57
|
"db.has_parameters": !!input.parameters
|
|
49
58
|
}
|
|
50
59
|
}, async (span) => {
|
|
60
|
+
let abortHandler;
|
|
61
|
+
let isAborted = false;
|
|
62
|
+
if (signal) {
|
|
63
|
+
abortHandler = () => {
|
|
64
|
+
if (!span.isRecording()) return;
|
|
65
|
+
isAborted = true;
|
|
66
|
+
span.setAttribute("cancelled", true);
|
|
67
|
+
span.setStatus({
|
|
68
|
+
code: SpanStatusCode.ERROR,
|
|
69
|
+
message: "Query cancelled by client"
|
|
70
|
+
});
|
|
71
|
+
span.end();
|
|
72
|
+
};
|
|
73
|
+
signal.addEventListener("abort", abortHandler, { once: true });
|
|
74
|
+
}
|
|
51
75
|
try {
|
|
52
|
-
if (!input.statement) throw
|
|
53
|
-
if (!input.warehouse_id) throw
|
|
76
|
+
if (!input.statement) throw ValidationError.missingField("statement");
|
|
77
|
+
if (!input.warehouse_id) throw ValidationError.missingField("warehouse_id");
|
|
54
78
|
const body = {
|
|
55
79
|
statement: input.statement,
|
|
56
80
|
parameters: input.parameters,
|
|
@@ -66,7 +90,7 @@ var SQLWarehouseConnector = class {
|
|
|
66
90
|
};
|
|
67
91
|
span.addEvent("statement.submitting", { "db.warehouse_id": input.warehouse_id });
|
|
68
92
|
const response = await workspaceClient.statementExecution.executeStatement(body, this._createContext(signal));
|
|
69
|
-
if (!response) throw
|
|
93
|
+
if (!response) throw ConnectionError.apiFailure("SQL Warehouse");
|
|
70
94
|
const status = response.status;
|
|
71
95
|
const statementId = response.statement_id;
|
|
72
96
|
span.setAttribute("db.statement_id", statementId);
|
|
@@ -84,27 +108,37 @@ var SQLWarehouseConnector = class {
|
|
|
84
108
|
case "SUCCEEDED":
|
|
85
109
|
result = this._transformDataArray(response);
|
|
86
110
|
break;
|
|
87
|
-
case "FAILED": throw
|
|
88
|
-
case "CANCELED": throw
|
|
89
|
-
case "CLOSED": throw
|
|
90
|
-
default: throw
|
|
111
|
+
case "FAILED": throw ExecutionError.statementFailed(status.error?.message);
|
|
112
|
+
case "CANCELED": throw ExecutionError.canceled();
|
|
113
|
+
case "CLOSED": throw ExecutionError.resultsClosed();
|
|
114
|
+
default: throw ExecutionError.unknownState(String(status?.state ?? "unknown"));
|
|
91
115
|
}
|
|
92
116
|
const resultData = result.result;
|
|
93
|
-
|
|
94
|
-
|
|
117
|
+
const rowCount = resultData?.data?.length ?? resultData?.data_array?.length ?? 0;
|
|
118
|
+
if (rowCount > 0) span.setAttribute("db.result.row_count", rowCount);
|
|
119
|
+
const duration = Date.now() - startTime;
|
|
120
|
+
logger.event()?.setContext("sql-warehouse", {
|
|
121
|
+
warehouse_id: input.warehouse_id,
|
|
122
|
+
rows_returned: rowCount,
|
|
123
|
+
query_duration_ms: duration
|
|
124
|
+
});
|
|
95
125
|
success = true;
|
|
96
|
-
span.setStatus({ code: SpanStatusCode.OK });
|
|
126
|
+
if (!isAborted) span.setStatus({ code: SpanStatusCode.OK });
|
|
97
127
|
return result;
|
|
98
128
|
} catch (error) {
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
129
|
+
if (!isAborted) {
|
|
130
|
+
span.recordException(error);
|
|
131
|
+
span.setStatus({
|
|
132
|
+
code: SpanStatusCode.ERROR,
|
|
133
|
+
message: error instanceof Error ? error.message : String(error)
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
if (error instanceof AppKitError) throw error;
|
|
137
|
+
throw ExecutionError.statementFailed(error instanceof Error ? error.message : String(error));
|
|
105
138
|
} finally {
|
|
139
|
+
if (abortHandler && signal) signal.removeEventListener("abort", abortHandler);
|
|
106
140
|
const duration = Date.now() - startTime;
|
|
107
|
-
span.end();
|
|
141
|
+
if (!isAborted) span.end();
|
|
108
142
|
const attributes = {
|
|
109
143
|
"db.warehouse_id": input.warehouse_id,
|
|
110
144
|
"db.catalog": input.catalog ?? "",
|
|
@@ -135,13 +169,13 @@ var SQLWarehouseConnector = class {
|
|
|
135
169
|
span.setAttribute("db.polling.current_attempt", pollCount);
|
|
136
170
|
const elapsedTime = Date.now() - startTime;
|
|
137
171
|
if (elapsedTime > timeout) {
|
|
138
|
-
const error =
|
|
172
|
+
const error = ExecutionError.statementFailed(`Polling timeout exceeded after ${timeout}ms (elapsed: ${elapsedTime}ms)`);
|
|
139
173
|
span.recordException(error);
|
|
140
174
|
span.setStatus({ code: SpanStatusCode.ERROR });
|
|
141
175
|
throw error;
|
|
142
176
|
}
|
|
143
177
|
if (signal?.aborted) {
|
|
144
|
-
const error =
|
|
178
|
+
const error = ExecutionError.canceled();
|
|
145
179
|
span.recordException(error);
|
|
146
180
|
span.setStatus({ code: SpanStatusCode.ERROR });
|
|
147
181
|
throw error;
|
|
@@ -152,7 +186,7 @@ var SQLWarehouseConnector = class {
|
|
|
152
186
|
"poll.elapsed_ms": elapsedTime
|
|
153
187
|
});
|
|
154
188
|
const response = await workspaceClient.statementExecution.getStatement({ statement_id: statementId }, this._createContext(signal));
|
|
155
|
-
if (!response) throw
|
|
189
|
+
if (!response) throw ConnectionError.apiFailure("SQL Warehouse");
|
|
156
190
|
const status = response.status;
|
|
157
191
|
span.addEvent("polling.status_check", {
|
|
158
192
|
"db.status": status?.state,
|
|
@@ -170,10 +204,10 @@ var SQLWarehouseConnector = class {
|
|
|
170
204
|
});
|
|
171
205
|
span.setStatus({ code: SpanStatusCode.OK });
|
|
172
206
|
return this._transformDataArray(response);
|
|
173
|
-
case "FAILED": throw
|
|
174
|
-
case "CANCELED": throw
|
|
175
|
-
case "CLOSED": throw
|
|
176
|
-
default: throw
|
|
207
|
+
case "FAILED": throw ExecutionError.statementFailed(status.error?.message);
|
|
208
|
+
case "CANCELED": throw ExecutionError.canceled();
|
|
209
|
+
case "CLOSED": throw ExecutionError.resultsClosed();
|
|
210
|
+
default: throw ExecutionError.unknownState(String(status?.state ?? "unknown"));
|
|
177
211
|
}
|
|
178
212
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
179
213
|
delay = Math.min(delay * 2, maxDelayBetweenPolls);
|
|
@@ -184,7 +218,8 @@ var SQLWarehouseConnector = class {
|
|
|
184
218
|
code: SpanStatusCode.ERROR,
|
|
185
219
|
message: error instanceof Error ? error.message : String(error)
|
|
186
220
|
});
|
|
187
|
-
throw error;
|
|
221
|
+
if (error instanceof AppKitError) throw error;
|
|
222
|
+
throw ExecutionError.statementFailed(error instanceof Error ? error.message : String(error));
|
|
188
223
|
} finally {
|
|
189
224
|
span.end();
|
|
190
225
|
}
|
|
@@ -242,7 +277,7 @@ var SQLWarehouseConnector = class {
|
|
|
242
277
|
const response = await workspaceClient.statementExecution.getStatement({ statement_id: jobId }, this._createContext(signal));
|
|
243
278
|
const chunks = response.result?.external_links;
|
|
244
279
|
const schema = response.manifest?.schema;
|
|
245
|
-
if (!chunks || !schema) throw
|
|
280
|
+
if (!chunks || !schema) throw ExecutionError.missingData("chunks or schema");
|
|
246
281
|
span.setAttribute("arrow.chunk_count", chunks.length);
|
|
247
282
|
const result = await this.arrowProcessor.processChunks(chunks, schema, signal);
|
|
248
283
|
span.setAttribute("arrow.data_size_bytes", result.data.length);
|
|
@@ -252,6 +287,10 @@ var SQLWarehouseConnector = class {
|
|
|
252
287
|
operation: "arrow.getData",
|
|
253
288
|
status: "success"
|
|
254
289
|
});
|
|
290
|
+
logger.event()?.setContext("sql-warehouse", {
|
|
291
|
+
arrow_data_size_bytes: result.data.length,
|
|
292
|
+
arrow_job_id: jobId
|
|
293
|
+
});
|
|
255
294
|
return result;
|
|
256
295
|
} catch (error) {
|
|
257
296
|
span.setStatus({
|
|
@@ -264,8 +303,9 @@ var SQLWarehouseConnector = class {
|
|
|
264
303
|
operation: "arrow.getData",
|
|
265
304
|
status: "error"
|
|
266
305
|
});
|
|
267
|
-
|
|
268
|
-
throw error;
|
|
306
|
+
logger.error("Failed Arrow job: %s %O", jobId, error);
|
|
307
|
+
if (error instanceof AppKitError) throw error;
|
|
308
|
+
throw ExecutionError.statementFailed(error instanceof Error ? error.message : String(error));
|
|
269
309
|
}
|
|
270
310
|
});
|
|
271
311
|
}
|