@lightdash/warehouses 0.2603.2 → 0.2604.1

package/dist/warehouseClients/DuckdbWarehouseClient.d.ts

@@ -20,6 +20,7 @@ export type DuckdbWarehouseClientArgs = {
     databasePath?: string;
     s3Config?: DuckdbS3SessionConfig;
     resourceLimits?: DuckdbResourceLimits;
+    bufferPoolSize?: string;
     logger?: DuckdbLogger;
 };
 export declare const mapFieldTypeFromTypeId: (typeId: number) => DimensionType;
@@ -29,13 +30,18 @@ export declare class DuckdbSqlBuilder extends WarehouseBaseSqlBuilder {
     getMetricSql(sql: string, metric: Metric): string;
     concatString(...args: string[]): string;
 }
+/** Reset shared state without closing — for use in tests with mocked instances. */
+export declare function resetSharedDuckdbStateForTesting(): void;
 export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreatePostgresCredentials> {
     private readonly databasePath;
     private readonly s3Config?;
     private readonly resourceLimits?;
+    private readonly bufferPoolSize?;
     private readonly logger?;
     constructor(args?: DuckdbWarehouseClientArgs);
+    close(): Promise<void>;
     private getSQLWithMetadata;
+    private connectWithRetry;
     private withSession;
     private bootstrapSession;
     private getBindValues;

package/dist/warehouseClients/DuckdbWarehouseClient.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,aAAa,EACb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA2BhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,MAAM,CAAC,EAAE,YAAY,CAAC;CACzB,CAAC;AAYF,eAAO,MAAM,sBAAsB,GAAI,QAAQ,MAAM,KAAG,aA+BvD,CAAC;AAEF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB;IAIrC,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C;AAED,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,yBAAyB,CAAC;IACrF,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;gBAE3B,IAAI,GAAE,yBAA8B;IAQhD,OAAO,CAAC,kBAAkB;YAQZ,WAAW;YA2CX,gBAAgB;IAmE9B,OAAO,CAAC,aAAa;mBA0BA,eAAe;IAoFpC,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAalC,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC;IA0CV,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,yBAAyB,CAAC,CAAC,mBAAmB,CAAC,CACtE;;;;;;IA8BC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMlC,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC;IAmBI,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,yBAAyB,CAAC,CAAC,UAAU,CAAC,CAC7D;;;;;;IAKC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrB,UAAU,CACZ,OAAO,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC/D,OAAO,CAAC,gBAAgB,CAAC;IAMtB,YAAY,CACd,OAAO,CAAC,EAAE,MAAM,EAChB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN;IAMK,SAAS,CACX,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,MAAM,EAChB,SAAS,CAAC,EAAE,MAAM,EAClB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC;CAK/B"}
+{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,aAAa,EACb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA2BhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,YAAY,CAAC;CACzB,CAAC;AAYF,eAAO,MAAM,sBAAsB,GAAI,QAAQ,MAAM,KAAG,aA+BvD,CAAC;AAEF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB;IAIrC,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C;AA4CD,mFAAmF;AACnF,wBAAgB,gCAAgC,IAAI,IAAI,CAIvD;AAED,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,yBAAyB,CAAC;IACrF,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAS;IAEzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;gBAE3B,IAAI,GAAE,yBAA8B;IAS1C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,kBAAkB;YAQZ,gBAAgB;YAoBhB,WAAW;YAwCX,gBAAgB;IA6F9B,OAAO,CAAC,aAAa;mBA0BA,eAAe;IAoFpC,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAalC,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC;IA0CV,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,yBAAyB,CAAC,CAAC,mBAAmB,CAAC,CACtE;;;;;;IA8BC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMlC,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC;IAmBI,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,yBAAyB,CAAC,CAAC,UAAU,CAAC,CAC7D;;;;;;IAKC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrB,UAAU,CACZ,OAAO,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC/D,OAAO,CAAC,gBAAgB,CAAC;IAMtB,YAAY,CACd,OAAO,CAAC,EAAE,MAAM,EAChB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN;IAMK,SAAS,CACX,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,MAAM,EAChB,SAAS,CAAC,EAAE,MAAM,EAClB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC;CAK/B"}

package/dist/warehouseClients/DuckdbWarehouseClient.js

@@ -4,6 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DuckdbWarehouseClient = exports.DuckdbSqlBuilder = exports.mapFieldTypeFromTypeId = void 0;
+exports.resetSharedDuckdbStateForTesting = resetSharedDuckdbStateForTesting;
 const node_api_1 = require("@duckdb/node-api");
 const common_1 = require("@lightdash/common");
 const promises_1 = __importDefault(require("fs/promises"));
@@ -75,24 +76,78 @@ class DuckdbSqlBuilder extends WarehouseBaseSqlBuilder_1.default {
     }
 }
 exports.DuckdbSqlBuilder = DuckdbSqlBuilder;
+/**
+ * Shared DuckDB instance — one per worker process.
+ * All DuckdbWarehouseClient instances share the same underlying DuckDB instance
+ * to maximize cache hits (parquet metadata, HTTP metadata, buffer pool).
+ */
+let sharedInstance = null;
+let httpfsInstalled = false;
+let cachesConfigured = false;
+async function getOrCreateSharedInstance(databasePath, logger) {
+    if (!sharedInstance) {
+        const t0 = performance.now();
+        sharedInstance = (await node_api_1.DuckDBInstance.create(databasePath));
+        const createMs = performance.now() - t0;
+        httpfsInstalled = false;
+        cachesConfigured = false;
+        logger?.info(`DuckDB shared instance created: path=${databasePath} createMs=${Math.round(createMs)}ms`);
+    }
+    return sharedInstance;
+}
+function clearSharedInstance(logger) {
+    if (sharedInstance) {
+        try {
+            sharedInstance.closeSync?.();
+        }
+        catch {
+            // best-effort cleanup
+        }
+        sharedInstance = null;
+        httpfsInstalled = false;
+        cachesConfigured = false;
+        logger?.info('DuckDB shared instance cleared');
+    }
+}
+/** Reset shared state without closing — for use in tests with mocked instances. */
+function resetSharedDuckdbStateForTesting() {
+    sharedInstance = null;
+    httpfsInstalled = false;
+    cachesConfigured = false;
+}
 class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
     constructor(args = {}) {
         super(DUCKDB_INTERNAL_CREDENTIALS, new DuckdbSqlBuilder());
         this.databasePath = args.databasePath ?? ':memory:';
         this.s3Config = args.s3Config;
         this.resourceLimits = args.resourceLimits;
+        this.bufferPoolSize = args.bufferPoolSize;
         this.logger = args.logger;
     }
+    async close() {
+        clearSharedInstance(this.logger);
+    }
     getSQLWithMetadata(sql, tags) {
         if (!tags) {
             return sql;
         }
         return `${sql}\n-- ${JSON.stringify(tags)}`;
     }
+    async connectWithRetry() {
+        const instance = await getOrCreateSharedInstance(this.databasePath, this.logger);
+        try {
+            return await instance.connect();
+        }
+        catch (firstError) {
+            this.logger?.info(`DuckDB connect failed, retrying with fresh instance: ${firstError}`);
+            clearSharedInstance(this.logger);
+            const freshInstance = await getOrCreateSharedInstance(this.databasePath, this.logger);
+            return freshInstance.connect();
+        }
+    }
     async withSession(callback) {
         const sessionStart = performance.now();
-        const instance = (await node_api_1.DuckDBInstance.create(this.databasePath));
-        const connection = await instance.connect();
+        const connection = await this.connectWithRetry();
         const connectMs = performance.now() - sessionStart;
         // Only create a temp dir when resource limits are set (spill to disk).
         const tempDir = this.resourceLimits
@@ -112,19 +167,36 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
         finally {
             connection.closeSync?.();
             connection.disconnectSync?.();
-            instance.closeSync?.();
+            // Note: we do NOT close the instance — it's shared across queries
             if (tempDir) {
                 await promises_1.default.rm(tempDir, { recursive: true, force: true }).catch(() => { });
             }
         }
     }
     async bootstrapSession(db, tempDir) {
-        const t0 = performance.now();
-        await db.run('INSTALL httpfs;');
-        const installMs = performance.now() - t0;
+        let installMs = 0;
+        if (!httpfsInstalled) {
+            const t0 = performance.now();
+            await db.run('INSTALL httpfs;');
+            installMs = performance.now() - t0;
+            httpfsInstalled = true;
+            this.logger?.info(`DuckDB httpfs installed (first use): ${Math.round(installMs)}ms`);
+        }
         const t1 = performance.now();
         await db.run('LOAD httpfs;');
         const loadMs = performance.now() - t1;
+        // Enable built-in caches — these are GLOBAL settings (instance-level, not
+        // connection-level), so they only need to be set once per shared instance.
+        if (!cachesConfigured) {
+            await db.run('SET enable_http_metadata_cache = true;');
+            await db.run('SET enable_external_file_cache = true;');
+            await db.run('SET parquet_metadata_cache = true;');
+            cachesConfigured = true;
+            if (this.bufferPoolSize) {
+                await db.run(`SET buffer_pool_size = '${this.bufferPoolSize}';`);
+            }
+            this.logger?.info(`DuckDB caches enabled: http_metadata=true external_file=true parquet_metadata=true buffer_pool_size=${this.bufferPoolSize ?? 'default'}`);
+        }
         if (this.resourceLimits && tempDir) {
             await db.run(`SET memory_limit = '${this.resourceLimits.memoryLimit}';`);
             await db.run(`SET temp_directory = '${tempDir}';`);

package/dist/warehouseClients/DuckdbWarehouseClient.test.js

@@ -138,6 +138,7 @@ describe('mapFieldTypeFromTypeId', () => {
 describe('DuckdbWarehouseClient', () => {
     beforeEach(() => {
         jest.clearAllMocks();
+        (0, DuckdbWarehouseClient_1.resetSharedDuckdbStateForTesting)();
     });
     it('should return query rows and mapped fields', async () => {
         const rows = [
@@ -213,7 +214,11 @@ describe('DuckdbWarehouseClient', () => {
         });
         await client.runQuery('SELECT 1 AS val', undefined, 'UTC');
         const runCalls = runMock.mock.calls.map((call) => call[0]);
+        expect(runCalls).toContain('INSTALL httpfs;');
         expect(runCalls).toContain('LOAD httpfs;');
+        expect(runCalls).toContain('SET enable_http_metadata_cache = true;');
+        expect(runCalls).toContain('SET enable_external_file_cache = true;');
+        expect(runCalls).toContain('SET parquet_metadata_cache = true;');
         expect(runCalls).toContain("SET s3_endpoint = 'localhost:9000';");
         expect(runCalls).toContain("SET s3_region = 'us-east-1';");
         expect(runCalls).toContain("SET TimeZone = 'UTC';");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lightdash/warehouses",
-  "version": "0.2603.2",
+  "version": "0.2604.1",
   "description": "Warehouse connectors for Lightdash",
   "license": "MIT",
   "repository": {
@@ -26,7 +26,7 @@
     "snowflake-sdk": "~2.3.4",
     "ssh2": "^1.14.0",
     "trino-client": "0.2.9",
-    "@lightdash/common": "0.2603.2"
+    "@lightdash/common": "0.2604.1"
   },
   "devDependencies": {
     "@types/node-fetch": "^2.6.13",