@lightdash/warehouses 0.2616.0 → 0.2616.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,aAAa,EACb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAsChE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,YAAY,CAAC;CACzB,CAAC;AAYF,eAAO,MAAM,sBAAsB,GAAI,QAAQ,MAAM,KAAG,aA+BvD,CAAC;AAEF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB;IAIrC,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C;
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,aAAa,EACb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAsChE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,YAAY,CAAC;CACzB,CAAC;AAYF,eAAO,MAAM,sBAAsB,GAAI,QAAQ,MAAM,KAAG,aA+BvD,CAAC;AAEF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB;IAIrC,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C;AA8CD,mFAAmF;AACnF,wBAAgB,gCAAgC,IAAI,IAAI,CAKvD;AA4BD,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,yBAAyB,CAAC;IACrF,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAS;IAEzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;gBAE3B,IAAI,GAAE,yBAA8B;IAS1C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,kBAAkB;YAQZ,gBAAgB;YAoBhB,WAAW;YAwCX,gBAAgB;IAsG9B,OAAO,CAAC,aAAa;mBA0BA,eAAe;IAoFpC,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAaxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAM/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;YAUrB,eAAe;YA8Bf,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC;IA4CV,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,yBAAyB,CAAC,CAAC,mBAAmB,CAAC,CACtE;;;;;;IA8BC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOlC,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC;IAoBI,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,yBAAyB,CAAC,CAAC,UAAU,CAAC,CAC7D;;;;;;IAKC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrB,UAAU,CACZ,OAAO,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC/D,OAAO,CAAC,gBAAgB,CAAC;IAMtB,YAAY,CACd,OAAO,CAAC,EAAE,MAAM,EAChB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN;IAMK,SAAS,CACX,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,MAAM,EAChB,SAAS,CAAC,EAAE,MAAM,EAClB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC;CAK/B"}
|
|
@@ -84,7 +84,6 @@ exports.DuckdbSqlBuilder = DuckdbSqlBuilder;
|
|
|
84
84
|
let sharedInstance = null;
|
|
85
85
|
let httpfsInstalled = false;
|
|
86
86
|
let cachesConfigured = false;
|
|
87
|
-
let configuredS3SecretFingerprint = null;
|
|
88
87
|
let sharedBootstrapQueue = Promise.resolve();
|
|
89
88
|
async function getOrCreateSharedInstance(databasePath, logger) {
|
|
90
89
|
if (!sharedInstance) {
|
|
@@ -108,7 +107,6 @@ function clearSharedInstance(logger) {
|
|
|
108
107
|
sharedInstance = null;
|
|
109
108
|
httpfsInstalled = false;
|
|
110
109
|
cachesConfigured = false;
|
|
111
|
-
configuredS3SecretFingerprint = null;
|
|
112
110
|
sharedBootstrapQueue = Promise.resolve();
|
|
113
111
|
logger?.info('DuckDB shared instance cleared');
|
|
114
112
|
}
|
|
@@ -118,7 +116,6 @@ function resetSharedDuckdbStateForTesting() {
|
|
|
118
116
|
sharedInstance = null;
|
|
119
117
|
httpfsInstalled = false;
|
|
120
118
|
cachesConfigured = false;
|
|
121
|
-
configuredS3SecretFingerprint = null;
|
|
122
119
|
sharedBootstrapQueue = Promise.resolve();
|
|
123
120
|
}
|
|
124
121
|
async function withSharedBootstrapLock(callback) {
|
|
@@ -126,16 +123,6 @@ async function withSharedBootstrapLock(callback) {
|
|
|
126
123
|
sharedBootstrapQueue = run.then(() => undefined, () => undefined);
|
|
127
124
|
return run;
|
|
128
125
|
}
|
|
129
|
-
function getS3SecretFingerprint(config) {
|
|
130
|
-
return JSON.stringify({
|
|
131
|
-
endpoint: config.endpoint,
|
|
132
|
-
region: config.region,
|
|
133
|
-
accessKey: config.accessKey,
|
|
134
|
-
secretKey: config.secretKey,
|
|
135
|
-
forcePathStyle: config.forcePathStyle,
|
|
136
|
-
useSsl: config.useSsl,
|
|
137
|
-
});
|
|
138
|
-
}
|
|
139
126
|
// DuckDB StatementType values — see duckdb/common/enums/statement_type.hpp
|
|
140
127
|
const ALLOWED_STATEMENT_TYPES_USER_SQL = new Set([1 /* SELECT */]);
|
|
141
128
|
const BLOCKED_STATEMENT_TYPES_INTERNAL_SQL = new Set([
|
|
@@ -206,10 +193,10 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
206
193
|
}
|
|
207
194
|
}
|
|
208
195
|
async bootstrapSession(db, tempDir) {
|
|
209
|
-
|
|
196
|
+
// INSTALL httpfs and global settings are instance-level — serialize and
|
|
197
|
+
// deduplicate them via the shared bootstrap lock.
|
|
198
|
+
const installMs = await withSharedBootstrapLock(async () => {
|
|
210
199
|
let nextInstallMs = 0;
|
|
211
|
-
let nextS3ConfigMs = 0;
|
|
212
|
-
let loadedDuringSharedBootstrap = false;
|
|
213
200
|
if (!httpfsInstalled) {
|
|
214
201
|
const t0 = performance.now();
|
|
215
202
|
await db.run('INSTALL httpfs;');
|
|
@@ -233,49 +220,12 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
233
220
|
}
|
|
234
221
|
this.logger?.info(`DuckDB caches enabled: http_metadata=true external_file=true parquet_metadata=true buffer_pool_size=${this.bufferPoolSize ?? 'default'}`);
|
|
235
222
|
}
|
|
236
|
-
|
|
237
|
-
const nextFingerprint = getS3SecretFingerprint(this.s3Config);
|
|
238
|
-
if (configuredS3SecretFingerprint !== nextFingerprint) {
|
|
239
|
-
const loadStart = performance.now();
|
|
240
|
-
await db.run('LOAD httpfs;');
|
|
241
|
-
loadedDuringSharedBootstrap = true;
|
|
242
|
-
const loadMs = performance.now() - loadStart;
|
|
243
|
-
const regionClause = this.s3Config.region
|
|
244
|
-
? `REGION '${this.escapeString(this.s3Config.region)}',`
|
|
245
|
-
: '';
|
|
246
|
-
const keyIdClause = this.s3Config.accessKey
|
|
247
|
-
? `KEY_ID '${this.escapeString(this.s3Config.accessKey)}',`
|
|
248
|
-
: '';
|
|
249
|
-
const secretClause = this.s3Config.secretKey
|
|
250
|
-
? `SECRET '${this.escapeString(this.s3Config.secretKey)}',`
|
|
251
|
-
: '';
|
|
252
|
-
const t2 = performance.now();
|
|
253
|
-
await db.run(`CREATE OR REPLACE SECRET __lightdash_s3 (
|
|
254
|
-
TYPE s3,
|
|
255
|
-
${keyIdClause}
|
|
256
|
-
${secretClause}
|
|
257
|
-
ENDPOINT '${this.escapeString(this.s3Config.endpoint)}',
|
|
258
|
-
${regionClause}
|
|
259
|
-
URL_STYLE '${this.s3Config.forcePathStyle ? 'path' : 'vhost'}',
|
|
260
|
-
USE_SSL ${this.s3Config.useSsl}
|
|
261
|
-
);`);
|
|
262
|
-
nextS3ConfigMs = performance.now() - t2;
|
|
263
|
-
configuredS3SecretFingerprint = nextFingerprint;
|
|
264
|
-
this.logger?.info(`DuckDB S3 secret configured: load_httpfs=${Math.round(loadMs)}ms s3_config=${Math.round(nextS3ConfigMs)}ms`);
|
|
265
|
-
}
|
|
266
|
-
}
|
|
267
|
-
return {
|
|
268
|
-
installMs: nextInstallMs,
|
|
269
|
-
s3ConfigMs: nextS3ConfigMs,
|
|
270
|
-
loadedHttpfsDuringSharedBootstrap: loadedDuringSharedBootstrap,
|
|
271
|
-
};
|
|
223
|
+
return nextInstallMs;
|
|
272
224
|
});
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
loadMs = performance.now() - t1;
|
|
278
|
-
}
|
|
225
|
+
// LOAD httpfs is per-connection — always run it.
|
|
226
|
+
const t1 = performance.now();
|
|
227
|
+
await db.run('LOAD httpfs;');
|
|
228
|
+
const loadMs = performance.now() - t1;
|
|
279
229
|
if (this.resourceLimits && tempDir) {
|
|
280
230
|
await db.run(`SET memory_limit = '${this.resourceLimits.memoryLimit}';`);
|
|
281
231
|
await db.run(`SET temp_directory = '${tempDir}';`);
|
|
@@ -285,6 +235,32 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
285
235
|
this.logger?.info(`DuckDB bootstrap timing: install_httpfs=${Math.round(installMs)}ms load_httpfs=${Math.round(loadMs)}ms`);
|
|
286
236
|
return;
|
|
287
237
|
}
|
|
238
|
+
// CREATE SECRET on every connection — DuckDB secrets may not reliably
|
|
239
|
+
// persist across connections on all DuckDB versions, so always set it.
|
|
240
|
+
// Serialize via the lock to avoid "Catalog write-write conflict on alter"
|
|
241
|
+
// when concurrent connections both run CREATE OR REPLACE SECRET.
|
|
242
|
+
const s3ConfigMs = await withSharedBootstrapLock(async () => {
|
|
243
|
+
const t2 = performance.now();
|
|
244
|
+
const regionClause = this.s3Config.region
|
|
245
|
+
? `REGION '${this.escapeString(this.s3Config.region)}',`
|
|
246
|
+
: '';
|
|
247
|
+
const keyIdClause = this.s3Config.accessKey
|
|
248
|
+
? `KEY_ID '${this.escapeString(this.s3Config.accessKey)}',`
|
|
249
|
+
: '';
|
|
250
|
+
const secretClause = this.s3Config.secretKey
|
|
251
|
+
? `SECRET '${this.escapeString(this.s3Config.secretKey)}',`
|
|
252
|
+
: '';
|
|
253
|
+
await db.run(`CREATE OR REPLACE SECRET __lightdash_s3 (
|
|
254
|
+
TYPE s3,
|
|
255
|
+
${keyIdClause}
|
|
256
|
+
${secretClause}
|
|
257
|
+
ENDPOINT '${this.escapeString(this.s3Config.endpoint)}',
|
|
258
|
+
${regionClause}
|
|
259
|
+
URL_STYLE '${this.s3Config.forcePathStyle ? 'path' : 'vhost'}',
|
|
260
|
+
USE_SSL ${this.s3Config.useSsl}
|
|
261
|
+
);`);
|
|
262
|
+
return performance.now() - t2;
|
|
263
|
+
});
|
|
288
264
|
this.logger?.info(`DuckDB bootstrap timing: install_httpfs=${Math.round(installMs)}ms load_httpfs=${Math.round(loadMs)}ms s3_config=${Math.round(s3ConfigMs)}ms`);
|
|
289
265
|
}
|
|
290
266
|
getBindValues(options) {
|
|
@@ -246,14 +246,19 @@ describe('DuckdbWarehouseClient', () => {
|
|
|
246
246
|
expect(runCalls).toContain("SET TimeZone = 'UTC';");
|
|
247
247
|
expect(streamMock).toHaveBeenCalledTimes(1);
|
|
248
248
|
});
|
|
249
|
-
it('should serialize
|
|
249
|
+
it('should serialize S3 secret creation across concurrent sessions without skipping', async () => {
|
|
250
250
|
let releaseSecretCreation;
|
|
251
251
|
const secretCreationBlocked = new Promise((resolve) => {
|
|
252
252
|
releaseSecretCreation = resolve;
|
|
253
253
|
});
|
|
254
|
+
let secretCreateCount = 0;
|
|
254
255
|
const runMock = jest.fn(async (sql) => {
|
|
255
256
|
if (sql.includes('CREATE OR REPLACE SECRET __lightdash_s3')) {
|
|
256
|
-
|
|
257
|
+
secretCreateCount += 1;
|
|
258
|
+
// Only block the first CREATE SECRET call
|
|
259
|
+
if (secretCreateCount === 1) {
|
|
260
|
+
await secretCreationBlocked;
|
|
261
|
+
}
|
|
257
262
|
}
|
|
258
263
|
});
|
|
259
264
|
const streamMock = jest.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
@@ -273,10 +278,13 @@ describe('DuckdbWarehouseClient', () => {
|
|
|
273
278
|
await new Promise((resolve) => {
|
|
274
279
|
setImmediate(resolve);
|
|
275
280
|
});
|
|
276
|
-
|
|
281
|
+
// First CREATE SECRET is blocked; second should not have started yet
|
|
282
|
+
// (serialized via the shared bootstrap lock)
|
|
283
|
+
expect(secretCreateCount).toBe(1);
|
|
277
284
|
releaseSecretCreation?.();
|
|
278
285
|
await Promise.all([firstQuery, secondQuery]);
|
|
279
|
-
|
|
286
|
+
// Both connections should create the secret (one each, serialized)
|
|
287
|
+
expect(runMock.mock.calls.filter(([sql]) => sql.includes('CREATE OR REPLACE SECRET __lightdash_s3'))).toHaveLength(2);
|
|
280
288
|
expect(streamMock).toHaveBeenCalledTimes(2);
|
|
281
289
|
});
|
|
282
290
|
it('should log structured DuckDB profile metrics with query tags', async () => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "0.2616.
|
|
3
|
+
"version": "0.2616.1",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
"snowflake-sdk": "~2.3.4",
|
|
27
27
|
"ssh2": "^1.14.0",
|
|
28
28
|
"trino-client": "0.2.9",
|
|
29
|
-
"@lightdash/common": "0.2616.
|
|
29
|
+
"@lightdash/common": "0.2616.1"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
32
|
"@types/node-fetch": "^2.6.13",
|