@lightdash/warehouses 0.2615.2 → 0.2615.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,aAAa,EACb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAsChE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,YAAY,CAAC;CACzB,CAAC;AAYF,eAAO,MAAM,sBAAsB,GAAI,QAAQ,MAAM,KAAG,aA+BvD,CAAC;AAEF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB;IAIrC,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C;
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,aAAa,EACb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAsChE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,YAAY,CAAC;CACzB,CAAC;AAYF,eAAO,MAAM,sBAAsB,GAAI,QAAQ,MAAM,KAAG,aA+BvD,CAAC;AAEF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB;IAIrC,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C;AAgDD,mFAAmF;AACnF,wBAAgB,gCAAgC,IAAI,IAAI,CAMvD;AAuCD,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,yBAAyB,CAAC;IACrF,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAS;IAEzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;gBAE3B,IAAI,GAAE,yBAA8B;IAS1C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,kBAAkB;YAQZ,gBAAgB;YAoBhB,WAAW;YAwCX,gBAAgB;IAuH9B,OAAO,CAAC,aAAa;mBA0BA,eAAe;IAoFpC,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAaxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAM/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;YAUrB,eAAe;YA8Bf,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC;IA4CV,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,yBAAyB,CAAC,CAAC,mBAAmB,CAAC,CACtE;;;;;;IA8BC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOlC,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC;IAoBI,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,yBAAyB,CAAC,CAAC,UAAU,CAAC,CAC7D;;;;;;IAKC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrB,UAAU,CACZ,OAAO,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC/D,OAAO,CAAC,gBAAgB,CAAC;IAMtB,YAAY,CACd,OAAO,CAAC,EAAE,MAAM,EAChB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN;IAMK,SAAS,CACX,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,MAAM,EAChB,SAAS,CAAC,EAAE,MAAM,EAClB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC;CAK/B"}
|
|
@@ -84,6 +84,8 @@ exports.DuckdbSqlBuilder = DuckdbSqlBuilder;
|
|
|
84
84
|
let sharedInstance = null;
|
|
85
85
|
let httpfsInstalled = false;
|
|
86
86
|
let cachesConfigured = false;
|
|
87
|
+
let configuredS3SecretFingerprint = null;
|
|
88
|
+
let sharedBootstrapQueue = Promise.resolve();
|
|
87
89
|
async function getOrCreateSharedInstance(databasePath, logger) {
|
|
88
90
|
if (!sharedInstance) {
|
|
89
91
|
const t0 = performance.now();
|
|
@@ -106,6 +108,8 @@ function clearSharedInstance(logger) {
|
|
|
106
108
|
sharedInstance = null;
|
|
107
109
|
httpfsInstalled = false;
|
|
108
110
|
cachesConfigured = false;
|
|
111
|
+
configuredS3SecretFingerprint = null;
|
|
112
|
+
sharedBootstrapQueue = Promise.resolve();
|
|
109
113
|
logger?.info('DuckDB shared instance cleared');
|
|
110
114
|
}
|
|
111
115
|
}
|
|
@@ -114,6 +118,23 @@ function resetSharedDuckdbStateForTesting() {
|
|
|
114
118
|
sharedInstance = null;
|
|
115
119
|
httpfsInstalled = false;
|
|
116
120
|
cachesConfigured = false;
|
|
121
|
+
configuredS3SecretFingerprint = null;
|
|
122
|
+
sharedBootstrapQueue = Promise.resolve();
|
|
123
|
+
}
|
|
124
|
+
async function withSharedBootstrapLock(callback) {
|
|
125
|
+
const run = sharedBootstrapQueue.then(callback, callback);
|
|
126
|
+
sharedBootstrapQueue = run.then(() => undefined, () => undefined);
|
|
127
|
+
return run;
|
|
128
|
+
}
|
|
129
|
+
function getS3SecretFingerprint(config) {
|
|
130
|
+
return JSON.stringify({
|
|
131
|
+
endpoint: config.endpoint,
|
|
132
|
+
region: config.region,
|
|
133
|
+
accessKey: config.accessKey,
|
|
134
|
+
secretKey: config.secretKey,
|
|
135
|
+
forcePathStyle: config.forcePathStyle,
|
|
136
|
+
useSsl: config.useSsl,
|
|
137
|
+
});
|
|
117
138
|
}
|
|
118
139
|
// DuckDB StatementType values — see duckdb/common/enums/statement_type.hpp
|
|
119
140
|
const ALLOWED_STATEMENT_TYPES_USER_SQL = new Set([1 /* SELECT */]);
|
|
@@ -185,32 +206,75 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
185
206
|
}
|
|
186
207
|
}
|
|
187
208
|
async bootstrapSession(db, tempDir) {
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
209
|
+
const { installMs, s3ConfigMs, loadedHttpfsDuringSharedBootstrap } = await withSharedBootstrapLock(async () => {
|
|
210
|
+
let nextInstallMs = 0;
|
|
211
|
+
let nextS3ConfigMs = 0;
|
|
212
|
+
let loadedDuringSharedBootstrap = false;
|
|
213
|
+
if (!httpfsInstalled) {
|
|
214
|
+
const t0 = performance.now();
|
|
215
|
+
await db.run('INSTALL httpfs;');
|
|
216
|
+
nextInstallMs = performance.now() - t0;
|
|
217
|
+
httpfsInstalled = true;
|
|
218
|
+
this.logger?.info(`DuckDB httpfs installed (first use): ${Math.round(nextInstallMs)}ms`);
|
|
219
|
+
}
|
|
220
|
+
// Enable built-in caches. These are global settings and should not
|
|
221
|
+
// be mutated concurrently on the shared instance.
|
|
222
|
+
if (!cachesConfigured) {
|
|
223
|
+
await db.run('SET enable_http_metadata_cache = true;');
|
|
224
|
+
await db.run('SET enable_external_file_cache = true;');
|
|
225
|
+
await db.run('SET parquet_metadata_cache = true;');
|
|
226
|
+
await db.run('SET allow_community_extensions = false;');
|
|
227
|
+
await db.run('SET autoinstall_known_extensions = false;');
|
|
228
|
+
await db.run('SET autoload_known_extensions = false;');
|
|
229
|
+
await db.run('SET allow_unredacted_secrets = false;');
|
|
230
|
+
cachesConfigured = true;
|
|
231
|
+
if (this.bufferPoolSize) {
|
|
232
|
+
await db.run(`SET buffer_pool_size = '${this.bufferPoolSize}';`);
|
|
233
|
+
}
|
|
234
|
+
this.logger?.info(`DuckDB caches enabled: http_metadata=true external_file=true parquet_metadata=true buffer_pool_size=${this.bufferPoolSize ?? 'default'}`);
|
|
212
235
|
}
|
|
213
|
-
this.
|
|
236
|
+
if (this.s3Config) {
|
|
237
|
+
const nextFingerprint = getS3SecretFingerprint(this.s3Config);
|
|
238
|
+
if (configuredS3SecretFingerprint !== nextFingerprint) {
|
|
239
|
+
const loadStart = performance.now();
|
|
240
|
+
await db.run('LOAD httpfs;');
|
|
241
|
+
loadedDuringSharedBootstrap = true;
|
|
242
|
+
const loadMs = performance.now() - loadStart;
|
|
243
|
+
const regionClause = this.s3Config.region
|
|
244
|
+
? `REGION '${this.escapeString(this.s3Config.region)}',`
|
|
245
|
+
: '';
|
|
246
|
+
const keyIdClause = this.s3Config.accessKey
|
|
247
|
+
? `KEY_ID '${this.escapeString(this.s3Config.accessKey)}',`
|
|
248
|
+
: '';
|
|
249
|
+
const secretClause = this.s3Config.secretKey
|
|
250
|
+
? `SECRET '${this.escapeString(this.s3Config.secretKey)}',`
|
|
251
|
+
: '';
|
|
252
|
+
const t2 = performance.now();
|
|
253
|
+
await db.run(`CREATE OR REPLACE SECRET __lightdash_s3 (
|
|
254
|
+
TYPE s3,
|
|
255
|
+
${keyIdClause}
|
|
256
|
+
${secretClause}
|
|
257
|
+
ENDPOINT '${this.escapeString(this.s3Config.endpoint)}',
|
|
258
|
+
${regionClause}
|
|
259
|
+
URL_STYLE '${this.s3Config.forcePathStyle ? 'path' : 'vhost'}',
|
|
260
|
+
USE_SSL ${this.s3Config.useSsl}
|
|
261
|
+
);`);
|
|
262
|
+
nextS3ConfigMs = performance.now() - t2;
|
|
263
|
+
configuredS3SecretFingerprint = nextFingerprint;
|
|
264
|
+
this.logger?.info(`DuckDB S3 secret configured: load_httpfs=${Math.round(loadMs)}ms s3_config=${Math.round(nextS3ConfigMs)}ms`);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
return {
|
|
268
|
+
installMs: nextInstallMs,
|
|
269
|
+
s3ConfigMs: nextS3ConfigMs,
|
|
270
|
+
loadedHttpfsDuringSharedBootstrap: loadedDuringSharedBootstrap,
|
|
271
|
+
};
|
|
272
|
+
});
|
|
273
|
+
let loadMs = 0;
|
|
274
|
+
if (!loadedHttpfsDuringSharedBootstrap) {
|
|
275
|
+
const t1 = performance.now();
|
|
276
|
+
await db.run('LOAD httpfs;');
|
|
277
|
+
loadMs = performance.now() - t1;
|
|
214
278
|
}
|
|
215
279
|
if (this.resourceLimits && tempDir) {
|
|
216
280
|
await db.run(`SET memory_limit = '${this.resourceLimits.memoryLimit}';`);
|
|
@@ -221,26 +285,6 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
221
285
|
this.logger?.info(`DuckDB bootstrap timing: install_httpfs=${Math.round(installMs)}ms load_httpfs=${Math.round(loadMs)}ms`);
|
|
222
286
|
return;
|
|
223
287
|
}
|
|
224
|
-
const t2 = performance.now();
|
|
225
|
-
const regionClause = this.s3Config.region
|
|
226
|
-
? `REGION '${this.escapeString(this.s3Config.region)}',`
|
|
227
|
-
: '';
|
|
228
|
-
const keyIdClause = this.s3Config.accessKey
|
|
229
|
-
? `KEY_ID '${this.escapeString(this.s3Config.accessKey)}',`
|
|
230
|
-
: '';
|
|
231
|
-
const secretClause = this.s3Config.secretKey
|
|
232
|
-
? `SECRET '${this.escapeString(this.s3Config.secretKey)}',`
|
|
233
|
-
: '';
|
|
234
|
-
await db.run(`CREATE OR REPLACE SECRET __lightdash_s3 (
|
|
235
|
-
TYPE s3,
|
|
236
|
-
${keyIdClause}
|
|
237
|
-
${secretClause}
|
|
238
|
-
ENDPOINT '${this.escapeString(this.s3Config.endpoint)}',
|
|
239
|
-
${regionClause}
|
|
240
|
-
URL_STYLE '${this.s3Config.forcePathStyle ? 'path' : 'vhost'}',
|
|
241
|
-
USE_SSL ${this.s3Config.useSsl}
|
|
242
|
-
);`);
|
|
243
|
-
const s3ConfigMs = performance.now() - t2;
|
|
244
288
|
this.logger?.info(`DuckDB bootstrap timing: install_httpfs=${Math.round(installMs)}ms load_httpfs=${Math.round(loadMs)}ms s3_config=${Math.round(s3ConfigMs)}ms`);
|
|
245
289
|
}
|
|
246
290
|
getBindValues(options) {
|
|
@@ -246,6 +246,39 @@ describe('DuckdbWarehouseClient', () => {
|
|
|
246
246
|
expect(runCalls).toContain("SET TimeZone = 'UTC';");
|
|
247
247
|
expect(streamMock).toHaveBeenCalledTimes(1);
|
|
248
248
|
});
|
|
249
|
+
it('should serialize shared S3 secret bootstrap across concurrent sessions', async () => {
|
|
250
|
+
let releaseSecretCreation;
|
|
251
|
+
const secretCreationBlocked = new Promise((resolve) => {
|
|
252
|
+
releaseSecretCreation = resolve;
|
|
253
|
+
});
|
|
254
|
+
const runMock = jest.fn(async (sql) => {
|
|
255
|
+
if (sql.includes('CREATE OR REPLACE SECRET __lightdash_s3')) {
|
|
256
|
+
await secretCreationBlocked;
|
|
257
|
+
}
|
|
258
|
+
});
|
|
259
|
+
const streamMock = jest.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
260
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
261
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
262
|
+
s3Config: {
|
|
263
|
+
endpoint: 'localhost:9000',
|
|
264
|
+
region: 'us-east-1',
|
|
265
|
+
accessKey: 'key',
|
|
266
|
+
secretKey: 'secret',
|
|
267
|
+
forcePathStyle: true,
|
|
268
|
+
useSsl: false,
|
|
269
|
+
},
|
|
270
|
+
});
|
|
271
|
+
const firstQuery = client.runQuery('SELECT 1 AS val');
|
|
272
|
+
const secondQuery = client.runQuery('SELECT 1 AS val');
|
|
273
|
+
await new Promise((resolve) => {
|
|
274
|
+
setImmediate(resolve);
|
|
275
|
+
});
|
|
276
|
+
expect(runMock.mock.calls.filter(([sql]) => sql.includes('CREATE OR REPLACE SECRET __lightdash_s3'))).toHaveLength(1);
|
|
277
|
+
releaseSecretCreation?.();
|
|
278
|
+
await Promise.all([firstQuery, secondQuery]);
|
|
279
|
+
expect(runMock.mock.calls.filter(([sql]) => sql.includes('CREATE OR REPLACE SECRET __lightdash_s3'))).toHaveLength(1);
|
|
280
|
+
expect(streamMock).toHaveBeenCalledTimes(2);
|
|
281
|
+
});
|
|
249
282
|
it('should log structured DuckDB profile metrics with query tags', async () => {
|
|
250
283
|
const runMock = jest.fn(async (sql) => {
|
|
251
284
|
const match = sql.match(/^PRAGMA profiling_output='(.+)';$/);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "0.2615.
|
|
3
|
+
"version": "0.2615.3",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
"snowflake-sdk": "~2.3.4",
|
|
27
27
|
"ssh2": "^1.14.0",
|
|
28
28
|
"trino-client": "0.2.9",
|
|
29
|
-
"@lightdash/common": "0.2615.
|
|
29
|
+
"@lightdash/common": "0.2615.3"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
32
|
"@types/node-fetch": "^2.6.13",
|