@lightdash/warehouses 0.2606.0 → 0.2607.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts +4 -0
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.js +88 -12
- package/dist/warehouseClients/DuckdbWarehouseClient.test.js +164 -3
- package/package.json +2 -2
|
@@ -47,6 +47,10 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreatePos
|
|
|
47
47
|
private getBindValues;
|
|
48
48
|
private static logQueryProfile;
|
|
49
49
|
private static getFieldsFromStreamResult;
|
|
50
|
+
private static stripSqlComments;
|
|
51
|
+
private static validateSqlFunctions;
|
|
52
|
+
private validateUserSql;
|
|
53
|
+
private validateInternalSql;
|
|
50
54
|
streamQuery(sql: string, streamCallback: (data: WarehouseResults) => void | Promise<void>, options?: {
|
|
51
55
|
values?: AnyType[];
|
|
52
56
|
queryParams?: Record<string, AnyType>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,aAAa,EACb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,aAAa,EACb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAsChE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,YAAY,CAAC;CACzB,CAAC;AAYF,eAAO,MAAM,sBAAsB,GAAI,QAAQ,MAAM,KAAG,aA+BvD,CAAC;AAEF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB;IAIrC,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C;AA4CD,mFAAmF;AACnF,wBAAgB,gCAAgC,IAAI,IAAI,CAIvD;AAiBD,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,yBAAyB,CAAC;IACrF,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAS;IAEzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;gBAE3B,IAAI,GAAE,yBAA8B;IAS1C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,kBAAkB;YAQZ,gBAAgB;YAoBhB,WAAW;YAwCX,gBAAgB;IA0F9B,OAAO,CAAC,aAAa;mBA0BA,eAAe;IAoFpC,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAaxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAM/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;YAUrB,eAAe;YA8Bf,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC;IA4CV,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,yBAAyB,CAAC,CAAC,mBAAmB,CAAC,CACtE;;;;;;IA8BC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOlC,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC;IAoBI,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,yBAAyB,CAAC,CAAC,UAAU,CAAC,CAC7D;;;;;;IAKC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrB,UAAU,CACZ,OAAO,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC/D,OAAO,CAAC,gBAAgB,CAAC;IAMtB,YAAY,CACd,OAAO,CAAC,EAAE,MAAM,EAChB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN;IAMK,SAAS,CACX,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,MAAM,EAChB,SAAS,CAAC,EAAE,MAAM,EAClB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC;CAK/B"}
|
|
@@ -115,6 +115,17 @@ function resetSharedDuckdbStateForTesting() {
|
|
|
115
115
|
httpfsInstalled = false;
|
|
116
116
|
cachesConfigured = false;
|
|
117
117
|
}
|
|
118
|
+
// DuckDB StatementType values — see duckdb/common/enums/statement_type.hpp
|
|
119
|
+
const ALLOWED_STATEMENT_TYPES_USER_SQL = new Set([1 /* SELECT */]);
|
|
120
|
+
const BLOCKED_STATEMENT_TYPES_INTERNAL_SQL = new Set([
|
|
121
|
+
12, // VARIABLE_SET
|
|
122
|
+
20, // SET
|
|
123
|
+
21, // LOAD
|
|
124
|
+
23, // EXTENSION (INSTALL)
|
|
125
|
+
25, // ATTACH
|
|
126
|
+
26, // DETACH
|
|
127
|
+
]);
|
|
128
|
+
const BLOCKED_FUNCTION_PATTERN = /\b(current_setting|duckdb_settings|duckdb_secrets)\s*\(/i;
|
|
118
129
|
class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
119
130
|
constructor(args = {}) {
|
|
120
131
|
super(DUCKDB_INTERNAL_CREDENTIALS, new DuckdbSqlBuilder());
|
|
@@ -191,6 +202,12 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
191
202
|
await db.run('SET enable_http_metadata_cache = true;');
|
|
192
203
|
await db.run('SET enable_external_file_cache = true;');
|
|
193
204
|
await db.run('SET parquet_metadata_cache = true;');
|
|
205
|
+
// Security: lock down after our extensions are installed/loaded
|
|
206
|
+
await db.run("SET disabled_filesystems = 'LocalFileSystem';");
|
|
207
|
+
await db.run('SET allow_community_extensions = false;');
|
|
208
|
+
await db.run('SET autoinstall_known_extensions = false;');
|
|
209
|
+
await db.run('SET autoload_known_extensions = false;');
|
|
210
|
+
await db.run('SET allow_unredacted_secrets = false;');
|
|
194
211
|
cachesConfigured = true;
|
|
195
212
|
if (this.bufferPoolSize) {
|
|
196
213
|
await db.run(`SET buffer_pool_size = '${this.bufferPoolSize}';`);
|
|
@@ -207,18 +224,24 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
207
224
|
return;
|
|
208
225
|
}
|
|
209
226
|
const t2 = performance.now();
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
227
|
+
const regionClause = this.s3Config.region
|
|
228
|
+
? `REGION '${this.escapeString(this.s3Config.region)}',`
|
|
229
|
+
: '';
|
|
230
|
+
const keyIdClause = this.s3Config.accessKey
|
|
231
|
+
? `KEY_ID '${this.escapeString(this.s3Config.accessKey)}',`
|
|
232
|
+
: '';
|
|
233
|
+
const secretClause = this.s3Config.secretKey
|
|
234
|
+
? `SECRET '${this.escapeString(this.s3Config.secretKey)}',`
|
|
235
|
+
: '';
|
|
236
|
+
await db.run(`CREATE OR REPLACE SECRET __lightdash_s3 (
|
|
237
|
+
TYPE s3,
|
|
238
|
+
${keyIdClause}
|
|
239
|
+
${secretClause}
|
|
240
|
+
ENDPOINT '${this.escapeString(this.s3Config.endpoint)}',
|
|
241
|
+
${regionClause}
|
|
242
|
+
URL_STYLE '${this.s3Config.forcePathStyle ? 'path' : 'vhost'}',
|
|
243
|
+
USE_SSL ${this.s3Config.useSsl}
|
|
244
|
+
);`);
|
|
222
245
|
const s3ConfigMs = performance.now() - t2;
|
|
223
246
|
this.logger?.info(`DuckDB bootstrap timing: install_httpfs=${Math.round(installMs)}ms load_httpfs=${Math.round(loadMs)}ms s3_config=${Math.round(s3ConfigMs)}ms`);
|
|
224
247
|
}
|
|
@@ -307,6 +330,56 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
307
330
|
}
|
|
308
331
|
return fields;
|
|
309
332
|
}
|
|
333
|
+
static stripSqlComments(sql) {
|
|
334
|
+
return sql
|
|
335
|
+
.replace(/--[^\n]*/g, '') // line comments
|
|
336
|
+
.replace(/\/\*[\s\S]*?\*\//g, ''); // block comments
|
|
337
|
+
}
|
|
338
|
+
static validateSqlFunctions(sql) {
|
|
339
|
+
const stripped = DuckdbWarehouseClient.stripSqlComments(sql);
|
|
340
|
+
const match = stripped.match(BLOCKED_FUNCTION_PATTERN);
|
|
341
|
+
if (match) {
|
|
342
|
+
throw new Error(`SQL validation error: function '${match[1]}' is not allowed`);
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
async validateUserSql(db, sql) {
|
|
346
|
+
const extracted = await db.extractStatements(sql);
|
|
347
|
+
if (extracted.count === 0) {
|
|
348
|
+
throw new Error('SQL validation error: empty SQL statement');
|
|
349
|
+
}
|
|
350
|
+
if (extracted.count > 1) {
|
|
351
|
+
throw new Error('SQL validation error: multiple SQL statements are not allowed');
|
|
352
|
+
}
|
|
353
|
+
const stmt = await extracted.prepare(0);
|
|
354
|
+
try {
|
|
355
|
+
if (!ALLOWED_STATEMENT_TYPES_USER_SQL.has(stmt.statementType)) {
|
|
356
|
+
throw new Error(`SQL validation error: only SELECT statements are allowed (got statement type ${stmt.statementType})`);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
finally {
|
|
360
|
+
stmt.destroySync();
|
|
361
|
+
}
|
|
362
|
+
DuckdbWarehouseClient.validateSqlFunctions(sql);
|
|
363
|
+
}
|
|
364
|
+
async validateInternalSql(db, sql) {
|
|
365
|
+
const extracted = await db.extractStatements(sql);
|
|
366
|
+
if (extracted.count === 0) {
|
|
367
|
+
throw new Error('SQL validation error: empty SQL statement');
|
|
368
|
+
}
|
|
369
|
+
for (let i = 0; i < extracted.count; i += 1) {
|
|
370
|
+
// eslint-disable-next-line no-await-in-loop
|
|
371
|
+
const stmt = await extracted.prepare(i);
|
|
372
|
+
try {
|
|
373
|
+
if (BLOCKED_STATEMENT_TYPES_INTERNAL_SQL.has(stmt.statementType)) {
|
|
374
|
+
throw new Error(`SQL validation error: statement type ${stmt.statementType} is not allowed in internal SQL`);
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
finally {
|
|
378
|
+
stmt.destroySync();
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
DuckdbWarehouseClient.validateSqlFunctions(sql);
|
|
382
|
+
}
|
|
310
383
|
async streamQuery(sql, streamCallback, options) {
|
|
311
384
|
await this.withSession(async (db) => {
|
|
312
385
|
if (options?.timezone) {
|
|
@@ -319,6 +392,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
319
392
|
await db.run("PRAGMA enable_profiling='json';");
|
|
320
393
|
await db.run(`PRAGMA profiling_output='${profilePath}';`);
|
|
321
394
|
}
|
|
395
|
+
await this.validateUserSql(db, sql);
|
|
322
396
|
const result = await db.stream(this.getSQLWithMetadata(sql, options?.tags), this.getBindValues(options));
|
|
323
397
|
const fields = DuckdbWarehouseClient.getFieldsFromStreamResult(result);
|
|
324
398
|
// eslint-disable-next-line no-restricted-syntax
|
|
@@ -353,6 +427,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
353
427
|
}
|
|
354
428
|
async runSql(sql) {
|
|
355
429
|
await this.withSession(async (db) => {
|
|
430
|
+
await this.validateInternalSql(db, sql);
|
|
356
431
|
await db.run(sql);
|
|
357
432
|
});
|
|
358
433
|
}
|
|
@@ -361,6 +436,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
361
436
|
let bootstrapMs = 0;
|
|
362
437
|
let queryMs = 0;
|
|
363
438
|
await this.withSession(async (db) => {
|
|
439
|
+
await this.validateInternalSql(db, sql);
|
|
364
440
|
bootstrapMs = performance.now() - totalStart;
|
|
365
441
|
const queryStart = performance.now();
|
|
366
442
|
await db.run(sql);
|
|
@@ -79,10 +79,18 @@ const getMockStreamResult = (chunks, columnTypeIds) => {
|
|
|
79
79
|
},
|
|
80
80
|
};
|
|
81
81
|
};
|
|
82
|
-
const
|
|
82
|
+
const createMockExtractStatements = (overrides) => jest.fn(async () => ({
|
|
83
|
+
count: overrides?.count ?? 1,
|
|
84
|
+
prepare: async () => ({
|
|
85
|
+
statementType: overrides?.statementType ?? 1, // SELECT
|
|
86
|
+
destroySync: jest.fn(),
|
|
87
|
+
}),
|
|
88
|
+
}));
|
|
89
|
+
const createMockConnection = (streamMock, runMock = jest.fn(), opts) => ({
|
|
83
90
|
connect: async () => ({
|
|
84
91
|
run: runMock,
|
|
85
92
|
stream: streamMock,
|
|
93
|
+
extractStatements: opts?.extractStatements ?? createMockExtractStatements(),
|
|
86
94
|
closeSync: jest.fn(),
|
|
87
95
|
disconnectSync: jest.fn(),
|
|
88
96
|
}),
|
|
@@ -219,8 +227,23 @@ describe('DuckdbWarehouseClient', () => {
|
|
|
219
227
|
expect(runCalls).toContain('SET enable_http_metadata_cache = true;');
|
|
220
228
|
expect(runCalls).toContain('SET enable_external_file_cache = true;');
|
|
221
229
|
expect(runCalls).toContain('SET parquet_metadata_cache = true;');
|
|
222
|
-
expect(runCalls).toContain("SET
|
|
223
|
-
expect(runCalls).toContain(
|
|
230
|
+
expect(runCalls).toContain("SET disabled_filesystems = 'LocalFileSystem';");
|
|
231
|
+
expect(runCalls).toContain('SET allow_community_extensions = false;');
|
|
232
|
+
expect(runCalls).toContain('SET autoinstall_known_extensions = false;');
|
|
233
|
+
expect(runCalls).toContain('SET autoload_known_extensions = false;');
|
|
234
|
+
expect(runCalls).toContain('SET allow_unredacted_secrets = false;');
|
|
235
|
+
// S3 credentials should use CREATE SECRET, not individual SET commands
|
|
236
|
+
const createSecretCall = runCalls.find((call) => call.includes('CREATE OR REPLACE SECRET'));
|
|
237
|
+
expect(createSecretCall).toBeDefined();
|
|
238
|
+
expect(createSecretCall).toContain("ENDPOINT 'localhost:9000'");
|
|
239
|
+
expect(createSecretCall).toContain("KEY_ID 'key'");
|
|
240
|
+
expect(createSecretCall).toContain("SECRET 'secret'");
|
|
241
|
+
expect(createSecretCall).toContain("REGION 'us-east-1'");
|
|
242
|
+
expect(createSecretCall).toContain("URL_STYLE 'path'");
|
|
243
|
+
expect(createSecretCall).toContain('USE_SSL false');
|
|
244
|
+
// Should NOT use individual SET s3_* commands
|
|
245
|
+
const s3SetCalls = runCalls.filter((call) => call.startsWith('SET s3_'));
|
|
246
|
+
expect(s3SetCalls).toHaveLength(0);
|
|
224
247
|
expect(runCalls).toContain("SET TimeZone = 'UTC';");
|
|
225
248
|
expect(streamMock).toHaveBeenCalledTimes(1);
|
|
226
249
|
});
|
|
@@ -269,4 +292,142 @@ describe('DuckdbWarehouseClient', () => {
|
|
|
269
292
|
scanAmplification: 9905024 / 68,
|
|
270
293
|
}));
|
|
271
294
|
});
|
|
295
|
+
describe('SQL security validation', () => {
|
|
296
|
+
it('should reject SET statements', async () => {
|
|
297
|
+
const streamMock = jest.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
298
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, jest.fn(), {
|
|
299
|
+
extractStatements: createMockExtractStatements({
|
|
300
|
+
statementType: 20, // SET
|
|
301
|
+
}),
|
|
302
|
+
}));
|
|
303
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
304
|
+
await expect(client.runQuery("SET s3_endpoint = 'attacker.com'")).rejects.toThrow('SQL validation error: only SELECT statements are allowed');
|
|
305
|
+
expect(streamMock).not.toHaveBeenCalled();
|
|
306
|
+
});
|
|
307
|
+
it('should reject COPY statements', async () => {
|
|
308
|
+
const streamMock = jest.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
309
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, jest.fn(), {
|
|
310
|
+
extractStatements: createMockExtractStatements({
|
|
311
|
+
statementType: 11, // COPY
|
|
312
|
+
}),
|
|
313
|
+
}));
|
|
314
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
315
|
+
await expect(client.runQuery("COPY t TO '/tmp/data.csv'")).rejects.toThrow('SQL validation error: only SELECT statements are allowed');
|
|
316
|
+
});
|
|
317
|
+
it('should reject ATTACH statements', async () => {
|
|
318
|
+
const streamMock = jest.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
319
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, jest.fn(), {
|
|
320
|
+
extractStatements: createMockExtractStatements({
|
|
321
|
+
statementType: 25, // ATTACH
|
|
322
|
+
}),
|
|
323
|
+
}));
|
|
324
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
325
|
+
await expect(client.runQuery("ATTACH DATABASE 'file.db'")).rejects.toThrow('SQL validation error: only SELECT statements are allowed');
|
|
326
|
+
});
|
|
327
|
+
it('should reject multiple statements', async () => {
|
|
328
|
+
const streamMock = jest.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
329
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, jest.fn(), {
|
|
330
|
+
extractStatements: createMockExtractStatements({
|
|
331
|
+
count: 2,
|
|
332
|
+
}),
|
|
333
|
+
}));
|
|
334
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
335
|
+
await expect(client.runQuery('SELECT 1; DROP TABLE users')).rejects.toThrow('SQL validation error: multiple SQL statements are not allowed');
|
|
336
|
+
});
|
|
337
|
+
it('should reject queries with current_setting()', async () => {
|
|
338
|
+
const streamMock = jest.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
339
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
|
340
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
341
|
+
await expect(client.runQuery("SELECT current_setting('s3_secret_access_key')")).rejects.toThrow("SQL validation error: function 'current_setting' is not allowed");
|
|
342
|
+
});
|
|
343
|
+
it('should reject queries with duckdb_settings()', async () => {
|
|
344
|
+
const streamMock = jest.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
345
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
|
346
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
347
|
+
await expect(client.runQuery('SELECT * FROM duckdb_settings()')).rejects.toThrow("SQL validation error: function 'duckdb_settings' is not allowed");
|
|
348
|
+
});
|
|
349
|
+
it('should reject queries with duckdb_secrets()', async () => {
|
|
350
|
+
const streamMock = jest.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
351
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
|
352
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
353
|
+
await expect(client.runQuery('SELECT * FROM duckdb_secrets()')).rejects.toThrow("SQL validation error: function 'duckdb_secrets' is not allowed");
|
|
354
|
+
});
|
|
355
|
+
it('should ignore blocked functions inside SQL comments', async () => {
|
|
356
|
+
const streamMock = jest.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
357
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
|
358
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
359
|
+
// Should NOT throw because current_setting is in a comment
|
|
360
|
+
const result = await client.runQuery("SELECT 1 -- current_setting('s3_secret_access_key')");
|
|
361
|
+
expect(result.rows).toEqual([{ val: 1 }]);
|
|
362
|
+
});
|
|
363
|
+
it('should allow COPY statements in runSql', async () => {
|
|
364
|
+
const streamMock = jest.fn();
|
|
365
|
+
const runMock = jest.fn();
|
|
366
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock, {
|
|
367
|
+
extractStatements: createMockExtractStatements({
|
|
368
|
+
statementType: 10, // COPY
|
|
369
|
+
}),
|
|
370
|
+
}));
|
|
371
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
372
|
+
await client.runSql("COPY table TO 's3://bucket/data.parquet' (FORMAT PARQUET)");
|
|
373
|
+
expect(runMock).toHaveBeenCalledWith("COPY table TO 's3://bucket/data.parquet' (FORMAT PARQUET)");
|
|
374
|
+
});
|
|
375
|
+
it('should reject SET statements in runSql', async () => {
|
|
376
|
+
const streamMock = jest.fn();
|
|
377
|
+
const runMock = jest.fn();
|
|
378
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock, {
|
|
379
|
+
extractStatements: createMockExtractStatements({
|
|
380
|
+
statementType: 20, // SET
|
|
381
|
+
}),
|
|
382
|
+
}));
|
|
383
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
384
|
+
await expect(client.runSql("SET s3_endpoint = 'attacker.com'")).rejects.toThrow('SQL validation error: statement type 20 is not allowed in internal SQL');
|
|
385
|
+
});
|
|
386
|
+
it('should reject ATTACH statements in runSql', async () => {
|
|
387
|
+
const streamMock = jest.fn();
|
|
388
|
+
const runMock = jest.fn();
|
|
389
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock, {
|
|
390
|
+
extractStatements: createMockExtractStatements({
|
|
391
|
+
statementType: 25, // ATTACH
|
|
392
|
+
}),
|
|
393
|
+
}));
|
|
394
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
395
|
+
await expect(client.runSql("ATTACH DATABASE 'file.db'")).rejects.toThrow('SQL validation error: statement type 25 is not allowed in internal SQL');
|
|
396
|
+
});
|
|
397
|
+
it('should reject LOAD statements in runSql', async () => {
|
|
398
|
+
const streamMock = jest.fn();
|
|
399
|
+
const runMock = jest.fn();
|
|
400
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock, {
|
|
401
|
+
extractStatements: createMockExtractStatements({
|
|
402
|
+
statementType: 21, // LOAD
|
|
403
|
+
}),
|
|
404
|
+
}));
|
|
405
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
406
|
+
await expect(client.runSql("LOAD 'malicious_extension'")).rejects.toThrow('SQL validation error: statement type 21 is not allowed in internal SQL');
|
|
407
|
+
});
|
|
408
|
+
it('should reject introspection functions in runSql', async () => {
|
|
409
|
+
const streamMock = jest.fn();
|
|
410
|
+
const runMock = jest.fn();
|
|
411
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
412
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
413
|
+
await expect(client.runSql("COPY (SELECT current_setting('s3_secret_access_key')) TO '/tmp/out.csv'")).rejects.toThrow("SQL validation error: function 'current_setting' is not allowed");
|
|
414
|
+
});
|
|
415
|
+
it('should allow valid SELECT queries', async () => {
|
|
416
|
+
const streamMock = jest.fn(async () => getMockStreamResult([[{ id: 1, name: 'test' }]], [DUCKDB_TYPE_IDS.INTEGER, DUCKDB_TYPE_IDS.VARCHAR]));
|
|
417
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
|
418
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
419
|
+
const result = await client.runQuery('SELECT id, name FROM users WHERE id = 1');
|
|
420
|
+
expect(result.rows).toEqual([{ id: 1, name: 'test' }]);
|
|
421
|
+
});
|
|
422
|
+
it('should reject EXPLAIN statements', async () => {
|
|
423
|
+
const streamMock = jest.fn(async () => getMockStreamResult([[{ explain_value: 'plan output' }]], [DUCKDB_TYPE_IDS.VARCHAR]));
|
|
424
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, jest.fn(), {
|
|
425
|
+
extractStatements: createMockExtractStatements({
|
|
426
|
+
statementType: 4, // EXPLAIN
|
|
427
|
+
}),
|
|
428
|
+
}));
|
|
429
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
430
|
+
await expect(client.runQuery('EXPLAIN SELECT * FROM users')).rejects.toThrow('SQL validation error: only SELECT statements are allowed');
|
|
431
|
+
});
|
|
432
|
+
});
|
|
272
433
|
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2607.1",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
"snowflake-sdk": "~2.3.4",
|
|
27
27
|
"ssh2": "^1.14.0",
|
|
28
28
|
"trino-client": "0.2.9",
|
|
29
|
-
"@lightdash/common": "0.
|
|
29
|
+
"@lightdash/common": "0.2607.1"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
32
|
"@types/node-fetch": "^2.6.13",
|