@lightdash/warehouses 1.55.0 → 1.56.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts +4 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.js +20 -5
- package/dist/warehouseClients/DuckdbWarehouseClient.test.js +66 -2
- package/package.json +2 -2
|
@@ -72,9 +72,10 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
72
72
|
private readonly logger?;
|
|
73
73
|
private readonly onQueryProfile?;
|
|
74
74
|
private readonly embeddedQueryTimeoutMs;
|
|
75
|
+
private allowsPreAggregateFileReads;
|
|
75
76
|
constructor(credentials?: CreateDuckdbCredentials | DuckdbConnectionCredentials, options?: DuckdbWarehouseClientOptions);
|
|
76
77
|
private static hashDucklakeConfig;
|
|
77
|
-
static createForPreAggregate(credentials
|
|
78
|
+
static createForPreAggregate(credentials: DuckdbS3Credentials, options?: DuckdbWarehouseClientOptions): DuckdbWarehouseClient;
|
|
78
79
|
private static getSharedInstanceSemaphore;
|
|
79
80
|
private getRequiredInstanceCacheKey;
|
|
80
81
|
private static tryAcquireEmbeddedConcurrency;
|
|
@@ -130,7 +131,9 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
130
131
|
private static stripSqlComments;
|
|
131
132
|
private static validateSqlFunctions;
|
|
132
133
|
private static validateUserSqlFileAccess;
|
|
134
|
+
private validateSelectSql;
|
|
133
135
|
private validateUserSql;
|
|
136
|
+
private validatePreAggregateSql;
|
|
134
137
|
private validateInternalSql;
|
|
135
138
|
streamQuery(sql: string, streamCallback: (data: WarehouseResults) => void | Promise<void>, options?: {
|
|
136
139
|
values?: AnyType[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAKA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAMb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAEhB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAM3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA6DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG,mBAAmB,CAAC;AAE9D,MAAM,MAAM,4BAA4B,GAAG;IACvC,kHAAkH;IAClH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;IAC9D,sBAAsB,CAAC,EAAE,MAAM,CAAC;CACnC,CAAC;AAEF,eAAO,MAAM,sBAAsB,WAAY,MAAM,KAAG,aA+BvD,CAAC;AAEF,eAAO,MAAM,kCAAkC,aACjC,MAAM,KACjB,eAAe,GAAG,SAOpB,CAAC;AAoCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB,CAEpC;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AAuJD,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,iCAAiC,CAAC;IAC7F,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAGhD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAE/C;IAEF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sCAAsC,CAG1D;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;IAEV,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAEhD,YACI,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B,EA8GzC;IAED,OAAO,CAAC,MAAM,CAAC,kBAAkB;IASjC,MAAM,CAAC,qBAAqB,CACxB,WAAW,
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAKA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAMb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAEhB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAM3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA6DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG,mBAAmB,CAAC;AAE9D,MAAM,MAAM,4BAA4B,GAAG;IACvC,kHAAkH;IAClH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;IAC9D,sBAAsB,CAAC,EAAE,MAAM,CAAC;CACnC,CAAC;AAEF,eAAO,MAAM,sBAAsB,WAAY,MAAM,KAAG,aA+BvD,CAAC;AAEF,eAAO,MAAM,kCAAkC,aACjC,MAAM,KACjB,eAAe,GAAG,SAOpB,CAAC;AAoCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB,CAEpC;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AAuJD,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,iCAAiC,CAAC;IAC7F,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAGhD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAE/C;IAEF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sCAAsC,CAG1D;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;IAEV,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAEhD,OAAO,CAAC,2BAA2B,CAAS;IAE5C,YACI,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B,EA8GzC;IAED,OAAO,CAAC,MAAM,CAAC,kBAAkB;IASjC,MAAM,CAAC,qBAAqB,CACxB,WAAW,EAAE,mBAAmB,EAChC,OAAO,CAAC,EAAE,4BAA4B,GACvC,qBAAqB,CAIvB;IAED,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAoBzC,OAAO,CAAC,2BAA2B;IAUnC,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAwC5C,OAAO,CAAC,kBAAkB;mBAQL,cAAc;mBAOd,mBAAmB;IAaxC,OAAO,CAAC,MAAM,CAAC,qBAAqB;mBAMf,uBAAuB;YAsB9B,aAAa;YAoBb,kCAAkC;mBAW3B,qBAAqB;mBA6ErB,uBAAuB;mBAgBvB,yBAAyB;mBAiFzB,mBAAmB;IA6BxC,mFAAmF;IACnF,MAAM,CAAC,gCAAgC,IAAI,IAAI,CAM9C;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAO3B;IAED,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CACZ;IAEnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAA+B;IAE3E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAA0B;IAEjE,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAIjC,OAAO,CAAC,MAAM,CAAC,UAAU;IAIzB,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAuB5C,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAqGzC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAMhC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAmBrC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IA2CrC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAqC/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAK;YAE9C,gBAAgB;IA+E9B,iEAAiE;YACnD,wBAAwB;IA+CtC,gFAAgF;YAClE,mBAAmB;YAkDnB,yBAAyB;YA4CzB,iBAAiB;IA2C/B,OAAO,CAAC,iBAAiB;IAMzB;;;;;;OAMG;YACW,WAAW;IAmBzB,+DAA+D;YACjD,iBAAiB;YAyEjB,yBAAyB;IA2BvC,OAAO,CAAC,4BAA4B;IAOpC,OAAO,CAAC,aAAa;YA0BP,eAAe;IAgG7B,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAaxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IA+D/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;IAUnC,OAAO,CAAC,MAAM,CAAC,yBAAyB;YAmB1B,iBAAiB;YA4BjB,eAAe;YASf,uBAAuB;YAQvB,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CAiEf;IAEK,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,mBAAmB,CAAC,CAC9E;;;;;;OA4BJ;IAEK,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKvC;IAEK,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC,CAkBD;IAEK,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,UAAU,CAAC,CACrE;;;;;OAGJ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAEK,UAAU,CACZ,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC,CA4C3B;IAEK,YAAY,CACd,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN,CAqBA;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC,CAS3B;CACJ"}
|
|
@@ -262,6 +262,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
262
262
|
DUCKDB_INTERNAL_CREDENTIALS;
|
|
263
263
|
}
|
|
264
264
|
super(effectiveCredentials, new DuckdbSqlBuilder(effectiveCredentials.startOfWeek));
|
|
265
|
+
this.allowsPreAggregateFileReads = false;
|
|
265
266
|
if (isS3Only) {
|
|
266
267
|
this.s3Config = credentials.s3Config;
|
|
267
268
|
}
|
|
@@ -321,7 +322,9 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
321
322
|
.slice(0, 16);
|
|
322
323
|
}
|
|
323
324
|
static createForPreAggregate(credentials, options) {
|
|
324
|
-
|
|
325
|
+
const client = new DuckdbWarehouseClient(credentials, options);
|
|
326
|
+
client.allowsPreAggregateFileReads = true;
|
|
327
|
+
return client;
|
|
325
328
|
}
|
|
326
329
|
static getSharedInstanceSemaphore(instanceCacheKey) {
|
|
327
330
|
const existingSemaphore = DuckdbWarehouseClient.sharedInstanceSemaphores.get(instanceCacheKey);
|
|
@@ -1220,9 +1223,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1220
1223
|
throw new Error('SQL validation error: file table paths are not allowed');
|
|
1221
1224
|
}
|
|
1222
1225
|
}
|
|
1223
|
-
async
|
|
1224
|
-
DuckdbWarehouseClient.validateSqlFunctions(sql);
|
|
1225
|
-
DuckdbWarehouseClient.validateUserSqlFileAccess(sql);
|
|
1226
|
+
async validateSelectSql(db, sql) {
|
|
1226
1227
|
const extracted = await db.extractStatements(sql);
|
|
1227
1228
|
if (extracted.count === 0) {
|
|
1228
1229
|
throw new Error('SQL validation error: empty SQL statement');
|
|
@@ -1240,6 +1241,15 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1240
1241
|
stmt.destroySync();
|
|
1241
1242
|
}
|
|
1242
1243
|
}
|
|
1244
|
+
async validateUserSql(db, sql) {
|
|
1245
|
+
DuckdbWarehouseClient.validateSqlFunctions(sql);
|
|
1246
|
+
DuckdbWarehouseClient.validateUserSqlFileAccess(sql);
|
|
1247
|
+
await this.validateSelectSql(db, sql);
|
|
1248
|
+
}
|
|
1249
|
+
async validatePreAggregateSql(db, sql) {
|
|
1250
|
+
DuckdbWarehouseClient.validateSqlFunctions(sql);
|
|
1251
|
+
await this.validateSelectSql(db, sql);
|
|
1252
|
+
}
|
|
1243
1253
|
async validateInternalSql(db, sql) {
|
|
1244
1254
|
const extracted = await db.extractStatements(sql);
|
|
1245
1255
|
if (extracted.count === 0) {
|
|
@@ -1276,7 +1286,12 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1276
1286
|
if (this.embeddedConfig) {
|
|
1277
1287
|
await db.run("SET disabled_filesystems = 'LocalFileSystem';");
|
|
1278
1288
|
}
|
|
1279
|
-
|
|
1289
|
+
if (this.allowsPreAggregateFileReads) {
|
|
1290
|
+
await this.validatePreAggregateSql(db, sql);
|
|
1291
|
+
}
|
|
1292
|
+
else {
|
|
1293
|
+
await this.validateUserSql(db, sql);
|
|
1294
|
+
}
|
|
1280
1295
|
reportPhase?.('session', performance.now() - sessionStart);
|
|
1281
1296
|
const queryStart = performance.now();
|
|
1282
1297
|
const result = await db.stream(this.getSQLWithMetadata(sql, options?.tags), this.getBindValues(options));
|
|
@@ -7,6 +7,15 @@ const common_1 = require("@lightdash/common");
|
|
|
7
7
|
const promises_1 = __importDefault(require("fs/promises"));
|
|
8
8
|
const DuckdbWarehouseClient_1 = require("./DuckdbWarehouseClient");
|
|
9
9
|
const createInstanceMock = vi.fn();
|
|
10
|
+
const duckdbS3Credentials = {
|
|
11
|
+
type: 'duckdb_s3',
|
|
12
|
+
s3Config: {
|
|
13
|
+
endpoint: 'localhost:9000',
|
|
14
|
+
region: 'us-east-1',
|
|
15
|
+
forcePathStyle: true,
|
|
16
|
+
useSsl: false,
|
|
17
|
+
},
|
|
18
|
+
};
|
|
10
19
|
// Must provide DuckDBTypeId since mapFieldTypeFromTypeId references it at runtime
|
|
11
20
|
const DUCKDB_TYPE_IDS = {
|
|
12
21
|
BOOLEAN: 1,
|
|
@@ -186,7 +195,7 @@ describe('DuckdbWarehouseClient', () => {
|
|
|
186
195
|
const chunk2 = [{ id: 3 }];
|
|
187
196
|
const streamMock = vi.fn(async () => getMockStreamResult([chunk1, chunk2], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
188
197
|
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
|
189
|
-
const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient
|
|
198
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
190
199
|
const streamCallback = vi.fn();
|
|
191
200
|
const result = await client.executeAsyncQuery({
|
|
192
201
|
sql: 'SELECT id FROM t',
|
|
@@ -204,7 +213,7 @@ describe('DuckdbWarehouseClient', () => {
|
|
|
204
213
|
it('should handle empty result set', async () => {
|
|
205
214
|
const streamMock = vi.fn(async () => getMockStreamResult([], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
206
215
|
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
|
207
|
-
const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient
|
|
216
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
208
217
|
const result = await client.runQuery('SELECT id FROM empty_table');
|
|
209
218
|
expect(result.rows).toEqual([]);
|
|
210
219
|
expect(result.fields).toEqual({});
|
|
@@ -653,6 +662,61 @@ describe('DuckdbWarehouseClient', () => {
|
|
|
653
662
|
expect(extractStatementsMock).not.toHaveBeenCalled();
|
|
654
663
|
expect(streamMock).not.toHaveBeenCalled();
|
|
655
664
|
});
|
|
665
|
+
it('should keep file readers blocked when only S3 is configured', async () => {
|
|
666
|
+
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
667
|
+
const extractStatementsMock = createMockExtractStatements();
|
|
668
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, vi.fn(), {
|
|
669
|
+
extractStatements: extractStatementsMock,
|
|
670
|
+
}));
|
|
671
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(duckdbS3Credentials);
|
|
672
|
+
await expect(client.runQuery("SELECT * FROM read_parquet('s3://bucket/data.parquet')")).rejects.toThrow("SQL validation error: function 'read_parquet' is not allowed");
|
|
673
|
+
expect(extractStatementsMock).not.toHaveBeenCalled();
|
|
674
|
+
expect(streamMock).not.toHaveBeenCalled();
|
|
675
|
+
});
|
|
676
|
+
it.each([
|
|
677
|
+
{
|
|
678
|
+
format: 'Parquet',
|
|
679
|
+
sql: "SELECT * FROM read_parquet('s3://bucket/data.parquet')",
|
|
680
|
+
},
|
|
681
|
+
{
|
|
682
|
+
format: 'inferred JSONL',
|
|
683
|
+
sql: "SELECT * FROM read_json_auto('s3://bucket/data.jsonl')",
|
|
684
|
+
},
|
|
685
|
+
{
|
|
686
|
+
format: 'typed JSONL',
|
|
687
|
+
sql: "SELECT * FROM read_json('s3://bucket/data.jsonl', format='newline_delimited')",
|
|
688
|
+
},
|
|
689
|
+
])('should allow pre-aggregate clients to read S3 $format files', async ({ sql }) => {
|
|
690
|
+
const rows = [{ val: 1 }];
|
|
691
|
+
const streamMock = vi.fn(async () => getMockStreamResult([rows], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
692
|
+
const extractStatementsMock = createMockExtractStatements();
|
|
693
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, vi.fn(), {
|
|
694
|
+
extractStatements: extractStatementsMock,
|
|
695
|
+
}));
|
|
696
|
+
const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate(duckdbS3Credentials);
|
|
697
|
+
const result = await client.runQuery(sql);
|
|
698
|
+
expect(result.rows).toEqual(rows);
|
|
699
|
+
expect(extractStatementsMock).toHaveBeenCalledTimes(1);
|
|
700
|
+
expect(streamMock).toHaveBeenCalledTimes(1);
|
|
701
|
+
});
|
|
702
|
+
it('should keep pre-aggregate queries read-only', async () => {
|
|
703
|
+
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
704
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, vi.fn(), {
|
|
705
|
+
extractStatements: createMockExtractStatements({
|
|
706
|
+
statementType: 11, // COPY
|
|
707
|
+
}),
|
|
708
|
+
}));
|
|
709
|
+
const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate(duckdbS3Credentials);
|
|
710
|
+
await expect(client.runQuery("COPY t TO 's3://bucket/data.parquet'")).rejects.toThrow('SQL validation error: only SELECT statements are allowed');
|
|
711
|
+
expect(streamMock).not.toHaveBeenCalled();
|
|
712
|
+
});
|
|
713
|
+
it('should keep secret introspection blocked for pre-aggregate queries', async () => {
|
|
714
|
+
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
715
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
|
716
|
+
const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate(duckdbS3Credentials);
|
|
717
|
+
await expect(client.runQuery("SELECT current_setting('s3_secret_access_key')")).rejects.toThrow("SQL validation error: function 'current_setting' is not allowed");
|
|
718
|
+
expect(streamMock).not.toHaveBeenCalled();
|
|
719
|
+
});
|
|
656
720
|
it('should reject user queries that use file table paths', async () => {
|
|
657
721
|
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
658
722
|
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.56.0",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
"snowflake-sdk": "2.3.4",
|
|
30
30
|
"ssh2": "1.14.0",
|
|
31
31
|
"trino-client": "0.2.9",
|
|
32
|
-
"@lightdash/common": "1.
|
|
32
|
+
"@lightdash/common": "1.56.0"
|
|
33
33
|
},
|
|
34
34
|
"devDependencies": {
|
|
35
35
|
"@types/big.js": "6.2.2",
|