@lightdash/warehouses 2.177.0 → 2.179.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts +20 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.js +88 -3
- package/dist/warehouseClients/DuckdbWarehouseClient.test.js +77 -0
- package/package.json +2 -2
|
@@ -35,7 +35,24 @@ export type DuckdbS3Credentials = {
|
|
|
35
35
|
type: 'duckdb_s3';
|
|
36
36
|
s3Config: DuckdbS3SessionConfig;
|
|
37
37
|
};
|
|
38
|
-
|
|
38
|
+
/** Server-owned manifest. Never accept this configuration from project APIs. */
|
|
39
|
+
export type DuckdbParquetSource = {
|
|
40
|
+
scope: string;
|
|
41
|
+
tables: {
|
|
42
|
+
name: string;
|
|
43
|
+
urls: string[];
|
|
44
|
+
}[];
|
|
45
|
+
httpAuth?: {
|
|
46
|
+
bearerToken: string;
|
|
47
|
+
};
|
|
48
|
+
s3Config?: DuckdbS3SessionConfig;
|
|
49
|
+
};
|
|
50
|
+
export type DuckdbParquetCredentials = {
|
|
51
|
+
type: 'duckdb_parquet';
|
|
52
|
+
/** Resolve again for every session so new files and refreshed credentials are visible. */
|
|
53
|
+
resolveSource: () => Promise<DuckdbParquetSource>;
|
|
54
|
+
};
|
|
55
|
+
export type DuckdbConnectionCredentials = DuckdbS3Credentials | DuckdbParquetCredentials;
|
|
39
56
|
export type DuckdbWarehouseClientOptions = {
|
|
40
57
|
/** Resource-constrained isolated sessions, used for materialization/parquet conversion and embedded databases. */
|
|
41
58
|
resourceLimits?: DuckdbResourceLimits;
|
|
@@ -76,6 +93,7 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
76
93
|
private static readonly sqlBuilder;
|
|
77
94
|
private readonly databasePath;
|
|
78
95
|
private readonly s3Config?;
|
|
96
|
+
private readonly parquetConfig?;
|
|
79
97
|
private readonly ducklakeConfig?;
|
|
80
98
|
private readonly embeddedConfig?;
|
|
81
99
|
private readonly resourceLimits?;
|
|
@@ -105,6 +123,7 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
105
123
|
private loadExtension;
|
|
106
124
|
private loadAwsExtensionForCredentialChain;
|
|
107
125
|
private static bootstrapQuerySession;
|
|
126
|
+
private bootstrapParquetViews;
|
|
108
127
|
private static bootstrapSharedInstance;
|
|
109
128
|
private static getOrCreateSharedInstance;
|
|
110
129
|
private static clearSharedInstance;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAMA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAMb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAGhB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAQ3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA6DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;IAChB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACpE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAMA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAMb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAGhB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAQ3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA6DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;IAChB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACpE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,gFAAgF;AAChF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;IAC3C,QAAQ,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IACnC,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACnC,IAAI,EAAE,gBAAgB,CAAC;IACvB,0FAA0F;IAC1F,aAAa,EAAE,MAAM,OAAO,CAAC,mBAAmB,CAAC,CAAC;CACrD,CAAC;AAEF,MAAM,MAAM,2BAA2B,GACjC,mBAAmB,GACnB,wBAAwB,CAAC;AAE/B,MAAM,MAAM,4BAA4B,GAAG;IACvC,kHAAkH;IAClH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;IAC9D,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,4BAA4B,CAAC,EAAE,MAAM,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,sBAAsB,WAAY,MAAM,KAAG,aA+BvD,CAAC;AAyBF,eAAO,MAAM,kCAAkC,aACjC,MAAM,KACjB,eAAe,GAAG,SAOpB,CAAC;AAoCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB,CAEpC;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AAmID,eAAO,MAAM,+BAA+B,yBAGzC,IAAI,CAAC,iCAAiC,EAAE,UAAU,GAAG,OAAO,CAAC,KAAG,MAOlE,CAAC;AAwFF,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,iCAAiC,CAAC;IAC7F,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAGhD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAE/C;IAEF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sCAAsC,CAG1D;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,8BAA8B,CAGlD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAA2B;IAE1D,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAU;IAE/C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;IAEV,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAEhD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAU;IAE9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,4BAA4B,CAAC,CAAS;IAEvD,OAAO,CAAC,2BAA2B,CAAS;IAE5C,OAAO,CAAC,gCAAgC,CAAS;IAEjD,YACI,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B,EA4IzC;IAED,OAAO,CAAC,MAAM,CAAC,kBAAkB;IASjC,MAAM,CAAC,qBAAqB,CACxB,WAAW,EAAE,mBAAmB,EAChC,OAAO,CAAC,EAAE,4BAA4B,GACvC,qBAAqB,CAIvB;IAED,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAoBzC,OAAO,CAAC,2BAA2B;mBAUd,6BAA6B;IA4DlD,OAAO,CAAC,MAAM,CAAC,iCAAiC;IAoBhD,OAAO,CAAC,kBAAkB;mBAQL,cAAc;mBAOd,mBAAmB;IAaxC,OAAO,CAAC,MAAM,CAAC,qBAAqB;mBAMf,uBAAuB;YAsB9B,aAAa;YAoBb,kCAAkC;mBAW3B,qBAAqB;YAwF5B,qBAAqB;mBAkFd,uBAAuB;mBAgBvB,yBAAyB;mBAiFzB,mBAAmB;IA6BxC,mFAAmF;IACnF,MAAM,CAAC,gCAAgC,IAAI,IAAI,CAS9C;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAO3B;IAED,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CACZ;IAEnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAA+B;IAE3E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAA0B;IAEjE,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAIjC,OAAO,CAAC,MAAM,CAAC,UAAU;IAIzB,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAuB5C,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAqGzC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAMhC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAmBrC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IA2CrC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IA2C/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAK;YAE9C,gBAAgB;IA+E9B,iEAAiE;YACnD,wBAAwB;IA+CtC,gFAAgF;YAClE,mBAAmB;YAkDnB,yBAAyB;YA4CzB,iBAAiB;IA2C/B,OAAO,CAAC,iBAAiB;IAMzB,OAAO,CAAC,YAAY;YAQN,WAAW;YAwEX,2BAA2B;IA4GzC,+DAA+D;YACjD,iBAAiB;YA8EjB,yBAAyB;IA2BvC,OAAO,CAAC,4BAA4B;IAOpC,OAAO,CAAC,aAAa;YA0BP,eAAe;IAgG7B,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAgBxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IA+D/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;IAYnC,MAAM,CAAC,yBAAyB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAiBlD;YAEa,iBAAiB;YA4BjB,eAAe;YASf,uBAAuB;YAQvB,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CAyFf;IAEK,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,mBAAmB,CAAC,CAC9E;;;;;;OA4BJ;IAEK,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKvC;IAEK,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC,CAkBD;IAEK,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,UAAU,CAAC,CACrE;;;OAGJ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAEK,UAAU,CACZ,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC,CA4C3B;IAEK,YAAY,CACd,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN,CAqBA;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC,CAS3B;CACJ"}
|
|
@@ -332,6 +332,11 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
332
332
|
const isS3Only = credentials &&
|
|
333
333
|
'type' in credentials &&
|
|
334
334
|
credentials.type === 'duckdb_s3';
|
|
335
|
+
const isParquet = credentials?.type === 'duckdb_parquet';
|
|
336
|
+
if (credentials?.type === common_1.WarehouseTypes.DUCKDB &&
|
|
337
|
+
credentials.connectionType === common_1.DuckdbConnectionType.ANALYTICS) {
|
|
338
|
+
throw new common_1.ParameterError('Analytics connections must be resolved by the internal project service');
|
|
339
|
+
}
|
|
335
340
|
const isDucklake = !isS3Only &&
|
|
336
341
|
credentials &&
|
|
337
342
|
'type' in credentials &&
|
|
@@ -343,7 +348,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
343
348
|
credentials.type === common_1.WarehouseTypes.DUCKDB &&
|
|
344
349
|
credentials.connectionType === common_1.DuckdbConnectionType.EMBEDDED;
|
|
345
350
|
let effectiveCredentials;
|
|
346
|
-
if (isS3Only) {
|
|
351
|
+
if (isS3Only || isParquet) {
|
|
347
352
|
effectiveCredentials = DUCKDB_INTERNAL_CREDENTIALS;
|
|
348
353
|
}
|
|
349
354
|
else if (isDucklake) {
|
|
@@ -377,6 +382,12 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
377
382
|
if (isS3Only) {
|
|
378
383
|
this.s3Config = credentials.s3Config;
|
|
379
384
|
}
|
|
385
|
+
if (isParquet) {
|
|
386
|
+
this.parquetConfig = credentials;
|
|
387
|
+
if (options?.instanceCacheKey || options?.enableInstanceCache) {
|
|
388
|
+
throw new common_1.ParameterError('Parquet project sessions cannot share a DuckDB instance');
|
|
389
|
+
}
|
|
390
|
+
}
|
|
380
391
|
if (isDucklake) {
|
|
381
392
|
this.ducklakeConfig =
|
|
382
393
|
credentials;
|
|
@@ -410,7 +421,13 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
410
421
|
...options?.resourceLimits,
|
|
411
422
|
}
|
|
412
423
|
: options?.resourceLimits;
|
|
413
|
-
this.sharedResourceLimits =
|
|
424
|
+
this.sharedResourceLimits = isParquet
|
|
425
|
+
? {
|
|
426
|
+
memoryLimit: '256MB',
|
|
427
|
+
threads: 2,
|
|
428
|
+
...options?.sharedResourceLimits,
|
|
429
|
+
}
|
|
430
|
+
: options?.sharedResourceLimits;
|
|
414
431
|
// DuckLake attaches a postgres catalog secret on every fresh DuckDB
|
|
415
432
|
// instance, and the postgres extension only pools 8 connections per
|
|
416
433
|
// instance — so parallel getFields() calls during project compile
|
|
@@ -422,7 +439,8 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
422
439
|
this.instanceCacheKey =
|
|
423
440
|
options?.instanceCacheKey ?? ducklakeAutoCacheKey;
|
|
424
441
|
this.logger = options?.logger;
|
|
425
|
-
this.enableQueryProfiling =
|
|
442
|
+
this.enableQueryProfiling =
|
|
443
|
+
!isParquet && (options?.enableQueryProfiling ?? false);
|
|
426
444
|
this.onQueryProfile = options?.onQueryProfile;
|
|
427
445
|
this.embeddedQueryTimeoutMs =
|
|
428
446
|
options?.embeddedQueryTimeoutMs ?? EMBEDDED_QUERY_TIMEOUT_MS;
|
|
@@ -593,6 +611,9 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
593
611
|
if (client.s3Config) {
|
|
594
612
|
await db.run(DuckdbWarehouseClient.buildS3SecretSql(client.s3Config));
|
|
595
613
|
}
|
|
614
|
+
if (client.parquetConfig) {
|
|
615
|
+
await client.bootstrapParquetViews(db);
|
|
616
|
+
}
|
|
596
617
|
if (client.ducklakeConfig) {
|
|
597
618
|
const stmts = DuckdbWarehouseClient.buildDucklakeAttachSql(client.ducklakeConfig);
|
|
598
619
|
// eslint-disable-next-line no-restricted-syntax
|
|
@@ -618,6 +639,67 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
618
639
|
httpfsMs,
|
|
619
640
|
};
|
|
620
641
|
}
|
|
642
|
+
async bootstrapParquetViews(db) {
|
|
643
|
+
const source = await this.parquetConfig.resolveSource();
|
|
644
|
+
const escape = DuckdbWarehouseClient.escapeDuckdbString;
|
|
645
|
+
const literal = (value) => `'${escape(value)}'`;
|
|
646
|
+
const scope = new URL(source.scope);
|
|
647
|
+
if (!['https:', 's3:'].includes(scope.protocol) ||
|
|
648
|
+
scope.username ||
|
|
649
|
+
scope.password ||
|
|
650
|
+
scope.search ||
|
|
651
|
+
scope.hash ||
|
|
652
|
+
!source.scope.endsWith('/') ||
|
|
653
|
+
scope.pathname === '/') {
|
|
654
|
+
throw new common_1.ParameterError('Parquet source requires a scoped remote prefix');
|
|
655
|
+
}
|
|
656
|
+
const names = new Set();
|
|
657
|
+
source.tables.forEach(({ name, urls }) => {
|
|
658
|
+
if (!/^[a-z][a-z0-9_]*$/.test(name) ||
|
|
659
|
+
names.has(name) ||
|
|
660
|
+
urls.length === 0) {
|
|
661
|
+
throw new common_1.ParameterError('Parquet tables require unique names and a non-empty file manifest');
|
|
662
|
+
}
|
|
663
|
+
names.add(name);
|
|
664
|
+
urls.forEach((url) => {
|
|
665
|
+
const parsed = new URL(url);
|
|
666
|
+
if (parsed.href !== url ||
|
|
667
|
+
!url.startsWith(source.scope) ||
|
|
668
|
+
!url.endsWith('.parquet') ||
|
|
669
|
+
parsed.search ||
|
|
670
|
+
parsed.hash ||
|
|
671
|
+
/[%*?[\]{}]/.test(parsed.pathname)) {
|
|
672
|
+
throw new common_1.ParameterError('Parquet file is outside the trusted source prefix');
|
|
673
|
+
}
|
|
674
|
+
});
|
|
675
|
+
});
|
|
676
|
+
if (source.httpAuth) {
|
|
677
|
+
if (scope.protocol !== 'https:') {
|
|
678
|
+
throw new common_1.ParameterError('HTTP credentials require an HTTPS source');
|
|
679
|
+
}
|
|
680
|
+
await db.run(`CREATE SECRET analytics_http (TYPE http, BEARER_TOKEN ${literal(source.httpAuth.bearerToken)}, SCOPE ${literal(source.scope)});`);
|
|
681
|
+
}
|
|
682
|
+
if (source.s3Config) {
|
|
683
|
+
if (DuckdbWarehouseClient.usesS3CredentialChain(source.s3Config)) {
|
|
684
|
+
await this.loadExtension(db, 'aws');
|
|
685
|
+
}
|
|
686
|
+
await db.run(DuckdbWarehouseClient.buildS3SecretSql({
|
|
687
|
+
...source.s3Config,
|
|
688
|
+
scope: [source.scope],
|
|
689
|
+
}));
|
|
690
|
+
}
|
|
691
|
+
// Restrict the engine, not just SQL validation. No globbing, arbitrary
|
|
692
|
+
// network reads, local files, or shared spill/cache directories.
|
|
693
|
+
await db.run("SET temp_directory = '';");
|
|
694
|
+
const files = source.tables.flatMap(({ urls }) => urls);
|
|
695
|
+
await db.run(`SET allowed_paths = [${files.map(literal).join(',')}];`);
|
|
696
|
+
await db.run('SET enable_external_access = false;');
|
|
697
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
698
|
+
for (const { name, urls } of source.tables) {
|
|
699
|
+
// eslint-disable-next-line no-await-in-loop
|
|
700
|
+
await db.run(`CREATE VIEW "${name}" AS SELECT * FROM read_parquet([${urls.map(literal).join(',')}], hive_partitioning = true, union_by_name = true);`);
|
|
701
|
+
}
|
|
702
|
+
}
|
|
621
703
|
static async bootstrapSharedInstance(instance, client) {
|
|
622
704
|
const db = await instance.connect();
|
|
623
705
|
try {
|
|
@@ -1120,6 +1202,9 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1120
1202
|
throw new common_1.WarehouseQueryError('External source query capacity is full. Try again shortly.');
|
|
1121
1203
|
}
|
|
1122
1204
|
try {
|
|
1205
|
+
if (this.parquetConfig) {
|
|
1206
|
+
return await this.withEphemeralQuerySession(callback);
|
|
1207
|
+
}
|
|
1123
1208
|
if (this.embeddedConfig) {
|
|
1124
1209
|
return await this.withDirectSession(callback, organizationUuid, onPhaseTiming);
|
|
1125
1210
|
}
|
|
@@ -142,6 +142,83 @@ const createMockConnection = (streamMock, runMock = vi.fn(), opts) => ({
|
|
|
142
142
|
}),
|
|
143
143
|
closeSync: vi.fn(),
|
|
144
144
|
});
|
|
145
|
+
describe('internal Parquet projects', () => {
|
|
146
|
+
const scope = 'https://storage.googleapis.com/example/events/compacted/org_id=org-a/';
|
|
147
|
+
const url = `${scope}stream=query_events/dt=2026-09-07/part.parquet`;
|
|
148
|
+
const source = () => ({
|
|
149
|
+
scope,
|
|
150
|
+
httpAuth: { bearerToken: 'test-token' },
|
|
151
|
+
tables: [{ name: 'query_events', urls: [url] }],
|
|
152
|
+
});
|
|
153
|
+
let run;
|
|
154
|
+
beforeEach(() => {
|
|
155
|
+
vi.clearAllMocks();
|
|
156
|
+
run = vi.fn();
|
|
157
|
+
createInstanceMock.mockResolvedValue(createMockConnection(vi.fn(async () => getMockStreamResult([[{ count: 2 }]], [5])), run));
|
|
158
|
+
});
|
|
159
|
+
it('binds trusted views, restricts external access and refreshes the manifest each session', async () => {
|
|
160
|
+
const resolveSource = vi.fn(async () => source());
|
|
161
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
162
|
+
type: 'duckdb_parquet',
|
|
163
|
+
resolveSource,
|
|
164
|
+
});
|
|
165
|
+
await client.runQuery('SELECT count(*) FROM query_events');
|
|
166
|
+
await client.runQuery('SELECT count(*) FROM query_events');
|
|
167
|
+
expect(resolveSource).toHaveBeenCalledTimes(2);
|
|
168
|
+
expect(createInstanceMock).toHaveBeenCalledTimes(2);
|
|
169
|
+
expect(run).toHaveBeenCalledWith(`SET allowed_paths = ['${url}'];`);
|
|
170
|
+
expect(run).toHaveBeenCalledWith('SET enable_external_access = false;');
|
|
171
|
+
expect(run).toHaveBeenCalledWith("SET temp_directory = '';");
|
|
172
|
+
expect(run).toHaveBeenCalledWith(expect.stringContaining('CREATE VIEW "query_events" AS SELECT * FROM read_parquet'));
|
|
173
|
+
expect(client.credentials).not.toHaveProperty('httpAuth');
|
|
174
|
+
});
|
|
175
|
+
it.each([
|
|
176
|
+
'https://storage.googleapis.com/example/events/compacted/org_id=org-b/file.parquet',
|
|
177
|
+
`${scope}../org-b/file.parquet`,
|
|
178
|
+
`${scope}%2e%2e/org-b/file.parquet`,
|
|
179
|
+
`${scope}*.parquet`,
|
|
180
|
+
'/tmp/private.parquet',
|
|
181
|
+
])('rejects files outside the exact manifest scope: %s', async (file) => {
|
|
182
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
183
|
+
type: 'duckdb_parquet',
|
|
184
|
+
resolveSource: async () => ({
|
|
185
|
+
...source(),
|
|
186
|
+
tables: [{ name: 'query_events', urls: [file] }],
|
|
187
|
+
}),
|
|
188
|
+
});
|
|
189
|
+
await expect(client.runQuery('SELECT * FROM query_events')).rejects.toThrow();
|
|
190
|
+
expect(run).not.toHaveBeenCalledWith(expect.stringContaining('CREATE SECRET'));
|
|
191
|
+
});
|
|
192
|
+
it.each([
|
|
193
|
+
{ name: 'x"; SELECT 1; --', urls: [url] },
|
|
194
|
+
{ name: 'query_events', urls: [] },
|
|
195
|
+
])('rejects arbitrary table SQL and empty manifests: $name', async (table) => {
|
|
196
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
197
|
+
type: 'duckdb_parquet',
|
|
198
|
+
resolveSource: async () => ({ ...source(), tables: [table] }),
|
|
199
|
+
});
|
|
200
|
+
await expect(client.runQuery('SELECT 1')).rejects.toThrow(/unique names/);
|
|
201
|
+
});
|
|
202
|
+
it('keeps user-supplied read_parquet forbidden', async () => {
|
|
203
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
204
|
+
type: 'duckdb_parquet',
|
|
205
|
+
resolveSource: async () => source(),
|
|
206
|
+
});
|
|
207
|
+
await expect(client.runQuery(`SELECT * FROM read_parquet('${url}')`)).rejects.toThrow(/not allowed/);
|
|
208
|
+
});
|
|
209
|
+
it('cannot create privileged readers from public project credentials or shared instances', () => {
|
|
210
|
+
expect(() => new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
211
|
+
type: common_1.WarehouseTypes.DUCKDB,
|
|
212
|
+
connectionType: common_1.DuckdbConnectionType.ANALYTICS,
|
|
213
|
+
database: 'memory',
|
|
214
|
+
schema: 'main',
|
|
215
|
+
})).toThrow(/internal project service/);
|
|
216
|
+
expect(() => new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
217
|
+
type: 'duckdb_parquet',
|
|
218
|
+
resolveSource: async () => source(),
|
|
219
|
+
}, { instanceCacheKey: 'shared' })).toThrow(/cannot share/);
|
|
220
|
+
});
|
|
221
|
+
});
|
|
145
222
|
describe('mapFieldTypeFromTypeId', () => {
|
|
146
223
|
it('should map date types', () => {
|
|
147
224
|
expect((0, DuckdbWarehouseClient_1.mapFieldTypeFromTypeId)(DUCKDB_TYPE_IDS.DATE)).toBe(common_1.DimensionType.DATE);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.179.0",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
"snowflake-sdk": "2.3.4",
|
|
29
29
|
"ssh2": "1.14.0",
|
|
30
30
|
"trino-client": "0.2.9",
|
|
31
|
-
"@lightdash/common": "2.
|
|
31
|
+
"@lightdash/common": "2.179.0"
|
|
32
32
|
},
|
|
33
33
|
"devDependencies": {
|
|
34
34
|
"oxlint": "1.75.0",
|