@lightdash/warehouses 2.178.0 → 2.180.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts +22 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.js +115 -3
- package/dist/warehouseClients/DuckdbWarehouseClient.test.js +113 -0
- package/package.json +2 -2
|
@@ -35,7 +35,26 @@ export type DuckdbS3Credentials = {
|
|
|
35
35
|
type: 'duckdb_s3';
|
|
36
36
|
s3Config: DuckdbS3SessionConfig;
|
|
37
37
|
};
|
|
38
|
-
|
|
38
|
+
/** Server-owned manifest. Never accept this configuration from project APIs. */
|
|
39
|
+
export type DuckdbParquetSource = {
|
|
40
|
+
scope: string;
|
|
41
|
+
tables: {
|
|
42
|
+
name: string;
|
|
43
|
+
urls: string[];
|
|
44
|
+
}[];
|
|
45
|
+
/** Exact server-signed GET URLs; never combine with bucket credentials. */
|
|
46
|
+
signedUrls?: boolean;
|
|
47
|
+
httpAuth?: {
|
|
48
|
+
bearerToken: string;
|
|
49
|
+
};
|
|
50
|
+
s3Config?: DuckdbS3SessionConfig;
|
|
51
|
+
};
|
|
52
|
+
export type DuckdbParquetCredentials = {
|
|
53
|
+
type: 'duckdb_parquet';
|
|
54
|
+
/** Resolve again for every session so new files and refreshed credentials are visible. */
|
|
55
|
+
resolveSource: () => Promise<DuckdbParquetSource>;
|
|
56
|
+
};
|
|
57
|
+
export type DuckdbConnectionCredentials = DuckdbS3Credentials | DuckdbParquetCredentials;
|
|
39
58
|
export type DuckdbWarehouseClientOptions = {
|
|
40
59
|
/** Resource-constrained isolated sessions, used for materialization/parquet conversion and embedded databases. */
|
|
41
60
|
resourceLimits?: DuckdbResourceLimits;
|
|
@@ -76,6 +95,7 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
76
95
|
private static readonly sqlBuilder;
|
|
77
96
|
private readonly databasePath;
|
|
78
97
|
private readonly s3Config?;
|
|
98
|
+
private readonly parquetConfig?;
|
|
79
99
|
private readonly ducklakeConfig?;
|
|
80
100
|
private readonly embeddedConfig?;
|
|
81
101
|
private readonly resourceLimits?;
|
|
@@ -105,6 +125,7 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
105
125
|
private loadExtension;
|
|
106
126
|
private loadAwsExtensionForCredentialChain;
|
|
107
127
|
private static bootstrapQuerySession;
|
|
128
|
+
private bootstrapParquetViews;
|
|
108
129
|
private static bootstrapSharedInstance;
|
|
109
130
|
private static getOrCreateSharedInstance;
|
|
110
131
|
private static clearSharedInstance;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAMA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAMb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAGhB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAQ3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA6DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;IAChB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACpE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAMA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAMb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAGhB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAQ3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA6DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;IAChB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACpE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,gFAAgF;AAChF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;IAC3C,2EAA2E;IAC3E,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IACnC,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACnC,IAAI,EAAE,gBAAgB,CAAC;IACvB,0FAA0F;IAC1F,aAAa,EAAE,MAAM,OAAO,CAAC,mBAAmB,CAAC,CAAC;CACrD,CAAC;AAEF,MAAM,MAAM,2BAA2B,GACjC,mBAAmB,GACnB,wBAAwB,CAAC;AAE/B,MAAM,MAAM,4BAA4B,GAAG;IACvC,kHAAkH;IAClH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;IAC9D,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,4BAA4B,CAAC,EAAE,MAAM,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,sBAAsB,WAAY,MAAM,KAAG,aA+BvD,CAAC;AAyBF,eAAO,MAAM,kCAAkC,aACjC,MAAM,KACjB,eAAe,GAAG,SAOpB,CAAC;AAoCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB,CAEpC;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AAmID,eAAO,MAAM,+BAA+B,yBAGzC,IAAI,CAAC,iCAAiC,EAAE,UAAU,GAAG,OAAO,CAAC,KAAG,MAOlE,CAAC;AAwFF,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,iCAAiC,CAAC;IAC7F,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAGhD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAE/C;IAEF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sCAAsC,CAG1D;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,8BAA8B,CAGlD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAA2B;IAE1D,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAU;IAE/C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;IAEV,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAEhD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAU;IAE9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,4BAA4B,CAAC,CAAS;IAEvD,OAAO,CAAC,2BAA2B,CAAS;IAE5C,OAAO,CAAC,gCAAgC,CAAS;IAEjD,YACI,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B,EA4IzC;IAED,OAAO,CAAC,MAAM,CAAC,kBAAkB;IASjC,MAAM,CAAC,qBAAqB,CACxB,WAAW,EAAE,mBAAmB,EAChC,OAAO,CAAC,EAAE,4BAA4B,GACvC,qBAAqB,CAIvB;IAED,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAoBzC,OAAO,CAAC,2BAA2B;mBAUd,6BAA6B;IA4DlD,OAAO,CAAC,MAAM,CAAC,iCAAiC;IAoBhD,OAAO,CAAC,kBAAkB;mBAQL,cAAc;mBAOd,mBAAmB;IAaxC,OAAO,CAAC,MAAM,CAAC,qBAAqB;mBAMf,uBAAuB;YAsB9B,aAAa;YAoBb,kCAAkC;mBAW3B,qBAAqB;YAwF5B,qBAAqB;mBAuGd,uBAAuB;mBAgBvB,yBAAyB;mBAiFzB,mBAAmB;IA6BxC,mFAAmF;IACnF,MAAM,CAAC,gCAAgC,IAAI,IAAI,CAS9C;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAO3B;IAED,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CACZ;IAEnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAA+B;IAE3E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAA0B;IAEjE,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAIjC,OAAO,CAAC,MAAM,CAAC,UAAU;IAIzB,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAuB5C,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAqGzC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAMhC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAmBrC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IA2CrC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IA2C/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAK;YAE9C,gBAAgB;IA+E9B,iEAAiE;YACnD,wBAAwB;IA+CtC,gFAAgF;YAClE,mBAAmB;YAkDnB,yBAAyB;YA4CzB,iBAAiB;IA2C/B,OAAO,CAAC,iBAAiB;IAMzB,OAAO,CAAC,YAAY;YAQN,WAAW;YA+EX,2BAA2B;IA4GzC,+DAA+D;YACjD,iBAAiB;YA8EjB,yBAAyB;IA2BvC,OAAO,CAAC,4BAA4B;IAOpC,OAAO,CAAC,aAAa;YA0BP,eAAe;IAgG7B,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAgBxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IA+D/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;IAYnC,MAAM,CAAC,yBAAyB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAiBlD;YAEa,iBAAiB;YA4BjB,eAAe;YAmBf,uBAAuB;YAQvB,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CAyFf;IAEK,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,mBAAmB,CAAC,CAC9E;;;;;;OA4BJ;IAEK,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKvC;IAEK,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC,CAkBD;IAEK,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,UAAU,CAAC,CACrE;;;OAGJ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAEK,UAAU,CACZ,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC,CA4C3B;IAEK,YAAY,CACd,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN,CAqBA;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC,CAS3B;CACJ"}
|
|
@@ -332,6 +332,11 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
332
332
|
const isS3Only = credentials &&
|
|
333
333
|
'type' in credentials &&
|
|
334
334
|
credentials.type === 'duckdb_s3';
|
|
335
|
+
const isParquet = credentials?.type === 'duckdb_parquet';
|
|
336
|
+
if (credentials?.type === common_1.WarehouseTypes.DUCKDB &&
|
|
337
|
+
credentials.connectionType === common_1.DuckdbConnectionType.ANALYTICS) {
|
|
338
|
+
throw new common_1.ParameterError('Analytics connections must be resolved by the internal project service');
|
|
339
|
+
}
|
|
335
340
|
const isDucklake = !isS3Only &&
|
|
336
341
|
credentials &&
|
|
337
342
|
'type' in credentials &&
|
|
@@ -343,7 +348,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
343
348
|
credentials.type === common_1.WarehouseTypes.DUCKDB &&
|
|
344
349
|
credentials.connectionType === common_1.DuckdbConnectionType.EMBEDDED;
|
|
345
350
|
let effectiveCredentials;
|
|
346
|
-
if (isS3Only) {
|
|
351
|
+
if (isS3Only || isParquet) {
|
|
347
352
|
effectiveCredentials = DUCKDB_INTERNAL_CREDENTIALS;
|
|
348
353
|
}
|
|
349
354
|
else if (isDucklake) {
|
|
@@ -377,6 +382,12 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
377
382
|
if (isS3Only) {
|
|
378
383
|
this.s3Config = credentials.s3Config;
|
|
379
384
|
}
|
|
385
|
+
if (isParquet) {
|
|
386
|
+
this.parquetConfig = credentials;
|
|
387
|
+
if (options?.instanceCacheKey || options?.enableInstanceCache) {
|
|
388
|
+
throw new common_1.ParameterError('Parquet project sessions cannot share a DuckDB instance');
|
|
389
|
+
}
|
|
390
|
+
}
|
|
380
391
|
if (isDucklake) {
|
|
381
392
|
this.ducklakeConfig =
|
|
382
393
|
credentials;
|
|
@@ -410,7 +421,13 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
410
421
|
...options?.resourceLimits,
|
|
411
422
|
}
|
|
412
423
|
: options?.resourceLimits;
|
|
413
|
-
this.sharedResourceLimits =
|
|
424
|
+
this.sharedResourceLimits = isParquet
|
|
425
|
+
? {
|
|
426
|
+
memoryLimit: '256MB',
|
|
427
|
+
threads: 2,
|
|
428
|
+
...options?.sharedResourceLimits,
|
|
429
|
+
}
|
|
430
|
+
: options?.sharedResourceLimits;
|
|
414
431
|
// DuckLake attaches a postgres catalog secret on every fresh DuckDB
|
|
415
432
|
// instance, and the postgres extension only pools 8 connections per
|
|
416
433
|
// instance — so parallel getFields() calls during project compile
|
|
@@ -422,7 +439,8 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
422
439
|
this.instanceCacheKey =
|
|
423
440
|
options?.instanceCacheKey ?? ducklakeAutoCacheKey;
|
|
424
441
|
this.logger = options?.logger;
|
|
425
|
-
this.enableQueryProfiling =
|
|
442
|
+
this.enableQueryProfiling =
|
|
443
|
+
!isParquet && (options?.enableQueryProfiling ?? false);
|
|
426
444
|
this.onQueryProfile = options?.onQueryProfile;
|
|
427
445
|
this.embeddedQueryTimeoutMs =
|
|
428
446
|
options?.embeddedQueryTimeoutMs ?? EMBEDDED_QUERY_TIMEOUT_MS;
|
|
@@ -593,6 +611,9 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
593
611
|
if (client.s3Config) {
|
|
594
612
|
await db.run(DuckdbWarehouseClient.buildS3SecretSql(client.s3Config));
|
|
595
613
|
}
|
|
614
|
+
if (client.parquetConfig) {
|
|
615
|
+
await client.bootstrapParquetViews(db);
|
|
616
|
+
}
|
|
596
617
|
if (client.ducklakeConfig) {
|
|
597
618
|
const stmts = DuckdbWarehouseClient.buildDucklakeAttachSql(client.ducklakeConfig);
|
|
598
619
|
// eslint-disable-next-line no-restricted-syntax
|
|
@@ -618,6 +639,83 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
618
639
|
httpfsMs,
|
|
619
640
|
};
|
|
620
641
|
}
|
|
642
|
+
async bootstrapParquetViews(db) {
|
|
643
|
+
const source = await this.parquetConfig.resolveSource();
|
|
644
|
+
const escape = DuckdbWarehouseClient.escapeDuckdbString;
|
|
645
|
+
const literal = (value) => `'${escape(value)}'`;
|
|
646
|
+
const scope = new URL(source.scope);
|
|
647
|
+
const localSignedSource = source.signedUrls &&
|
|
648
|
+
scope.protocol === 'http:' &&
|
|
649
|
+
['localhost', '127.0.0.1', '[::1]'].includes(scope.hostname);
|
|
650
|
+
if ((!['https:', 's3:'].includes(scope.protocol) &&
|
|
651
|
+
!localSignedSource) ||
|
|
652
|
+
scope.username ||
|
|
653
|
+
scope.password ||
|
|
654
|
+
scope.search ||
|
|
655
|
+
scope.hash ||
|
|
656
|
+
!source.scope.endsWith('/') ||
|
|
657
|
+
scope.pathname === '/') {
|
|
658
|
+
throw new common_1.ParameterError('Parquet source requires a scoped remote prefix');
|
|
659
|
+
}
|
|
660
|
+
if (source.signedUrls &&
|
|
661
|
+
(source.httpAuth || source.s3Config || scope.protocol === 's3:')) {
|
|
662
|
+
throw new common_1.ParameterError('Signed Parquet sources cannot carry bucket credentials');
|
|
663
|
+
}
|
|
664
|
+
const names = new Set();
|
|
665
|
+
source.tables.forEach(({ name, urls }) => {
|
|
666
|
+
if (!/^[a-z][a-z0-9_]*$/.test(name) ||
|
|
667
|
+
names.has(name) ||
|
|
668
|
+
urls.length === 0) {
|
|
669
|
+
throw new common_1.ParameterError('Parquet tables require unique names and a non-empty file manifest');
|
|
670
|
+
}
|
|
671
|
+
names.add(name);
|
|
672
|
+
urls.forEach((url) => {
|
|
673
|
+
const parsed = new URL(url);
|
|
674
|
+
if (parsed.href !== url ||
|
|
675
|
+
!url.startsWith(source.scope) ||
|
|
676
|
+
!parsed.pathname.endsWith('.parquet') ||
|
|
677
|
+
(!source.signedUrls && parsed.search) ||
|
|
678
|
+
(source.signedUrls &&
|
|
679
|
+
(!parsed.searchParams.has('X-Amz-Signature') ||
|
|
680
|
+
!/^[a-zA-Z0-9_./=%-]+$/.test(parsed.pathname) ||
|
|
681
|
+
/%(?!3D)/i.test(parsed.pathname))) ||
|
|
682
|
+
parsed.hash ||
|
|
683
|
+
/[*?[\]{}]/.test(parsed.pathname) ||
|
|
684
|
+
(!source.signedUrls && parsed.pathname.includes('%'))) {
|
|
685
|
+
throw new common_1.ParameterError('Parquet file is outside the trusted source prefix');
|
|
686
|
+
}
|
|
687
|
+
});
|
|
688
|
+
});
|
|
689
|
+
if (source.httpAuth) {
|
|
690
|
+
if (scope.protocol !== 'https:') {
|
|
691
|
+
throw new common_1.ParameterError('HTTP credentials require an HTTPS source');
|
|
692
|
+
}
|
|
693
|
+
await db.run(`CREATE SECRET analytics_http (TYPE http, BEARER_TOKEN ${literal(source.httpAuth.bearerToken)}, SCOPE ${literal(source.scope)});`);
|
|
694
|
+
}
|
|
695
|
+
if (source.s3Config) {
|
|
696
|
+
if (DuckdbWarehouseClient.usesS3CredentialChain(source.s3Config)) {
|
|
697
|
+
await this.loadExtension(db, 'aws');
|
|
698
|
+
}
|
|
699
|
+
await db.run(DuckdbWarehouseClient.buildS3SecretSql({
|
|
700
|
+
...source.s3Config,
|
|
701
|
+
scope: [source.scope],
|
|
702
|
+
}));
|
|
703
|
+
}
|
|
704
|
+
// Restrict the engine, not just SQL validation. No globbing, arbitrary
|
|
705
|
+
// network reads, local files, or shared spill/cache directories.
|
|
706
|
+
await db.run("SET temp_directory = '';");
|
|
707
|
+
await db.run('SET enable_http_metadata_cache = false;');
|
|
708
|
+
await db.run('SET enable_external_file_cache = false;');
|
|
709
|
+
await db.run('SET parquet_metadata_cache = false;');
|
|
710
|
+
const files = source.tables.flatMap(({ urls }) => urls);
|
|
711
|
+
await db.run(`SET allowed_paths = [${files.map(literal).join(',')}];`);
|
|
712
|
+
await db.run('SET enable_external_access = false;');
|
|
713
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
714
|
+
for (const { name, urls } of source.tables) {
|
|
715
|
+
// eslint-disable-next-line no-await-in-loop
|
|
716
|
+
await db.run(`CREATE VIEW "${name}" AS SELECT * FROM read_parquet([${urls.map(literal).join(',')}], hive_partitioning = true, union_by_name = true);`);
|
|
717
|
+
}
|
|
718
|
+
}
|
|
621
719
|
static async bootstrapSharedInstance(instance, client) {
|
|
622
720
|
const db = await instance.connect();
|
|
623
721
|
try {
|
|
@@ -1120,6 +1218,16 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1120
1218
|
throw new common_1.WarehouseQueryError('External source query capacity is full. Try again shortly.');
|
|
1121
1219
|
}
|
|
1122
1220
|
try {
|
|
1221
|
+
if (this.parquetConfig) {
|
|
1222
|
+
try {
|
|
1223
|
+
return await this.withEphemeralQuerySession(callback);
|
|
1224
|
+
}
|
|
1225
|
+
catch (error) {
|
|
1226
|
+
if (error instanceof common_1.ParameterError)
|
|
1227
|
+
throw error;
|
|
1228
|
+
throw new common_1.WarehouseQueryError('Internal analytics query failed. Check storage access and query permissions.');
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1123
1231
|
if (this.embeddedConfig) {
|
|
1124
1232
|
return await this.withDirectSession(callback, organizationUuid, onPhaseTiming);
|
|
1125
1233
|
}
|
|
@@ -1497,6 +1605,10 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1497
1605
|
}
|
|
1498
1606
|
}
|
|
1499
1607
|
async validateUserSql(db, sql) {
|
|
1608
|
+
if (this.parquetConfig &&
|
|
1609
|
+
/\b(duckdb_\w+|pragma_\w+|sqlite_\w+|pg_\w+|information_schema)\b/i.test(DuckdbWarehouseClient.stripSqlComments(sql))) {
|
|
1610
|
+
throw new common_1.ParameterError('Internal analytics catalog access is not allowed');
|
|
1611
|
+
}
|
|
1500
1612
|
DuckdbWarehouseClient.validateSqlFunctions(sql);
|
|
1501
1613
|
DuckdbWarehouseClient.validateUserSqlFileAccess(sql);
|
|
1502
1614
|
await this.validateSelectSql(db, sql);
|
|
@@ -142,6 +142,119 @@ const createMockConnection = (streamMock, runMock = vi.fn(), opts) => ({
|
|
|
142
142
|
}),
|
|
143
143
|
closeSync: vi.fn(),
|
|
144
144
|
});
|
|
145
|
+
describe('internal Parquet projects', () => {
|
|
146
|
+
const scope = 'https://storage.googleapis.com/example/events/compacted/org_id=org-a/';
|
|
147
|
+
const url = `${scope}stream=query_events/dt=2026-09-07/part.parquet`;
|
|
148
|
+
const source = () => ({
|
|
149
|
+
scope,
|
|
150
|
+
httpAuth: { bearerToken: 'test-token' },
|
|
151
|
+
tables: [{ name: 'query_events', urls: [url] }],
|
|
152
|
+
});
|
|
153
|
+
let run;
|
|
154
|
+
beforeEach(() => {
|
|
155
|
+
vi.clearAllMocks();
|
|
156
|
+
run = vi.fn();
|
|
157
|
+
createInstanceMock.mockResolvedValue(createMockConnection(vi.fn(async () => getMockStreamResult([[{ count: 2 }]], [5])), run));
|
|
158
|
+
});
|
|
159
|
+
it('binds trusted views, restricts external access and refreshes the manifest each session', async () => {
|
|
160
|
+
const resolveSource = vi.fn(async () => source());
|
|
161
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
162
|
+
type: 'duckdb_parquet',
|
|
163
|
+
resolveSource,
|
|
164
|
+
});
|
|
165
|
+
await client.runQuery('SELECT count(*) FROM query_events');
|
|
166
|
+
await client.runQuery('SELECT count(*) FROM query_events');
|
|
167
|
+
expect(resolveSource).toHaveBeenCalledTimes(2);
|
|
168
|
+
expect(createInstanceMock).toHaveBeenCalledTimes(2);
|
|
169
|
+
expect(run).toHaveBeenCalledWith(`SET allowed_paths = ['${url}'];`);
|
|
170
|
+
expect(run).toHaveBeenCalledWith('SET enable_external_access = false;');
|
|
171
|
+
expect(run).toHaveBeenCalledWith("SET temp_directory = '';");
|
|
172
|
+
expect(run).toHaveBeenCalledWith(expect.stringContaining('CREATE VIEW "query_events" AS SELECT * FROM read_parquet'));
|
|
173
|
+
expect(client.credentials).not.toHaveProperty('httpAuth');
|
|
174
|
+
});
|
|
175
|
+
it.each([
|
|
176
|
+
'https://storage.googleapis.com/example/events/compacted/org_id=org-b/file.parquet',
|
|
177
|
+
`${scope}../org-b/file.parquet`,
|
|
178
|
+
`${scope}%2e%2e/org-b/file.parquet`,
|
|
179
|
+
`${scope}*.parquet`,
|
|
180
|
+
'/tmp/private.parquet',
|
|
181
|
+
])('rejects files outside the exact manifest scope: %s', async (file) => {
|
|
182
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
183
|
+
type: 'duckdb_parquet',
|
|
184
|
+
resolveSource: async () => ({
|
|
185
|
+
...source(),
|
|
186
|
+
tables: [{ name: 'query_events', urls: [file] }],
|
|
187
|
+
}),
|
|
188
|
+
});
|
|
189
|
+
await expect(client.runQuery('SELECT * FROM query_events')).rejects.toThrow();
|
|
190
|
+
expect(run).not.toHaveBeenCalledWith(expect.stringContaining('CREATE SECRET'));
|
|
191
|
+
});
|
|
192
|
+
it.each([
|
|
193
|
+
{ name: 'x"; SELECT 1; --', urls: [url] },
|
|
194
|
+
{ name: 'query_events', urls: [] },
|
|
195
|
+
])('rejects arbitrary table SQL and empty manifests: $name', async (table) => {
|
|
196
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
197
|
+
type: 'duckdb_parquet',
|
|
198
|
+
resolveSource: async () => ({ ...source(), tables: [table] }),
|
|
199
|
+
});
|
|
200
|
+
await expect(client.runQuery('SELECT 1')).rejects.toThrow(/unique names/);
|
|
201
|
+
});
|
|
202
|
+
it('keeps user-supplied read_parquet forbidden', async () => {
|
|
203
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
204
|
+
type: 'duckdb_parquet',
|
|
205
|
+
resolveSource: async () => source(),
|
|
206
|
+
});
|
|
207
|
+
await expect(client.runQuery(`SELECT * FROM read_parquet('${url}')`)).rejects.toThrow(/query permissions/);
|
|
208
|
+
});
|
|
209
|
+
it('allows exact signed GET URLs without configuring bucket credentials', async () => {
|
|
210
|
+
const signedScope = scope.replace('org_id=', 'org_id%3D');
|
|
211
|
+
const signedUrl = `${url.replace(/=/g, '%3D')}?X-Amz-Signature=test&X-Amz-Expires=900`;
|
|
212
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
213
|
+
type: 'duckdb_parquet',
|
|
214
|
+
resolveSource: async () => ({
|
|
215
|
+
scope: signedScope,
|
|
216
|
+
signedUrls: true,
|
|
217
|
+
tables: [{ name: 'query_events', urls: [signedUrl] }],
|
|
218
|
+
}),
|
|
219
|
+
});
|
|
220
|
+
await client.runQuery('SELECT count(*) FROM query_events');
|
|
221
|
+
expect(run).toHaveBeenCalledWith(`SET allowed_paths = ['${signedUrl}'];`);
|
|
222
|
+
expect(run).not.toHaveBeenCalledWith(expect.stringContaining('CREATE SECRET'));
|
|
223
|
+
expect(run).toHaveBeenCalledWith('SET enable_external_file_cache = false;');
|
|
224
|
+
});
|
|
225
|
+
it.each([
|
|
226
|
+
'SELECT sql FROM duckdb_views()',
|
|
227
|
+
'SELECT * FROM "information_schema"."views"',
|
|
228
|
+
'SELECT * FROM sqlite_master',
|
|
229
|
+
"SELECT * FROM pragma_storage_info('query_events')",
|
|
230
|
+
])('blocks metadata queries that could disclose signed URLs: %s', async (sql) => {
|
|
231
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
232
|
+
type: 'duckdb_parquet',
|
|
233
|
+
resolveSource: async () => source(),
|
|
234
|
+
});
|
|
235
|
+
await expect(client.runQuery(sql)).rejects.toThrow(/catalog access/);
|
|
236
|
+
});
|
|
237
|
+
it('redacts signed URLs from native query failures', async () => {
|
|
238
|
+
run.mockRejectedValue(new Error(`${url}?X-Amz-Signature=private`));
|
|
239
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
240
|
+
type: 'duckdb_parquet',
|
|
241
|
+
resolveSource: async () => source(),
|
|
242
|
+
});
|
|
243
|
+
await expect(client.runQuery('SELECT count(*) FROM query_events')).rejects.toThrow(/^Internal analytics query failed\. Check storage access and query permissions\.$/);
|
|
244
|
+
});
|
|
245
|
+
it('cannot create privileged readers from public project credentials or shared instances', () => {
|
|
246
|
+
expect(() => new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
247
|
+
type: common_1.WarehouseTypes.DUCKDB,
|
|
248
|
+
connectionType: common_1.DuckdbConnectionType.ANALYTICS,
|
|
249
|
+
database: 'memory',
|
|
250
|
+
schema: 'main',
|
|
251
|
+
})).toThrow(/internal project service/);
|
|
252
|
+
expect(() => new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
253
|
+
type: 'duckdb_parquet',
|
|
254
|
+
resolveSource: async () => source(),
|
|
255
|
+
}, { instanceCacheKey: 'shared' })).toThrow(/cannot share/);
|
|
256
|
+
});
|
|
257
|
+
});
|
|
145
258
|
describe('mapFieldTypeFromTypeId', () => {
|
|
146
259
|
it('should map date types', () => {
|
|
147
260
|
expect((0, DuckdbWarehouseClient_1.mapFieldTypeFromTypeId)(DUCKDB_TYPE_IDS.DATE)).toBe(common_1.DimensionType.DATE);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.180.0",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
"snowflake-sdk": "2.3.4",
|
|
29
29
|
"ssh2": "1.14.0",
|
|
30
30
|
"trino-client": "0.2.9",
|
|
31
|
-
"@lightdash/common": "2.
|
|
31
|
+
"@lightdash/common": "2.180.0"
|
|
32
32
|
},
|
|
33
33
|
"devDependencies": {
|
|
34
34
|
"oxlint": "1.75.0",
|