@lightdash/warehouses 1.227.0 → 1.227.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts +7 -0
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.js +50 -13
- package/dist/warehouseClients/DuckdbWarehouseClient.test.js +44 -0
- package/package.json +2 -2
|
@@ -8,6 +8,8 @@ export type DuckdbS3SessionConfig = {
|
|
|
8
8
|
secretKey?: string;
|
|
9
9
|
forcePathStyle: boolean;
|
|
10
10
|
useSsl: boolean;
|
|
11
|
+
/** Limit this DuckDB secret to one trusted S3 URI or prefix. */
|
|
12
|
+
scope?: string;
|
|
11
13
|
};
|
|
12
14
|
export type DuckdbResourceLimits = {
|
|
13
15
|
memoryLimit?: string;
|
|
@@ -50,6 +52,8 @@ export type DuckdbWarehouseClientOptions = {
|
|
|
50
52
|
embeddedQueryTimeoutMs?: number;
|
|
51
53
|
enableInstanceCache?: boolean;
|
|
52
54
|
projectUuid?: string;
|
|
55
|
+
/** Process-local per-organization cap for isolated S3 query clients. */
|
|
56
|
+
organizationConcurrencyLimit?: number;
|
|
53
57
|
};
|
|
54
58
|
export declare const mapFieldTypeFromTypeId: (typeId: number) => DimensionType;
|
|
55
59
|
export declare const getDuckdbTimestampDomainFromString: (typeName: string) => TimestampDomain | undefined;
|
|
@@ -66,6 +70,7 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
66
70
|
private static readonly sharedInstanceSemaphores;
|
|
67
71
|
private static readonly embeddedConcurrencyBudget;
|
|
68
72
|
private static readonly embeddedOrganizationConcurrencyBudgets;
|
|
73
|
+
private static readonly organizationConcurrencyBudgets;
|
|
69
74
|
private static readonly sqlBuilder;
|
|
70
75
|
private readonly databasePath;
|
|
71
76
|
private readonly s3Config?;
|
|
@@ -80,6 +85,7 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
80
85
|
private readonly embeddedQueryTimeoutMs;
|
|
81
86
|
private readonly enableInstanceCache;
|
|
82
87
|
private readonly projectUuid?;
|
|
88
|
+
private readonly organizationConcurrencyLimit?;
|
|
83
89
|
private allowsPreAggregateFileReads;
|
|
84
90
|
private hasWarnedAboutMotherduckTimezone;
|
|
85
91
|
constructor(credentials?: CreateDuckdbCredentials | DuckdbConnectionCredentials, options?: DuckdbWarehouseClientOptions);
|
|
@@ -88,6 +94,7 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
88
94
|
private static getSharedInstanceSemaphore;
|
|
89
95
|
private getRequiredInstanceCacheKey;
|
|
90
96
|
private static tryAcquireEmbeddedConcurrency;
|
|
97
|
+
private static tryAcquireOrganizationConcurrency;
|
|
91
98
|
private getSQLWithMetadata;
|
|
92
99
|
private static hardenInstance;
|
|
93
100
|
private static applyResourceLimits;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAKA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAMb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAEhB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAQ3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA6DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAKA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAMb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAEhB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAQ3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA6DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;IAChB,gEAAgE;IAChE,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACpE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG,mBAAmB,CAAC;AAE9D,MAAM,MAAM,4BAA4B,GAAG;IACvC,kHAAkH;IAClH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;IAC9D,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,4BAA4B,CAAC,EAAE,MAAM,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,sBAAsB,WAAY,MAAM,KAAG,aA+BvD,CAAC;AAEF,eAAO,MAAM,kCAAkC,aACjC,MAAM,KACjB,eAAe,GAAG,SAOpB,CAAC;AAoCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB,CAEpC;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AAkFD,eAAO,MAAM,+BAA+B,yBAGzC,IAAI,CAAC,iCAAiC,EAAE,UAAU,GAAG,OAAO,CAAC,KAAG,MAOlE,CAAC;AA8DF,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,iCAAiC,CAAC;IAC7F,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAGhD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAE/C;IAEF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sCAAsC,CAG1D;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,8BAA8B,CAGlD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAU;IAE/C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;IAEV,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAEhD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAU;IAE9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,4BAA4B,CAAC,CAAS;IAEvD,OAAO,CAAC,2BAA2B,CAAS;IAE5C,OAAO,CAAC,gCAAgC,CAAS;IAEjD,YACI,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B,EAmHzC;IAED,OAAO,CAAC,MAAM,CAAC,kBAAkB;IASjC,MAAM,CAAC,qBAAqB,CACxB,WAAW,EAAE,mBAAmB,EAChC,OAAO,CAAC,EAAE,4BAA4B,GACvC,qBAAqB,CAIvB;IAED,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAoBzC,OAAO,CAAC,2BAA2B;IAUnC,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAwC5C,OAAO,CAAC,MAAM,CAAC,iCAAiC;IAoBhD,OAAO,CAAC,kBAAkB;mBAQL,cAAc;mBAOd,mBAAmB;IAaxC,OAAO,CAAC,MAAM,CAAC,qBAAqB;mBAMf,uBAAuB;YAsB9B,aAAa;YAoBb,kCAAkC;mBAW3B,qBAAqB;mBA6ErB,uBAAuB;mBAgBvB,yBAAyB;mBAiFzB,mBAAmB;IA6BxC,mFAAmF;IACnF,MAAM,CAAC,gCAAgC,IAAI,IAAI,CAM9C;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAO3B;IAED,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CACZ;IAEnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAA+B;IAE3E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAA0B;IAEjE,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAIjC,OAAO,CAAC,MAAM,CAAC,UAAU;IAIzB,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAuB5C,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAqGzC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAMhC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAmBrC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IA2CrC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAyC/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAK;YAE9C,gBAAgB;IA+E9B,iEAAiE;YACnD,wBAAwB;IA+CtC,gFAAgF;YAClE,mBAAmB;YAkDnB,yBAAyB;YA4CzB,iBAAiB;IA2C/B,OAAO,CAAC,iBAAiB;IAMzB,OAAO,CAAC,YAAY;YAQN,WAAW;YAqEX,2BAA2B;IA4GzC,+DAA+D;YACjD,iBAAiB;YA8EjB,yBAAyB;IA2BvC,OAAO,CAAC,4BAA4B;IAOpC,OAAO,CAAC,aAAa;YA0BP,eAAe;IAgG7B,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAaxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IA+D/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;IAYnC,MAAM,CAAC,yBAAyB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAiBlD;YAEa,iBAAiB;YA4BjB,eAAe;YASf,uBAAuB;YAQvB,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CAyFf;IAEK,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,mBAAmB,CAAC,CAC9E;;;;;;OA4BJ;IAEK,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKvC;IAEK,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC,CAkBD;IAEK,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,UAAU,CAAC,CACrE;;;;;OAGJ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAEK,UAAU,CACZ,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC,CA4C3B;IAEK,YAAY,CACd,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN,CAqBA;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC,CAS3B;CACJ"}
|
|
@@ -357,6 +357,8 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
357
357
|
options?.embeddedQueryTimeoutMs ?? EMBEDDED_QUERY_TIMEOUT_MS;
|
|
358
358
|
this.enableInstanceCache = options?.enableInstanceCache ?? false;
|
|
359
359
|
this.projectUuid = options?.projectUuid;
|
|
360
|
+
this.organizationConcurrencyLimit =
|
|
361
|
+
options?.organizationConcurrencyLimit;
|
|
360
362
|
}
|
|
361
363
|
static hashDucklakeConfig(ducklake) {
|
|
362
364
|
return (0, crypto_1.createHash)('sha256')
|
|
@@ -406,6 +408,20 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
406
408
|
}
|
|
407
409
|
};
|
|
408
410
|
}
|
|
411
|
+
static tryAcquireOrganizationConcurrency(organizationUuid, limit) {
|
|
412
|
+
const key = `${limit}:${organizationUuid}`;
|
|
413
|
+
const budget = DuckdbWarehouseClient.organizationConcurrencyBudgets.get(key) ??
|
|
414
|
+
new ConcurrencyBudget(limit);
|
|
415
|
+
DuckdbWarehouseClient.organizationConcurrencyBudgets.set(key, budget);
|
|
416
|
+
if (!budget.tryAcquire())
|
|
417
|
+
return undefined;
|
|
418
|
+
return () => {
|
|
419
|
+
budget.release();
|
|
420
|
+
if (budget.isIdle()) {
|
|
421
|
+
DuckdbWarehouseClient.organizationConcurrencyBudgets.delete(key);
|
|
422
|
+
}
|
|
423
|
+
};
|
|
424
|
+
}
|
|
409
425
|
getSQLWithMetadata(sql, tags) {
|
|
410
426
|
if (!tags) {
|
|
411
427
|
return sql;
|
|
@@ -800,6 +816,9 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
800
816
|
const secretClause = s3Config.secretKey
|
|
801
817
|
? `SECRET '${escape(s3Config.secretKey)}',`
|
|
802
818
|
: '';
|
|
819
|
+
const scopeClause = s3Config.scope
|
|
820
|
+
? `SCOPE '${escape(s3Config.scope)}',`
|
|
821
|
+
: '';
|
|
803
822
|
return `CREATE OR REPLACE SECRET __lightdash_s3 (
|
|
804
823
|
TYPE s3,
|
|
805
824
|
${providerClause}
|
|
@@ -807,6 +826,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
807
826
|
${secretClause}
|
|
808
827
|
ENDPOINT '${escape(s3Config.endpoint)}',
|
|
809
828
|
${regionClause}
|
|
829
|
+
${scopeClause}
|
|
810
830
|
URL_STYLE '${s3Config.forcePathStyle ? 'path' : 'vhost'}',
|
|
811
831
|
USE_SSL ${s3Config.useSsl}
|
|
812
832
|
);`;
|
|
@@ -996,23 +1016,39 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
996
1016
|
this.databasePath.startsWith('md:'));
|
|
997
1017
|
}
|
|
998
1018
|
async withSession(callback, organizationUuid, retryable = () => true, onPhaseTiming) {
|
|
999
|
-
|
|
1000
|
-
|
|
1019
|
+
const releaseOrganizationConcurrency = !this.embeddedConfig &&
|
|
1020
|
+
organizationUuid &&
|
|
1021
|
+
this.organizationConcurrencyLimit
|
|
1022
|
+
? DuckdbWarehouseClient.tryAcquireOrganizationConcurrency(organizationUuid, this.organizationConcurrencyLimit)
|
|
1023
|
+
: undefined;
|
|
1024
|
+
if (!this.embeddedConfig &&
|
|
1025
|
+
organizationUuid &&
|
|
1026
|
+
this.organizationConcurrencyLimit &&
|
|
1027
|
+
!releaseOrganizationConcurrency) {
|
|
1028
|
+
throw new common_1.WarehouseQueryError('External source query capacity is full. Try again shortly.');
|
|
1001
1029
|
}
|
|
1002
|
-
|
|
1003
|
-
if (this.
|
|
1004
|
-
this.
|
|
1005
|
-
return this.withMotherduckCachedSession(callback, retryable, onPhaseTiming);
|
|
1030
|
+
try {
|
|
1031
|
+
if (this.embeddedConfig) {
|
|
1032
|
+
return await this.withDirectSession(callback, organizationUuid, onPhaseTiming);
|
|
1006
1033
|
}
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1034
|
+
if (this.isMotherduck()) {
|
|
1035
|
+
if (this.enableInstanceCache &&
|
|
1036
|
+
this.credentials.requireUserCredentials !== true) {
|
|
1037
|
+
return await this.withMotherduckCachedSession(callback, retryable, onPhaseTiming);
|
|
1038
|
+
}
|
|
1039
|
+
return await this.withDirectSession(callback, organizationUuid, onPhaseTiming);
|
|
1040
|
+
}
|
|
1041
|
+
if (this.hasResourceLimits()) {
|
|
1042
|
+
return await this.withIsolatedSession(callback);
|
|
1043
|
+
}
|
|
1044
|
+
if (this.instanceCacheKey) {
|
|
1045
|
+
return await this.withSharedSession(callback);
|
|
1046
|
+
}
|
|
1047
|
+
return await this.withEphemeralQuerySession(callback);
|
|
1011
1048
|
}
|
|
1012
|
-
|
|
1013
|
-
|
|
1049
|
+
finally {
|
|
1050
|
+
releaseOrganizationConcurrency?.();
|
|
1014
1051
|
}
|
|
1015
|
-
return this.withEphemeralQuerySession(callback);
|
|
1016
1052
|
}
|
|
1017
1053
|
async withMotherduckCachedSession(callback, retryable, onPhaseTiming) {
|
|
1018
1054
|
let activeEntryId;
|
|
@@ -1575,6 +1611,7 @@ DuckdbWarehouseClient.sharedInstanceResourceLimits = new Map();
|
|
|
1575
1611
|
DuckdbWarehouseClient.sharedInstanceSemaphores = new Map();
|
|
1576
1612
|
DuckdbWarehouseClient.embeddedConcurrencyBudget = new ConcurrencyBudget(EMBEDDED_MAX_CONCURRENT_QUERIES);
|
|
1577
1613
|
DuckdbWarehouseClient.embeddedOrganizationConcurrencyBudgets = new Map();
|
|
1614
|
+
DuckdbWarehouseClient.organizationConcurrencyBudgets = new Map();
|
|
1578
1615
|
DuckdbWarehouseClient.sqlBuilder = new DuckdbSqlBuilder();
|
|
1579
1616
|
DuckdbWarehouseClient.DUCKLAKE_CATALOG_SECRET = '__lightdash_ducklake_catalog';
|
|
1580
1617
|
DuckdbWarehouseClient.DUCKLAKE_DATA_SECRET = '__lightdash_ducklake_data';
|
|
@@ -638,6 +638,50 @@ describe('DuckdbWarehouseClient', () => {
|
|
|
638
638
|
expect(secretSql).not.toContain('KEY_ID');
|
|
639
639
|
expect(secretSql).not.toContain("SECRET '");
|
|
640
640
|
});
|
|
641
|
+
it('scopes S3 credentials to the trusted external-source object', async () => {
|
|
642
|
+
const runMock = vi.fn();
|
|
643
|
+
createInstanceMock.mockResolvedValue(createMockConnection(vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER])), runMock));
|
|
644
|
+
const scope = 's3://bucket/external-sources/project/source/table/v1.parquet';
|
|
645
|
+
const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate({
|
|
646
|
+
type: 'duckdb_s3',
|
|
647
|
+
s3Config: {
|
|
648
|
+
endpoint: 's3.amazonaws.com',
|
|
649
|
+
forcePathStyle: false,
|
|
650
|
+
useSsl: true,
|
|
651
|
+
scope,
|
|
652
|
+
},
|
|
653
|
+
});
|
|
654
|
+
await client.runQuery('SELECT 1');
|
|
655
|
+
const secretSql = runMock.mock.calls
|
|
656
|
+
.map(([sql]) => sql)
|
|
657
|
+
.find((sql) => sql.includes('CREATE OR REPLACE SECRET __lightdash_s3'));
|
|
658
|
+
expect(secretSql).toContain(`SCOPE '${scope}'`);
|
|
659
|
+
});
|
|
660
|
+
it('caps isolated S3 queries per organization', async () => {
|
|
661
|
+
let resolveStream = () => { };
|
|
662
|
+
const streamMock = vi.fn(() => new Promise((resolve) => {
|
|
663
|
+
resolveStream = resolve;
|
|
664
|
+
}));
|
|
665
|
+
createInstanceMock.mockImplementation(async () => createMockConnection(streamMock));
|
|
666
|
+
const createClient = () => DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate({
|
|
667
|
+
type: 'duckdb_s3',
|
|
668
|
+
s3Config: {
|
|
669
|
+
endpoint: 's3.amazonaws.com',
|
|
670
|
+
forcePathStyle: false,
|
|
671
|
+
useSsl: true,
|
|
672
|
+
scope: 's3://bucket/external-sources/object.parquet',
|
|
673
|
+
},
|
|
674
|
+
}, {
|
|
675
|
+
resourceLimits: { memoryLimit: '128MB', threads: 1 },
|
|
676
|
+
organizationConcurrencyLimit: 1,
|
|
677
|
+
});
|
|
678
|
+
const tags = { organization_uuid: 'external-source-org' };
|
|
679
|
+
const first = createClient().runQuery('SELECT 1', tags);
|
|
680
|
+
await vi.waitFor(() => expect(streamMock).toHaveBeenCalledOnce());
|
|
681
|
+
await expect(createClient().runQuery('SELECT 2', tags)).rejects.toThrow('External source query capacity is full');
|
|
682
|
+
resolveStream(getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
683
|
+
await first;
|
|
684
|
+
});
|
|
641
685
|
it('should load bundled extensions without runtime installs', async () => {
|
|
642
686
|
const accessMock = vi.spyOn(promises_1.default, 'access').mockResolvedValue();
|
|
643
687
|
try {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "1.227.
|
|
3
|
+
"version": "1.227.1",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
"snowflake-sdk": "2.3.4",
|
|
29
29
|
"ssh2": "1.14.0",
|
|
30
30
|
"trino-client": "0.2.9",
|
|
31
|
-
"@lightdash/common": "1.227.
|
|
31
|
+
"@lightdash/common": "1.227.1"
|
|
32
32
|
},
|
|
33
33
|
"devDependencies": {
|
|
34
34
|
"oxlint": "1.75.0",
|