@lightdash/warehouses 2.616.0 → 2.616.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -96,6 +96,12 @@ export type DuckdbWarehouseClientOptions = {
|
|
|
96
96
|
projectUuid?: string;
|
|
97
97
|
/** Process-local per-organization cap for isolated S3 query clients. */
|
|
98
98
|
organizationConcurrencyLimit?: number;
|
|
99
|
+
/**
|
|
100
|
+
* Confine the instance to its storage bucket(s) and a private temp
|
|
101
|
+
* directory. Serve sessions only read object storage, so they have no
|
|
102
|
+
* use for anything else on disk or on the network.
|
|
103
|
+
*/
|
|
104
|
+
objectStorageOnly?: boolean;
|
|
99
105
|
};
|
|
100
106
|
export declare const mapFieldTypeFromTypeId: (typeId: number) => DimensionType;
|
|
101
107
|
export declare const getDuckdbTimestampDomainFromString: (typeName: string) => TimestampDomain | undefined;
|
|
@@ -117,7 +123,13 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
117
123
|
* the replacement while the token is unchanged.
|
|
118
124
|
*/
|
|
119
125
|
private static readonly sharedInstanceGcsTokens;
|
|
120
|
-
|
|
126
|
+
/**
|
|
127
|
+
* The S3 URIs each shared instance's secret is scoped to. Sessions that
|
|
128
|
+
* need a URI outside it widen the secret in place; it is never narrowed
|
|
129
|
+
* while other sessions may still read through it.
|
|
130
|
+
*/
|
|
131
|
+
private static readonly sharedInstanceS3Scopes;
|
|
132
|
+
private static readonly sharedInstanceS3SecretLocks;
|
|
121
133
|
private static readonly embeddedConcurrencyBudget;
|
|
122
134
|
private static readonly embeddedOrganizationConcurrencyBudgets;
|
|
123
135
|
private static readonly organizationConcurrencyBudgets;
|
|
@@ -137,6 +149,7 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
137
149
|
private readonly enableInstanceCache;
|
|
138
150
|
private readonly projectUuid?;
|
|
139
151
|
private readonly organizationConcurrencyLimit?;
|
|
152
|
+
private readonly objectStorageOnly;
|
|
140
153
|
private allowsPreAggregateFileReads;
|
|
141
154
|
private hasWarnedAboutMotherduckTimezone;
|
|
142
155
|
constructor(credentials?: CreateDuckdbCredentials | DuckdbConnectionCredentials, options?: DuckdbWarehouseClientOptions);
|
|
@@ -148,6 +161,13 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
148
161
|
private static tryAcquireOrganizationConcurrency;
|
|
149
162
|
private getSQLWithMetadata;
|
|
150
163
|
private static hardenInstance;
|
|
164
|
+
/**
|
|
165
|
+
* Limit the instance to its storage bucket(s) and a private temp
|
|
166
|
+
* directory for spill. Runs last in bootstrap: extensions are already
|
|
167
|
+
* loaded and the lockdown cannot be lifted afterwards, while the storage
|
|
168
|
+
* secret can still be replaced on the running instance.
|
|
169
|
+
*/
|
|
170
|
+
private static restrictToObjectStorage;
|
|
151
171
|
private static applyResourceLimits;
|
|
152
172
|
private static usesS3CredentialChain;
|
|
153
173
|
private static getBundledExtensionPath;
|
|
@@ -176,12 +196,17 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
176
196
|
private static configureS3Secret;
|
|
177
197
|
private static getGcsAccessToken;
|
|
178
198
|
private static createGcsSecret;
|
|
199
|
+
/** URIs this session needs that the shared instance's secret does not yet cover. */
|
|
200
|
+
private static getMissingSharedS3Scope;
|
|
201
|
+
private static getSharedS3SecretToken;
|
|
179
202
|
/**
|
|
180
|
-
*
|
|
181
|
-
*
|
|
182
|
-
*
|
|
203
|
+
* Keeps a shared instance's S3 secret current for this session: the
|
|
204
|
+
* GCS bearer token the provider now holds, and a scope covering every
|
|
205
|
+
* URI the session reads. Secrets belong to the instance, so the secret
|
|
206
|
+
* is only replaced when something changed, once per instance under a
|
|
207
|
+
* lock, and the scope only ever widens (union of what sessions needed).
|
|
183
208
|
*/
|
|
184
|
-
private static
|
|
209
|
+
private static ensureSharedS3Secret;
|
|
185
210
|
private static buildS3SecretSql;
|
|
186
211
|
private static readonly CONNECT_RETRIES_BEFORE_RECREATE;
|
|
187
212
|
private connectWithRetry;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAMA,OAAO,EACH,OAAO,
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAMA,OAAO,EACH,OAAO,EAEP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAOb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAChB,eAAe,EAGf,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAY3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA8DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;IAChB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB,GAAG,CACE;IAAE,QAAQ,CAAC,EAAE,SAAS,CAAC;IAAC,cAAc,CAAC,EAAE,KAAK,CAAA;CAAE,GAChD;IACI,QAAQ,EAAE,WAAW,CAAC;IACtB,0EAA0E;IAC1E,cAAc,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IACtC,KAAK,EAAE,MAAM,EAAE,CAAC;CACnB,CACN,CAAC;AAOF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACpE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS,GAAG,QAAQ,CAAC;CACpE,EAAE,CAAC;AAEJ,gFAAgF;AAChF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,kFAAkF;IAClF,WAAW,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAAC,OAAO,CAAC,EAAE,oBAAoB,CAAA;KAAE,EAAE,CAAC;IAC3E,4EAA4E;IAC5E,WAAW,CAAC,EAAE;QACV,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,oBAAoB,CAAC;KACjC,EAAE,CAAC;IACJ,2EAA2E;IAC3E,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IACnC,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACnC,IAAI,EAAE,gBAAgB,CAAC;IACvB,0FAA0F;IAC1F,aAAa,EAAE,CACX,gBAAgB,CAAC,EAAE,SAAS,MAAM,EAAE,KACnC,OAAO,CAAC,mBAAmB,CAAC,CAAC;CACrC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GACjC,mBAAmB,GACnB,wBAAwB,CAAC;AAE/B,MAAM,MAAM,4BAA4B,GAAG;IACvC,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,kHAAkH;IAClH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;IAC9D,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,4BAA4B,CAAC,EAAE,MAAM,CAAC;IACtC;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC/B,CAAC;AAEF,eAAO,MAAM,sBAAsB,WAAY,MAAM,KAAG,aA+BvD,CAAC;AAyBF,eAAO,MAAM,kCAAkC,aACjC,MAAM,KACjB,eAAe,GAAG,SAOpB,CAAC;AAoCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB,CAEpC;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AAmID,eAAO,MAAM,+BAA+B,yBAGzC,IAAI,CAAC,iCAAiC,EAAE,UAAU,GAAG,OAAO,CAAC,KAAG,MAOlE,CAAC;AAiGF,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,iCAAiC,CAAC;IAC7F,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAGhD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ;;;;;OAKG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAA6B;IAE5E;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sBAAsB,CAG1C;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,2BAA2B,CAG/C;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAE/C;IAEF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sCAAsC,CAG1D;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,8BAA8B,CAGlD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAA2B;IAE1D,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAU;IAE/C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;IAEV,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAEhD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAU;IAE9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,4BAA4B,CAAC,CAAS;IAEvD,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAU;IAE5C,OAAO,CAAC,2BAA2B,CAAS;IAE5C,OAAO,CAAC,gCAAgC,CAAS;IAEjD,YACI,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B,EAiJzC;IAED,OAAO,CAAC,MAAM,CAAC,kBAAkB;IASjC,MAAM,CAAC,qBAAqB,CACxB,WAAW,EAAE,mBAAmB,EAChC,OAAO,CAAC,EAAE,4BAA4B,GACvC,qBAAqB,CAIvB;IAED,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAoBzC,OAAO,CAAC,2BAA2B;mBAUd,6BAA6B;IA4DlD,OAAO,CAAC,MAAM,CAAC,iCAAiC;IAoBhD,OAAO,CAAC,kBAAkB;mBAQL,cAAc;IAOnC;;;;;OAKG;mBACkB,uBAAuB;mBAmCvB,mBAAmB;IAaxC,OAAO,CAAC,MAAM,CAAC,qBAAqB;mBASf,uBAAuB;YAsB9B,aAAa;YAoBb,kCAAkC;mBAW3B,qBAAqB;IA4F1C,2EAA2E;mBACtD,qBAAqB;YA8B5B,qBAAqB;mBAiNd,uBAAuB;mBAgBvB,yBAAyB;mBA6FzB,mBAAmB;IAmCxC,mFAAmF;IACnF,MAAM,CAAC,gCAAgC,IAAI,IAAI,CAY9C;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAO3B;IAED,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CACZ;IAEnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAA+B;IAE3E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAA0B;IAEjE,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAIjC,OAAO,CAAC,MAAM,CAAC,UAAU;IAIzB,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAuB5C,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAqGzC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAMhC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAmBrC,OAAO,CAAC,MAAM,CAAC,sBAAsB;mBA2ChB,iBAAiB;mBAejB,iBAAiB;mBAiBjB,eAAe;IA+BpC,oFAAoF;IACpF,OAAO,CAAC,MAAM,CAAC,uBAAuB;mBAUjB,sBAAsB;IAQ3C;;;;;;OAMG;mBACkB,oBAAoB;IAkFzC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IA6C/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAK;YAE9C,gBAAgB;IA+E9B,iEAAiE;YACnD,wBAAwB;IA+CtC,gFAAgF;YAClE,mBAAmB;YAkDnB,yBAAyB;YAyDzB,iBAAiB;IAkD/B,OAAO,CAAC,iBAAiB;IAMzB,OAAO,CAAC,YAAY;YAQN,WAAW;YAmFX,2BAA2B;IA4GzC,+DAA+D;YACjD,iBAAiB;YA8EjB,yBAAyB;IA2BvC,OAAO,CAAC,4BAA4B;IAOpC,OAAO,CAAC,aAAa;YA0BP,eAAe;IAgG7B,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAgBxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IA+D/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;IAYnC,MAAM,CAAC,yBAAyB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAiBlD;YAEa,iBAAiB;YA4BjB,eAAe;YAmBf,uBAAuB;YAQvB,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CA6Ff;IAEK,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,mBAAmB,CAAC,CAC9E;;;;;;;OA4BJ;IAEK,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKvC;IAEK,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC,CAkBD;IAEK,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,UAAU,CAAC,CACrE;;;OAGJ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAEK,UAAU,CACZ,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC,CA4C3B;IAEK,YAAY,CACd,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,eAAe,CAAC,CAsB1B;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC,CAS3B;CACJ"}
|
|
@@ -453,6 +453,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
453
453
|
this.projectUuid = options?.projectUuid;
|
|
454
454
|
this.organizationConcurrencyLimit =
|
|
455
455
|
options?.organizationConcurrencyLimit;
|
|
456
|
+
this.objectStorageOnly = options?.objectStorageOnly ?? false;
|
|
456
457
|
}
|
|
457
458
|
static hashDucklakeConfig(ducklake) {
|
|
458
459
|
return (0, crypto_1.createHash)('sha256')
|
|
@@ -543,6 +544,34 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
543
544
|
await db.run('SET autoload_known_extensions = false;');
|
|
544
545
|
await db.run('SET allow_unredacted_secrets = false;');
|
|
545
546
|
}
|
|
547
|
+
/**
|
|
548
|
+
* Limit the instance to its storage bucket(s) and a private temp
|
|
549
|
+
* directory for spill. Runs last in bootstrap: extensions are already
|
|
550
|
+
* loaded and the lockdown cannot be lifted afterwards, while the storage
|
|
551
|
+
* secret can still be replaced on the running instance.
|
|
552
|
+
*/
|
|
553
|
+
static async restrictToObjectStorage(db, client) {
|
|
554
|
+
const escape = DuckdbWarehouseClient.escapeDuckdbString;
|
|
555
|
+
const buckets = new Set((client.s3Config?.scope ?? []).map((uri) => {
|
|
556
|
+
const { protocol, host } = new URL(uri);
|
|
557
|
+
return `${protocol}//${host}/`;
|
|
558
|
+
}));
|
|
559
|
+
if (buckets.size === 0) {
|
|
560
|
+
throw new Error('DuckDB object-storage sessions need a storage scope');
|
|
561
|
+
}
|
|
562
|
+
const tempDirectory = client.instanceCacheKey
|
|
563
|
+
? path_1.default.join(os_1.default.tmpdir(), `duckdb-serve-${(0, crypto_1.createHash)('sha256')
|
|
564
|
+
.update(client.instanceCacheKey)
|
|
565
|
+
.digest('hex')
|
|
566
|
+
.slice(0, 16)}`)
|
|
567
|
+
: await promises_1.default.mkdtemp(path_1.default.join(os_1.default.tmpdir(), 'duckdb-serve-'));
|
|
568
|
+
await promises_1.default.mkdir(tempDirectory, { recursive: true });
|
|
569
|
+
await db.run(`SET temp_directory = '${escape(tempDirectory)}';`);
|
|
570
|
+
await db.run(`SET allowed_directories = [${[`${tempDirectory}/`, ...buckets]
|
|
571
|
+
.map((directory) => `'${escape(directory)}'`)
|
|
572
|
+
.join(', ')}];`);
|
|
573
|
+
await db.run('SET enable_external_access = false;');
|
|
574
|
+
}
|
|
546
575
|
static async applyResourceLimits(db, resourceLimits) {
|
|
547
576
|
if (resourceLimits?.memoryLimit) {
|
|
548
577
|
await db.run(`SET memory_limit = '${resourceLimits.memoryLimit}';`);
|
|
@@ -629,6 +658,9 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
629
658
|
}
|
|
630
659
|
}
|
|
631
660
|
await DuckdbWarehouseClient.hardenInstance(db);
|
|
661
|
+
if (client.objectStorageOnly) {
|
|
662
|
+
await DuckdbWarehouseClient.restrictToObjectStorage(db, client);
|
|
663
|
+
}
|
|
632
664
|
const bootstrapMs = performance.now() - bootstrapStart;
|
|
633
665
|
client.logger?.info(`DuckDB query bootstrap complete: cacheKey=${client.instanceCacheKey ?? 'none'} bootstrap=${(0, common_1.formatMilliseconds)(bootstrapMs)}ms httpfs=${(0, common_1.formatMilliseconds)(httpfsMs)}ms memory_limit=${client.sharedResourceLimits?.memoryLimit ?? 'default'} threads=${client.sharedResourceLimits?.threads ?? 'default'} s3=${client.s3Config ? 'configured' : 'none'} ducklake=${client.ducklakeConfig ? 'configured' : 'none'} shared=${client.instanceCacheKey ? 'true' : 'false'}`, {
|
|
634
666
|
shared: !!client.instanceCacheKey,
|
|
@@ -876,6 +908,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
876
908
|
};
|
|
877
909
|
}
|
|
878
910
|
DuckdbWarehouseClient.sharedInstanceGcsTokens.delete(instanceCacheKey);
|
|
911
|
+
DuckdbWarehouseClient.sharedInstanceS3Scopes.delete(instanceCacheKey);
|
|
879
912
|
const createStart = performance.now();
|
|
880
913
|
const instance = await node_api_1.DuckDBInstance.create(':memory:');
|
|
881
914
|
const instanceCreateMs = performance.now() - createStart;
|
|
@@ -890,6 +923,9 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
890
923
|
}
|
|
891
924
|
DuckdbWarehouseClient.sharedInstances.set(instanceCacheKey, instance);
|
|
892
925
|
DuckdbWarehouseClient.sharedInstanceResourceLimits.set(instanceCacheKey, client.sharedResourceLimits);
|
|
926
|
+
if (client.s3Config?.scope) {
|
|
927
|
+
DuckdbWarehouseClient.sharedInstanceS3Scopes.set(instanceCacheKey, [...client.s3Config.scope]);
|
|
928
|
+
}
|
|
893
929
|
client.logger?.info(`DuckDB shared instance initialized: cacheKey=${instanceCacheKey} semaphore_wait=${(0, common_1.formatMilliseconds)(semaphoreWaitMs)}ms instance_create=${(0, common_1.formatMilliseconds)(instanceCreateMs)}ms bootstrap=${(0, common_1.formatMilliseconds)(bootstrapMs)}ms`, {
|
|
894
930
|
instanceCacheKey,
|
|
895
931
|
semaphoreWaitMs,
|
|
@@ -923,6 +959,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
923
959
|
DuckdbWarehouseClient.sharedInstances.delete(instanceCacheKey);
|
|
924
960
|
DuckdbWarehouseClient.sharedInstanceResourceLimits.delete(instanceCacheKey);
|
|
925
961
|
DuckdbWarehouseClient.sharedInstanceGcsTokens.delete(instanceCacheKey);
|
|
962
|
+
DuckdbWarehouseClient.sharedInstanceS3Scopes.delete(instanceCacheKey);
|
|
926
963
|
logger?.info(`DuckDB shared instance cleared: ${instanceCacheKey}`);
|
|
927
964
|
}
|
|
928
965
|
}
|
|
@@ -936,7 +973,8 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
936
973
|
DuckdbWarehouseClient.sharedInstanceResourceLimits.clear();
|
|
937
974
|
DuckdbWarehouseClient.sharedInstanceSemaphores.clear();
|
|
938
975
|
DuckdbWarehouseClient.sharedInstanceGcsTokens.clear();
|
|
939
|
-
DuckdbWarehouseClient.
|
|
976
|
+
DuckdbWarehouseClient.sharedInstanceS3Scopes.clear();
|
|
977
|
+
DuckdbWarehouseClient.sharedInstanceS3SecretLocks.clear();
|
|
940
978
|
DuckdbWarehouseClient.embeddedConcurrencyBudget.reset();
|
|
941
979
|
DuckdbWarehouseClient.embeddedOrganizationConcurrencyBudgets.forEach((budget) => budget.reset());
|
|
942
980
|
DuckdbWarehouseClient.embeddedOrganizationConcurrencyBudgets.clear();
|
|
@@ -1150,17 +1188,34 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1150
1188
|
throw new Error(`Unable to configure GCP OAuth for DuckDB storage: ${reason}`);
|
|
1151
1189
|
}
|
|
1152
1190
|
}
|
|
1191
|
+
/** URIs this session needs that the shared instance's secret does not yet cover. */
|
|
1192
|
+
static getMissingSharedS3Scope(instanceCacheKey, s3Config) {
|
|
1193
|
+
const installed = DuckdbWarehouseClient.sharedInstanceS3Scopes.get(instanceCacheKey);
|
|
1194
|
+
if (!installed || !s3Config.scope)
|
|
1195
|
+
return [];
|
|
1196
|
+
return s3Config.scope.filter((uri) => !installed.includes(uri));
|
|
1197
|
+
}
|
|
1198
|
+
static async getSharedS3SecretToken(s3Config) {
|
|
1199
|
+
return s3Config.authMode === 'gcp_oauth'
|
|
1200
|
+
? DuckdbWarehouseClient.getGcsAccessToken(s3Config)
|
|
1201
|
+
: null;
|
|
1202
|
+
}
|
|
1153
1203
|
/**
|
|
1154
|
-
*
|
|
1155
|
-
*
|
|
1156
|
-
*
|
|
1204
|
+
* Keeps a shared instance's S3 secret current for this session: the
|
|
1205
|
+
* GCS bearer token the provider now holds, and a scope covering every
|
|
1206
|
+
* URI the session reads. Secrets belong to the instance, so the secret
|
|
1207
|
+
* is only replaced when something changed, once per instance under a
|
|
1208
|
+
* lock, and the scope only ever widens (union of what sessions needed).
|
|
1157
1209
|
*/
|
|
1158
|
-
static async
|
|
1210
|
+
static async ensureSharedS3Secret(instanceCacheKey, db, s3Config) {
|
|
1159
1211
|
const tokens = DuckdbWarehouseClient.sharedInstanceGcsTokens;
|
|
1160
|
-
const
|
|
1161
|
-
|
|
1212
|
+
const scopes = DuckdbWarehouseClient.sharedInstanceS3Scopes;
|
|
1213
|
+
const needsReplacement = (token) => (token !== null && tokens.get(instanceCacheKey) !== token) ||
|
|
1214
|
+
DuckdbWarehouseClient.getMissingSharedS3Scope(instanceCacheKey, s3Config).length > 0;
|
|
1215
|
+
const token = await DuckdbWarehouseClient.getSharedS3SecretToken(s3Config);
|
|
1216
|
+
if (!needsReplacement(token))
|
|
1162
1217
|
return;
|
|
1163
|
-
const locks = DuckdbWarehouseClient.
|
|
1218
|
+
const locks = DuckdbWarehouseClient.sharedInstanceS3SecretLocks;
|
|
1164
1219
|
let lock = locks.get(instanceCacheKey);
|
|
1165
1220
|
if (!lock) {
|
|
1166
1221
|
lock = new AsyncSemaphore();
|
|
@@ -1168,14 +1223,41 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1168
1223
|
}
|
|
1169
1224
|
await lock.acquire();
|
|
1170
1225
|
try {
|
|
1171
|
-
// Re-read
|
|
1172
|
-
//
|
|
1173
|
-
// fetched above must not overwrite it.
|
|
1174
|
-
const current = await DuckdbWarehouseClient.
|
|
1175
|
-
if (
|
|
1226
|
+
// Re-read under the lock. Another session may have installed a
|
|
1227
|
+
// newer token or a wider scope while this one waited, and what
|
|
1228
|
+
// was fetched above must not overwrite it.
|
|
1229
|
+
const current = await DuckdbWarehouseClient.getSharedS3SecretToken(s3Config);
|
|
1230
|
+
if (!needsReplacement(current))
|
|
1176
1231
|
return;
|
|
1177
|
-
|
|
1178
|
-
|
|
1232
|
+
const installedScope = scopes.get(instanceCacheKey);
|
|
1233
|
+
const scope = installedScope
|
|
1234
|
+
? [
|
|
1235
|
+
...installedScope,
|
|
1236
|
+
...DuckdbWarehouseClient.getMissingSharedS3Scope(instanceCacheKey, s3Config),
|
|
1237
|
+
]
|
|
1238
|
+
: s3Config.scope;
|
|
1239
|
+
switch (s3Config.authMode) {
|
|
1240
|
+
case 'gcp_oauth': {
|
|
1241
|
+
if (current === null) {
|
|
1242
|
+
throw new Error('DuckDB GCP OAuth requires an access token');
|
|
1243
|
+
}
|
|
1244
|
+
await DuckdbWarehouseClient.createGcsSecret(db, { ...s3Config, scope: scope ?? s3Config.scope }, current);
|
|
1245
|
+
tokens.set(instanceCacheKey, current);
|
|
1246
|
+
break;
|
|
1247
|
+
}
|
|
1248
|
+
case 'default':
|
|
1249
|
+
case undefined:
|
|
1250
|
+
await db.run(DuckdbWarehouseClient.buildS3SecretSql({
|
|
1251
|
+
...s3Config,
|
|
1252
|
+
scope,
|
|
1253
|
+
}));
|
|
1254
|
+
break;
|
|
1255
|
+
default:
|
|
1256
|
+
return (0, common_1.assertUnreachable)(s3Config, 'Unsupported DuckDB storage auth mode');
|
|
1257
|
+
}
|
|
1258
|
+
if (scope) {
|
|
1259
|
+
scopes.set(instanceCacheKey, scope);
|
|
1260
|
+
}
|
|
1179
1261
|
}
|
|
1180
1262
|
finally {
|
|
1181
1263
|
lock.release();
|
|
@@ -1381,8 +1463,8 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1381
1463
|
const sessionStart = performance.now();
|
|
1382
1464
|
const sharedConnection = await this.connectWithRetry();
|
|
1383
1465
|
try {
|
|
1384
|
-
if (this.s3Config
|
|
1385
|
-
await DuckdbWarehouseClient.
|
|
1466
|
+
if (this.s3Config) {
|
|
1467
|
+
await DuckdbWarehouseClient.ensureSharedS3Secret(this.getRequiredInstanceCacheKey(), sharedConnection.connection, this.s3Config);
|
|
1386
1468
|
}
|
|
1387
1469
|
await DuckdbWarehouseClient.applyResourceLimits(sharedConnection.connection, DuckdbWarehouseClient.sharedInstanceResourceLimits.get(this.getRequiredInstanceCacheKey()));
|
|
1388
1470
|
await DuckdbWarehouseClient.hardenInstance(sharedConnection.connection);
|
|
@@ -1866,9 +1948,12 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1866
1948
|
}
|
|
1867
1949
|
}
|
|
1868
1950
|
}
|
|
1951
|
+
// Profiles are written to a local file, which object-storage
|
|
1952
|
+
// and embedded sessions do not use
|
|
1869
1953
|
const profilePath = this.logger &&
|
|
1870
1954
|
this.enableQueryProfiling &&
|
|
1871
1955
|
!this.embeddedConfig &&
|
|
1956
|
+
!this.objectStorageOnly &&
|
|
1872
1957
|
!this.isMotherduck()
|
|
1873
1958
|
? path_1.default.join(os_1.default.tmpdir(), `duckdb-profile-${Date.now()}-${Math.random().toString(36).slice(2)}.json`)
|
|
1874
1959
|
: undefined;
|
|
@@ -2034,7 +2119,13 @@ DuckdbWarehouseClient.sharedInstanceSemaphores = new Map();
|
|
|
2034
2119
|
* the replacement while the token is unchanged.
|
|
2035
2120
|
*/
|
|
2036
2121
|
DuckdbWarehouseClient.sharedInstanceGcsTokens = new Map();
|
|
2037
|
-
|
|
2122
|
+
/**
|
|
2123
|
+
* The S3 URIs each shared instance's secret is scoped to. Sessions that
|
|
2124
|
+
* need a URI outside it widen the secret in place; it is never narrowed
|
|
2125
|
+
* while other sessions may still read through it.
|
|
2126
|
+
*/
|
|
2127
|
+
DuckdbWarehouseClient.sharedInstanceS3Scopes = new Map();
|
|
2128
|
+
DuckdbWarehouseClient.sharedInstanceS3SecretLocks = new Map();
|
|
2038
2129
|
DuckdbWarehouseClient.embeddedConcurrencyBudget = new ConcurrencyBudget(EMBEDDED_MAX_CONCURRENT_QUERIES);
|
|
2039
2130
|
DuckdbWarehouseClient.embeddedOrganizationConcurrencyBudgets = new Map();
|
|
2040
2131
|
DuckdbWarehouseClient.organizationConcurrencyBudgets = new Map();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "2.616.
|
|
3
|
+
"version": "2.616.2",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"@databricks/sql": "1.13.0",
|
|
22
22
|
"@duckdb/node-api": "1.5.2-r.2",
|
|
23
23
|
"@google-cloud/bigquery": "8.3.1",
|
|
24
|
-
"@lightdash/common": "2.616.
|
|
24
|
+
"@lightdash/common": "2.616.2",
|
|
25
25
|
"node-fetch": "2.7.0",
|
|
26
26
|
"pg": "8.13.1",
|
|
27
27
|
"pg-cursor": "2.10.0",
|