@lightdash/warehouses 2.469.0 → 2.470.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -107,6 +107,14 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
107
107
|
private static readonly sharedInstances;
|
|
108
108
|
private static readonly sharedInstanceResourceLimits;
|
|
109
109
|
private static readonly sharedInstanceSemaphores;
|
|
110
|
+
/**
|
|
111
|
+
* The Google access token each shared instance's GCS secret currently
|
|
112
|
+
* holds. Secrets belong to the instance, not the connection, so concurrent
|
|
113
|
+
* sessions replacing the same secret race in the catalog. Sessions skip
|
|
114
|
+
* the replacement while the token is unchanged.
|
|
115
|
+
*/
|
|
116
|
+
private static readonly sharedInstanceGcsTokens;
|
|
117
|
+
private static readonly sharedInstanceGcsSecretLocks;
|
|
110
118
|
private static readonly embeddedConcurrencyBudget;
|
|
111
119
|
private static readonly embeddedOrganizationConcurrencyBudgets;
|
|
112
120
|
private static readonly organizationConcurrencyBudgets;
|
|
@@ -163,6 +171,14 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
163
171
|
private static buildDucklakeSecretSql;
|
|
164
172
|
private static buildDucklakeAttachSql;
|
|
165
173
|
private static configureS3Secret;
|
|
174
|
+
private static getGcsAccessToken;
|
|
175
|
+
private static createGcsSecret;
|
|
176
|
+
/**
|
|
177
|
+
* Points a shared instance's GCS secret at the current token. The token
|
|
178
|
+
* provider caches tokens, so this only replaces the secret when the token
|
|
179
|
+
* rotates, once per instance under a lock.
|
|
180
|
+
*/
|
|
181
|
+
private static ensureSharedGcsSecret;
|
|
166
182
|
private static buildS3SecretSql;
|
|
167
183
|
private static readonly CONNECT_RETRIES_BEFORE_RECREATE;
|
|
168
184
|
private connectWithRetry;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAMA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAOb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAChB,eAAe,EAGf,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAY3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA8DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;IAChB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB,GAAG,CACE;IAAE,QAAQ,CAAC,EAAE,SAAS,CAAC;IAAC,cAAc,CAAC,EAAE,KAAK,CAAA;CAAE,GAChD;IACI,QAAQ,EAAE,WAAW,CAAC;IACtB,0EAA0E;IAC1E,cAAc,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IACtC,KAAK,EAAE,MAAM,EAAE,CAAC;CACnB,CACN,CAAC;
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAMA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAOb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAChB,eAAe,EAGf,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAY3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA8DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;IAChB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB,GAAG,CACE;IAAE,QAAQ,CAAC,EAAE,SAAS,CAAC;IAAC,cAAc,CAAC,EAAE,KAAK,CAAA;CAAE,GAChD;IACI,QAAQ,EAAE,WAAW,CAAC;IACtB,0EAA0E;IAC1E,cAAc,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IACtC,KAAK,EAAE,MAAM,EAAE,CAAC;CACnB,CACN,CAAC;AAOF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACpE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS,GAAG,QAAQ,CAAC;CACpE,EAAE,CAAC;AAEJ,gFAAgF;AAChF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAAC,OAAO,CAAC,EAAE,oBAAoB,CAAA;KAAE,EAAE,CAAC;IAC3E,4EAA4E;IAC5E,WAAW,CAAC,EAAE;QACV,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,oBAAoB,CAAC;KACjC,EAAE,CAAC;IACJ,2EAA2E;IAC3E,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IACnC,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACnC,IAAI,EAAE,gBAAgB,CAAC;IACvB,0FAA0F;IAC1F,aAAa,EAAE,CACX,gBAAgB,CAAC,EAAE,SAAS,MAAM,EAAE,KACnC,OAAO,CAAC,mBAAmB,CAAC,CAAC;CACrC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GACjC,mBAAmB,GACnB,wBAAwB,CAAC;AAE/B,MAAM,MAAM,4BAA4B,GAAG;IACvC,kHAAkH;IAClH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;IAC9D,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,4BAA4B,CAAC,EAAE,MAAM,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,sBAAsB,WAAY,MAAM,KAAG,aA+BvD,CAAC;AAyBF,eAAO,MAAM,kCAAkC,aACjC,MAAM,KACjB,eAAe,GAAG,SAOpB,CAAC;AAoCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB,CAEpC;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AAmID,eAAO,MAAM,+BAA+B,yBAGzC,IAAI,CAAC,iCAAiC,EAAE,UAAU,GAAG,OAAO,CAAC,KAAG,MAOlE,CAAC;AAiGF,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,iCAAiC,CAAC;IAC7F,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAGhD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ;;;;;OAKG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAA6B;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAGhD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAE/C;IAEF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sCAAsC,CAG1D;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,8BAA8B,CAGlD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAA2B;IAE1D,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAU;IAE/C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;IAEV,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAEhD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAU;IAE9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,4BAA4B,CAAC,CAAS;IAEvD,OAAO,CAAC,2BAA2B,CAAS;IAE5C,OAAO,CAAC,gCAAgC,CAAS;IAEjD,YACI,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B,EA+IzC;IAED,OAAO,CAAC,MAAM,CAAC,kBAAkB;IASjC,MAAM,CAAC,qBAAqB,CACxB,WAAW,EAAE,mBAAmB,EAChC,OAAO,CAAC,EAAE,4BAA4B,GACvC,qBAAqB,CAIvB;IAED,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAoBzC,OAAO,CAAC,2BAA2B;mBAUd,6BAA6B;IA4DlD,OAAO,CAAC,MAAM,CAAC,iCAAiC;IAoBhD,OAAO,CAAC,kBAAkB;mBAQL,cAAc;mBAOd,mBAAmB;IAaxC,OAAO,CAAC,MAAM,CAAC,qBAAqB;mBASf,uBAAuB;YAsB9B,aAAa;YAoBb,kCAAkC;mBAW3B,qBAAqB;IAuF1C,2EAA2E;mBACtD,qBAAqB;YA8B5B,qBAAqB;mBA2Jd,uBAAuB;mBAgBvB,yBAAyB;mBAoFzB,mBAAmB;IAgCxC,mFAAmF;IACnF,MAAM,CAAC,gCAAgC,IAAI,IAAI,CAW9C;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAO3B;IAED,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CACZ;IAEnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAA+B;IAE3E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAA0B;IAEjE,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAIjC,OAAO,CAAC,MAAM,CAAC,UAAU;IAIzB,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAuB5C,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAqGzC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAMhC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAmBrC,OAAO,CAAC,MAAM,CAAC,sBAAsB;mBA2ChB,iBAAiB;mBAejB,iBAAiB;mBAiBjB,eAAe;IA+BpC;;;;OAIG;mBACkB,qBAAqB;IA8B1C,OAAO,CAAC,MAAM,CAAC,gBAAgB;IA6C/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAK;YAE9C,gBAAgB;IA+E9B,iEAAiE;YACnD,wBAAwB;IA+CtC,gFAAgF;YAClE,mBAAmB;YAkDnB,yBAAyB;YA8CzB,iBAAiB;IAkD/B,OAAO,CAAC,iBAAiB;IAMzB,OAAO,CAAC,YAAY;YAQN,WAAW;YAmFX,2BAA2B;IA4GzC,+DAA+D;YACjD,iBAAiB;YA8EjB,yBAAyB;IA2BvC,OAAO,CAAC,4BAA4B;IAOpC,OAAO,CAAC,aAAa;YA0BP,eAAe;IAgG7B,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAgBxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IA+D/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;IAYnC,MAAM,CAAC,yBAAyB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAiBlD;YAEa,iBAAiB;YA4BjB,eAAe;YAmBf,uBAAuB;YAQvB,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CA0Ff;IAEK,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,mBAAmB,CAAC,CAC9E;;;;;;OA4BJ;IAEK,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKvC;IAEK,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC,CAkBD;IAEK,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,UAAU,CAAC,CACrE;;;OAGJ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAEK,UAAU,CACZ,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC,CA4C3B;IAEK,YAAY,CACd,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,eAAe,CAAC,CAsB1B;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC,CAS3B;CACJ"}
|
|
@@ -833,6 +833,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
833
833
|
bootstrapMs: 0,
|
|
834
834
|
};
|
|
835
835
|
}
|
|
836
|
+
DuckdbWarehouseClient.sharedInstanceGcsTokens.delete(instanceCacheKey);
|
|
836
837
|
const createStart = performance.now();
|
|
837
838
|
const instance = await node_api_1.DuckDBInstance.create(':memory:');
|
|
838
839
|
const instanceCreateMs = performance.now() - createStart;
|
|
@@ -879,6 +880,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
879
880
|
}
|
|
880
881
|
DuckdbWarehouseClient.sharedInstances.delete(instanceCacheKey);
|
|
881
882
|
DuckdbWarehouseClient.sharedInstanceResourceLimits.delete(instanceCacheKey);
|
|
883
|
+
DuckdbWarehouseClient.sharedInstanceGcsTokens.delete(instanceCacheKey);
|
|
882
884
|
logger?.info(`DuckDB shared instance cleared: ${instanceCacheKey}`);
|
|
883
885
|
}
|
|
884
886
|
}
|
|
@@ -891,6 +893,8 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
891
893
|
DuckdbWarehouseClient.sharedInstances.clear();
|
|
892
894
|
DuckdbWarehouseClient.sharedInstanceResourceLimits.clear();
|
|
893
895
|
DuckdbWarehouseClient.sharedInstanceSemaphores.clear();
|
|
896
|
+
DuckdbWarehouseClient.sharedInstanceGcsTokens.clear();
|
|
897
|
+
DuckdbWarehouseClient.sharedInstanceGcsSecretLocks.clear();
|
|
894
898
|
DuckdbWarehouseClient.embeddedConcurrencyBudget.reset();
|
|
895
899
|
DuckdbWarehouseClient.embeddedOrganizationConcurrencyBudgets.forEach((budget) => budget.reset());
|
|
896
900
|
DuckdbWarehouseClient.embeddedOrganizationConcurrencyBudgets.clear();
|
|
@@ -1063,37 +1067,78 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1063
1067
|
return stmts;
|
|
1064
1068
|
}
|
|
1065
1069
|
static async configureS3Secret(db, s3Config) {
|
|
1066
|
-
const escape = (v) => DuckdbWarehouseClient.sqlBuilder.escapeString(v);
|
|
1067
1070
|
if (s3Config.authMode === 'gcp_oauth') {
|
|
1068
|
-
|
|
1069
|
-
throw new Error('DuckDB GCP OAuth requires a storage scope');
|
|
1070
|
-
}
|
|
1071
|
-
let token;
|
|
1072
|
-
try {
|
|
1073
|
-
token = await s3Config.getAccessToken();
|
|
1074
|
-
if (!token)
|
|
1075
|
-
throw new Error('Empty access token');
|
|
1076
|
-
}
|
|
1077
|
-
catch {
|
|
1078
|
-
throw new Error('Unable to obtain a Google access token for DuckDB storage');
|
|
1079
|
-
}
|
|
1080
|
-
try {
|
|
1081
|
-
await db.run(`CREATE OR REPLACE SECRET __lightdash_s3 (
|
|
1082
|
-
TYPE gcs,
|
|
1083
|
-
BEARER_TOKEN $1,
|
|
1084
|
-
ENDPOINT '${escape(s3Config.endpoint)}',
|
|
1085
|
-
SCOPE (${s3Config.scope.map((uri) => `'${escape(uri)}'`).join(', ')}),
|
|
1086
|
-
URL_STYLE '${s3Config.forcePathStyle ? 'path' : 'vhost'}',
|
|
1087
|
-
USE_SSL ${s3Config.useSsl}
|
|
1088
|
-
);`, [token]);
|
|
1089
|
-
}
|
|
1090
|
-
catch {
|
|
1091
|
-
throw new Error('Unable to configure GCP OAuth for DuckDB storage');
|
|
1092
|
-
}
|
|
1071
|
+
await DuckdbWarehouseClient.createGcsSecret(db, s3Config, await DuckdbWarehouseClient.getGcsAccessToken(s3Config));
|
|
1093
1072
|
return;
|
|
1094
1073
|
}
|
|
1095
1074
|
await db.run(DuckdbWarehouseClient.buildS3SecretSql(s3Config));
|
|
1096
1075
|
}
|
|
1076
|
+
static async getGcsAccessToken(s3Config) {
|
|
1077
|
+
if (!s3Config.scope?.length) {
|
|
1078
|
+
throw new Error('DuckDB GCP OAuth requires a storage scope');
|
|
1079
|
+
}
|
|
1080
|
+
try {
|
|
1081
|
+
const token = await s3Config.getAccessToken();
|
|
1082
|
+
if (!token)
|
|
1083
|
+
throw new Error('Empty access token');
|
|
1084
|
+
return token;
|
|
1085
|
+
}
|
|
1086
|
+
catch {
|
|
1087
|
+
throw new Error('Unable to obtain a Google access token for DuckDB storage');
|
|
1088
|
+
}
|
|
1089
|
+
}
|
|
1090
|
+
static async createGcsSecret(db, s3Config, token) {
|
|
1091
|
+
const escape = (v) => DuckdbWarehouseClient.sqlBuilder.escapeString(v);
|
|
1092
|
+
try {
|
|
1093
|
+
await db.run(`CREATE OR REPLACE SECRET __lightdash_s3 (
|
|
1094
|
+
TYPE gcs,
|
|
1095
|
+
BEARER_TOKEN $1,
|
|
1096
|
+
ENDPOINT '${escape(s3Config.endpoint)}',
|
|
1097
|
+
SCOPE (${s3Config.scope.map((uri) => `'${escape(uri)}'`).join(', ')}),
|
|
1098
|
+
URL_STYLE '${s3Config.forcePathStyle ? 'path' : 'vhost'}',
|
|
1099
|
+
USE_SSL ${s3Config.useSsl}
|
|
1100
|
+
);`, [token]);
|
|
1101
|
+
}
|
|
1102
|
+
catch (error) {
|
|
1103
|
+
// Keep DuckDB's reason, but never the token: a failed statement
|
|
1104
|
+
// can echo its parameters.
|
|
1105
|
+
const reason = (0, common_1.getErrorMessage)(error)
|
|
1106
|
+
.split(token)
|
|
1107
|
+
.join('[redacted]');
|
|
1108
|
+
throw new Error(`Unable to configure GCP OAuth for DuckDB storage: ${reason}`);
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
/**
|
|
1112
|
+
* Points a shared instance's GCS secret at the current token. The token
|
|
1113
|
+
* provider caches tokens, so this only replaces the secret when the token
|
|
1114
|
+
* rotates, once per instance under a lock.
|
|
1115
|
+
*/
|
|
1116
|
+
static async ensureSharedGcsSecret(instanceCacheKey, db, s3Config) {
|
|
1117
|
+
const tokens = DuckdbWarehouseClient.sharedInstanceGcsTokens;
|
|
1118
|
+
const token = await DuckdbWarehouseClient.getGcsAccessToken(s3Config);
|
|
1119
|
+
if (tokens.get(instanceCacheKey) === token)
|
|
1120
|
+
return;
|
|
1121
|
+
const locks = DuckdbWarehouseClient.sharedInstanceGcsSecretLocks;
|
|
1122
|
+
let lock = locks.get(instanceCacheKey);
|
|
1123
|
+
if (!lock) {
|
|
1124
|
+
lock = new AsyncSemaphore();
|
|
1125
|
+
locks.set(instanceCacheKey, lock);
|
|
1126
|
+
}
|
|
1127
|
+
await lock.acquire();
|
|
1128
|
+
try {
|
|
1129
|
+
// Re-read the token under the lock. Another session may have
|
|
1130
|
+
// installed a newer one while this one waited, and the token
|
|
1131
|
+
// fetched above must not overwrite it.
|
|
1132
|
+
const current = await DuckdbWarehouseClient.getGcsAccessToken(s3Config);
|
|
1133
|
+
if (tokens.get(instanceCacheKey) === current)
|
|
1134
|
+
return;
|
|
1135
|
+
await DuckdbWarehouseClient.createGcsSecret(db, s3Config, current);
|
|
1136
|
+
tokens.set(instanceCacheKey, current);
|
|
1137
|
+
}
|
|
1138
|
+
finally {
|
|
1139
|
+
lock.release();
|
|
1140
|
+
}
|
|
1141
|
+
}
|
|
1097
1142
|
static buildS3SecretSql(s3Config) {
|
|
1098
1143
|
const escape = (v) => DuckdbWarehouseClient.sqlBuilder.escapeString(v);
|
|
1099
1144
|
const usesStaticCredentials = !DuckdbWarehouseClient.usesS3CredentialChain(s3Config);
|
|
@@ -1287,7 +1332,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1287
1332
|
const sharedConnection = await this.connectWithRetry();
|
|
1288
1333
|
try {
|
|
1289
1334
|
if (this.s3Config?.authMode === 'gcp_oauth') {
|
|
1290
|
-
await DuckdbWarehouseClient.
|
|
1335
|
+
await DuckdbWarehouseClient.ensureSharedGcsSecret(this.getRequiredInstanceCacheKey(), sharedConnection.connection, this.s3Config);
|
|
1291
1336
|
}
|
|
1292
1337
|
await DuckdbWarehouseClient.applyResourceLimits(sharedConnection.connection, DuckdbWarehouseClient.sharedInstanceResourceLimits.get(this.getRequiredInstanceCacheKey()));
|
|
1293
1338
|
await DuckdbWarehouseClient.hardenInstance(sharedConnection.connection);
|
|
@@ -1932,6 +1977,14 @@ exports.DuckdbWarehouseClient = DuckdbWarehouseClient;
|
|
|
1932
1977
|
DuckdbWarehouseClient.sharedInstances = new Map();
|
|
1933
1978
|
DuckdbWarehouseClient.sharedInstanceResourceLimits = new Map();
|
|
1934
1979
|
DuckdbWarehouseClient.sharedInstanceSemaphores = new Map();
|
|
1980
|
+
/**
|
|
1981
|
+
* The Google access token each shared instance's GCS secret currently
|
|
1982
|
+
* holds. Secrets belong to the instance, not the connection, so concurrent
|
|
1983
|
+
* sessions replacing the same secret race in the catalog. Sessions skip
|
|
1984
|
+
* the replacement while the token is unchanged.
|
|
1985
|
+
*/
|
|
1986
|
+
DuckdbWarehouseClient.sharedInstanceGcsTokens = new Map();
|
|
1987
|
+
DuckdbWarehouseClient.sharedInstanceGcsSecretLocks = new Map();
|
|
1935
1988
|
DuckdbWarehouseClient.embeddedConcurrencyBudget = new ConcurrencyBudget(EMBEDDED_MAX_CONCURRENT_QUERIES);
|
|
1936
1989
|
DuckdbWarehouseClient.embeddedOrganizationConcurrencyBudgets = new Map();
|
|
1937
1990
|
DuckdbWarehouseClient.organizationConcurrencyBudgets = new Map();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.470.1",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"@databricks/sql": "1.13.0",
|
|
22
22
|
"@duckdb/node-api": "1.5.2-r.2",
|
|
23
23
|
"@google-cloud/bigquery": "8.3.1",
|
|
24
|
-
"@lightdash/common": "2.
|
|
24
|
+
"@lightdash/common": "2.470.1",
|
|
25
25
|
"node-fetch": "2.7.0",
|
|
26
26
|
"pg": "8.13.1",
|
|
27
27
|
"pg-cursor": "2.10.0",
|