@lightdash/warehouses 2.399.1 → 2.399.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo.build +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.js +6 -3
- package/dist/warehouseClients/duckdbEmbeddedBundle.d.ts +27 -0
- package/dist/warehouseClients/duckdbEmbeddedBundle.d.ts.map +1 -0
- package/dist/warehouseClients/duckdbEmbeddedBundle.js +127 -0
- package/package.json +2 -2
package/dist/index.d.ts
CHANGED
|
@@ -6,6 +6,7 @@ export * from './warehouseClients/AthenaWarehouseClient';
|
|
|
6
6
|
export * from './warehouseClients/BigqueryWarehouseClient';
|
|
7
7
|
export * from './warehouseClients/DatabricksWarehouseClient';
|
|
8
8
|
export * from './warehouseClients/DuckdbWarehouseClient';
|
|
9
|
+
export * from './warehouseClients/duckdbEmbeddedBundle';
|
|
9
10
|
export { MotherduckInstanceCache } from './warehouseClients/MotherduckInstanceCache';
|
|
10
11
|
export type { MotherduckCacheEvent } from './warehouseClients/MotherduckInstanceCache';
|
|
11
12
|
export * from './warehouseClients/ListedDatabasesPostgresWarehouseClient';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,SAAS,CAAC;AACxB,cAAc,qCAAqC,CAAC;AACpD,cAAc,kCAAkC,CAAC;AACjD,cAAc,0CAA0C,CAAC;AACzD,cAAc,4CAA4C,CAAC;AAC3D,cAAc,8CAA8C,CAAC;AAC7D,cAAc,0CAA0C,CAAC;AACzD,OAAO,EAAE,uBAAuB,EAAE,MAAM,4CAA4C,CAAC;AACrF,YAAY,EAAE,oBAAoB,EAAE,MAAM,4CAA4C,CAAC;AACvF,cAAc,2DAA2D,CAAC;AAC1E,cAAc,4CAA4C,CAAC;AAC3D,cAAc,4CAA4C,CAAC;AAC3D,cAAc,2CAA2C,CAAC;AAC1D,cAAc,6CAA6C,CAAC;AAC5D,cAAc,yCAAyC,CAAC;AACxD,cAAc,+BAA+B,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,SAAS,CAAC;AACxB,cAAc,qCAAqC,CAAC;AACpD,cAAc,kCAAkC,CAAC;AACjD,cAAc,0CAA0C,CAAC;AACzD,cAAc,4CAA4C,CAAC;AAC3D,cAAc,8CAA8C,CAAC;AAC7D,cAAc,0CAA0C,CAAC;AACzD,cAAc,yCAAyC,CAAC;AACxD,OAAO,EAAE,uBAAuB,EAAE,MAAM,4CAA4C,CAAC;AACrF,YAAY,EAAE,oBAAoB,EAAE,MAAM,4CAA4C,CAAC;AACvF,cAAc,2DAA2D,CAAC;AAC1E,cAAc,4CAA4C,CAAC;AAC3D,cAAc,4CAA4C,CAAC;AAC3D,cAAc,2CAA2C,CAAC;AAC1D,cAAc,6CAA6C,CAAC;AAC5D,cAAc,yCAAyC,CAAC;AACxD,cAAc,+BAA+B,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -23,6 +23,7 @@ __exportStar(require("./warehouseClients/AthenaWarehouseClient"), exports);
|
|
|
23
23
|
__exportStar(require("./warehouseClients/BigqueryWarehouseClient"), exports);
|
|
24
24
|
__exportStar(require("./warehouseClients/DatabricksWarehouseClient"), exports);
|
|
25
25
|
__exportStar(require("./warehouseClients/DuckdbWarehouseClient"), exports);
|
|
26
|
+
__exportStar(require("./warehouseClients/duckdbEmbeddedBundle"), exports);
|
|
26
27
|
var MotherduckInstanceCache_1 = require("./warehouseClients/MotherduckInstanceCache");
|
|
27
28
|
Object.defineProperty(exports, "MotherduckInstanceCache", { enumerable: true, get: function () { return MotherduckInstanceCache_1.MotherduckInstanceCache; } });
|
|
28
29
|
__exportStar(require("./warehouseClients/ListedDatabasesPostgresWarehouseClient"), exports);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAMA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAOb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAChB,eAAe,EAGf,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAMA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAOb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAChB,eAAe,EAGf,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAY3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA8DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;IAChB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB,GAAG,CACE;IAAE,QAAQ,CAAC,EAAE,SAAS,CAAC;IAAC,cAAc,CAAC,EAAE,KAAK,CAAA;CAAE,GAChD;IACI,QAAQ,EAAE,WAAW,CAAC;IACtB,0EAA0E;IAC1E,cAAc,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IACtC,KAAK,EAAE,MAAM,EAAE,CAAC;CACnB,CACN,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACpE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,gFAAgF;AAChF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;IAC3C,4EAA4E;IAC5E,WAAW,CAAC,EAAE;QACV,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE;YACL,IAAI,EAAE,MAAM,CAAC;YACb,IAAI,EAAE,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS,GAAG,QAAQ,CAAC;SACpE,EAAE,CAAC;KACP,EAAE,CAAC;IACJ,2EAA2E;IAC3E,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IACnC,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACnC,IAAI,EAAE,gBAAgB,CAAC;IACvB,0FAA0F;IAC1F,aAAa,EAAE,MAAM,OAAO,CAAC,mBAAmB,CAAC,CAAC;CACrD,CAAC;AAEF,MAAM,MAAM,2BAA2B,GACjC,mBAAmB,GACnB,wBAAwB,CAAC;AAE/B,MAAM,MAAM,4BAA4B,GAAG;IACvC,kHAAkH;IAClH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;IAC9D,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,4BAA4B,CAAC,EAAE,MAAM,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,sBAAsB,WAAY,MAAM,KAAG,aA+BvD,CAAC;AAyBF,eAAO,MAAM,kCAAkC,aACjC,MAAM,KACjB,eAAe,GAAG,SAOpB,CAAC;AAoCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB,CAEpC;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AAmID,eAAO,MAAM,+BAA+B,yBAGzC,IAAI,CAAC,iCAAiC,EAAE,UAAU,GAAG,OAAO,CAAC,KAAG,MAOlE,CAAC;AAiGF,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,iCAAiC,CAAC;IAC7F,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAGhD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAE/C;IAEF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sCAAsC,CAG1D;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,8BAA8B,CAGlD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAA2B;IAE1D,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAU;IAE/C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;IAEV,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAEhD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAU;IAE9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,4BAA4B,CAAC,CAAS;IAEvD,OAAO,CAAC,2BAA2B,CAAS;IAE5C,OAAO,CAAC,gCAAgC,CAAS;IAEjD,YACI,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B,EA+IzC;IAED,OAAO,CAAC,MAAM,CAAC,kBAAkB;IASjC,MAAM,CAAC,qBAAqB,CACxB,WAAW,EAAE,mBAAmB,EAChC,OAAO,CAAC,EAAE,4BAA4B,GACvC,qBAAqB,CAIvB;IAED,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAoBzC,OAAO,CAAC,2BAA2B;mBAUd,6BAA6B;IA4DlD,OAAO,CAAC,MAAM,CAAC,iCAAiC;IAoBhD,OAAO,CAAC,kBAAkB;mBAQL,cAAc;mBAOd,mBAAmB;IAaxC,OAAO,CAAC,MAAM,CAAC,qBAAqB;mBASf,uBAAuB;YAsB9B,aAAa;YAoBb,kCAAkC;mBAW3B,qBAAqB;YAuF5B,qBAAqB;mBAoJd,uBAAuB;mBAgBvB,yBAAyB;mBAiFzB,mBAAmB;IA6BxC,mFAAmF;IACnF,MAAM,CAAC,gCAAgC,IAAI,IAAI,CAS9C;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAO3B;IAED,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CACZ;IAEnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAA+B;IAE3E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAA0B;IAEjE,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAIjC,OAAO,CAAC,MAAM,CAAC,UAAU;IAIzB,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAuB5C,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAqGzC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAMhC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAmBrC,OAAO,CAAC,MAAM,CAAC,sBAAsB;mBA2ChB,iBAAiB;IAyCtC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IA6C/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAK;YAE9C,gBAAgB;IA+E9B,iEAAiE;YACnD,wBAAwB;IA+CtC,gFAAgF;YAClE,mBAAmB;YAkDnB,yBAAyB;YA8CzB,iBAAiB;IAiD/B,OAAO,CAAC,iBAAiB;IAMzB,OAAO,CAAC,YAAY;YAQN,WAAW;YAmFX,2BAA2B;IA4GzC,+DAA+D;YACjD,iBAAiB;YA8EjB,yBAAyB;IA2BvC,OAAO,CAAC,4BAA4B;IAOpC,OAAO,CAAC,aAAa;YA0BP,eAAe;IAgG7B,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAgBxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IA+D/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;IAYnC,MAAM,CAAC,yBAAyB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAiBlD;YAEa,iBAAiB;YA4BjB,eAAe;YAmBf,uBAAuB;YAQvB,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CA0Ff;IAEK,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,mBAAmB,CAAC,CAC9E;;;;;;OA4BJ;IAEK,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKvC;IAEK,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC,CAkBD;IAEK,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,UAAU,CAAC,CACrE;;;OAGJ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAEK,UAAU,CACZ,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC,CA4C3B;IAEK,YAAY,CACd,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,eAAe,CAAC,CAsB1B;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC,CAS3B;CACJ"}
|
|
@@ -44,6 +44,7 @@ const fs_1 = __importDefault(require("fs"));
|
|
|
44
44
|
const promises_1 = __importDefault(require("fs/promises"));
|
|
45
45
|
const os_1 = __importDefault(require("os"));
|
|
46
46
|
const path_1 = __importDefault(require("path"));
|
|
47
|
+
const duckdbEmbeddedBundle_1 = require("./duckdbEmbeddedBundle");
|
|
47
48
|
const MotherduckErrorClassifier_1 = require("./MotherduckErrorClassifier");
|
|
48
49
|
const MotherduckInstanceCache = __importStar(require("./MotherduckInstanceCache"));
|
|
49
50
|
const WarehouseBaseClient_1 = __importDefault(require("./WarehouseBaseClient"));
|
|
@@ -299,7 +300,7 @@ const getPositiveIntegerEnvironmentVariable = (name, defaultValue) => {
|
|
|
299
300
|
const EMBEDDED_MAX_CONCURRENT_QUERIES = getPositiveIntegerEnvironmentVariable('PLAYGROUND_MAX_CONCURRENT_QUERIES', 8);
|
|
300
301
|
const EMBEDDED_MAX_CONCURRENT_QUERIES_PER_ORGANIZATION = getPositiveIntegerEnvironmentVariable('PLAYGROUND_MAX_CONCURRENT_QUERIES_PER_ORGANIZATION', 4);
|
|
301
302
|
const EMBEDDED_CONCURRENCY_ACQUIRE_TIMEOUT_MS = getPositiveIntegerEnvironmentVariable('PLAYGROUND_CONCURRENCY_ACQUIRE_TIMEOUT_MS', 15000);
|
|
302
|
-
const resolveEmbeddedDatabasePath = (dataset) => {
|
|
303
|
+
const resolveEmbeddedDatabasePath = (dataset, bundleVersion) => {
|
|
303
304
|
if (!EMBEDDED_DATASET_PATTERN.test(dataset)) {
|
|
304
305
|
throw new common_1.ParameterError('Embedded DuckDB dataset must contain only lowercase letters, numbers, hyphens, and underscores');
|
|
305
306
|
}
|
|
@@ -311,7 +312,9 @@ const resolveEmbeddedDatabasePath = (dataset) => {
|
|
|
311
312
|
catch {
|
|
312
313
|
throw new common_1.ParameterError(`Embedded DuckDB dataset "${dataset}" was not found in ${baseDirectory}`);
|
|
313
314
|
}
|
|
314
|
-
const databasePath =
|
|
315
|
+
const databasePath = bundleVersion
|
|
316
|
+
? (0, duckdbEmbeddedBundle_1.resolveEmbeddedBundleDatabasePath)(baseDirectory, dataset, bundleVersion)
|
|
317
|
+
: path_1.default.join(baseDirectory, (0, duckdbEmbeddedBundle_1.getEmbeddedDatabaseFileName)(dataset));
|
|
315
318
|
let realDatabasePath;
|
|
316
319
|
try {
|
|
317
320
|
realDatabasePath = fs_1.default.realpathSync(databasePath);
|
|
@@ -399,7 +402,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
399
402
|
// Project DuckDB credentials map to MotherDuck only. The in-memory
|
|
400
403
|
// internal credentials remain available for pre-aggregate helper flows.
|
|
401
404
|
if (this.embeddedConfig) {
|
|
402
|
-
this.databasePath = resolveEmbeddedDatabasePath(this.embeddedConfig.dataset);
|
|
405
|
+
this.databasePath = resolveEmbeddedDatabasePath(this.embeddedConfig.dataset, this.embeddedConfig.bundleVersion);
|
|
403
406
|
}
|
|
404
407
|
else if (this.ducklakeConfig ||
|
|
405
408
|
effectiveCredentials.database === ':memory:') {
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
export declare const EMBEDDED_BUNDLE_CHECKSUMS_FILE = "SHA256SUMS";
|
|
2
|
+
export declare const EMBEDDED_BUNDLE_EXPLORES_FILE = "explores.json";
|
|
3
|
+
export declare const EMBEDDED_BUNDLE_PREVIOUS_DIRECTORY = "previous";
|
|
4
|
+
export type EmbeddedBundle = {
|
|
5
|
+
directory: string;
|
|
6
|
+
version: string;
|
|
7
|
+
checksums: ReadonlyMap<string, string>;
|
|
8
|
+
};
|
|
9
|
+
export declare const getEmbeddedDatabaseFileName: (dataset: string) => string;
|
|
10
|
+
export declare const parseEmbeddedBundleChecksums: (text: string) => Map<string, string>;
|
|
11
|
+
/** Identifies a database and the explores compiled from it, so a change to either gives a new version. */
|
|
12
|
+
export declare const getEmbeddedBundleVersion: (checksums: ReadonlyMap<string, string>, dataset: string) => string;
|
|
13
|
+
export declare const readEmbeddedBundle: (directory: string, dataset: string) => EmbeddedBundle | null;
|
|
14
|
+
export declare const getFileSha256: (filePath: string) => string;
|
|
15
|
+
/** Returns the path of a bundle file after checking it against the bundle's checksum. */
|
|
16
|
+
export declare const verifyEmbeddedBundleFile: (bundle: EmbeddedBundle, fileName: string) => string;
|
|
17
|
+
export declare const readEmbeddedBundles: (baseDirectory: string, dataset: string) => {
|
|
18
|
+
current: EmbeddedBundle | null;
|
|
19
|
+
previous: EmbeddedBundle | null;
|
|
20
|
+
};
|
|
21
|
+
/**
|
|
22
|
+
* Finds the database for a bundle version among the current bundle and the
|
|
23
|
+
* one retained from the previous release. Two versions can share a database
|
|
24
|
+
* file, so the match is by checksum rather than by directory.
|
|
25
|
+
*/
|
|
26
|
+
export declare const resolveEmbeddedBundleDatabasePath: (baseDirectory: string, dataset: string, bundleVersion: string) => string;
|
|
27
|
+
//# sourceMappingURL=duckdbEmbeddedBundle.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"duckdbEmbeddedBundle.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/duckdbEmbeddedBundle.ts"],"names":[],"mappings":"AAKA,eAAO,MAAM,8BAA8B,eAAe,CAAC;AAC3D,eAAO,MAAM,6BAA6B,kBAAkB,CAAC;AAC7D,eAAO,MAAM,kCAAkC,aAAa,CAAC;AAE7D,MAAM,MAAM,cAAc,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC1C,CAAC;AAIF,eAAO,MAAM,2BAA2B,YAAa,MAAM,WACpC,CAAC;AAExB,eAAO,MAAM,4BAA4B,SAC/B,MAAM,KACb,GAAG,CAAC,MAAM,EAAE,MAAM,CAoBpB,CAAC;AAEF,0GAA0G;AAC1G,eAAO,MAAM,wBAAwB,cACtB,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,WAC7B,MAAM,KAChB,MAeF,CAAC;AAEF,eAAO,MAAM,kBAAkB,cAChB,MAAM,WACR,MAAM,KAChB,cAAc,GAAG,IAiBnB,CAAC;AAIF,eAAO,MAAM,aAAa,aAAc,MAAM,KAAG,MAUhD,CAAC;AAKF,yFAAyF;AACzF,eAAO,MAAM,wBAAwB,WACzB,cAAc,YACZ,MAAM,KACjB,MAoBF,CAAC;AAEF,eAAO,MAAM,mBAAmB,kBACb,MAAM,WACZ,MAAM,KAChB;IAAE,OAAO,EAAE,cAAc,GAAG,IAAI,CAAC;IAAC,QAAQ,EAAE,cAAc,GAAG,IAAI,CAAA;CAMlE,CAAC;AAEH;;;;GAIG;AACH,eAAO,MAAM,iCAAiC,kBAC3B,MAAM,WACZ,MAAM,iBACA,MAAM,KACtB,MA2BF,CAAC"}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.resolveEmbeddedBundleDatabasePath = exports.readEmbeddedBundles = exports.verifyEmbeddedBundleFile = exports.getFileSha256 = exports.readEmbeddedBundle = exports.getEmbeddedBundleVersion = exports.parseEmbeddedBundleChecksums = exports.getEmbeddedDatabaseFileName = exports.EMBEDDED_BUNDLE_PREVIOUS_DIRECTORY = exports.EMBEDDED_BUNDLE_EXPLORES_FILE = exports.EMBEDDED_BUNDLE_CHECKSUMS_FILE = void 0;
|
|
7
|
+
const common_1 = require("@lightdash/common");
|
|
8
|
+
const crypto_1 = require("crypto");
|
|
9
|
+
const fs_1 = __importDefault(require("fs"));
|
|
10
|
+
const path_1 = __importDefault(require("path"));
|
|
11
|
+
exports.EMBEDDED_BUNDLE_CHECKSUMS_FILE = 'SHA256SUMS';
|
|
12
|
+
exports.EMBEDDED_BUNDLE_EXPLORES_FILE = 'explores.json';
|
|
13
|
+
exports.EMBEDDED_BUNDLE_PREVIOUS_DIRECTORY = 'previous';
|
|
14
|
+
const CHECKSUM_LINE = /^([0-9a-f]{64}) {2}([A-Za-z0-9._-]+)$/;
|
|
15
|
+
const getEmbeddedDatabaseFileName = (dataset) => `${dataset}.duckdb`;
|
|
16
|
+
exports.getEmbeddedDatabaseFileName = getEmbeddedDatabaseFileName;
|
|
17
|
+
const parseEmbeddedBundleChecksums = (text) => {
|
|
18
|
+
const checksums = new Map();
|
|
19
|
+
text.split('\n')
|
|
20
|
+
.filter((line) => line !== '')
|
|
21
|
+
.forEach((line) => {
|
|
22
|
+
const match = CHECKSUM_LINE.exec(line);
|
|
23
|
+
if (!match) {
|
|
24
|
+
throw new common_1.WarehouseConnectionError(`${exports.EMBEDDED_BUNDLE_CHECKSUMS_FILE} has an invalid line: ${line}`);
|
|
25
|
+
}
|
|
26
|
+
const [, digest, fileName] = match;
|
|
27
|
+
if (checksums.has(fileName)) {
|
|
28
|
+
throw new common_1.WarehouseConnectionError(`${exports.EMBEDDED_BUNDLE_CHECKSUMS_FILE} lists ${fileName} more than once`);
|
|
29
|
+
}
|
|
30
|
+
checksums.set(fileName, digest);
|
|
31
|
+
});
|
|
32
|
+
return checksums;
|
|
33
|
+
};
|
|
34
|
+
exports.parseEmbeddedBundleChecksums = parseEmbeddedBundleChecksums;
|
|
35
|
+
/** Identifies a database and the explores compiled from it, so a change to either gives a new version. */
|
|
36
|
+
const getEmbeddedBundleVersion = (checksums, dataset) => {
|
|
37
|
+
const databaseFile = (0, exports.getEmbeddedDatabaseFileName)(dataset);
|
|
38
|
+
const exploresDigest = checksums.get(exports.EMBEDDED_BUNDLE_EXPLORES_FILE);
|
|
39
|
+
const databaseDigest = checksums.get(databaseFile);
|
|
40
|
+
if (!exploresDigest || !databaseDigest) {
|
|
41
|
+
throw new common_1.WarehouseConnectionError(`${exports.EMBEDDED_BUNDLE_CHECKSUMS_FILE} must list ${exports.EMBEDDED_BUNDLE_EXPLORES_FILE} and ${databaseFile}`);
|
|
42
|
+
}
|
|
43
|
+
return (0, crypto_1.createHash)('sha256')
|
|
44
|
+
.update(`${exploresDigest} ${exports.EMBEDDED_BUNDLE_EXPLORES_FILE}\n${databaseDigest} ${databaseFile}\n`)
|
|
45
|
+
.digest('hex')
|
|
46
|
+
.slice(0, 16);
|
|
47
|
+
};
|
|
48
|
+
exports.getEmbeddedBundleVersion = getEmbeddedBundleVersion;
|
|
49
|
+
const readEmbeddedBundle = (directory, dataset) => {
|
|
50
|
+
let text;
|
|
51
|
+
try {
|
|
52
|
+
text = fs_1.default.readFileSync(path_1.default.join(directory, exports.EMBEDDED_BUNDLE_CHECKSUMS_FILE), 'utf8');
|
|
53
|
+
}
|
|
54
|
+
catch (error) {
|
|
55
|
+
if (error.code === 'ENOENT')
|
|
56
|
+
return null;
|
|
57
|
+
throw error;
|
|
58
|
+
}
|
|
59
|
+
const checksums = (0, exports.parseEmbeddedBundleChecksums)(text);
|
|
60
|
+
return {
|
|
61
|
+
directory,
|
|
62
|
+
version: (0, exports.getEmbeddedBundleVersion)(checksums, dataset),
|
|
63
|
+
checksums,
|
|
64
|
+
};
|
|
65
|
+
};
|
|
66
|
+
exports.readEmbeddedBundle = readEmbeddedBundle;
|
|
67
|
+
const fileDigests = new Map();
|
|
68
|
+
const getFileSha256 = (filePath) => {
|
|
69
|
+
const stat = fs_1.default.statSync(filePath);
|
|
70
|
+
const statKey = `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeMs}`;
|
|
71
|
+
const cached = fileDigests.get(filePath);
|
|
72
|
+
if (cached?.statKey === statKey)
|
|
73
|
+
return cached.digest;
|
|
74
|
+
const digest = (0, crypto_1.createHash)('sha256')
|
|
75
|
+
.update(fs_1.default.readFileSync(filePath))
|
|
76
|
+
.digest('hex');
|
|
77
|
+
fileDigests.set(filePath, { statKey, digest });
|
|
78
|
+
return digest;
|
|
79
|
+
};
|
|
80
|
+
exports.getFileSha256 = getFileSha256;
|
|
81
|
+
const fileMatchesDigest = (filePath, digest) => fs_1.default.existsSync(filePath) && (0, exports.getFileSha256)(filePath) === digest;
|
|
82
|
+
/** Returns the path of a bundle file after checking it against the bundle's checksum. */
|
|
83
|
+
const verifyEmbeddedBundleFile = (bundle, fileName) => {
|
|
84
|
+
const expected = bundle.checksums.get(fileName);
|
|
85
|
+
if (!expected) {
|
|
86
|
+
throw new common_1.WarehouseConnectionError(`${exports.EMBEDDED_BUNDLE_CHECKSUMS_FILE} in ${bundle.directory} does not list ${fileName}`);
|
|
87
|
+
}
|
|
88
|
+
const filePath = path_1.default.join(bundle.directory, fileName);
|
|
89
|
+
if (!fs_1.default.existsSync(filePath)) {
|
|
90
|
+
throw new common_1.WarehouseConnectionError(`${fileName} is missing from ${bundle.directory}`);
|
|
91
|
+
}
|
|
92
|
+
const actual = (0, exports.getFileSha256)(filePath);
|
|
93
|
+
if (actual !== expected) {
|
|
94
|
+
throw new common_1.WarehouseConnectionError(`${fileName} in ${bundle.directory} does not match ${exports.EMBEDDED_BUNDLE_CHECKSUMS_FILE}: expected ${expected}, found ${actual}`);
|
|
95
|
+
}
|
|
96
|
+
return filePath;
|
|
97
|
+
};
|
|
98
|
+
exports.verifyEmbeddedBundleFile = verifyEmbeddedBundleFile;
|
|
99
|
+
const readEmbeddedBundles = (baseDirectory, dataset) => ({
|
|
100
|
+
current: (0, exports.readEmbeddedBundle)(baseDirectory, dataset),
|
|
101
|
+
previous: (0, exports.readEmbeddedBundle)(path_1.default.join(baseDirectory, exports.EMBEDDED_BUNDLE_PREVIOUS_DIRECTORY), dataset),
|
|
102
|
+
});
|
|
103
|
+
exports.readEmbeddedBundles = readEmbeddedBundles;
|
|
104
|
+
/**
|
|
105
|
+
* Finds the database for a bundle version among the current bundle and the
|
|
106
|
+
* one retained from the previous release. Two versions can share a database
|
|
107
|
+
* file, so the match is by checksum rather than by directory.
|
|
108
|
+
*/
|
|
109
|
+
const resolveEmbeddedBundleDatabasePath = (baseDirectory, dataset, bundleVersion) => {
|
|
110
|
+
const { current, previous } = (0, exports.readEmbeddedBundles)(baseDirectory, dataset);
|
|
111
|
+
const available = [current, previous].filter((bundle) => bundle !== null);
|
|
112
|
+
const bundle = available.find(({ version }) => version === bundleVersion);
|
|
113
|
+
if (!bundle) {
|
|
114
|
+
throw new common_1.WarehouseConnectionError(`Sample data version mismatch: the project uses version ${bundleVersion}, but this server has ${available.map(({ version }) => version).join(' and ') ||
|
|
115
|
+
'no versioned sample data'}`);
|
|
116
|
+
}
|
|
117
|
+
const databaseFile = (0, exports.getEmbeddedDatabaseFileName)(dataset);
|
|
118
|
+
const digest = bundle.checksums.get(databaseFile);
|
|
119
|
+
const databasePath = [bundle, ...available]
|
|
120
|
+
.map(({ directory }) => path_1.default.join(directory, databaseFile))
|
|
121
|
+
.find((candidate) => digest ? fileMatchesDigest(candidate, digest) : false);
|
|
122
|
+
if (!databasePath) {
|
|
123
|
+
throw new common_1.WarehouseConnectionError(`Sample data version ${bundleVersion} needs ${databaseFile} with checksum ${digest}, but no file on this server matches it`);
|
|
124
|
+
}
|
|
125
|
+
return databasePath;
|
|
126
|
+
};
|
|
127
|
+
exports.resolveEmbeddedBundleDatabasePath = resolveEmbeddedBundleDatabasePath;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "2.399.
|
|
3
|
+
"version": "2.399.3",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"@databricks/sql": "1.13.0",
|
|
22
22
|
"@duckdb/node-api": "1.5.2-r.2",
|
|
23
23
|
"@google-cloud/bigquery": "8.3.1",
|
|
24
|
-
"@lightdash/common": "2.399.
|
|
24
|
+
"@lightdash/common": "2.399.3",
|
|
25
25
|
"node-fetch": "2.7.0",
|
|
26
26
|
"pg": "8.13.1",
|
|
27
27
|
"pg-cursor": "2.10.0",
|