@lightdash/warehouses 2.475.0 → 2.475.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -49,6 +49,8 @@ export type DuckdbParquetColumns = {
|
|
|
49
49
|
}[];
|
|
50
50
|
/** Server-owned manifest. Never accept this configuration from project APIs. */
|
|
51
51
|
export type DuckdbParquetSource = {
|
|
52
|
+
/** Trusted provider bytes for exact manifest URLs; staged privately per query. */
|
|
53
|
+
fileBuffers?: ReadonlyMap<string, Uint8Array>;
|
|
52
54
|
scope: string;
|
|
53
55
|
tables: {
|
|
54
56
|
name: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAMA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAOb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAChB,eAAe,EAGf,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAY3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA8DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;IAChB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB,GAAG,CACE;IAAE,QAAQ,CAAC,EAAE,SAAS,CAAC;IAAC,cAAc,CAAC,EAAE,KAAK,CAAA;CAAE,GAChD;IACI,QAAQ,EAAE,WAAW,CAAC;IACtB,0EAA0E;IAC1E,cAAc,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IACtC,KAAK,EAAE,MAAM,EAAE,CAAC;CACnB,CACN,CAAC;AAOF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACpE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS,GAAG,QAAQ,CAAC;CACpE,EAAE,CAAC;AAEJ,gFAAgF;AAChF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAAC,OAAO,CAAC,EAAE,oBAAoB,CAAA;KAAE,EAAE,CAAC;IAC3E,4EAA4E;IAC5E,WAAW,CAAC,EAAE;QACV,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,oBAAoB,CAAC;KACjC,EAAE,CAAC;IACJ,2EAA2E;IAC3E,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IACnC,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACnC,IAAI,EAAE,gBAAgB,CAAC;IACvB,0FAA0F;IAC1F,aAAa,EAAE,CACX,gBAAgB,CAAC,EAAE,SAAS,MAAM,EAAE,KACnC,OAAO,CAAC,mBAAmB,CAAC,CAAC;CACrC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GACjC,mBAAmB,GACnB,wBAAwB,CAAC;AAE/B,MAAM,MAAM,4BAA4B,GAAG;IACvC,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,kHAAkH;IAClH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;IAC9D,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,4BAA4B,CAAC,EAAE,MAAM,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,sBAAsB,WAAY,MAAM,KAAG,aA+BvD,CAAC;AAyBF,eAAO,MAAM,kCAAkC,aACjC,MAAM,KACjB,eAAe,GAAG,SAOpB,CAAC;AAoCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB,CAEpC;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AAmID,eAAO,MAAM,+BAA+B,yBAGzC,IAAI,CAAC,iCAAiC,EAAE,UAAU,GAAG,OAAO,CAAC,KAAG,MAOlE,CAAC;AAiGF,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,iCAAiC,CAAC;IAC7F,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAGhD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ;;;;;OAKG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAA6B;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAGhD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAE/C;IAEF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sCAAsC,CAG1D;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,8BAA8B,CAGlD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAA2B;IAE1D,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAU;IAE/C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;IAEV,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAEhD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAU;IAE9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,4BAA4B,CAAC,CAAS;IAEvD,OAAO,CAAC,2BAA2B,CAAS;IAE5C,OAAO,CAAC,gCAAgC,CAAS;IAEjD,YACI,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B,EAgJzC;IAED,OAAO,CAAC,MAAM,CAAC,kBAAkB;IASjC,MAAM,CAAC,qBAAqB,CACxB,WAAW,EAAE,mBAAmB,EAChC,OAAO,CAAC,EAAE,4BAA4B,GACvC,qBAAqB,CAIvB;IAED,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAoBzC,OAAO,CAAC,2BAA2B;mBAUd,6BAA6B;IA4DlD,OAAO,CAAC,MAAM,CAAC,iCAAiC;IAoBhD,OAAO,CAAC,kBAAkB;mBAQL,cAAc;mBAOd,mBAAmB;IAaxC,OAAO,CAAC,MAAM,CAAC,qBAAqB;mBASf,uBAAuB;YAsB9B,aAAa;YAoBb,kCAAkC;mBAW3B,qBAAqB;
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAMA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAOb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAChB,eAAe,EAGf,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAY3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA8DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;IAChB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB,GAAG,CACE;IAAE,QAAQ,CAAC,EAAE,SAAS,CAAC;IAAC,cAAc,CAAC,EAAE,KAAK,CAAA;CAAE,GAChD;IACI,QAAQ,EAAE,WAAW,CAAC;IACtB,0EAA0E;IAC1E,cAAc,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IACtC,KAAK,EAAE,MAAM,EAAE,CAAC;CACnB,CACN,CAAC;AAOF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACpE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS,GAAG,QAAQ,CAAC;CACpE,EAAE,CAAC;AAEJ,gFAAgF;AAChF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,kFAAkF;IAClF,WAAW,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAAC,OAAO,CAAC,EAAE,oBAAoB,CAAA;KAAE,EAAE,CAAC;IAC3E,4EAA4E;IAC5E,WAAW,CAAC,EAAE;QACV,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,oBAAoB,CAAC;KACjC,EAAE,CAAC;IACJ,2EAA2E;IAC3E,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IACnC,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACnC,IAAI,EAAE,gBAAgB,CAAC;IACvB,0FAA0F;IAC1F,aAAa,EAAE,CACX,gBAAgB,CAAC,EAAE,SAAS,MAAM,EAAE,KACnC,OAAO,CAAC,mBAAmB,CAAC,CAAC;CACrC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GACjC,mBAAmB,GACnB,wBAAwB,CAAC;AAE/B,MAAM,MAAM,4BAA4B,GAAG;IACvC,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,kHAAkH;IAClH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;IAC9D,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,4BAA4B,CAAC,EAAE,MAAM,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,sBAAsB,WAAY,MAAM,KAAG,aA+BvD,CAAC;AAyBF,eAAO,MAAM,kCAAkC,aACjC,MAAM,KACjB,eAAe,GAAG,SAOpB,CAAC;AAoCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB,CAEpC;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AAmID,eAAO,MAAM,+BAA+B,yBAGzC,IAAI,CAAC,iCAAiC,EAAE,UAAU,GAAG,OAAO,CAAC,KAAG,MAOlE,CAAC;AAiGF,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,iCAAiC,CAAC;IAC7F,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAGhD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ;;;;;OAKG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAA6B;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAGhD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAE/C;IAEF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sCAAsC,CAG1D;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,8BAA8B,CAGlD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAA2B;IAE1D,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAU;IAE/C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;IAEV,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAEhD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAU;IAE9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,4BAA4B,CAAC,CAAS;IAEvD,OAAO,CAAC,2BAA2B,CAAS;IAE5C,OAAO,CAAC,gCAAgC,CAAS;IAEjD,YACI,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B,EAgJzC;IAED,OAAO,CAAC,MAAM,CAAC,kBAAkB;IASjC,MAAM,CAAC,qBAAqB,CACxB,WAAW,EAAE,mBAAmB,EAChC,OAAO,CAAC,EAAE,4BAA4B,GACvC,qBAAqB,CAIvB;IAED,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAoBzC,OAAO,CAAC,2BAA2B;mBAUd,6BAA6B;IA4DlD,OAAO,CAAC,MAAM,CAAC,iCAAiC;IAoBhD,OAAO,CAAC,kBAAkB;mBAQL,cAAc;mBAOd,mBAAmB;IAaxC,OAAO,CAAC,MAAM,CAAC,qBAAqB;mBASf,uBAAuB;YAsB9B,aAAa;YAoBb,kCAAkC;mBAW3B,qBAAqB;IAwF1C,2EAA2E;mBACtD,qBAAqB;YA8B5B,qBAAqB;mBAiNd,uBAAuB;mBAgBvB,yBAAyB;mBAoFzB,mBAAmB;IAgCxC,mFAAmF;IACnF,MAAM,CAAC,gCAAgC,IAAI,IAAI,CAW9C;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAO3B;IAED,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CACZ;IAEnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAA+B;IAE3E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAA0B;IAEjE,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAIjC,OAAO,CAAC,MAAM,CAAC,UAAU;IAIzB,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAuB5C,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAqGzC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAMhC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAmBrC,OAAO,CAAC,MAAM,CAAC,sBAAsB;mBA2ChB,iBAAiB;mBAejB,iBAAiB;mBAiBjB,eAAe;IA+BpC;;;;OAIG;mBACkB,qBAAqB;IA8B1C,OAAO,CAAC,MAAM,CAAC,gBAAgB;IA6C/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAK;YAE9C,gBAAgB;IA+E9B,iEAAiE;YACnD,wBAAwB;IA+CtC,gFAAgF;YAClE,mBAAmB;YAkDnB,yBAAyB;YAyDzB,iBAAiB;IAkD/B,OAAO,CAAC,iBAAiB;IAMzB,OAAO,CAAC,YAAY;YAQN,WAAW;YAmFX,2BAA2B;IA4GzC,+DAA+D;YACjD,iBAAiB;YA8EjB,yBAAyB;IA2BvC,OAAO,CAAC,4BAA4B;IAOpC,OAAO,CAAC,aAAa;YA0BP,eAAe;IAgG7B,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAgBxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IA+D/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;IAYnC,MAAM,CAAC,yBAAyB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAiBlD;YAEa,iBAAiB;YA4BjB,eAAe;YAmBf,uBAAuB;YAQvB,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CA0Ff;IAEK,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,mBAAmB,CAAC,CAC9E;;;;;;OA4BJ;IAEK,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKvC;IAEK,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC,CAkBD;IAEK,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,UAAU,CAAC,CACrE;;;OAGJ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAEK,UAAU,CACZ,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC,CA4C3B;IAEK,YAAY,CACd,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,eAAe,CAAC,CAsB1B;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC,CAS3B;CACJ"}
|
|
@@ -586,7 +586,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
586
586
|
await this.loadExtension(db, 'aws');
|
|
587
587
|
}
|
|
588
588
|
}
|
|
589
|
-
static async bootstrapQuerySession(db, client, querySql) {
|
|
589
|
+
static async bootstrapQuerySession(db, client, querySql, parquetDirectory) {
|
|
590
590
|
const bootstrapStart = performance.now();
|
|
591
591
|
const httpfsStart = performance.now();
|
|
592
592
|
if (!client.ducklakeConfig) {
|
|
@@ -618,7 +618,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
618
618
|
await DuckdbWarehouseClient.configureS3Secret(db, client.s3Config);
|
|
619
619
|
}
|
|
620
620
|
if (client.parquetConfig) {
|
|
621
|
-
await client.bootstrapParquetViews(db, querySql);
|
|
621
|
+
await client.bootstrapParquetViews(db, querySql, parquetDirectory);
|
|
622
622
|
}
|
|
623
623
|
if (client.ducklakeConfig) {
|
|
624
624
|
const stmts = DuckdbWarehouseClient.buildDucklakeAttachSql(client.ducklakeConfig);
|
|
@@ -677,10 +677,17 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
677
677
|
parser.closeSync();
|
|
678
678
|
}
|
|
679
679
|
}
|
|
680
|
-
async bootstrapParquetViews(db, querySql) {
|
|
680
|
+
async bootstrapParquetViews(db, querySql, parquetDirectory) {
|
|
681
681
|
const references = await DuckdbWarehouseClient.getParquetQueryTables(querySql);
|
|
682
682
|
const referencedTables = references ? new Set(references) : undefined;
|
|
683
|
-
const
|
|
683
|
+
const resolved = await this.parquetConfig.resolveSource(references);
|
|
684
|
+
const source = {
|
|
685
|
+
...resolved,
|
|
686
|
+
tables: resolved.tables.map((table) => ({
|
|
687
|
+
...table,
|
|
688
|
+
urls: [...table.urls],
|
|
689
|
+
})),
|
|
690
|
+
};
|
|
684
691
|
const escape = DuckdbWarehouseClient.escapeDuckdbString;
|
|
685
692
|
const literal = (value) => `'${escape(value)}'`;
|
|
686
693
|
const scope = new URL(source.scope);
|
|
@@ -775,6 +782,41 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
775
782
|
await db.run('SET enable_http_metadata_cache = true;');
|
|
776
783
|
await db.run('SET enable_external_file_cache = true;');
|
|
777
784
|
await db.run('SET parquet_metadata_cache = true;');
|
|
785
|
+
// Only validated manifest URLs can supply bytes. Each query owns a
|
|
786
|
+
// private directory; never allow the shared cache or another query's paths.
|
|
787
|
+
if (source.signedUrls && source.fileBuffers && parquetDirectory) {
|
|
788
|
+
let stagedBytes = 0;
|
|
789
|
+
for (const table of source.tables) {
|
|
790
|
+
const buffers = table.urls.map((url) => source.fileBuffers.get(url));
|
|
791
|
+
const size = buffers.reduce((total, buffer) => total + (buffer?.byteLength ?? 0), 0);
|
|
792
|
+
// DuckDB requires consistent Hive paths within each read_parquet.
|
|
793
|
+
// Never mix remote and local files in one table, including on failure.
|
|
794
|
+
if (buffers.every((buffer) => buffer) &&
|
|
795
|
+
stagedBytes + size <= 8 * 1024 * 1024) {
|
|
796
|
+
stagedBytes += size;
|
|
797
|
+
const localPaths = [];
|
|
798
|
+
try {
|
|
799
|
+
for (let i = 0; i < table.urls.length; i += 1) {
|
|
800
|
+
const localPath = path_1.default.join(parquetDirectory, decodeURIComponent(new URL(table.urls[i]).pathname));
|
|
801
|
+
// eslint-disable-next-line no-await-in-loop
|
|
802
|
+
await promises_1.default.mkdir(path_1.default.dirname(localPath), {
|
|
803
|
+
recursive: true,
|
|
804
|
+
});
|
|
805
|
+
// eslint-disable-next-line no-await-in-loop
|
|
806
|
+
await promises_1.default.writeFile(localPath, buffers[i], {
|
|
807
|
+
mode: 0o600,
|
|
808
|
+
flag: 'wx',
|
|
809
|
+
});
|
|
810
|
+
localPaths.push(localPath);
|
|
811
|
+
}
|
|
812
|
+
table.urls = localPaths;
|
|
813
|
+
}
|
|
814
|
+
catch {
|
|
815
|
+
// Keep the entire table remote if temporary storage is unavailable.
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
}
|
|
778
820
|
const files = source.tables.flatMap(({ urls }) => urls);
|
|
779
821
|
await db.run(`SET allowed_paths = [${files.map(literal).join(',')}];`);
|
|
780
822
|
await db.run('SET enable_external_access = false;');
|
|
@@ -1306,8 +1348,14 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1306
1348
|
const connectStart = performance.now();
|
|
1307
1349
|
const connection = await instance.connect();
|
|
1308
1350
|
const connectMs = performance.now() - connectStart;
|
|
1351
|
+
let parquetDirectory;
|
|
1309
1352
|
try {
|
|
1310
|
-
|
|
1353
|
+
if (this.parquetConfig) {
|
|
1354
|
+
parquetDirectory = await promises_1.default
|
|
1355
|
+
.mkdtemp(path_1.default.join(os_1.default.tmpdir(), 'lightdash-analytics-query-'))
|
|
1356
|
+
.catch(() => undefined);
|
|
1357
|
+
}
|
|
1358
|
+
const bootstrapTiming = await DuckdbWarehouseClient.bootstrapQuerySession(connection, this, querySql, parquetDirectory);
|
|
1311
1359
|
const queryStart = performance.now();
|
|
1312
1360
|
const result = await callback(connection);
|
|
1313
1361
|
const queryMs = performance.now() - queryStart;
|
|
@@ -1325,6 +1373,8 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1325
1373
|
connection.closeSync?.();
|
|
1326
1374
|
connection.disconnectSync?.();
|
|
1327
1375
|
instance.closeSync?.();
|
|
1376
|
+
if (parquetDirectory)
|
|
1377
|
+
await promises_1.default.rm(parquetDirectory, { recursive: true, force: true });
|
|
1328
1378
|
}
|
|
1329
1379
|
}
|
|
1330
1380
|
async withSharedSession(callback) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "2.475.
|
|
3
|
+
"version": "2.475.2",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"@databricks/sql": "1.13.0",
|
|
22
22
|
"@duckdb/node-api": "1.5.2-r.2",
|
|
23
23
|
"@google-cloud/bigquery": "8.3.1",
|
|
24
|
-
"@lightdash/common": "2.475.
|
|
24
|
+
"@lightdash/common": "2.475.2",
|
|
25
25
|
"node-fetch": "2.7.0",
|
|
26
26
|
"pg": "8.13.1",
|
|
27
27
|
"pg-cursor": "2.10.0",
|