@lightdash/warehouses 3.62.0 → 3.63.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -134,6 +134,7 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
134
134
|
private static readonly sharedInstanceS3Scopes;
|
|
135
135
|
private static readonly sharedInstanceS3SecretLocks;
|
|
136
136
|
private static readonly embeddedConcurrencyBudget;
|
|
137
|
+
private static readonly parquetConcurrencyBudget;
|
|
137
138
|
private static readonly embeddedOrganizationConcurrencyBudgets;
|
|
138
139
|
private static readonly organizationConcurrencyBudgets;
|
|
139
140
|
private static readonly sqlBuilder;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAMA,OAAO,EACH,OAAO,EAEP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAOb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAChB,eAAe,EAGf,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAY3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA8DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;IAChB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB,GAAG,CACE;IAAE,QAAQ,CAAC,EAAE,SAAS,CAAC;IAAC,cAAc,CAAC,EAAE,KAAK,CAAA;CAAE,GAChD;IACI,QAAQ,EAAE,WAAW,CAAC;IACtB,0EAA0E;IAC1E,cAAc,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IACtC,KAAK,EAAE,MAAM,EAAE,CAAC;CACnB,CACN,CAAC;AAOF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACpE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,uEAAuE;IACvE,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS,GAAG,QAAQ,CAAC;CACpE,EAAE,CAAC;AAEJ,gFAAgF;AAChF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,kFAAkF;IAClF,WAAW,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAAC,OAAO,CAAC,EAAE,oBAAoB,CAAA;KAAE,EAAE,CAAC;IAC3E,4EAA4E;IAC5E,WAAW,CAAC,EAAE;QACV,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,oBAAoB,CAAC;KACjC,EAAE,CAAC;IACJ,2EAA2E;IAC3E,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IACnC,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACnC,IAAI,EAAE,gBAAgB,CAAC;IACvB,0FAA0F;IAC1F,aAAa,EAAE,CACX,gBAAgB,CAAC,EAAE,SAAS,MAAM,EAAE,KACnC,OAAO,CAAC,mBAAmB,CAAC,CAAC;CACrC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GACjC,mBAAmB,GACnB,wBAAwB,CAAC;AAE/B,MAAM,MAAM,4BAA4B,GAAG;IACvC,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,kHAAkH;IAClH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;IAC9D,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,4BAA4B,CAAC,EAAE,MAAM,CAAC;IACtC;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC/B,CAAC;AAEF,eAAO,MAAM,sBAAsB,WAAY,MAAM,KAAG,aA+BvD,CAAC;AAyBF,eAAO,MAAM,kCAAkC,aACjC,MAAM,KACjB,eAAe,GAAG,SAOpB,CAAC;AAEF,eAAO,MAAM,8BAA8B,aAC7B,MAAM,KACjB,aAYF,CAAC;AAoCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB,CAEpC;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAMA,OAAO,EACH,OAAO,EAEP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAOb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAChB,eAAe,EAGf,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAY3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA8DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;IAChB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB,GAAG,CACE;IAAE,QAAQ,CAAC,EAAE,SAAS,CAAC;IAAC,cAAc,CAAC,EAAE,KAAK,CAAA;CAAE,GAChD;IACI,QAAQ,EAAE,WAAW,CAAC;IACtB,0EAA0E;IAC1E,cAAc,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IACtC,KAAK,EAAE,MAAM,EAAE,CAAC;CACnB,CACN,CAAC;AAOF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACpE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,uEAAuE;IACvE,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS,GAAG,QAAQ,CAAC;CACpE,EAAE,CAAC;AAEJ,gFAAgF;AAChF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,kFAAkF;IAClF,WAAW,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAAC,OAAO,CAAC,EAAE,oBAAoB,CAAA;KAAE,EAAE,CAAC;IAC3E,4EAA4E;IAC5E,WAAW,CAAC,EAAE;QACV,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,oBAAoB,CAAC;KACjC,EAAE,CAAC;IACJ,2EAA2E;IAC3E,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IACnC,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACnC,IAAI,EAAE,gBAAgB,CAAC;IACvB,0FAA0F;IAC1F,aAAa,EAAE,CACX,gBAAgB,CAAC,EAAE,SAAS,MAAM,EAAE,KACnC,OAAO,CAAC,mBAAmB,CAAC,CAAC;CACrC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GACjC,mBAAmB,GACnB,wBAAwB,CAAC;AAE/B,MAAM,MAAM,4BAA4B,GAAG;IACvC,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,kHAAkH;IAClH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;IAC9D,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,4BAA4B,CAAC,EAAE,MAAM,CAAC;IACtC;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC/B,CAAC;AAEF,eAAO,MAAM,sBAAsB,WAAY,MAAM,KAAG,aA+BvD,CAAC;AAyBF,eAAO,MAAM,kCAAkC,aACjC,MAAM,KACjB,eAAe,GAAG,SAOpB,CAAC;AAEF,eAAO,MAAM,8BAA8B,aAC7B,MAAM,KACjB,aAYF,CAAC;AAoCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB,CAEpC;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AA+ID,eAAO,MAAM,+BAA+B,yBAGzC,IAAI,CAAC,iCAAiC,EAAE,UAAU,GAAG,OAAO,CAAC,KAAG,MAOlE,CAAC;AAiGF,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,iCAAiC,CAAC;IAC7F,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAGhD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ;;;;;OAKG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAA6B;IAE5E;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sBAAsB,CAG1C;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,2BAA2B,CAG/C;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAE/C;IAEF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAA4B;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sCAAsC,CAG1D;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,8BAA8B,CAGlD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAA2B;IAE1D,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAU;IAE/C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;IAEV,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAEhD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAU;IAE9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,4BAA4B,CAAC,CAAS;IAEvD,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAU;IAE5C,OAAO,CAAC,2BAA2B,CAAS;IAE5C,OAAO,CAAC,gCAAgC,CAAS;IAEjD,YACI,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B,EAgJzC;IAED,OAAO,CAAC,MAAM,CAAC,kBAAkB;IASjC,MAAM,CAAC,qBAAqB,CACxB,WAAW,EAAE,mBAAmB,EAChC,OAAO,CAAC,EAAE,4BAA4B,GACvC,qBAAqB,CAIvB;IAED,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAoBzC,OAAO,CAAC,2BAA2B;mBAUd,6BAA6B;IA4DlD,OAAO,CAAC,MAAM,CAAC,iCAAiC;IAoBhD,OAAO,CAAC,kBAAkB;mBAQL,cAAc;IAOnC;;;;;OAKG;mBACkB,uBAAuB;mBAmCvB,mBAAmB;IAaxC,OAAO,CAAC,MAAM,CAAC,qBAAqB;mBASf,uBAAuB;YAsB9B,aAAa;YAoBb,kCAAkC;mBAW3B,qBAAqB;IAiG1C,2EAA2E;mBACtD,qBAAqB;YA8B5B,qBAAqB;mBAuNd,uBAAuB;mBAgBvB,yBAAyB;mBA6FzB,mBAAmB;IAmCxC,mFAAmF;IACnF,MAAM,CAAC,gCAAgC,IAAI,IAAI,CAa9C;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAO3B;IAED,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CACZ;IAEnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAA+B;IAE3E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAA0B;IAEjE,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAIjC,OAAO,CAAC,MAAM,CAAC,UAAU;IAIzB,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAuB5C,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAqGzC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAMhC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAmBrC,OAAO,CAAC,MAAM,CAAC,sBAAsB;mBA2ChB,iBAAiB;mBAejB,iBAAiB;mBAiBjB,eAAe;IA+BpC,oFAAoF;IACpF,OAAO,CAAC,MAAM,CAAC,uBAAuB;mBAUjB,sBAAsB;IAQ3C;;;;;;OAMG;mBACkB,oBAAoB;IAkFzC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IA6C/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAK;YAE9C,gBAAgB;IA+E9B,iEAAiE;YACnD,wBAAwB;IA+CtC,gFAAgF;YAClE,mBAAmB;YAwDnB,yBAAyB;YAmHzB,iBAAiB;IAkD/B,OAAO,CAAC,iBAAiB;IAMzB,OAAO,CAAC,YAAY;YAQN,WAAW;YAuFX,2BAA2B;IA4GzC,+DAA+D;YACjD,iBAAiB;YA8EjB,yBAAyB;IA2BvC,OAAO,CAAC,4BAA4B;IAOpC,OAAO,CAAC,aAAa;YA0BP,eAAe;IA4G7B,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAgBxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IA+D/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;IAYnC,MAAM,CAAC,yBAAyB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAiBlD;YAEa,iBAAiB;YA4BjB,eAAe;YAmCf,uBAAuB;YAQvB,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CA6Ff;IAEK,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,mBAAmB,CAAC,CAC9E;;;;;;;OA4BJ;IAEK,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKvC;IAEK,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC,CAkBD;IAEK,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,UAAU,CAAC,CACrE;;;OAGJ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAEK,UAAU,CACZ,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC,CA4C3B;IAEK,YAAY,CACd,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,eAAe,CAAC,CAsB1B;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC,CAS3B;CACJ"}
|
|
@@ -231,16 +231,18 @@ class ConcurrencyBudget {
|
|
|
231
231
|
const pendingAcquire = {
|
|
232
232
|
resolve,
|
|
233
233
|
};
|
|
234
|
-
|
|
235
|
-
const
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
234
|
+
if (timeoutMs !== undefined) {
|
|
235
|
+
const timeout = setTimeout(() => {
|
|
236
|
+
const index = this.pendingAcquires.indexOf(pendingAcquire);
|
|
237
|
+
if (index === -1) {
|
|
238
|
+
return;
|
|
239
|
+
}
|
|
240
|
+
this.pendingAcquires.splice(index, 1);
|
|
241
|
+
resolve(false);
|
|
242
|
+
}, timeoutMs);
|
|
243
|
+
timeout.unref();
|
|
244
|
+
pendingAcquire.timeout = timeout;
|
|
245
|
+
}
|
|
244
246
|
this.pendingAcquires.push(pendingAcquire);
|
|
245
247
|
});
|
|
246
248
|
}
|
|
@@ -272,6 +274,12 @@ class ConcurrencyBudget {
|
|
|
272
274
|
});
|
|
273
275
|
}
|
|
274
276
|
}
|
|
277
|
+
const PARQUET_QUERY_TIMEOUT_MS = 120000;
|
|
278
|
+
class ParquetQueryTimeoutError extends common_1.WarehouseQueryError {
|
|
279
|
+
constructor() {
|
|
280
|
+
super('Analytics query exceeded the 120 second execution limit and was stopped. Try a narrower date range.');
|
|
281
|
+
}
|
|
282
|
+
}
|
|
275
283
|
// DuckDB StatementType values — see duckdb/common/enums/statement_type.hpp
|
|
276
284
|
const ALLOWED_STATEMENT_TYPES_USER_SQL = new Set([1 /* SELECT */]);
|
|
277
285
|
const BLOCKED_STATEMENT_TYPES_INTERNAL_SQL = new Set([
|
|
@@ -441,9 +449,8 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
441
449
|
this.sharedResourceLimits = isParquet
|
|
442
450
|
? {
|
|
443
451
|
memoryLimit: '256MB',
|
|
444
|
-
//
|
|
445
|
-
|
|
446
|
-
threads: 4,
|
|
452
|
+
// Parallel JSON decoders can exhaust this budget before spilling.
|
|
453
|
+
threads: 1,
|
|
447
454
|
...options?.sharedResourceLimits,
|
|
448
455
|
}
|
|
449
456
|
: options?.sharedResourceLimits;
|
|
@@ -675,6 +682,10 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
675
682
|
if (client.objectStorageOnly) {
|
|
676
683
|
await DuckdbWarehouseClient.restrictToObjectStorage(db, client);
|
|
677
684
|
}
|
|
685
|
+
if (client.parquetConfig) {
|
|
686
|
+
await db.run("SET allowed_configs = ['TimeZone'];");
|
|
687
|
+
await db.run('SET lock_configuration = true;');
|
|
688
|
+
}
|
|
678
689
|
const bootstrapMs = performance.now() - bootstrapStart;
|
|
679
690
|
client.logger?.info(`DuckDB query bootstrap complete: cacheKey=${client.instanceCacheKey ?? 'none'} bootstrap=${(0, common_1.formatMilliseconds)(bootstrapMs)}ms httpfs=${(0, common_1.formatMilliseconds)(httpfsMs)}ms memory_limit=${client.sharedResourceLimits?.memoryLimit ?? 'default'} threads=${client.sharedResourceLimits?.threads ?? 'default'} s3=${client.s3Config ? 'configured' : 'none'} ducklake=${client.ducklakeConfig ? 'configured' : 'none'} shared=${client.instanceCacheKey ? 'true' : 'false'}`, {
|
|
680
691
|
shared: !!client.instanceCacheKey,
|
|
@@ -818,9 +829,16 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
818
829
|
scope: [source.scope],
|
|
819
830
|
});
|
|
820
831
|
}
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
832
|
+
const spillDirectory = parquetDirectory
|
|
833
|
+
? path_1.default.join(parquetDirectory, 'spill')
|
|
834
|
+
: '';
|
|
835
|
+
if (spillDirectory)
|
|
836
|
+
await promises_1.default.mkdir(spillDirectory, { mode: 0o700 });
|
|
837
|
+
// DuckDB allowlists this private child when external access is disabled.
|
|
838
|
+
// User SQL still cannot call file functions or change this configuration.
|
|
839
|
+
await db.run("SET max_temp_directory_size = '2GB';");
|
|
840
|
+
await db.run(`SET temp_directory = ${literal(spillDirectory)};`);
|
|
841
|
+
await db.run('SET allowed_directories = [];');
|
|
824
842
|
// These caches live only in this query's private, memory-limited
|
|
825
843
|
// instance, which is closed in withEphemeralQuerySession's finally.
|
|
826
844
|
// Reuse metadata between view binding, validation and execution without
|
|
@@ -990,6 +1008,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
990
1008
|
DuckdbWarehouseClient.sharedInstanceS3Scopes.clear();
|
|
991
1009
|
DuckdbWarehouseClient.sharedInstanceS3SecretLocks.clear();
|
|
992
1010
|
DuckdbWarehouseClient.embeddedConcurrencyBudget.reset();
|
|
1011
|
+
DuckdbWarehouseClient.parquetConcurrencyBudget.reset();
|
|
993
1012
|
DuckdbWarehouseClient.embeddedOrganizationConcurrencyBudgets.forEach((budget) => budget.reset());
|
|
994
1013
|
DuckdbWarehouseClient.embeddedOrganizationConcurrencyBudgets.clear();
|
|
995
1014
|
}
|
|
@@ -1440,22 +1459,44 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1440
1459
|
}
|
|
1441
1460
|
async withEphemeralQuerySession(callback, querySql) {
|
|
1442
1461
|
const sessionStart = performance.now();
|
|
1443
|
-
const
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1462
|
+
const budget = this.parquetConfig
|
|
1463
|
+
? DuckdbWarehouseClient.parquetConcurrencyBudget
|
|
1464
|
+
: undefined;
|
|
1465
|
+
if (budget && !(await budget.acquire())) {
|
|
1466
|
+
throw new common_1.WarehouseQueryError('Analytics query capacity was reset. Try again.');
|
|
1467
|
+
}
|
|
1468
|
+
let instance;
|
|
1469
|
+
let connection;
|
|
1449
1470
|
let parquetDirectory;
|
|
1471
|
+
let queryFailed = false;
|
|
1472
|
+
let deadlineExceeded = false;
|
|
1473
|
+
let timeout;
|
|
1450
1474
|
try {
|
|
1475
|
+
const instanceCreateStart = performance.now();
|
|
1476
|
+
instance = await node_api_1.DuckDBInstance.create(':memory:');
|
|
1477
|
+
const instanceCreateMs = performance.now() - instanceCreateStart;
|
|
1478
|
+
const connectStart = performance.now();
|
|
1479
|
+
connection = await instance.connect();
|
|
1480
|
+
const connectMs = performance.now() - connectStart;
|
|
1481
|
+
if (this.parquetConfig) {
|
|
1482
|
+
timeout = setTimeout(() => {
|
|
1483
|
+
deadlineExceeded = true;
|
|
1484
|
+
connection?.interrupt();
|
|
1485
|
+
}, PARQUET_QUERY_TIMEOUT_MS);
|
|
1486
|
+
timeout.unref();
|
|
1487
|
+
}
|
|
1451
1488
|
if (this.parquetConfig) {
|
|
1452
1489
|
parquetDirectory = await promises_1.default
|
|
1453
1490
|
.mkdtemp(path_1.default.join(os_1.default.tmpdir(), 'lightdash-analytics-query-'))
|
|
1454
1491
|
.catch(() => undefined);
|
|
1455
1492
|
}
|
|
1456
1493
|
const bootstrapTiming = await DuckdbWarehouseClient.bootstrapQuerySession(connection, this, querySql, parquetDirectory);
|
|
1494
|
+
if (deadlineExceeded)
|
|
1495
|
+
throw new Error('Analytics query deadline exceeded');
|
|
1457
1496
|
const queryStart = performance.now();
|
|
1458
1497
|
const result = await callback(connection);
|
|
1498
|
+
if (deadlineExceeded)
|
|
1499
|
+
throw new Error('Analytics query deadline exceeded');
|
|
1459
1500
|
const queryMs = performance.now() - queryStart;
|
|
1460
1501
|
const totalMs = performance.now() - sessionStart;
|
|
1461
1502
|
this.logger?.info(`DuckDB ephemeral query session complete: instance_create=${(0, common_1.formatMilliseconds)(instanceCreateMs)}ms connect=${(0, common_1.formatMilliseconds)(connectMs)}ms bootstrap=${(0, common_1.formatMilliseconds)(bootstrapTiming.bootstrapMs)}ms query=${(0, common_1.formatMilliseconds)(queryMs)}ms total=${(0, common_1.formatMilliseconds)(totalMs)}ms`, {
|
|
@@ -1467,12 +1508,50 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1467
1508
|
});
|
|
1468
1509
|
return result;
|
|
1469
1510
|
}
|
|
1511
|
+
catch (error) {
|
|
1512
|
+
queryFailed = true;
|
|
1513
|
+
if (deadlineExceeded) {
|
|
1514
|
+
throw new ParquetQueryTimeoutError();
|
|
1515
|
+
}
|
|
1516
|
+
throw error;
|
|
1517
|
+
}
|
|
1470
1518
|
finally {
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1519
|
+
if (timeout)
|
|
1520
|
+
clearTimeout(timeout);
|
|
1521
|
+
const cleanupErrors = [];
|
|
1522
|
+
try {
|
|
1523
|
+
for (const close of [
|
|
1524
|
+
() => connection?.closeSync?.(),
|
|
1525
|
+
() => connection?.disconnectSync?.(),
|
|
1526
|
+
() => instance?.closeSync?.(),
|
|
1527
|
+
]) {
|
|
1528
|
+
try {
|
|
1529
|
+
close();
|
|
1530
|
+
}
|
|
1531
|
+
catch (error) {
|
|
1532
|
+
cleanupErrors.push(error);
|
|
1533
|
+
}
|
|
1534
|
+
}
|
|
1535
|
+
if (parquetDirectory) {
|
|
1536
|
+
await promises_1.default
|
|
1537
|
+
.rm(parquetDirectory, {
|
|
1538
|
+
recursive: true,
|
|
1539
|
+
force: true,
|
|
1540
|
+
maxRetries: 3,
|
|
1541
|
+
retryDelay: 100,
|
|
1542
|
+
})
|
|
1543
|
+
.catch((error) => cleanupErrors.push(error));
|
|
1544
|
+
}
|
|
1545
|
+
if (cleanupErrors.length > 0) {
|
|
1546
|
+
this.logger?.warn?.('DuckDB query cleanup failed');
|
|
1547
|
+
// eslint-disable-next-line no-unsafe-finally -- Preserve primary errors, fail successful queries only.
|
|
1548
|
+
if (!queryFailed)
|
|
1549
|
+
throw cleanupErrors[0];
|
|
1550
|
+
}
|
|
1551
|
+
}
|
|
1552
|
+
finally {
|
|
1553
|
+
budget?.release();
|
|
1554
|
+
}
|
|
1476
1555
|
}
|
|
1477
1556
|
}
|
|
1478
1557
|
async withSharedSession(callback) {
|
|
@@ -1531,7 +1610,8 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1531
1610
|
return await this.withEphemeralQuerySession(callback, querySql);
|
|
1532
1611
|
}
|
|
1533
1612
|
catch (error) {
|
|
1534
|
-
if (error instanceof common_1.ParameterError
|
|
1613
|
+
if (error instanceof common_1.ParameterError ||
|
|
1614
|
+
error instanceof ParquetQueryTimeoutError)
|
|
1535
1615
|
throw error;
|
|
1536
1616
|
throw new common_1.WarehouseQueryError('Internal analytics query failed. Check storage access and query permissions.');
|
|
1537
1617
|
}
|
|
@@ -1919,8 +1999,18 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1919
1999
|
}
|
|
1920
2000
|
}
|
|
1921
2001
|
async validateUserSql(db, sql) {
|
|
2002
|
+
const stripped = DuckdbWarehouseClient.stripSqlComments(sql);
|
|
2003
|
+
if (this.parquetConfig && /\bU&"/i.test(stripped)) {
|
|
2004
|
+
throw new common_1.ParameterError('Escaped identifiers are not supported in analytics queries');
|
|
2005
|
+
}
|
|
2006
|
+
if (this.parquetConfig &&
|
|
2007
|
+
stripped
|
|
2008
|
+
.match(/\$\$[\s\S]*?\$\$|'(?:[^']|'')*'|"(?:[^"]|"")*"/g)
|
|
2009
|
+
?.some((token) => token.startsWith('"') && /[\\/]/.test(token))) {
|
|
2010
|
+
throw new common_1.ParameterError('File table paths are not allowed in analytics queries');
|
|
2011
|
+
}
|
|
1922
2012
|
if (this.parquetConfig &&
|
|
1923
|
-
/\b(duckdb_\w+|pragma_\w+|sqlite_\w+|pg_\w+|information_schema)\b/i.test(
|
|
2013
|
+
/\b(duckdb_\w+|pragma_\w+|sqlite_\w+|pg_\w+|information_schema)\b/i.test(stripped)) {
|
|
1924
2014
|
throw new common_1.ParameterError('Internal analytics catalog access is not allowed');
|
|
1925
2015
|
}
|
|
1926
2016
|
DuckdbWarehouseClient.validateSqlFunctions(sql);
|
|
@@ -2149,6 +2239,7 @@ DuckdbWarehouseClient.sharedInstanceGcsTokens = new Map();
|
|
|
2149
2239
|
DuckdbWarehouseClient.sharedInstanceS3Scopes = new Map();
|
|
2150
2240
|
DuckdbWarehouseClient.sharedInstanceS3SecretLocks = new Map();
|
|
2151
2241
|
DuckdbWarehouseClient.embeddedConcurrencyBudget = new ConcurrencyBudget(EMBEDDED_MAX_CONCURRENT_QUERIES);
|
|
2242
|
+
DuckdbWarehouseClient.parquetConcurrencyBudget = new ConcurrencyBudget(2);
|
|
2152
2243
|
DuckdbWarehouseClient.embeddedOrganizationConcurrencyBudgets = new Map();
|
|
2153
2244
|
DuckdbWarehouseClient.organizationConcurrencyBudgets = new Map();
|
|
2154
2245
|
DuckdbWarehouseClient.sqlBuilder = new DuckdbSqlBuilder();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.63.1",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"@databricks/sql": "1.13.0",
|
|
22
22
|
"@duckdb/node-api": "1.5.2-r.2",
|
|
23
23
|
"@google-cloud/bigquery": "8.3.1",
|
|
24
|
-
"@lightdash/common": "3.
|
|
24
|
+
"@lightdash/common": "3.63.1",
|
|
25
25
|
"node-fetch": "2.7.0",
|
|
26
26
|
"pg": "8.13.1",
|
|
27
27
|
"pg-cursor": "2.10.0",
|