@lightdash/warehouses 1.55.0 → 1.56.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (16) hide show
  1. package/dist/.tsbuildinfo +1 -1
  2. package/dist/warehouseClients/DuckdbWarehouseClient.d.ts +4 -1
  3. package/dist/warehouseClients/DuckdbWarehouseClient.d.ts.map +1 -1
  4. package/dist/warehouseClients/DuckdbWarehouseClient.js +20 -5
  5. package/dist/warehouseClients/DuckdbWarehouseClient.test.js +66 -2
  6. package/dist/warehouseClients/PgQueryStream.d.ts +1 -1
  7. package/dist/warehouseClients/PgQueryStream.d.ts.map +1 -1
  8. package/dist/warehouseClients/PgQueryStream.js +2 -1
  9. package/dist/warehouseClients/PostgresWarehouseClient.d.ts +9 -0
  10. package/dist/warehouseClients/PostgresWarehouseClient.d.ts.map +1 -1
  11. package/dist/warehouseClients/PostgresWarehouseClient.js +33 -11
  12. package/dist/warehouseClients/PostgresWarehouseClient.test.js +24 -0
  13. package/dist/warehouseClients/RedshiftWarehouseClient.test.d.ts +2 -0
  14. package/dist/warehouseClients/RedshiftWarehouseClient.test.d.ts.map +1 -0
  15. package/dist/warehouseClients/RedshiftWarehouseClient.test.js +33 -0
  16. package/package.json +2 -2
package/dist/warehouseClients/DuckdbWarehouseClient.d.ts CHANGED
@@ -72,9 +72,10 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
72
72
  private readonly logger?;
73
73
  private readonly onQueryProfile?;
74
74
  private readonly embeddedQueryTimeoutMs;
75
+ private allowsPreAggregateFileReads;
75
76
  constructor(credentials?: CreateDuckdbCredentials | DuckdbConnectionCredentials, options?: DuckdbWarehouseClientOptions);
76
77
  private static hashDucklakeConfig;
77
- static createForPreAggregate(credentials?: DuckdbConnectionCredentials, options?: DuckdbWarehouseClientOptions): DuckdbWarehouseClient;
78
+ static createForPreAggregate(credentials: DuckdbS3Credentials, options?: DuckdbWarehouseClientOptions): DuckdbWarehouseClient;
78
79
  private static getSharedInstanceSemaphore;
79
80
  private getRequiredInstanceCacheKey;
80
81
  private static tryAcquireEmbeddedConcurrency;
@@ -130,7 +131,9 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
130
131
  private static stripSqlComments;
131
132
  private static validateSqlFunctions;
132
133
  private static validateUserSqlFileAccess;
134
+ private validateSelectSql;
133
135
  private validateUserSql;
136
+ private validatePreAggregateSql;
134
137
  private validateInternalSql;
135
138
  streamQuery(sql: string, streamCallback: (data: WarehouseResults) => void | Promise<void>, options?: {
136
139
  values?: AnyType[];
package/dist/warehouseClients/DuckdbWarehouseClient.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAKA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAMb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAEhB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAM3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA6DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG,mBAAmB,CAAC;AAE9D,MAAM,MAAM,4BAA4B,GAAG;IACvC,kHAAkH;IAClH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;IAC9D,sBAAsB,CAAC,EAAE,MAAM,CAAC;CACnC,CAAC;AAEF,eAAO,MAAM,sBAAsB,WAAY,MAAM,KAAG,aA+BvD,CAAC;AAEF,eAAO,MAAM,kCAAkC,aACjC,MAAM,KACjB,eAAe,GAAG,SAOpB,CAAC;AAoCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB,CAEpC;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AAuJD,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,iCAAiC,CAAC;IAC7F,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAGhD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAE/C;IAEF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sCAAsC,CAG1D;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;IAEV,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAEhD,YACI,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B,EA8GzC;IAED,OAAO,CAAC,MAAM,CAAC,kBAAkB;IASjC,MAAM,CAAC,qBAAqB,CACxB,WAAW,CAAC,EAAE,2BAA2B,EACzC,OAAO,CAAC,EAAE,4BAA4B,GACvC,qBAAqB,CAEvB;IAED,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAoBzC,OAAO,CAAC,2BAA2B;IAUnC,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAwC5C,OAAO,CAAC,kBAAkB;mBAQL,cAAc;mBAOd,mBAAmB;IAaxC,OAAO,CAAC,MAAM,CAAC,qBAAqB;mBAMf,uBAAuB;YAsB9B,aAAa;YAoBb,kCAAkC;mBAW3B,qBAAqB;mBA6ErB,uBAAuB;mBAgBvB,yBAAyB;mBAiFzB,mBAAmB;IA6BxC,mFAAmF;IACnF,MAAM,CAAC,gCAAgC,IAAI,IAAI,CAM9C;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAO3B;IAED,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CACZ;IAEnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAA+B;IAE3E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAA0B;IAEjE,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAIjC,OAAO,CAAC,MAAM,CAAC,UAAU;IAIzB,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAuB5C,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAqGzC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAMhC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAmBrC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IA2CrC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAqC/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAK;YAE9C,gBAAgB;IA+E9B,iEAAiE;YACnD,wBAAwB;IA+CtC,gFAAgF;YAClE,mBAAmB;YAkDnB,yBAAyB;YA4CzB,iBAAiB;IA2C/B,OAAO,CAAC,iBAAiB;IAMzB;;;;;;OAMG;YACW,WAAW;IAmBzB,+DAA+D;YACjD,iBAAiB;YAyEjB,yBAAyB;IA2BvC,OAAO,CAAC,4BAA4B;IAOpC,OAAO,CAAC,aAAa;YA0BP,eAAe;IAgG7B,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAaxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IA+D/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;IAUnC,OAAO,CAAC,MAAM,CAAC,yBAAyB;YAmB1B,eAAe;YA+Bf,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CA6Df;IAEK,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,mBAAmB,CAAC,CAC9E;;;;;;OA4BJ;IAEK,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKvC;IAEK,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC,CAkBD;IAEK,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,UAAU,CAAC,CACrE;;;;;OAGJ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAEK,UAAU,CACZ,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC,CA4C3B;IAEK,YAAY,CACd,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN,CAqBA;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC,CAS3B;CACJ"}
1
+ {"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAKA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAMb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAEhB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAM3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA6DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG,mBAAmB,CAAC;AAE9D,MAAM,MAAM,4BAA4B,GAAG;IACvC,kHAAkH;IAClH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;IAC9D,sBAAsB,CAAC,EAAE,MAAM,CAAC;CACnC,CAAC;AAEF,eAAO,MAAM,sBAAsB,WAAY,MAAM,KAAG,aA+BvD,CAAC;AAEF,eAAO,MAAM,kCAAkC,aACjC,MAAM,KACjB,eAAe,GAAG,SAOpB,CAAC;AAoCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB,CAEpC;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AAuJD,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,iCAAiC,CAAC;IAC7F,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAGhD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAE/C;IAEF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sCAAsC,CAG1D;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;IAEV,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAEhD,OAAO,CAAC,2BAA2B,CAAS;IAE5C,YACI,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B,EA8GzC;IAED,OAAO,CAAC,MAAM,CAAC,kBAAkB;IASjC,MAAM,CAAC,qBAAqB,CACxB,WAAW,EAAE,mBAAmB,EAChC,OAAO,CAAC,EAAE,4BAA4B,GACvC,qBAAqB,CAIvB;IAED,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAoBzC,OAAO,CAAC,2BAA2B;IAUnC,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAwC5C,OAAO,CAAC,kBAAkB;mBAQL,cAAc;mBAOd,mBAAmB;IAaxC,OAAO,CAAC,MAAM,CAAC,qBAAqB;mBAMf,uBAAuB;YAsB9B,aAAa;YAoBb,kCAAkC;mBAW3B,qBAAqB;mBA6ErB,uBAAuB;mBAgBvB,yBAAyB;mBAiFzB,mBAAmB;IA6BxC,mFAAmF;IACnF,MAAM,CAAC,gCAAgC,IAAI,IAAI,CAM9C;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAO3B;IAED,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CACZ;IAEnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAA+B;IAE3E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAA0B;IAEjE,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAIjC,OAAO,CAAC,MAAM,CAAC,UAAU;IAIzB,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAuB5C,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAqGzC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAMhC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAmBrC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IA2CrC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAqC/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAK;YAE9C,gBAAgB;IA+E9B,iEAAiE;YACnD,wBAAwB;IA+CtC,gFAAgF;YAClE,mBAAmB;YAkDnB,yBAAyB;YA4CzB,iBAAiB;IA2C/B,OAAO,CAAC,iBAAiB;IAMzB;;;;;;OAMG;YACW,WAAW;IAmBzB,+DAA+D;YACjD,iBAAiB;YAyEjB,yBAAyB;IA2BvC,OAAO,CAAC,4BAA4B;IAOpC,OAAO,CAAC,aAAa;YA0BP,eAAe;IAgG7B,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAaxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IA+D/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;IAUnC,OAAO,CAAC,MAAM,CAAC,yBAAyB;YAmB1B,iBAAiB;YA4BjB,eAAe;YASf,uBAAuB;YAQvB,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CAiEf;IAEK,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,mBAAmB,CAAC,CAC9E;;;;;;OA4BJ;IAEK,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKvC;IAEK,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC,CAkBD;IAEK,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,UAAU,CAAC,CACrE;;;;;OAGJ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAEK,UAAU,CACZ,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC,CA4C3B;IAEK,YAAY,CACd,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN,CAqBA;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC,CAS3B;CACJ"}
package/dist/warehouseClients/DuckdbWarehouseClient.js CHANGED
@@ -262,6 +262,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
262
262
  DUCKDB_INTERNAL_CREDENTIALS;
263
263
  }
264
264
  super(effectiveCredentials, new DuckdbSqlBuilder(effectiveCredentials.startOfWeek));
265
+ this.allowsPreAggregateFileReads = false;
265
266
  if (isS3Only) {
266
267
  this.s3Config = credentials.s3Config;
267
268
  }
@@ -321,7 +322,9 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
321
322
  .slice(0, 16);
322
323
  }
323
324
  static createForPreAggregate(credentials, options) {
324
- return new DuckdbWarehouseClient(credentials, options);
325
+ const client = new DuckdbWarehouseClient(credentials, options);
326
+ client.allowsPreAggregateFileReads = true;
327
+ return client;
325
328
  }
326
329
  static getSharedInstanceSemaphore(instanceCacheKey) {
327
330
  const existingSemaphore = DuckdbWarehouseClient.sharedInstanceSemaphores.get(instanceCacheKey);
@@ -1220,9 +1223,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
1220
1223
  throw new Error('SQL validation error: file table paths are not allowed');
1221
1224
  }
1222
1225
  }
1223
- async validateUserSql(db, sql) {
1224
- DuckdbWarehouseClient.validateSqlFunctions(sql);
1225
- DuckdbWarehouseClient.validateUserSqlFileAccess(sql);
1226
+ async validateSelectSql(db, sql) {
1226
1227
  const extracted = await db.extractStatements(sql);
1227
1228
  if (extracted.count === 0) {
1228
1229
  throw new Error('SQL validation error: empty SQL statement');
@@ -1240,6 +1241,15 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
1240
1241
  stmt.destroySync();
1241
1242
  }
1242
1243
  }
1244
+ async validateUserSql(db, sql) {
1245
+ DuckdbWarehouseClient.validateSqlFunctions(sql);
1246
+ DuckdbWarehouseClient.validateUserSqlFileAccess(sql);
1247
+ await this.validateSelectSql(db, sql);
1248
+ }
1249
+ async validatePreAggregateSql(db, sql) {
1250
+ DuckdbWarehouseClient.validateSqlFunctions(sql);
1251
+ await this.validateSelectSql(db, sql);
1252
+ }
1243
1253
  async validateInternalSql(db, sql) {
1244
1254
  const extracted = await db.extractStatements(sql);
1245
1255
  if (extracted.count === 0) {
@@ -1276,7 +1286,12 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
1276
1286
  if (this.embeddedConfig) {
1277
1287
  await db.run("SET disabled_filesystems = 'LocalFileSystem';");
1278
1288
  }
1279
- await this.validateUserSql(db, sql);
1289
+ if (this.allowsPreAggregateFileReads) {
1290
+ await this.validatePreAggregateSql(db, sql);
1291
+ }
1292
+ else {
1293
+ await this.validateUserSql(db, sql);
1294
+ }
1280
1295
  reportPhase?.('session', performance.now() - sessionStart);
1281
1296
  const queryStart = performance.now();
1282
1297
  const result = await db.stream(this.getSQLWithMetadata(sql, options?.tags), this.getBindValues(options));
package/dist/warehouseClients/DuckdbWarehouseClient.test.js CHANGED
@@ -7,6 +7,15 @@ const common_1 = require("@lightdash/common");
7
7
  const promises_1 = __importDefault(require("fs/promises"));
8
8
  const DuckdbWarehouseClient_1 = require("./DuckdbWarehouseClient");
9
9
  const createInstanceMock = vi.fn();
10
+ const duckdbS3Credentials = {
11
+ type: 'duckdb_s3',
12
+ s3Config: {
13
+ endpoint: 'localhost:9000',
14
+ region: 'us-east-1',
15
+ forcePathStyle: true,
16
+ useSsl: false,
17
+ },
18
+ };
10
19
  // Must provide DuckDBTypeId since mapFieldTypeFromTypeId references it at runtime
11
20
  const DUCKDB_TYPE_IDS = {
12
21
  BOOLEAN: 1,
@@ -186,7 +195,7 @@ describe('DuckdbWarehouseClient', () => {
186
195
  const chunk2 = [{ id: 3 }];
187
196
  const streamMock = vi.fn(async () => getMockStreamResult([chunk1, chunk2], [DUCKDB_TYPE_IDS.INTEGER]));
188
197
  createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
189
- const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate();
198
+ const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
190
199
  const streamCallback = vi.fn();
191
200
  const result = await client.executeAsyncQuery({
192
201
  sql: 'SELECT id FROM t',
@@ -204,7 +213,7 @@ describe('DuckdbWarehouseClient', () => {
204
213
  it('should handle empty result set', async () => {
205
214
  const streamMock = vi.fn(async () => getMockStreamResult([], [DUCKDB_TYPE_IDS.INTEGER]));
206
215
  createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
207
- const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate();
216
+ const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
208
217
  const result = await client.runQuery('SELECT id FROM empty_table');
209
218
  expect(result.rows).toEqual([]);
210
219
  expect(result.fields).toEqual({});
@@ -653,6 +662,61 @@ describe('DuckdbWarehouseClient', () => {
653
662
  expect(extractStatementsMock).not.toHaveBeenCalled();
654
663
  expect(streamMock).not.toHaveBeenCalled();
655
664
  });
665
+ it('should keep file readers blocked when only S3 is configured', async () => {
666
+ const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
667
+ const extractStatementsMock = createMockExtractStatements();
668
+ createInstanceMock.mockResolvedValue(createMockConnection(streamMock, vi.fn(), {
669
+ extractStatements: extractStatementsMock,
670
+ }));
671
+ const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(duckdbS3Credentials);
672
+ await expect(client.runQuery("SELECT * FROM read_parquet('s3://bucket/data.parquet')")).rejects.toThrow("SQL validation error: function 'read_parquet' is not allowed");
673
+ expect(extractStatementsMock).not.toHaveBeenCalled();
674
+ expect(streamMock).not.toHaveBeenCalled();
675
+ });
676
+ it.each([
677
+ {
678
+ format: 'Parquet',
679
+ sql: "SELECT * FROM read_parquet('s3://bucket/data.parquet')",
680
+ },
681
+ {
682
+ format: 'inferred JSONL',
683
+ sql: "SELECT * FROM read_json_auto('s3://bucket/data.jsonl')",
684
+ },
685
+ {
686
+ format: 'typed JSONL',
687
+ sql: "SELECT * FROM read_json('s3://bucket/data.jsonl', format='newline_delimited')",
688
+ },
689
+ ])('should allow pre-aggregate clients to read S3 $format files', async ({ sql }) => {
690
+ const rows = [{ val: 1 }];
691
+ const streamMock = vi.fn(async () => getMockStreamResult([rows], [DUCKDB_TYPE_IDS.INTEGER]));
692
+ const extractStatementsMock = createMockExtractStatements();
693
+ createInstanceMock.mockResolvedValue(createMockConnection(streamMock, vi.fn(), {
694
+ extractStatements: extractStatementsMock,
695
+ }));
696
+ const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate(duckdbS3Credentials);
697
+ const result = await client.runQuery(sql);
698
+ expect(result.rows).toEqual(rows);
699
+ expect(extractStatementsMock).toHaveBeenCalledTimes(1);
700
+ expect(streamMock).toHaveBeenCalledTimes(1);
701
+ });
702
+ it('should keep pre-aggregate queries read-only', async () => {
703
+ const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
704
+ createInstanceMock.mockResolvedValue(createMockConnection(streamMock, vi.fn(), {
705
+ extractStatements: createMockExtractStatements({
706
+ statementType: 11, // COPY
707
+ }),
708
+ }));
709
+ const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate(duckdbS3Credentials);
710
+ await expect(client.runQuery("COPY t TO 's3://bucket/data.parquet'")).rejects.toThrow('SQL validation error: only SELECT statements are allowed');
711
+ expect(streamMock).not.toHaveBeenCalled();
712
+ });
713
+ it('should keep secret introspection blocked for pre-aggregate queries', async () => {
714
+ const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
715
+ createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
716
+ const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate(duckdbS3Credentials);
717
+ await expect(client.runQuery("SELECT current_setting('s3_secret_access_key')")).rejects.toThrow("SQL validation error: function 'current_setting' is not allowed");
718
+ expect(streamMock).not.toHaveBeenCalled();
719
+ });
656
720
  it('should reject user queries that use file table paths', async () => {
657
721
  const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
658
722
  createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
package/dist/warehouseClients/PgQueryStream.d.ts CHANGED
@@ -22,5 +22,5 @@ declare class QueryStream extends Readable implements Submittable {
22
22
  _destroy(_err: Error, cb: Function): void;
23
23
  _read(size: number): void;
24
24
  }
25
- export = QueryStream;
25
+ export default QueryStream;
26
26
  //# sourceMappingURL=PgQueryStream.d.ts.map
package/dist/warehouseClients/PgQueryStream.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"PgQueryStream.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/PgQueryStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAe,WAAW,EAAE,MAAM,IAAI,CAAC;AAE1D,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAElC,UAAU,iBAAiB;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,OAAO,CAAC;CACnB;AAID,cAAM,WAAY,SAAQ,QAAS,YAAW,WAAW;IACrD,MAAM,EAAE,OAAO,CAAC;IAEhB,OAAO,EAAE,OAAO,CAAC;IAEjB,oBAAoB,EAAE,QAAQ,CAAC;IAE/B,aAAa,EAAE,QAAQ,CAAC;IAExB,qBAAqB,EAAE,QAAQ,CAAC;IAEhC,qBAAqB,EAAE,QAAQ,CAAC;IAEhC,mBAAmB,EAAE,QAAQ,CAAC;IAE9B,WAAW,EAAE,QAAQ,CAAC;IAEtB,gBAAgB,EAAE,QAAQ,CAAC;IAE3B,YACI,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,OAAO,EAAE,EAClB,MAAM,GAAE,iBAAsB,EA+BjC;IAEM,MAAM,CAAC,UAAU,EAAE,UAAU,GAAG,IAAI,CAE1C;IAGM,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,QAIxC;IAIM,KAAK,CAAC,IAAI,EAAE,MAAM,QAexB;CACJ;SAEQ,WAAW"}
1
+ {"version":3,"file":"PgQueryStream.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/PgQueryStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAe,WAAW,EAAE,MAAM,IAAI,CAAC;AAE1D,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAElC,UAAU,iBAAiB;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,OAAO,CAAC;CACnB;AAID,cAAM,WAAY,SAAQ,QAAS,YAAW,WAAW;IACrD,MAAM,EAAE,OAAO,CAAC;IAEhB,OAAO,EAAE,OAAO,CAAC;IAEjB,oBAAoB,EAAE,QAAQ,CAAC;IAE/B,aAAa,EAAE,QAAQ,CAAC;IAExB,qBAAqB,EAAE,QAAQ,CAAC;IAEhC,qBAAqB,EAAE,QAAQ,CAAC;IAEhC,mBAAmB,EAAE,QAAQ,CAAC;IAE9B,WAAW,EAAE,QAAQ,CAAC;IAEtB,gBAAgB,EAAE,QAAQ,CAAC;IAE3B,YACI,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,OAAO,EAAE,EAClB,MAAM,GAAE,iBAAsB,EA+BjC;IAEM,MAAM,CAAC,UAAU,EAAE,UAAU,GAAG,IAAI,CAE1C;IAGM,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,QAIxC;IAIM,KAAK,CAAC,IAAI,EAAE,MAAM,QAexB;CACJ;eAEc,WAAW"}
package/dist/warehouseClients/PgQueryStream.js CHANGED
@@ -2,6 +2,7 @@
2
2
  var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
5
6
  const pg_cursor_1 = __importDefault(require("pg-cursor"));
6
7
  const stream_1 = require("stream");
7
8
  // Note: this is a copy of the QueryStream class from pg-query-stream with the following changes:
@@ -54,4 +55,4 @@ class QueryStream extends stream_1.Readable {
54
55
  });
55
56
  }
56
57
  }
57
- module.exports = QueryStream;
58
+ exports.default = QueryStream;
package/dist/warehouseClients/PostgresWarehouseClient.d.ts CHANGED
@@ -56,9 +56,16 @@ export declare class PostgresSqlBuilder extends WarehouseBaseSqlBuilder {
56
56
  getMetricSql(sql: string, metric: Metric): string;
57
57
  concatString(...args: string[]): string;
58
58
  }
59
+ type CatalogQueryFilters = {
60
+ databaseFilters: string[];
61
+ schemaFilters: string[];
62
+ tableFilters: string[];
63
+ values: string[] | undefined;
64
+ };
59
65
  export declare class PostgresClient<T extends CreatePostgresLikeCredentials> extends WarehouseBaseClient<T> {
60
66
  config: pg.PoolConfig;
61
67
  constructor(credentials: T, config: pg.PoolConfig);
68
+ protected getCatalogQueryFilters(databases: Set<string>, schemas: Set<string>, tables: Set<string>): CatalogQueryFilters;
62
69
  private getSQLWithMetadata;
63
70
  static convertQueryResultFields(fields: QueryResult<AnyType>['fields']): WarehouseResults['fields'];
64
71
  private static getNoticeError;
@@ -82,6 +89,8 @@ export declare class PostgresClient<T extends CreatePostgresLikeCredentials> ext
82
89
  parseError(error: pg.DatabaseError, query?: string): WarehouseQueryError;
83
90
  }
84
91
  export declare class PostgresWarehouseClient extends PostgresClient<CreatePostgresCredentials> {
92
+ protected getCatalogQueryFilters(databases: Set<string>, schemas: Set<string>, tables: Set<string>): CatalogQueryFilters;
85
93
  constructor(credentials: CreatePostgresCredentials);
86
94
  }
95
+ export {};
87
96
  //# sourceMappingURL=PostgresWarehouseClient.d.ts.map
package/dist/warehouseClients/PostgresWarehouseClient.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"PostgresWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/PostgresWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,6BAA6B,EAG7B,MAAM,EAIN,mBAAmB,EACnB,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAG3B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAc,WAAW,EAAS,MAAM,IAAI,CAAC;AAKpD,OAAO,mBAAmB,CAAC;AAE3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAKhE,oBAAY,aAAa;IACrB,OAAO,YAAY;IACnB,GAAG,QAAQ;IACX,IAAI,SAAS;IACb,IAAI,SAAS;IACb,IAAI,SAAS;IACb,KAAK,UAAU;IACf,WAAW,gBAAgB;IAC3B,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,SAAS,cAAc;IACvB,MAAM,WAAW;IACjB,QAAQ,aAAa;IACrB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,gBAAgB,qBAAqB;IACrC,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,IAAI,SAAS;IACb,KAAK,UAAU;IACf,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,iBAAiB,sBAAsB;IACvC,QAAQ,aAAa;IACrB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,OAAO,WAAW;IAClB,sBAAsB,2BAA2B;IACjD,SAAS,cAAc;IACvB,YAAY,gBAAgB;IAC5B,2BAA2B,gCAAgC;IAC3D,wBAAwB,6BAA6B;CACxD;AA4CD,eAAO,MAAM,0BAA0B,SAC7B,MAAM,KACb,eAAe,GAAG,SAWpB,CAAC;AA6CF,qBAAa,kBAAmB,SAAQ,uBAAuB;IAC3D,IAAI,iBAA2B;IAE/B,cAAc,IAAI,mBAAmB,CAEpC;IAED,wBAAwB,IAAI,MAAM,CAEjC;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAiBlC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAahD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AAED,qBAAa,cAAc,CACvB,CAAC,SAAS,6BAA6B,CACzC,SAAQ,mBAAmB,CAAC,CAAC,CAAC;IAC5B,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC;IAEtB,YAAY,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,UAAU,EAGhD;IAED,OAAO,CAAC,kBAAkB;IAQ1B,MAAM,CAAC,wBAAwB,CAC3B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,GACvC,gBAAgB,CAAC,QAAQ,CAAC,CAO5B;IAED,OAAO,CAAC,MAAM,CAAC,cAAc;IAavB,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CA0Of;IAEK,UAAU,CACZ,QAAQ,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,gCA0GN;IAEK,YAAY;;;;SAuBjB;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CA0B3B;IAED,UAAU,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,EAAE,KAAK,GAAE,MAAW,uBAsDrD;CACJ;AA0CD,qBAAa,uBAAwB,SAAQ,cAAc,CAAC,yBAAyB,CAAC;IAClF,YAAY,WAAW,EAAE,yBAAyB,EAUjD;CACJ"}
1
+ {"version":3,"file":"PostgresWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/PostgresWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,6BAA6B,EAG7B,MAAM,EAIN,mBAAmB,EACnB,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAG3B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAc,WAAW,EAAS,MAAM,IAAI,CAAC;AAKpD,OAAO,mBAAmB,CAAC;AAE3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAKhE,oBAAY,aAAa;IACrB,OAAO,YAAY;IACnB,GAAG,QAAQ;IACX,IAAI,SAAS;IACb,IAAI,SAAS;IACb,IAAI,SAAS;IACb,KAAK,UAAU;IACf,WAAW,gBAAgB;IAC3B,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,SAAS,cAAc;IACvB,MAAM,WAAW;IACjB,QAAQ,aAAa;IACrB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,gBAAgB,qBAAqB;IACrC,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,IAAI,SAAS;IACb,KAAK,UAAU;IACf,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,iBAAiB,sBAAsB;IACvC,QAAQ,aAAa;IACrB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,OAAO,WAAW;IAClB,sBAAsB,2BAA2B;IACjD,SAAS,cAAc;IACvB,YAAY,gBAAgB;IAC5B,2BAA2B,gCAAgC;IAC3D,wBAAwB,6BAA6B;CACxD;AA4CD,eAAO,MAAM,0BAA0B,SAC7B,MAAM,KACb,eAAe,GAAG,SAWpB,CAAC;AA6CF,qBAAa,kBAAmB,SAAQ,uBAAuB;IAC3D,IAAI,iBAA2B;IAE/B,cAAc,IAAI,mBAAmB,CAEpC;IAED,wBAAwB,IAAI,MAAM,CAEjC;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAiBlC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAahD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AAED,KAAK,mBAAmB,GAAG;IACvB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;CAChC,CAAC;AAEF,qBAAa,cAAc,CACvB,CAAC,SAAS,6BAA6B,CACzC,SAAQ,mBAAmB,CAAC,CAAC,CAAC;IAC5B,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC;IAEtB,YAAY,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,UAAU,EAGhD;IAED,SAAS,CAAC,sBAAsB,CAC5B,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,EACtB,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,EACpB,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,GACpB,mBAAmB,CAOrB;IAED,OAAO,CAAC,kBAAkB;IAQ1B,MAAM,CAAC,wBAAwB,CAC3B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,GACvC,gBAAgB,CAAC,QAAQ,CAAC,CAO5B;IAED,OAAO,CAAC,MAAM,CAAC,cAAc;IAavB,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CA0Of;IAEK,UAAU,CACZ,QAAQ,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,gCAmHN;IAEK,YAAY;;;;SAuBjB;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CA0B3B;IAED,UAAU,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,EAAE,KAAK,GAAE,MAAW,uBAsDrD;CACJ;AA0CD,qBAAa,uBAAwB,SAAQ,cAAc,CAAC,yBAAyB,CAAC;IAClF,SAAS,CAAC,sBAAsB,CAC5B,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,EACtB,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,EACpB,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,GACpB,mBAAmB,CAarB;IAED,YAAY,WAAW,EAAE,yBAAyB,EAUjD;CACJ"}
package/dist/warehouseClients/PostgresWarehouseClient.js CHANGED
@@ -235,6 +235,14 @@ class PostgresClient extends WarehouseBaseClient_1.default {
235
235
  super(credentials, new PostgresSqlBuilder(credentials.startOfWeek));
236
236
  this.config = config;
237
237
  }
238
+ getCatalogQueryFilters(databases, schemas, tables) {
239
+ return {
240
+ databaseFilters: [...databases].map((value) => `'${value}'`),
241
+ schemaFilters: [...schemas].map((value) => `'${value}'`),
242
+ tableFilters: [...tables].map((value) => `'${value}'`),
243
+ values: undefined,
244
+ };
245
+ }
238
246
  getSQLWithMetadata(sql, tags) {
239
247
  let alteredQuery = sql;
240
248
  if (tags) {
@@ -437,15 +445,18 @@ class PostgresClient extends WarehouseBaseClient_1.default {
437
445
  }
438
446
  async getCatalog(requests) {
439
447
  const { databases, schemas, tables } = requests.reduce((acc, { database, schema, table }) => ({
440
- databases: acc.databases.add(`'${database}'`),
441
- schemas: acc.schemas.add(`'${schema}'`),
442
- tables: acc.tables.add(`'${table}'`),
448
+ databases: acc.databases.add(database),
449
+ schemas: acc.schemas.add(schema),
450
+ tables: acc.tables.add(table),
443
451
  }), {
444
452
  databases: new Set(),
445
453
  schemas: new Set(),
446
454
  tables: new Set(),
447
455
  });
448
- if (databases.size <= 0 || schemas.size <= 0 || tables.size <= 0) {
456
+ const { databaseFilters, schemaFilters, tableFilters, values } = this.getCatalogQueryFilters(databases, schemas, tables);
457
+ if (databaseFilters.length <= 0 ||
458
+ schemaFilters.length <= 0 ||
459
+ tableFilters.length <= 0) {
449
460
  return {};
450
461
  }
451
462
  const { rows: pgVersionRows } = await this.runQuery('SELECT version()');
@@ -462,9 +473,9 @@ class PostgresClient extends WarehouseBaseClient_1.default {
462
473
  column_name,
463
474
  data_type
464
475
  FROM information_schema.columns
465
- WHERE table_catalog IN (${Array.from(databases)})
466
- AND table_schema IN (${Array.from(schemas)})
467
- AND table_name IN (${Array.from(tables)})
476
+ WHERE table_catalog IN (${databaseFilters})
477
+ AND table_schema IN (${schemaFilters})
478
+ AND table_name IN (${tableFilters})
468
479
  ${supportsMatviews
469
480
  ? `
470
481
 
@@ -480,13 +491,15 @@ class PostgresClient extends WarehouseBaseClient_1.default {
480
491
  JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid
481
492
  JOIN pg_catalog.pg_matviews mv ON n.nspname = mv.schemaname AND c.relname = mv.matviewname
482
493
  WHERE c.relkind = 'm'
483
- AND current_database() IN (${Array.from(databases)})
484
- AND n.nspname IN (${Array.from(schemas)})
485
- AND c.relname IN (${Array.from(tables)})
494
+ AND current_database() IN (${databaseFilters})
495
+ AND n.nspname IN (${schemaFilters})
496
+ AND c.relname IN (${tableFilters})
486
497
  AND a.attnum > 0
487
498
  AND NOT a.attisdropped`
488
499
  : ''}`;
489
- const { rows } = await this.runQuery(query);
500
+ const { rows } = values === undefined
501
+ ? await this.runQuery(query)
502
+ : await this.runQuery(query, {}, undefined, values);
490
503
  const catalog = rows.reduce((acc, { table_catalog, table_schema, table_name, column_name, data_type, }) => {
491
504
  const match = requests.find(({ database, schema, table }) => database === table_catalog &&
492
505
  schema === table_schema &&
@@ -639,6 +652,15 @@ const getSSLConfigFromMode = ({ sslcert, sslkey, sslmode, sslrootcert, }) => {
639
652
  }
640
653
  };
641
654
  class PostgresWarehouseClient extends PostgresClient {
655
+ getCatalogQueryFilters(databases, schemas, tables) {
656
+ const values = [...databases, ...schemas, ...tables];
657
+ return {
658
+ databaseFilters: [...databases].map((_, index) => `$${index + 1}`),
659
+ schemaFilters: [...schemas].map((_, index) => `$${databases.size + index + 1}`),
660
+ tableFilters: [...tables].map((_, index) => `$${databases.size + schemas.size + index + 1}`),
661
+ values,
662
+ };
663
+ }
642
664
  constructor(credentials) {
643
665
  const ssl = getSSLConfigFromMode(credentials);
644
666
  super(credentials, {
package/dist/warehouseClients/PostgresWarehouseClient.test.js CHANGED
@@ -147,6 +147,30 @@ describe('PostgresWarehouseClient', () => {
147
147
  const warehouse = new PostgresWarehouseClient_1.PostgresWarehouseClient(PostgresWarehouseClient_mock_1.credentials);
148
148
  expect(await warehouse.getCatalog([])).toEqual({});
149
149
  });
150
+ it('binds catalog filter values as query parameters', async () => {
151
+ const warehouse = new PostgresWarehouseClient_1.PostgresWarehouseClient(PostgresWarehouseClient_mock_1.credentials);
152
+ const database = "missing') OR (SELECT CAST(current_database() AS integer)=0) OR ('x'='x";
153
+ const runQuery = vi
154
+ .spyOn(warehouse, 'runQuery')
155
+ .mockResolvedValueOnce({
156
+ rows: [{ version: 'PostgreSQL 15.4' }],
157
+ fields: {},
158
+ })
159
+ .mockResolvedValueOnce({ rows: [], fields: {} });
160
+ await warehouse.getCatalog([
161
+ { database, schema: 'public', table: 'orders' },
162
+ ]);
163
+ const catalogQuery = runQuery.mock.calls[1][0];
164
+ expect(catalogQuery).toContain('table_catalog IN ($1)');
165
+ expect(catalogQuery).toContain('table_schema IN ($2)');
166
+ expect(catalogQuery).toContain('table_name IN ($3)');
167
+ expect(catalogQuery).not.toContain(database);
168
+ expect(runQuery.mock.calls[1][3]).toEqual([
169
+ database,
170
+ 'public',
171
+ 'orders',
172
+ ]);
173
+ });
150
174
  });
151
175
  describe('PostgresWarehouseClient statement timeout', () => {
152
176
  const mockPoolWithQuery = (queryMock) => {
package/dist/warehouseClients/RedshiftWarehouseClient.test.d.ts ADDED
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=RedshiftWarehouseClient.test.d.ts.map
package/dist/warehouseClients/RedshiftWarehouseClient.test.d.ts.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"RedshiftWarehouseClient.test.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/RedshiftWarehouseClient.test.ts"],"names":[],"mappings":""}
package/dist/warehouseClients/RedshiftWarehouseClient.test.js ADDED
@@ -0,0 +1,33 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ const common_1 = require("@lightdash/common");
4
+ const RedshiftWarehouseClient_1 = require("./RedshiftWarehouseClient");
5
+ describe('RedshiftWarehouseClient', () => {
6
+ it('keeps catalog filters in the Redshift query', async () => {
7
+ const warehouse = new RedshiftWarehouseClient_1.RedshiftWarehouseClient({
8
+ type: common_1.WarehouseTypes.REDSHIFT,
9
+ host: 'localhost',
10
+ user: 'analytics',
11
+ password: 'password',
12
+ port: 5439,
13
+ dbname: 'warehouse',
14
+ schema: 'public',
15
+ authenticationType: common_1.RedshiftAuthenticationType.PASSWORD,
16
+ });
17
+ const runQuery = vi
18
+ .spyOn(warehouse, 'runQuery')
19
+ .mockResolvedValueOnce({
20
+ rows: [{ version: 'PostgreSQL 8.0.2' }],
21
+ fields: {},
22
+ })
23
+ .mockResolvedValueOnce({ rows: [], fields: {} });
24
+ await warehouse.getCatalog([
25
+ { database: 'warehouse', schema: 'public', table: 'orders' },
26
+ ]);
27
+ const catalogQuery = runQuery.mock.calls[1][0];
28
+ expect(catalogQuery).toContain("table_catalog IN ('warehouse')");
29
+ expect(catalogQuery).toContain("table_schema IN ('public')");
30
+ expect(catalogQuery).toContain("table_name IN ('orders')");
31
+ expect(runQuery.mock.calls[1][3]).toBeUndefined();
32
+ });
33
+ });
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@lightdash/warehouses",
3
- "version": "1.55.0",
3
+ "version": "1.56.1",
4
4
  "description": "Warehouse connectors for Lightdash",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -29,7 +29,7 @@
29
29
  "snowflake-sdk": "2.3.4",
30
30
  "ssh2": "1.14.0",
31
31
  "trino-client": "0.2.9",
32
- "@lightdash/common": "1.55.0"
32
+ "@lightdash/common": "1.56.1"
33
33
  },
34
34
  "devDependencies": {
35
35
  "@types/big.js": "6.2.2",