@lightdash/warehouses 2.71.1 → 2.71.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/warehouseClientFromCredentials.d.ts +3 -1
- package/dist/warehouseClientFromCredentials.d.ts.map +1 -1
- package/dist/warehouseClientFromCredentials.js +1 -1
- package/dist/warehouseClients/ClickhouseWarehouseClient.d.ts +6 -1
- package/dist/warehouseClients/ClickhouseWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/ClickhouseWarehouseClient.js +10 -2
- package/dist/warehouseClients/ClickhouseWarehouseClient.test.js +12 -0
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.js +72 -7
- package/dist/warehouseClients/DuckdbWarehouseClient.test.js +98 -17
- package/package.json +2 -2
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { CreateWarehouseCredentials } from '@lightdash/common';
|
|
2
2
|
import { WarehouseClient } from './types';
|
|
3
|
+
import { type ClickhouseWarehouseClientOptions } from './warehouseClients/ClickhouseWarehouseClient';
|
|
3
4
|
import { type DuckdbWarehouseClientOptions } from './warehouseClients/DuckdbWarehouseClient';
|
|
4
|
-
export
|
|
5
|
+
export type WarehouseClientOptions = DuckdbWarehouseClientOptions & ClickhouseWarehouseClientOptions;
|
|
6
|
+
export declare const warehouseClientFromCredentials: (credentials: CreateWarehouseCredentials, options?: WarehouseClientOptions) => WarehouseClient;
|
|
5
7
|
//# sourceMappingURL=warehouseClientFromCredentials.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"warehouseClientFromCredentials.d.ts","sourceRoot":"","sources":["../src/warehouseClientFromCredentials.ts"],"names":[],"mappings":"AAAA,OAAO,EAEH,0BAA0B,EAG7B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"warehouseClientFromCredentials.d.ts","sourceRoot":"","sources":["../src/warehouseClientFromCredentials.ts"],"names":[],"mappings":"AAAA,OAAO,EAEH,0BAA0B,EAG7B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAG1C,OAAO,EAEH,KAAK,gCAAgC,EACxC,MAAM,8CAA8C,CAAC;AAEtD,OAAO,EAEH,KAAK,4BAA4B,EACpC,MAAM,0CAA0C,CAAC;AAMlD,MAAM,MAAM,sBAAsB,GAAG,4BAA4B,GAC7D,gCAAgC,CAAC;AAErC,eAAO,MAAM,8BAA8B,gBAC1B,0BAA0B,YAC7B,sBAAsB,KACjC,eA4BF,CAAC"}
|
|
@@ -26,7 +26,7 @@ const warehouseClientFromCredentials = (credentials, options) => {
|
|
|
26
26
|
case common_1.WarehouseTypes.TRINO:
|
|
27
27
|
return new TrinoWarehouseClient_1.TrinoWarehouseClient(credentials);
|
|
28
28
|
case common_1.WarehouseTypes.CLICKHOUSE:
|
|
29
|
-
return new ClickhouseWarehouseClient_1.ClickhouseWarehouseClient(credentials);
|
|
29
|
+
return new ClickhouseWarehouseClient_1.ClickhouseWarehouseClient(credentials, options);
|
|
30
30
|
case common_1.WarehouseTypes.ATHENA:
|
|
31
31
|
return new AthenaWarehouseClient_1.AthenaWarehouseClient(credentials);
|
|
32
32
|
case common_1.WarehouseTypes.DUCKDB:
|
|
@@ -57,9 +57,14 @@ export declare class ClickhouseSqlBuilder extends WarehouseBaseSqlBuilder {
|
|
|
57
57
|
getNullSafeEqualJoinSql(left: string, right: string): string;
|
|
58
58
|
buildArray(elements: string[]): string;
|
|
59
59
|
}
|
|
60
|
+
export declare const getMaxOpenConnections: (maxOpenConnections?: number) => number;
|
|
61
|
+
export type ClickhouseWarehouseClientOptions = {
|
|
62
|
+
/** Upper bound of concurrent queries sharing this client; sizes the HTTP socket pool. */
|
|
63
|
+
maxOpenConnections?: number;
|
|
64
|
+
};
|
|
60
65
|
export declare class ClickhouseWarehouseClient extends WarehouseBaseClient<CreateClickhouseCredentials> {
|
|
61
66
|
client: ClickHouseClient;
|
|
62
|
-
constructor(credentials: CreateClickhouseCredentials);
|
|
67
|
+
constructor(credentials: CreateClickhouseCredentials, options?: ClickhouseWarehouseClientOptions);
|
|
63
68
|
streamQuery(sql: string, streamCallback: (data: WarehouseResults) => void | Promise<void>, options: {
|
|
64
69
|
queryParams?: Record<string, AnyType>;
|
|
65
70
|
tags?: Record<string, string>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ClickhouseWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/ClickhouseWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAGH,KAAK,gBAAgB,EAExB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACH,OAAO,EACP,2BAA2B,EAC3B,aAAa,EAEb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAGhB,gBAAgB,EAChB,cAAc,EACd,KAAK,eAAe,EACvB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAM5C,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAEhE,oBAAY,eAAe;IACvB,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,IAAI,SAAS;IACb,KAAK,UAAU;IACf,KAAK,UAAU;IACf,KAAK,UAAU;IACf,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,SAAS,cAAc;IACvB,SAAS,cAAc;IACvB,UAAU,eAAe;IACzB,UAAU,eAAe;IACzB,IAAI,SAAS;IACb,MAAM,WAAW;IACjB,WAAW,gBAAgB;IAC3B,IAAI,SAAS;IACb,IAAI,SAAS;IACb,MAAM,WAAW;IACjB,QAAQ,aAAa;IACrB,UAAU,eAAe;IACzB,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,KAAK,UAAU;IACf,KAAK,UAAU;IACf,GAAG,QAAQ;IACX,QAAQ,aAAa;IACrB,cAAc,mBAAmB;IACjC,IAAI,SAAS;IACb,IAAI,SAAS;CAChB;AAED,UAAU,SAAS;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACjB;AAmBD,eAAO,MAAM,4BAA4B,SAC/B,eAAe,GAAG,MAAM,KAC/B,eAAe,GAAG,SAQpB,CAAC;AAEF,eAAO,MAAM,8BAA8B,SACjC,eAAe,GAAG,MAAM,KAC/B,aA+BF,CAAC;AAyCF,qBAAa,oBAAqB,SAAQ,uBAAuB;IAC7D,QAAQ,CAAC,IAAI,6BAA6B;IAE1C,cAAc,IAAI,mBAAmB,CAEpC;IAED,wBAAwB,IAAI,MAAM,CAEjC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAiBlC;IAED,eAAe,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAGlC;IAED,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAI5D;IAED,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM,CAGR;IAED,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGrC;IAED,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAK3D;IAED,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAGrC;CACJ;
|
|
1
|
+
{"version":3,"file":"ClickhouseWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/ClickhouseWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAGH,KAAK,gBAAgB,EAExB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACH,OAAO,EACP,2BAA2B,EAC3B,aAAa,EAEb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAGhB,gBAAgB,EAChB,cAAc,EACd,KAAK,eAAe,EACvB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAM5C,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAEhE,oBAAY,eAAe;IACvB,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,IAAI,SAAS;IACb,KAAK,UAAU;IACf,KAAK,UAAU;IACf,KAAK,UAAU;IACf,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,SAAS,cAAc;IACvB,SAAS,cAAc;IACvB,UAAU,eAAe;IACzB,UAAU,eAAe;IACzB,IAAI,SAAS;IACb,MAAM,WAAW;IACjB,WAAW,gBAAgB;IAC3B,IAAI,SAAS;IACb,IAAI,SAAS;IACb,MAAM,WAAW;IACjB,QAAQ,aAAa;IACrB,UAAU,eAAe;IACzB,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,KAAK,UAAU;IACf,KAAK,UAAU;IACf,GAAG,QAAQ;IACX,QAAQ,aAAa;IACrB,cAAc,mBAAmB;IACjC,IAAI,SAAS;IACb,IAAI,SAAS;CAChB;AAED,UAAU,SAAS;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACjB;AAmBD,eAAO,MAAM,4BAA4B,SAC/B,eAAe,GAAG,MAAM,KAC/B,eAAe,GAAG,SAQpB,CAAC;AAEF,eAAO,MAAM,8BAA8B,SACjC,eAAe,GAAG,MAAM,KAC/B,aA+BF,CAAC;AAyCF,qBAAa,oBAAqB,SAAQ,uBAAuB;IAC7D,QAAQ,CAAC,IAAI,6BAA6B;IAE1C,cAAc,IAAI,mBAAmB,CAEpC;IAED,wBAAwB,IAAI,MAAM,CAEjC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAiBlC;IAED,eAAe,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAGlC;IAED,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAI5D;IAED,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM,CAGR;IAED,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGrC;IAED,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAK3D;IAED,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAGrC;CACJ;AAMD,eAAO,MAAM,qBAAqB,wBAAyB,MAAM,KAAG,MAG9B,CAAC;AAEvC,MAAM,MAAM,gCAAgC,GAAG;IAC3C,yFAAyF;IACzF,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC/B,CAAC;AAEF,qBAAa,yBAA0B,SAAQ,mBAAmB,CAAC,2BAA2B,CAAC;IAC3F,MAAM,EAAE,gBAAgB,CAAC;IAEzB,YACI,WAAW,EAAE,2BAA2B,EACxC,OAAO,CAAC,EAAE,gCAAgC,EAiB7C;IAEK,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC,CA2Gf;IAEK,UAAU,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAmCjE;IAEK,SAAS,CACX,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAyB3B;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EAAE,oCAAoC;IACvD,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CA6B3B;IAEK,YAAY;;;;SAmBjB;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAS1B;CACJ"}
|
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.ClickhouseWarehouseClient = exports.ClickhouseSqlBuilder = exports.convertDataTypeToDimensionType = exports.getClickhouseTimestampDomain = exports.ClickhouseTypes = void 0;
|
|
6
|
+
exports.ClickhouseWarehouseClient = exports.getMaxOpenConnections = exports.ClickhouseSqlBuilder = exports.convertDataTypeToDimensionType = exports.getClickhouseTimestampDomain = exports.ClickhouseTypes = void 0;
|
|
7
7
|
const client_1 = require("@clickhouse/client");
|
|
8
8
|
const common_1 = require("@lightdash/common");
|
|
9
9
|
const processPromisesInBatches_1 = require("../utils/processPromisesInBatches");
|
|
@@ -186,8 +186,15 @@ class ClickhouseSqlBuilder extends WarehouseBaseSqlBuilder_1.default {
|
|
|
186
186
|
}
|
|
187
187
|
}
|
|
188
188
|
exports.ClickhouseSqlBuilder = ClickhouseSqlBuilder;
|
|
189
|
+
const DEFAULT_MAX_OPEN_CONNECTIONS = 10;
|
|
190
|
+
// The client is cached per project and shared by all concurrent query jobs;
|
|
191
|
+
// when jobs outnumber sockets they queue and report inflated exec times.
|
|
192
|
+
const getMaxOpenConnections = (maxOpenConnections) => Number.isInteger(maxOpenConnections)
|
|
193
|
+
? Math.max(maxOpenConnections, DEFAULT_MAX_OPEN_CONNECTIONS)
|
|
194
|
+
: DEFAULT_MAX_OPEN_CONNECTIONS;
|
|
195
|
+
exports.getMaxOpenConnections = getMaxOpenConnections;
|
|
189
196
|
class ClickhouseWarehouseClient extends WarehouseBaseClient_1.default {
|
|
190
|
-
constructor(credentials) {
|
|
197
|
+
constructor(credentials, options) {
|
|
191
198
|
super(credentials, new ClickhouseSqlBuilder(credentials.startOfWeek));
|
|
192
199
|
const protocol = credentials.secure ? 'https' : 'http';
|
|
193
200
|
const url = `${protocol}://${credentials.host}:${credentials.port}`;
|
|
@@ -197,6 +204,7 @@ class ClickhouseWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
197
204
|
password: credentials.password,
|
|
198
205
|
database: credentials.schema, // In clickhouse schema = database
|
|
199
206
|
request_timeout: (credentials.timeoutSeconds || 30) * 1000,
|
|
207
|
+
max_open_connections: (0, exports.getMaxOpenConnections)(options?.maxOpenConnections),
|
|
200
208
|
});
|
|
201
209
|
}
|
|
202
210
|
async streamQuery(sql, streamCallback, options) {
|
|
@@ -2,6 +2,18 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const common_1 = require("@lightdash/common");
|
|
4
4
|
const ClickhouseWarehouseClient_1 = require("./ClickhouseWarehouseClient");
|
|
5
|
+
describe('getMaxOpenConnections', () => {
|
|
6
|
+
it('defaults to 10 when concurrency is unknown', () => {
|
|
7
|
+
expect((0, ClickhouseWarehouseClient_1.getMaxOpenConnections)()).toBe(10);
|
|
8
|
+
expect((0, ClickhouseWarehouseClient_1.getMaxOpenConnections)(NaN)).toBe(10);
|
|
9
|
+
});
|
|
10
|
+
it('matches the given concurrency when larger than the default', () => {
|
|
11
|
+
expect((0, ClickhouseWarehouseClient_1.getMaxOpenConnections)(100)).toBe(100);
|
|
12
|
+
});
|
|
13
|
+
it('never shrinks below the default', () => {
|
|
14
|
+
expect((0, ClickhouseWarehouseClient_1.getMaxOpenConnections)(1)).toBe(10);
|
|
15
|
+
});
|
|
16
|
+
});
|
|
5
17
|
describe('ClickhouseSqlBuilder', () => {
|
|
6
18
|
const builder = new ClickhouseWarehouseClient_1.ClickhouseSqlBuilder();
|
|
7
19
|
it('builds arrays with [...] syntax, not ARRAY[...] which ClickHouse rejects', () => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAKA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAMb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAEhB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAQ3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA6DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;IAChB,gEAAgE;IAChE,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACpE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG,mBAAmB,CAAC;AAE9D,MAAM,MAAM,4BAA4B,GAAG;IACvC,kHAAkH;IAClH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;IAC9D,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,4BAA4B,CAAC,EAAE,MAAM,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,sBAAsB,WAAY,MAAM,KAAG,aA+BvD,CAAC;AAEF,eAAO,MAAM,kCAAkC,aACjC,MAAM,KACjB,eAAe,GAAG,SAOpB,CAAC;AAoCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB,CAEpC;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAKA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAMb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAEhB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAQ3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA6DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;IAChB,gEAAgE;IAChE,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACpE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG,mBAAmB,CAAC;AAE9D,MAAM,MAAM,4BAA4B,GAAG;IACvC,kHAAkH;IAClH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;IAC9D,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,4BAA4B,CAAC,EAAE,MAAM,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,sBAAsB,WAAY,MAAM,KAAG,aA+BvD,CAAC;AAEF,eAAO,MAAM,kCAAkC,aACjC,MAAM,KACjB,eAAe,GAAG,SAOpB,CAAC;AAoCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB,CAEpC;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AAmID,eAAO,MAAM,+BAA+B,yBAGzC,IAAI,CAAC,iCAAiC,EAAE,UAAU,GAAG,OAAO,CAAC,KAAG,MAOlE,CAAC;AAwFF,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,iCAAiC,CAAC;IAC7F,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAGhD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAE/C;IAEF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sCAAsC,CAG1D;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,8BAA8B,CAGlD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAU;IAE/C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;IAEV,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAEhD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAU;IAE9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,4BAA4B,CAAC,CAAS;IAEvD,OAAO,CAAC,2BAA2B,CAAS;IAE5C,OAAO,CAAC,gCAAgC,CAAS;IAEjD,YACI,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B,EAmHzC;IAED,OAAO,CAAC,MAAM,CAAC,kBAAkB;IASjC,MAAM,CAAC,qBAAqB,CACxB,WAAW,EAAE,mBAAmB,EAChC,OAAO,CAAC,EAAE,4BAA4B,GACvC,qBAAqB,CAIvB;IAED,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAoBzC,OAAO,CAAC,2BAA2B;mBAUd,6BAA6B;IA4DlD,OAAO,CAAC,MAAM,CAAC,iCAAiC;IAoBhD,OAAO,CAAC,kBAAkB;mBAQL,cAAc;mBAOd,mBAAmB;IAaxC,OAAO,CAAC,MAAM,CAAC,qBAAqB;mBAMf,uBAAuB;YAsB9B,aAAa;YAoBb,kCAAkC;mBAW3B,qBAAqB;mBA6ErB,uBAAuB;mBAgBvB,yBAAyB;mBAiFzB,mBAAmB;IA6BxC,mFAAmF;IACnF,MAAM,CAAC,gCAAgC,IAAI,IAAI,CAS9C;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAO3B;IAED,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CACZ;IAEnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAA+B;IAE3E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAA0B;IAEjE,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAIjC,OAAO,CAAC,MAAM,CAAC,UAAU;IAIzB,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAuB5C,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAqGzC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAMhC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAmBrC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IA2CrC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAyC/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAK;YAE9C,gBAAgB;IA+E9B,iEAAiE;YACnD,wBAAwB;IA+CtC,gFAAgF;YAClE,mBAAmB;YAkDnB,yBAAyB;YA4CzB,iBAAiB;IA2C/B,OAAO,CAAC,iBAAiB;IAMzB,OAAO,CAAC,YAAY;YAQN,WAAW;YAqEX,2BAA2B;IA4GzC,+DAA+D;YACjD,iBAAiB;YA8EjB,yBAAyB;IA2BvC,OAAO,CAAC,4BAA4B;IAOpC,OAAO,CAAC,aAAa;YA0BP,eAAe;IAgG7B,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAaxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IA+D/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;IAYnC,MAAM,CAAC,yBAAyB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAiBlD;YAEa,iBAAiB;YA4BjB,eAAe;YASf,uBAAuB;YAQvB,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CAyFf;IAEK,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,mBAAmB,CAAC,CAC9E;;;;;;OA4BJ;IAEK,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKvC;IAEK,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC,CAkBD;IAEK,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,UAAU,CAAC,CACrE;;;;;OAGJ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAEK,UAAU,CACZ,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC,CA4C3B;IAEK,YAAY,CACd,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN,CAqBA;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC,CAS3B;CACJ"}
|
|
@@ -177,6 +177,7 @@ class ConcurrencyBudget {
|
|
|
177
177
|
constructor(limit) {
|
|
178
178
|
this.limit = limit;
|
|
179
179
|
this.active = 0;
|
|
180
|
+
this.pendingAcquires = [];
|
|
180
181
|
}
|
|
181
182
|
tryAcquire() {
|
|
182
183
|
if (this.active >= this.limit) {
|
|
@@ -185,7 +186,39 @@ class ConcurrencyBudget {
|
|
|
185
186
|
this.active += 1;
|
|
186
187
|
return true;
|
|
187
188
|
}
|
|
189
|
+
acquire(timeoutMs) {
|
|
190
|
+
if (this.tryAcquire()) {
|
|
191
|
+
return Promise.resolve(true);
|
|
192
|
+
}
|
|
193
|
+
return new Promise((resolve) => {
|
|
194
|
+
const pendingAcquire = {
|
|
195
|
+
resolve,
|
|
196
|
+
};
|
|
197
|
+
const timeout = setTimeout(() => {
|
|
198
|
+
const index = this.pendingAcquires.indexOf(pendingAcquire);
|
|
199
|
+
if (index === -1) {
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
this.pendingAcquires.splice(index, 1);
|
|
203
|
+
resolve(false);
|
|
204
|
+
}, timeoutMs);
|
|
205
|
+
timeout.unref();
|
|
206
|
+
pendingAcquire.timeout = timeout;
|
|
207
|
+
this.pendingAcquires.push(pendingAcquire);
|
|
208
|
+
});
|
|
209
|
+
}
|
|
188
210
|
release() {
|
|
211
|
+
if (this.active === 0) {
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
const pendingAcquire = this.pendingAcquires.shift();
|
|
215
|
+
if (pendingAcquire) {
|
|
216
|
+
if (pendingAcquire.timeout) {
|
|
217
|
+
clearTimeout(pendingAcquire.timeout);
|
|
218
|
+
}
|
|
219
|
+
pendingAcquire.resolve(true);
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
189
222
|
this.active -= 1;
|
|
190
223
|
}
|
|
191
224
|
isIdle() {
|
|
@@ -193,6 +226,13 @@ class ConcurrencyBudget {
|
|
|
193
226
|
}
|
|
194
227
|
reset() {
|
|
195
228
|
this.active = 0;
|
|
229
|
+
const pendingAcquires = this.pendingAcquires.splice(0);
|
|
230
|
+
pendingAcquires.forEach(({ resolve, timeout }) => {
|
|
231
|
+
if (timeout) {
|
|
232
|
+
clearTimeout(timeout);
|
|
233
|
+
}
|
|
234
|
+
resolve(false);
|
|
235
|
+
});
|
|
196
236
|
}
|
|
197
237
|
}
|
|
198
238
|
// DuckDB StatementType values — see duckdb/common/enums/statement_type.hpp
|
|
@@ -226,8 +266,17 @@ const EMBEDDED_RESOURCE_LIMITS = {
|
|
|
226
266
|
threads: 1,
|
|
227
267
|
};
|
|
228
268
|
const EMBEDDED_QUERY_TIMEOUT_MS = 10000;
|
|
229
|
-
const
|
|
230
|
-
const
|
|
269
|
+
const getPositiveIntegerEnvironmentVariable = (name, defaultValue) => {
|
|
270
|
+
const environmentValue = process.env[name]?.trim();
|
|
271
|
+
if (!environmentValue || !/^\d+$/.test(environmentValue)) {
|
|
272
|
+
return defaultValue;
|
|
273
|
+
}
|
|
274
|
+
const value = Number(environmentValue);
|
|
275
|
+
return Number.isSafeInteger(value) && value > 0 ? value : defaultValue;
|
|
276
|
+
};
|
|
277
|
+
const EMBEDDED_MAX_CONCURRENT_QUERIES = getPositiveIntegerEnvironmentVariable('PLAYGROUND_MAX_CONCURRENT_QUERIES', 8);
|
|
278
|
+
const EMBEDDED_MAX_CONCURRENT_QUERIES_PER_ORGANIZATION = getPositiveIntegerEnvironmentVariable('PLAYGROUND_MAX_CONCURRENT_QUERIES_PER_ORGANIZATION', 4);
|
|
279
|
+
const EMBEDDED_CONCURRENCY_ACQUIRE_TIMEOUT_MS = getPositiveIntegerEnvironmentVariable('PLAYGROUND_CONCURRENCY_ACQUIRE_TIMEOUT_MS', 15000);
|
|
231
280
|
const resolveEmbeddedDatabasePath = (dataset) => {
|
|
232
281
|
if (!EMBEDDED_DATASET_PATTERN.test(dataset)) {
|
|
233
282
|
throw new common_1.ParameterError('Embedded DuckDB dataset must contain only lowercase letters, numbers, hyphens, and underscores');
|
|
@@ -386,21 +435,36 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
386
435
|
}
|
|
387
436
|
return this.instanceCacheKey;
|
|
388
437
|
}
|
|
389
|
-
static tryAcquireEmbeddedConcurrency(organizationUuid) {
|
|
390
|
-
|
|
438
|
+
static async tryAcquireEmbeddedConcurrency(organizationUuid) {
|
|
439
|
+
const acquireDeadline = Date.now() + EMBEDDED_CONCURRENCY_ACQUIRE_TIMEOUT_MS;
|
|
440
|
+
const globalAcquired = await DuckdbWarehouseClient.embeddedConcurrencyBudget.acquire(EMBEDDED_CONCURRENCY_ACQUIRE_TIMEOUT_MS);
|
|
441
|
+
if (!globalAcquired) {
|
|
391
442
|
return undefined;
|
|
392
443
|
}
|
|
393
444
|
if (!organizationUuid) {
|
|
394
|
-
|
|
445
|
+
let released = false;
|
|
446
|
+
return () => {
|
|
447
|
+
if (released) {
|
|
448
|
+
return;
|
|
449
|
+
}
|
|
450
|
+
released = true;
|
|
451
|
+
DuckdbWarehouseClient.embeddedConcurrencyBudget.release();
|
|
452
|
+
};
|
|
395
453
|
}
|
|
396
454
|
const organizationBudget = DuckdbWarehouseClient.embeddedOrganizationConcurrencyBudgets.get(organizationUuid) ??
|
|
397
455
|
new ConcurrencyBudget(EMBEDDED_MAX_CONCURRENT_QUERIES_PER_ORGANIZATION);
|
|
398
456
|
DuckdbWarehouseClient.embeddedOrganizationConcurrencyBudgets.set(organizationUuid, organizationBudget);
|
|
399
|
-
|
|
457
|
+
const organizationAcquired = await organizationBudget.acquire(Math.max(0, acquireDeadline - Date.now()));
|
|
458
|
+
if (!organizationAcquired) {
|
|
400
459
|
DuckdbWarehouseClient.embeddedConcurrencyBudget.release();
|
|
401
460
|
return undefined;
|
|
402
461
|
}
|
|
462
|
+
let released = false;
|
|
403
463
|
return () => {
|
|
464
|
+
if (released) {
|
|
465
|
+
return;
|
|
466
|
+
}
|
|
467
|
+
released = true;
|
|
404
468
|
organizationBudget.release();
|
|
405
469
|
DuckdbWarehouseClient.embeddedConcurrencyBudget.release();
|
|
406
470
|
if (organizationBudget.isIdle()) {
|
|
@@ -625,6 +689,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
625
689
|
DuckdbWarehouseClient.sharedInstanceResourceLimits.clear();
|
|
626
690
|
DuckdbWarehouseClient.sharedInstanceSemaphores.clear();
|
|
627
691
|
DuckdbWarehouseClient.embeddedConcurrencyBudget.reset();
|
|
692
|
+
DuckdbWarehouseClient.embeddedOrganizationConcurrencyBudgets.forEach((budget) => budget.reset());
|
|
628
693
|
DuckdbWarehouseClient.embeddedOrganizationConcurrencyBudgets.clear();
|
|
629
694
|
}
|
|
630
695
|
async close() {
|
|
@@ -1127,7 +1192,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1127
1192
|
async withDirectSession(callback, organizationUuid, onPhaseTiming) {
|
|
1128
1193
|
const sessionStart = performance.now();
|
|
1129
1194
|
const releaseEmbeddedConcurrency = this.embeddedConfig
|
|
1130
|
-
? DuckdbWarehouseClient.tryAcquireEmbeddedConcurrency(organizationUuid)
|
|
1195
|
+
? await DuckdbWarehouseClient.tryAcquireEmbeddedConcurrency(organizationUuid)
|
|
1131
1196
|
: undefined;
|
|
1132
1197
|
if (this.embeddedConfig && !releaseEmbeddedConcurrency) {
|
|
1133
1198
|
throw new common_1.WarehouseQueryError('Playground query capacity is full. Try again shortly.');
|
|
@@ -200,6 +200,9 @@ describe('DuckdbWarehouseClient', () => {
|
|
|
200
200
|
maxEntries: 8,
|
|
201
201
|
});
|
|
202
202
|
});
|
|
203
|
+
afterEach(() => {
|
|
204
|
+
vi.useRealTimers();
|
|
205
|
+
});
|
|
203
206
|
it('should return query rows and mapped fields', async () => {
|
|
204
207
|
const rows = [
|
|
205
208
|
{
|
|
@@ -862,9 +865,9 @@ describe('DuckdbWarehouseClient', () => {
|
|
|
862
865
|
expect(runMock).toHaveBeenCalledWith("SET disabled_filesystems = 'LocalFileSystem';");
|
|
863
866
|
expect(runMock).not.toHaveBeenCalledWith("PRAGMA enable_profiling='json';");
|
|
864
867
|
});
|
|
865
|
-
it('
|
|
868
|
+
it('runs a queued embedded query once a same-organization holder releases', async () => {
|
|
866
869
|
const pendingStreams = [];
|
|
867
|
-
const streamMock = vi.fn(() => new Promise((resolve) => {
|
|
870
|
+
const streamMock = vi.fn((_sql) => new Promise((resolve) => {
|
|
868
871
|
pendingStreams.push(resolve);
|
|
869
872
|
}));
|
|
870
873
|
createInstanceMock.mockImplementation(async () => createMockConnection(streamMock));
|
|
@@ -872,32 +875,110 @@ describe('DuckdbWarehouseClient', () => {
|
|
|
872
875
|
type: common_1.WarehouseTypes.DUCKDB,
|
|
873
876
|
connectionType: common_1.DuckdbConnectionType.EMBEDDED,
|
|
874
877
|
dataset: 'jaffle_shop',
|
|
875
|
-
});
|
|
876
|
-
const
|
|
877
|
-
const
|
|
878
|
-
await vi.waitFor(() => expect(streamMock).toHaveBeenCalledTimes(
|
|
879
|
-
|
|
880
|
-
|
|
878
|
+
}, { embeddedQueryTimeoutMs: 60000 });
|
|
879
|
+
const tags = { organization_uuid: 'organization-1' };
|
|
880
|
+
const queries = Array.from({ length: 6 }, (_, index) => createClient().runQuery(`SELECT ${index + 1} AS val`, tags));
|
|
881
|
+
await vi.waitFor(() => expect(streamMock).toHaveBeenCalledTimes(4));
|
|
882
|
+
expect(createInstanceMock).toHaveBeenCalledTimes(4);
|
|
883
|
+
pendingStreams[0](getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
884
|
+
await queries[0];
|
|
885
|
+
await vi.waitFor(() => expect(streamMock).toHaveBeenCalledTimes(5));
|
|
886
|
+
expect(createInstanceMock).toHaveBeenCalledTimes(5);
|
|
887
|
+
expect(streamMock.mock.calls[4]?.[0]).toContain('SELECT 5 AS val');
|
|
888
|
+
pendingStreams[1](getMockStreamResult([[{ val: 2 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
889
|
+
await queries[1];
|
|
890
|
+
await vi.waitFor(() => expect(streamMock).toHaveBeenCalledTimes(6));
|
|
891
|
+
expect(createInstanceMock).toHaveBeenCalledTimes(6);
|
|
892
|
+
expect(streamMock.mock.calls[5]?.[0]).toContain('SELECT 6 AS val');
|
|
881
893
|
pendingStreams.forEach((resolve) => resolve(getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER])));
|
|
882
|
-
await Promise.all(
|
|
894
|
+
await Promise.all(queries.slice(1));
|
|
883
895
|
});
|
|
884
|
-
it('
|
|
885
|
-
|
|
896
|
+
it('times out a queued embedded query with the existing capacity error', async () => {
|
|
897
|
+
vi.useFakeTimers();
|
|
898
|
+
const pendingStreams = [];
|
|
886
899
|
const streamMock = vi.fn(() => new Promise((resolve) => {
|
|
887
|
-
|
|
900
|
+
pendingStreams.push(resolve);
|
|
888
901
|
}));
|
|
889
902
|
createInstanceMock.mockImplementation(async () => createMockConnection(streamMock));
|
|
903
|
+
const createClient = () => new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
904
|
+
type: common_1.WarehouseTypes.DUCKDB,
|
|
905
|
+
connectionType: common_1.DuckdbConnectionType.EMBEDDED,
|
|
906
|
+
dataset: 'jaffle_shop',
|
|
907
|
+
}, { embeddedQueryTimeoutMs: 60000 });
|
|
908
|
+
const tags = { organization_uuid: 'organization-1' };
|
|
909
|
+
const activeQueries = Array.from({ length: 4 }, (_, index) => createClient().runQuery(`SELECT ${index + 1} AS val`, tags));
|
|
910
|
+
await vi.waitFor(() => expect(streamMock).toHaveBeenCalledTimes(4));
|
|
911
|
+
const queuedQuery = createClient().runQuery('SELECT 5 AS val', tags);
|
|
912
|
+
const capacityError = queuedQuery.catch((error) => error);
|
|
913
|
+
await vi.advanceTimersByTimeAsync(15000);
|
|
914
|
+
const error = await capacityError;
|
|
915
|
+
expect(error).toBeInstanceOf(common_1.WarehouseQueryError);
|
|
916
|
+
expect(error).toMatchObject({
|
|
917
|
+
message: 'Playground query capacity is full. Try again shortly.',
|
|
918
|
+
});
|
|
919
|
+
expect(createInstanceMock).toHaveBeenCalledTimes(4);
|
|
920
|
+
pendingStreams.forEach((resolve) => resolve(getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER])));
|
|
921
|
+
await Promise.all(activeQueries);
|
|
922
|
+
});
|
|
923
|
+
it('returns embedded capacity when a query throws', async () => {
|
|
924
|
+
const pendingStreams = [];
|
|
925
|
+
const streamMock = vi.fn((sql) => {
|
|
926
|
+
if (sql.includes('SELECT 4')) {
|
|
927
|
+
return Promise.reject(new Error('Query failed'));
|
|
928
|
+
}
|
|
929
|
+
return new Promise((resolve) => {
|
|
930
|
+
pendingStreams.push(resolve);
|
|
931
|
+
});
|
|
932
|
+
});
|
|
933
|
+
createInstanceMock.mockImplementation(async () => createMockConnection(streamMock));
|
|
890
934
|
const createClient = () => new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
891
935
|
type: common_1.WarehouseTypes.DUCKDB,
|
|
892
936
|
connectionType: common_1.DuckdbConnectionType.EMBEDDED,
|
|
893
937
|
dataset: 'jaffle_shop',
|
|
894
938
|
});
|
|
895
939
|
const tags = { organization_uuid: 'organization-1' };
|
|
896
|
-
const
|
|
897
|
-
|
|
898
|
-
await expect(
|
|
899
|
-
|
|
900
|
-
await
|
|
940
|
+
const activeQueries = Array.from({ length: 3 }, (_, index) => createClient().runQuery(`SELECT ${index + 1} AS val`, tags));
|
|
941
|
+
const failedQuery = createClient().runQuery('SELECT 4 AS val', tags);
|
|
942
|
+
await expect(failedQuery).rejects.toThrow('Query failed');
|
|
943
|
+
const nextQuery = createClient().runQuery('SELECT 5 AS val', tags);
|
|
944
|
+
await vi.waitFor(() => expect(streamMock).toHaveBeenCalledTimes(5));
|
|
945
|
+
pendingStreams.forEach((resolve) => resolve(getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER])));
|
|
946
|
+
await Promise.all([...activeQueries, nextQuery]);
|
|
947
|
+
});
|
|
948
|
+
it('returns the global slot when organization capacity times out', async () => {
|
|
949
|
+
vi.useFakeTimers();
|
|
950
|
+
const pendingStreams = [];
|
|
951
|
+
const streamMock = vi.fn(() => new Promise((resolve) => {
|
|
952
|
+
pendingStreams.push(resolve);
|
|
953
|
+
}));
|
|
954
|
+
createInstanceMock.mockImplementation(async () => createMockConnection(streamMock));
|
|
955
|
+
const createClient = () => new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
956
|
+
type: common_1.WarehouseTypes.DUCKDB,
|
|
957
|
+
connectionType: common_1.DuckdbConnectionType.EMBEDDED,
|
|
958
|
+
dataset: 'jaffle_shop',
|
|
959
|
+
}, { embeddedQueryTimeoutMs: 60000 });
|
|
960
|
+
const organizationOneTags = {
|
|
961
|
+
organization_uuid: 'organization-1',
|
|
962
|
+
};
|
|
963
|
+
const organizationTwoTags = {
|
|
964
|
+
organization_uuid: 'organization-2',
|
|
965
|
+
};
|
|
966
|
+
const organizationOneQueries = Array.from({ length: 4 }, (_, index) => createClient().runQuery(`SELECT ${index + 1} AS val`, organizationOneTags));
|
|
967
|
+
await vi.waitFor(() => expect(streamMock).toHaveBeenCalledTimes(4));
|
|
968
|
+
const timedOutQuery = createClient().runQuery('SELECT 5 AS val', organizationOneTags);
|
|
969
|
+
const capacityError = timedOutQuery.catch((error) => error);
|
|
970
|
+
await vi.advanceTimersByTimeAsync(15000);
|
|
971
|
+
await expect(capacityError).resolves.toMatchObject({
|
|
972
|
+
name: 'WarehouseQueryError',
|
|
973
|
+
message: 'Playground query capacity is full. Try again shortly.',
|
|
974
|
+
});
|
|
975
|
+
const organizationTwoQueries = Array.from({ length: 4 }, (_, index) => createClient().runQuery(`SELECT ${index + 6} AS val`, organizationTwoTags));
|
|
976
|
+
await vi.waitFor(() => expect(streamMock).toHaveBeenCalledTimes(8));
|
|
977
|
+
pendingStreams.forEach((resolve) => resolve(getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER])));
|
|
978
|
+
await Promise.all([
|
|
979
|
+
...organizationOneQueries,
|
|
980
|
+
...organizationTwoQueries,
|
|
981
|
+
]);
|
|
901
982
|
});
|
|
902
983
|
it('logs structured profiles for non-embedded queries and reports metrics', async () => {
|
|
903
984
|
const runMock = vi.fn(async (sql) => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "2.71.
|
|
3
|
+
"version": "2.71.3",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
"snowflake-sdk": "2.3.4",
|
|
29
29
|
"ssh2": "1.14.0",
|
|
30
30
|
"trino-client": "0.2.9",
|
|
31
|
-
"@lightdash/common": "2.71.
|
|
31
|
+
"@lightdash/common": "2.71.3"
|
|
32
32
|
},
|
|
33
33
|
"devDependencies": {
|
|
34
34
|
"oxlint": "1.75.0",
|