@lightdash/warehouses 0.2592.4 → 0.2594.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -9,9 +9,14 @@ export type DuckdbS3SessionConfig = {
|
|
|
9
9
|
forcePathStyle: boolean;
|
|
10
10
|
useSsl: boolean;
|
|
11
11
|
};
|
|
12
|
+
export type DuckdbResourceLimits = {
|
|
13
|
+
memoryLimit: string;
|
|
14
|
+
threads: number;
|
|
15
|
+
};
|
|
12
16
|
export type DuckdbWarehouseClientArgs = {
|
|
13
17
|
databasePath?: string;
|
|
14
18
|
s3Config?: DuckdbS3SessionConfig;
|
|
19
|
+
resourceLimits?: DuckdbResourceLimits;
|
|
15
20
|
};
|
|
16
21
|
export declare const mapFieldTypeFromTypeId: (typeId: number) => DimensionType;
|
|
17
22
|
export declare class DuckdbSqlBuilder extends WarehouseBaseSqlBuilder {
|
|
@@ -23,6 +28,7 @@ export declare class DuckdbSqlBuilder extends WarehouseBaseSqlBuilder {
|
|
|
23
28
|
export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreatePostgresCredentials> {
|
|
24
29
|
private readonly databasePath;
|
|
25
30
|
private readonly s3Config?;
|
|
31
|
+
private readonly resourceLimits?;
|
|
26
32
|
constructor(args?: DuckdbWarehouseClientArgs);
|
|
27
33
|
private getSQLWithMetadata;
|
|
28
34
|
private withSession;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,aAAa,EACb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,aAAa,EACb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA2BhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,cAAc,CAAC,EAAE,oBAAoB,CAAC;CACzC,CAAC;AAYF,eAAO,MAAM,sBAAsB,GAAI,QAAQ,MAAM,KAAG,aA+BvD,CAAC;AAEF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB;IAIrC,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C;AAED,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,yBAAyB,CAAC;IACrF,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;gBAE3C,IAAI,GAAE,yBAA8B;IAOhD,OAAO,CAAC,kBAAkB;YAQZ,WAAW;YA4BX,gBAAgB;IAqD9B,OAAO,CAAC,aAAa;IA0BrB,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAalC,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC;IAsBV,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,yBAAyB,CAAC,CAAC,mBAAmB,CAAC,CACtE;;;;;;IA8BC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMlC,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,yBAAyB,CAAC,CAAC,UAAU,CAAC,CAC7D;;;;;;IAKC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrB,UAAU,CACZ,OAAO,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC/D,OAAO,CAAC,gBAAgB,CAAC;IAMtB,YAAY,CACd,OAAO,CAAC,EAAE,MAAM,EAChB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN;IAMK,SAAS,CACX,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,MAAM,EAChB,SAAS,CAAC,EAAE,MAAM,EAClB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC;CAK/B"}
|
|
@@ -6,6 +6,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
6
|
exports.DuckdbWarehouseClient = exports.DuckdbSqlBuilder = exports.mapFieldTypeFromTypeId = void 0;
|
|
7
7
|
const node_api_1 = require("@duckdb/node-api");
|
|
8
8
|
const common_1 = require("@lightdash/common");
|
|
9
|
+
const promises_1 = __importDefault(require("fs/promises"));
|
|
10
|
+
const os_1 = __importDefault(require("os"));
|
|
11
|
+
const path_1 = __importDefault(require("path"));
|
|
9
12
|
const WarehouseBaseClient_1 = __importDefault(require("./WarehouseBaseClient"));
|
|
10
13
|
const WarehouseBaseSqlBuilder_1 = __importDefault(require("./WarehouseBaseSqlBuilder"));
|
|
11
14
|
const DUCKDB_INTERNAL_CREDENTIALS = {
|
|
@@ -77,6 +80,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
77
80
|
super(DUCKDB_INTERNAL_CREDENTIALS, new DuckdbSqlBuilder());
|
|
78
81
|
this.databasePath = args.databasePath ?? ':memory:';
|
|
79
82
|
this.s3Config = args.s3Config;
|
|
83
|
+
this.resourceLimits = args.resourceLimits;
|
|
80
84
|
}
|
|
81
85
|
getSQLWithMetadata(sql, tags) {
|
|
82
86
|
if (!tags) {
|
|
@@ -87,19 +91,31 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
87
91
|
async withSession(callback) {
|
|
88
92
|
const instance = (await node_api_1.DuckDBInstance.create(this.databasePath));
|
|
89
93
|
const connection = await instance.connect();
|
|
94
|
+
// Only create a temp dir when resource limits are set (spill to disk).
|
|
95
|
+
const tempDir = this.resourceLimits
|
|
96
|
+
? await promises_1.default.mkdtemp(path_1.default.join(os_1.default.tmpdir(), 'duckdb-temp-'))
|
|
97
|
+
: undefined;
|
|
90
98
|
try {
|
|
91
|
-
await this.bootstrapSession(connection);
|
|
99
|
+
await this.bootstrapSession(connection, tempDir);
|
|
92
100
|
return await callback(connection);
|
|
93
101
|
}
|
|
94
102
|
finally {
|
|
95
103
|
connection.closeSync?.();
|
|
96
104
|
connection.disconnectSync?.();
|
|
97
105
|
instance.closeSync?.();
|
|
106
|
+
if (tempDir) {
|
|
107
|
+
await promises_1.default.rm(tempDir, { recursive: true, force: true }).catch(() => { });
|
|
108
|
+
}
|
|
98
109
|
}
|
|
99
110
|
}
|
|
100
|
-
async bootstrapSession(db) {
|
|
111
|
+
async bootstrapSession(db, tempDir) {
|
|
101
112
|
await db.run('INSTALL httpfs;');
|
|
102
113
|
await db.run('LOAD httpfs;');
|
|
114
|
+
if (this.resourceLimits && tempDir) {
|
|
115
|
+
await db.run(`SET memory_limit = '${this.resourceLimits.memoryLimit}';`);
|
|
116
|
+
await db.run(`SET temp_directory = '${tempDir}';`);
|
|
117
|
+
await db.run(`SET threads = ${this.resourceLimits.threads};`);
|
|
118
|
+
}
|
|
103
119
|
if (!this.s3Config) {
|
|
104
120
|
return;
|
|
105
121
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2594.0",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
"snowflake-sdk": "~2.3.4",
|
|
27
27
|
"ssh2": "^1.14.0",
|
|
28
28
|
"trino-client": "0.2.9",
|
|
29
|
-
"@lightdash/common": "0.
|
|
29
|
+
"@lightdash/common": "0.2594.0"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
32
|
"@types/node-fetch": "^2.6.13",
|