@lightdash/warehouses 0.2593.0 → 0.2595.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -13,10 +13,14 @@ export type DuckdbResourceLimits = {
|
|
|
13
13
|
memoryLimit: string;
|
|
14
14
|
threads: number;
|
|
15
15
|
};
|
|
16
|
+
export type DuckdbLogger = {
|
|
17
|
+
info: (message: string, metadata?: Record<string, unknown>) => void;
|
|
18
|
+
};
|
|
16
19
|
export type DuckdbWarehouseClientArgs = {
|
|
17
20
|
databasePath?: string;
|
|
18
21
|
s3Config?: DuckdbS3SessionConfig;
|
|
19
22
|
resourceLimits?: DuckdbResourceLimits;
|
|
23
|
+
logger?: DuckdbLogger;
|
|
20
24
|
};
|
|
21
25
|
export declare const mapFieldTypeFromTypeId: (typeId: number) => DimensionType;
|
|
22
26
|
export declare class DuckdbSqlBuilder extends WarehouseBaseSqlBuilder {
|
|
@@ -29,6 +33,7 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreatePos
|
|
|
29
33
|
private readonly databasePath;
|
|
30
34
|
private readonly s3Config?;
|
|
31
35
|
private readonly resourceLimits?;
|
|
36
|
+
private readonly logger?;
|
|
32
37
|
constructor(args?: DuckdbWarehouseClientArgs);
|
|
33
38
|
private getSQLWithMetadata;
|
|
34
39
|
private withSession;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,aAAa,EACb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA2BhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,cAAc,CAAC,EAAE,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,aAAa,EACb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA2BhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,MAAM,CAAC,EAAE,YAAY,CAAC;CACzB,CAAC;AAYF,eAAO,MAAM,sBAAsB,GAAI,QAAQ,MAAM,KAAG,aA+BvD,CAAC;AAEF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB;IAIrC,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C;AAED,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,yBAAyB,CAAC;IACrF,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;gBAE3B,IAAI,GAAE,yBAA8B;IAQhD,OAAO,CAAC,kBAAkB;YAQZ,WAAW;YA8CX,gBAAgB;IAsE9B,OAAO,CAAC,aAAa;IA0BrB,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAalC,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC;IAsBV,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,yBAAyB,CAAC,CAAC,mBAAmB,CAAC,CACtE;;;;;;IA8BC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMlC,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,yBAAyB,CAAC,CAAC,UAAU,CAAC,CAC7D;;;;;;IAKC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrB,UAAU,CACZ,OAAO,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC/D,OAAO,CAAC,gBAAgB,CAAC;IAMtB,YAAY,CACd,OAAO,CAAC,EAAE,MAAM,EAChB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN;IAMK,SAAS,CACX,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,MAAM,EAChB,SAAS,CAAC,EAAE,MAAM,EAClB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC;CAK/B"}
|
|
@@ -81,6 +81,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
81
81
|
this.databasePath = args.databasePath ?? ':memory:';
|
|
82
82
|
this.s3Config = args.s3Config;
|
|
83
83
|
this.resourceLimits = args.resourceLimits;
|
|
84
|
+
this.logger = args.logger;
|
|
84
85
|
}
|
|
85
86
|
getSQLWithMetadata(sql, tags) {
|
|
86
87
|
if (!tags) {
|
|
@@ -89,15 +90,29 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
89
90
|
return `${sql}\n-- ${JSON.stringify(tags)}`;
|
|
90
91
|
}
|
|
91
92
|
async withSession(callback) {
|
|
93
|
+
const sessionStart = performance.now();
|
|
92
94
|
const instance = (await node_api_1.DuckDBInstance.create(this.databasePath));
|
|
93
95
|
const connection = await instance.connect();
|
|
96
|
+
const connectMs = performance.now() - sessionStart;
|
|
94
97
|
// Only create a temp dir when resource limits are set (spill to disk).
|
|
95
98
|
const tempDir = this.resourceLimits
|
|
96
99
|
? await promises_1.default.mkdtemp(path_1.default.join(os_1.default.tmpdir(), 'duckdb-temp-'))
|
|
97
100
|
: undefined;
|
|
98
101
|
try {
|
|
102
|
+
const bootstrapStart = performance.now();
|
|
99
103
|
await this.bootstrapSession(connection, tempDir);
|
|
100
|
-
|
|
104
|
+
const bootstrapMs = performance.now() - bootstrapStart;
|
|
105
|
+
const queryStart = performance.now();
|
|
106
|
+
const result = await callback(connection);
|
|
107
|
+
const queryMs = performance.now() - queryStart;
|
|
108
|
+
const totalMs = performance.now() - sessionStart;
|
|
109
|
+
this.logger?.info('DuckDB session timing', {
|
|
110
|
+
connectMs: Math.round(connectMs),
|
|
111
|
+
bootstrapMs: Math.round(bootstrapMs),
|
|
112
|
+
queryMs: Math.round(queryMs),
|
|
113
|
+
totalMs: Math.round(totalMs),
|
|
114
|
+
});
|
|
115
|
+
return result;
|
|
101
116
|
}
|
|
102
117
|
finally {
|
|
103
118
|
connection.closeSync?.();
|
|
@@ -109,16 +124,25 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
109
124
|
}
|
|
110
125
|
}
|
|
111
126
|
async bootstrapSession(db, tempDir) {
|
|
127
|
+
const t0 = performance.now();
|
|
112
128
|
await db.run('INSTALL httpfs;');
|
|
129
|
+
const installMs = performance.now() - t0;
|
|
130
|
+
const t1 = performance.now();
|
|
113
131
|
await db.run('LOAD httpfs;');
|
|
132
|
+
const loadMs = performance.now() - t1;
|
|
114
133
|
if (this.resourceLimits && tempDir) {
|
|
115
134
|
await db.run(`SET memory_limit = '${this.resourceLimits.memoryLimit}';`);
|
|
116
135
|
await db.run(`SET temp_directory = '${tempDir}';`);
|
|
117
136
|
await db.run(`SET threads = ${this.resourceLimits.threads};`);
|
|
118
137
|
}
|
|
119
138
|
if (!this.s3Config) {
|
|
139
|
+
this.logger?.info('DuckDB bootstrap timing', {
|
|
140
|
+
installHttpfsMs: Math.round(installMs),
|
|
141
|
+
loadHttpfsMs: Math.round(loadMs),
|
|
142
|
+
});
|
|
120
143
|
return;
|
|
121
144
|
}
|
|
145
|
+
const t2 = performance.now();
|
|
122
146
|
await db.run(`SET s3_endpoint = '${this.escapeString(this.s3Config.endpoint)}';`);
|
|
123
147
|
if (this.s3Config.region) {
|
|
124
148
|
await db.run(`SET s3_region = '${this.escapeString(this.s3Config.region)}';`);
|
|
@@ -131,6 +155,12 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
131
155
|
}
|
|
132
156
|
await db.run(`SET s3_use_ssl = ${this.s3Config.useSsl};`);
|
|
133
157
|
await db.run(`SET s3_url_style = '${this.s3Config.forcePathStyle ? 'path' : 'vhost'}';`);
|
|
158
|
+
const s3ConfigMs = performance.now() - t2;
|
|
159
|
+
this.logger?.info('DuckDB bootstrap timing', {
|
|
160
|
+
installHttpfsMs: Math.round(installMs),
|
|
161
|
+
loadHttpfsMs: Math.round(loadMs),
|
|
162
|
+
s3ConfigMs: Math.round(s3ConfigMs),
|
|
163
|
+
});
|
|
134
164
|
}
|
|
135
165
|
getBindValues(options) {
|
|
136
166
|
const hasValues = !!options?.values && options.values.length > 0;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2595.0",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
"snowflake-sdk": "~2.3.4",
|
|
27
27
|
"ssh2": "^1.14.0",
|
|
28
28
|
"trino-client": "0.2.9",
|
|
29
|
-
"@lightdash/common": "0.
|
|
29
|
+
"@lightdash/common": "0.2595.0"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
32
|
"@types/node-fetch": "^2.6.13",
|