@lightdash/warehouses 0.3019.2 → 0.3021.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts +16 -4
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.js +274 -24
- package/dist/warehouseClients/DuckdbWarehouseClient.test.js +138 -0
- package/package.json +3 -3
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AnyType, CreateDuckdbCredentials, DimensionType, Metric, SupportedDbtAdapter, WarehouseCatalog, WarehouseResults } from '@lightdash/common';
|
|
1
|
+
import { AnyType, CreateDuckdbCredentials, CreateDuckdbMotherduckCredentials, DimensionType, Metric, SupportedDbtAdapter, WarehouseCatalog, WarehouseResults } from '@lightdash/common';
|
|
2
2
|
import WarehouseBaseClient from './WarehouseBaseClient';
|
|
3
3
|
import WarehouseBaseSqlBuilder from './WarehouseBaseSqlBuilder';
|
|
4
4
|
export type DuckdbS3SessionConfig = {
|
|
@@ -58,18 +58,20 @@ export type DuckdbWarehouseClientArgs = {
|
|
|
58
58
|
databasePath?: string;
|
|
59
59
|
s3Config?: DuckdbS3SessionConfig;
|
|
60
60
|
};
|
|
61
|
-
export declare class DuckdbWarehouseClient extends WarehouseBaseClient<
|
|
61
|
+
export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuckdbMotherduckCredentials> {
|
|
62
62
|
private static readonly sharedInstances;
|
|
63
63
|
private static readonly sharedInstanceSemaphores;
|
|
64
64
|
private static readonly sqlBuilder;
|
|
65
65
|
private readonly databasePath;
|
|
66
66
|
private readonly s3Config?;
|
|
67
|
+
private readonly ducklakeConfig?;
|
|
67
68
|
private readonly resourceLimits?;
|
|
68
69
|
private readonly sharedResourceLimits?;
|
|
69
70
|
private readonly instanceCacheKey?;
|
|
70
71
|
private readonly logger?;
|
|
71
72
|
private readonly onQueryProfile?;
|
|
72
73
|
constructor(credentials?: CreateDuckdbCredentials | DuckdbConnectionCredentials, options?: DuckdbWarehouseClientOptions);
|
|
74
|
+
private static hashDucklakeConfig;
|
|
73
75
|
static createForPreAggregate(credentials?: DuckdbConnectionCredentials, options?: DuckdbWarehouseClientOptions): DuckdbWarehouseClient;
|
|
74
76
|
private static getSharedInstanceSemaphore;
|
|
75
77
|
private getRequiredInstanceCacheKey;
|
|
@@ -84,6 +86,16 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
84
86
|
/** Reset shared state without closing — for use in tests with mocked instances. */
|
|
85
87
|
static resetSharedDuckdbStateForTesting(): void;
|
|
86
88
|
close(): Promise<void>;
|
|
89
|
+
private static readonly DUCKLAKE_CATALOG_SECRET;
|
|
90
|
+
private static readonly DUCKLAKE_DATA_SECRET;
|
|
91
|
+
private static readonly DUCKLAKE_SECRET;
|
|
92
|
+
private static escapeDuckdbString;
|
|
93
|
+
private static quoteIdent;
|
|
94
|
+
private static buildDucklakeCatalogSecretSql;
|
|
95
|
+
private static buildDucklakeDataSecretSql;
|
|
96
|
+
private static catalogUsesSecret;
|
|
97
|
+
private static buildDucklakeSecretSql;
|
|
98
|
+
private static buildDucklakeAttachSql;
|
|
87
99
|
private static buildS3SecretSql;
|
|
88
100
|
private static readonly CONNECT_RETRIES_BEFORE_RECREATE;
|
|
89
101
|
private connectWithRetry;
|
|
@@ -117,7 +129,7 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
117
129
|
tags?: Record<string, string>;
|
|
118
130
|
timezone?: string;
|
|
119
131
|
}): Promise<void>;
|
|
120
|
-
executeAsyncQuery(...args: Parameters<WarehouseBaseClient<
|
|
132
|
+
executeAsyncQuery(...args: Parameters<WarehouseBaseClient<CreateDuckdbMotherduckCredentials>['executeAsyncQuery']>): Promise<{
|
|
121
133
|
durationMs: number;
|
|
122
134
|
queryId: string | null;
|
|
123
135
|
queryMetadata: import("@lightdash/common").WarehouseQueryMetadata | null;
|
|
@@ -129,7 +141,7 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
129
141
|
queryMs: number;
|
|
130
142
|
totalMs: number;
|
|
131
143
|
}>;
|
|
132
|
-
runQuery(...args: Parameters<WarehouseBaseClient<
|
|
144
|
+
runQuery(...args: Parameters<WarehouseBaseClient<CreateDuckdbMotherduckCredentials>['runQuery']>): Promise<{
|
|
133
145
|
fields: Record<string, {
|
|
134
146
|
type: DimensionType;
|
|
135
147
|
}>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,uBAAuB,
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,uBAAuB,EAEvB,iCAAiC,EACjC,aAAa,EAMb,MAAM,EAIN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAK3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA4DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG,mBAAmB,CAAC;AAG9D,MAAM,MAAM,wBAAwB,GAAG,mBAAmB,CAAC;AAE3D,MAAM,MAAM,4BAA4B,GAAG;IACvC,2FAA2F;IAC3F,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;CACjE,CAAC;AAEF,eAAO,MAAM,sBAAsB,GAAI,QAAQ,MAAM,KAAG,aA+BvD,CAAC;AAoCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB;IAIrC,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C;AA+CD,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CACpC,CAAC;AAEF,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,iCAAiC,CAAC;IAC7F,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;gBAGN,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B;IAqF1C,OAAO,CAAC,MAAM,CAAC,kBAAkB;IASjC,MAAM,CAAC,qBAAqB,CACxB,WAAW,CAAC,EAAE,2BAA2B,EACzC,OAAO,CAAC,EAAE,4BAA4B,GACvC,qBAAqB;IAIxB,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAoBzC,OAAO,CAAC,2BAA2B;IAUnC,OAAO,CAAC,kBAAkB;mBAQL,cAAc;IAWnC,OAAO,CAAC,MAAM,CAAC,qBAAqB;mBAMf,kCAAkC;mBAUlC,qBAAqB;mBAqFrB,uBAAuB;mBAgBvB,yBAAyB;IA6E9C,OAAO,CAAC,MAAM,CAAC,mBAAmB;IAoBlC,mFAAmF;IACnF,MAAM,CAAC,gCAAgC,IAAI,IAAI;IAKzC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAS5B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CACZ;IAEnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAA+B;IAE3E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAA0B;IAEjE,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAIjC,OAAO,CAAC,MAAM,CAAC,UAAU;IAIzB,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAuB5C,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAqGzC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAMhC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAmBrC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IA2CrC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAqC/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAK;YAE9C,gBAAgB;IA+E9B,iEAAiE;YACnD,wBAAwB;IAqDtC,gFAAgF;YAClE,mBAAmB;YAkDnB,yBAAyB;YA4CzB,iBAAiB;IAkC/B,OAAO,CAAC,iBAAiB;IAMzB;;;;;;OAMG;YACW,WAAW;IAkBzB,+DAA+D;YACjD,iBAAiB;IA0C/B,OAAO,CAAC,aAAa;YA0BP,eAAe;IAgG7B,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAaxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAM/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;YAUrB,eAAe;YA8Bf,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC;IA4CV,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,mBAAmB,CAAC,CAC9E;;;;;;IA8BC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOlC,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC;IAoBI,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,UAAU,CAAC,CACrE;;;;;;IAKC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrB,UAAU,CACZ,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC;IAuCtB,YAAY,CACd,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN;IAuBK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC;CAU/B"}
|
|
@@ -6,6 +6,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
6
|
exports.DuckdbWarehouseClient = exports.DuckdbSqlBuilder = exports.mapFieldTypeFromTypeId = void 0;
|
|
7
7
|
const node_api_1 = require("@duckdb/node-api");
|
|
8
8
|
const common_1 = require("@lightdash/common");
|
|
9
|
+
const crypto_1 = require("crypto");
|
|
9
10
|
const promises_1 = __importDefault(require("fs/promises"));
|
|
10
11
|
const os_1 = __importDefault(require("os"));
|
|
11
12
|
const path_1 = __importDefault(require("path"));
|
|
@@ -70,6 +71,7 @@ const mapFieldTypeFromString = (typeName) => {
|
|
|
70
71
|
};
|
|
71
72
|
const DUCKDB_INTERNAL_CREDENTIALS = {
|
|
72
73
|
type: common_1.WarehouseTypes.DUCKDB,
|
|
74
|
+
connectionType: common_1.DuckdbConnectionType.MOTHERDUCK,
|
|
73
75
|
database: ':memory:',
|
|
74
76
|
schema: 'main',
|
|
75
77
|
token: '',
|
|
@@ -138,22 +140,52 @@ const BLOCKED_STATEMENT_TYPES_INTERNAL_SQL = new Set([
|
|
|
138
140
|
const BLOCKED_FUNCTION_PATTERN = /\b(current_setting|duckdb_settings|duckdb_secrets)\s*\(/i;
|
|
139
141
|
class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
140
142
|
constructor(credentials, options) {
|
|
141
|
-
const
|
|
143
|
+
const isS3Only = credentials &&
|
|
142
144
|
'type' in credentials &&
|
|
143
|
-
credentials.type === 'duckdb_s3'
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
DUCKDB_INTERNAL_CREDENTIALS);
|
|
147
|
-
super(effectiveCredentials, new DuckdbSqlBuilder(effectiveCredentials.startOfWeek));
|
|
148
|
-
// Determine s3Config from either the old DuckdbConnectionCredentials or options
|
|
149
|
-
if (credentials &&
|
|
145
|
+
credentials.type === 'duckdb_s3';
|
|
146
|
+
const isDucklake = !isS3Only &&
|
|
147
|
+
credentials &&
|
|
150
148
|
'type' in credentials &&
|
|
151
|
-
credentials.type ===
|
|
149
|
+
credentials.type === common_1.WarehouseTypes.DUCKDB &&
|
|
150
|
+
credentials.connectionType === common_1.DuckdbConnectionType.DUCKLAKE;
|
|
151
|
+
let effectiveCredentials;
|
|
152
|
+
if (isS3Only) {
|
|
153
|
+
effectiveCredentials = DUCKDB_INTERNAL_CREDENTIALS;
|
|
154
|
+
}
|
|
155
|
+
else if (isDucklake) {
|
|
156
|
+
const ducklake = credentials;
|
|
157
|
+
// DuckLake is attached on top of an in-memory base instance.
|
|
158
|
+
// Surface the catalog alias as the DuckDB database name so the
|
|
159
|
+
// existing information_schema queries line up.
|
|
160
|
+
effectiveCredentials = {
|
|
161
|
+
type: common_1.WarehouseTypes.DUCKDB,
|
|
162
|
+
connectionType: common_1.DuckdbConnectionType.MOTHERDUCK,
|
|
163
|
+
database: ducklake.catalogAlias ?? 'ducklake',
|
|
164
|
+
schema: ducklake.schema,
|
|
165
|
+
token: '',
|
|
166
|
+
threads: ducklake.threads,
|
|
167
|
+
requireUserCredentials: ducklake.requireUserCredentials,
|
|
168
|
+
startOfWeek: ducklake.startOfWeek,
|
|
169
|
+
dataTimezone: ducklake.dataTimezone,
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
else {
|
|
173
|
+
effectiveCredentials =
|
|
174
|
+
credentials ??
|
|
175
|
+
DUCKDB_INTERNAL_CREDENTIALS;
|
|
176
|
+
}
|
|
177
|
+
super(effectiveCredentials, new DuckdbSqlBuilder(effectiveCredentials.startOfWeek));
|
|
178
|
+
if (isS3Only) {
|
|
152
179
|
this.s3Config = credentials.s3Config;
|
|
153
180
|
}
|
|
181
|
+
if (isDucklake) {
|
|
182
|
+
this.ducklakeConfig =
|
|
183
|
+
credentials;
|
|
184
|
+
}
|
|
154
185
|
// Project DuckDB credentials map to MotherDuck only. The in-memory
|
|
155
186
|
// internal credentials remain available for pre-aggregate helper flows.
|
|
156
|
-
if (
|
|
187
|
+
if (this.ducklakeConfig ||
|
|
188
|
+
effectiveCredentials.database === ':memory:') {
|
|
157
189
|
this.databasePath = ':memory:';
|
|
158
190
|
}
|
|
159
191
|
else {
|
|
@@ -165,10 +197,25 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
165
197
|
}
|
|
166
198
|
this.resourceLimits = options?.resourceLimits;
|
|
167
199
|
this.sharedResourceLimits = options?.sharedResourceLimits;
|
|
168
|
-
|
|
200
|
+
// DuckLake attaches a postgres catalog secret on every fresh DuckDB
|
|
201
|
+
// instance, and the postgres extension only pools 8 connections per
|
|
202
|
+
// instance — so parallel getFields() calls during project compile
|
|
203
|
+
// exhaust the pool. Default DuckLake clients to a shared warm instance
|
|
204
|
+
// keyed on the credentials so the attach runs once.
|
|
205
|
+
const ducklakeAutoCacheKey = this.ducklakeConfig
|
|
206
|
+
? `ducklake:${DuckdbWarehouseClient.hashDucklakeConfig(this.ducklakeConfig)}`
|
|
207
|
+
: undefined;
|
|
208
|
+
this.instanceCacheKey =
|
|
209
|
+
options?.instanceCacheKey ?? ducklakeAutoCacheKey;
|
|
169
210
|
this.logger = options?.logger;
|
|
170
211
|
this.onQueryProfile = options?.onQueryProfile;
|
|
171
212
|
}
|
|
213
|
+
static hashDucklakeConfig(ducklake) {
|
|
214
|
+
return (0, crypto_1.createHash)('sha256')
|
|
215
|
+
.update(JSON.stringify(ducklake))
|
|
216
|
+
.digest('hex')
|
|
217
|
+
.slice(0, 16);
|
|
218
|
+
}
|
|
172
219
|
static createForPreAggregate(credentials, options) {
|
|
173
220
|
return new DuckdbWarehouseClient(credentials, options);
|
|
174
221
|
}
|
|
@@ -193,10 +240,11 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
193
240
|
}
|
|
194
241
|
return `${sql}\n-- ${JSON.stringify(tags)}`;
|
|
195
242
|
}
|
|
196
|
-
static async hardenInstance(db) {
|
|
243
|
+
static async hardenInstance(db, options) {
|
|
197
244
|
await db.run('SET allow_community_extensions = false;');
|
|
198
|
-
|
|
199
|
-
await db.run(
|
|
245
|
+
const autoload = options?.allowKnownExtensionAutoload ?? false;
|
|
246
|
+
await db.run(`SET autoinstall_known_extensions = ${autoload};`);
|
|
247
|
+
await db.run(`SET autoload_known_extensions = ${autoload};`);
|
|
200
248
|
await db.run('SET allow_unredacted_secrets = false;');
|
|
201
249
|
}
|
|
202
250
|
static usesS3CredentialChain(s3Config) {
|
|
@@ -211,14 +259,27 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
211
259
|
static async bootstrapQuerySession(db, client) {
|
|
212
260
|
const bootstrapStart = performance.now();
|
|
213
261
|
const httpfsStart = performance.now();
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
262
|
+
if (!client.ducklakeConfig) {
|
|
263
|
+
// For DuckLake mode, httpfs and the ducklake/postgres/mysql/azure
|
|
264
|
+
// extensions are autoloaded by ATTACH — no explicit INSTALL/LOAD.
|
|
265
|
+
await db.run('INSTALL httpfs;');
|
|
266
|
+
await db.run('LOAD httpfs;');
|
|
267
|
+
await DuckdbWarehouseClient.loadAwsExtensionForCredentialChain(db, client.s3Config);
|
|
268
|
+
}
|
|
217
269
|
const httpfsMs = performance.now() - httpfsStart;
|
|
218
270
|
await db.run('SET enable_http_metadata_cache = true;');
|
|
219
271
|
await db.run('SET enable_external_file_cache = true;');
|
|
220
272
|
await db.run('SET parquet_metadata_cache = true;');
|
|
221
|
-
|
|
273
|
+
if (client.ducklakeConfig) {
|
|
274
|
+
// Parallel getFields() calls during compile all funnel through the
|
|
275
|
+
// attached postgres catalog. The duckdb-postgres extension caps
|
|
276
|
+
// the per-attach pool at 8 by default, which the compile easily
|
|
277
|
+
// exhausts on projects with >8 models.
|
|
278
|
+
await db.run('SET pg_connection_limit = 64;');
|
|
279
|
+
}
|
|
280
|
+
await DuckdbWarehouseClient.hardenInstance(db, {
|
|
281
|
+
allowKnownExtensionAutoload: !!client.ducklakeConfig,
|
|
282
|
+
});
|
|
222
283
|
if (client.sharedResourceLimits?.memoryLimit) {
|
|
223
284
|
await db.run(`SET memory_limit = '${client.sharedResourceLimits.memoryLimit}';`);
|
|
224
285
|
}
|
|
@@ -228,8 +289,16 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
228
289
|
if (client.s3Config) {
|
|
229
290
|
await db.run(DuckdbWarehouseClient.buildS3SecretSql(client.s3Config));
|
|
230
291
|
}
|
|
292
|
+
if (client.ducklakeConfig) {
|
|
293
|
+
const stmts = DuckdbWarehouseClient.buildDucklakeAttachSql(client.ducklakeConfig);
|
|
294
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
295
|
+
for (const stmt of stmts) {
|
|
296
|
+
// eslint-disable-next-line no-await-in-loop
|
|
297
|
+
await db.run(stmt);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
231
300
|
const bootstrapMs = performance.now() - bootstrapStart;
|
|
232
|
-
client.logger?.info(`DuckDB query bootstrap complete: cacheKey=${client.instanceCacheKey ?? 'none'} bootstrap=${(0, common_1.formatMilliseconds)(bootstrapMs)}ms httpfs=${(0, common_1.formatMilliseconds)(httpfsMs)}ms memory_limit=${client.sharedResourceLimits?.memoryLimit ?? 'default'} threads=${client.sharedResourceLimits?.threads ?? 'default'} s3=${client.s3Config ? 'configured' : 'none'} shared=${client.instanceCacheKey ? 'true' : 'false'}`, {
|
|
301
|
+
client.logger?.info(`DuckDB query bootstrap complete: cacheKey=${client.instanceCacheKey ?? 'none'} bootstrap=${(0, common_1.formatMilliseconds)(bootstrapMs)}ms httpfs=${(0, common_1.formatMilliseconds)(httpfsMs)}ms memory_limit=${client.sharedResourceLimits?.memoryLimit ?? 'default'} threads=${client.sharedResourceLimits?.threads ?? 'default'} s3=${client.s3Config ? 'configured' : 'none'} ducklake=${client.ducklakeConfig ? 'configured' : 'none'} shared=${client.instanceCacheKey ? 'true' : 'false'}`, {
|
|
233
302
|
shared: !!client.instanceCacheKey,
|
|
234
303
|
instanceCacheKey: client.instanceCacheKey,
|
|
235
304
|
bootstrapMs,
|
|
@@ -237,6 +306,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
237
306
|
memoryLimit: client.sharedResourceLimits?.memoryLimit ?? 'default',
|
|
238
307
|
threads: client.sharedResourceLimits?.threads ?? 'default',
|
|
239
308
|
s3Configured: !!client.s3Config,
|
|
309
|
+
ducklakeConfigured: !!client.ducklakeConfig,
|
|
240
310
|
});
|
|
241
311
|
return {
|
|
242
312
|
bootstrapMs,
|
|
@@ -335,6 +405,168 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
335
405
|
DuckdbWarehouseClient.clearSharedInstance(this.instanceCacheKey, this.logger);
|
|
336
406
|
}
|
|
337
407
|
}
|
|
408
|
+
static escapeDuckdbString(v) {
|
|
409
|
+
return DuckdbWarehouseClient.sqlBuilder.escapeString(v);
|
|
410
|
+
}
|
|
411
|
+
static quoteIdent(name) {
|
|
412
|
+
return `"${name.replace(/"/g, '""')}"`;
|
|
413
|
+
}
|
|
414
|
+
static buildDucklakeCatalogSecretSql(ducklake) {
|
|
415
|
+
const e = DuckdbWarehouseClient.escapeDuckdbString;
|
|
416
|
+
const { catalog } = ducklake;
|
|
417
|
+
switch (catalog.type) {
|
|
418
|
+
case common_1.DucklakeCatalogType.POSTGRES:
|
|
419
|
+
return `CREATE OR REPLACE SECRET ${DuckdbWarehouseClient.DUCKLAKE_CATALOG_SECRET} (
|
|
420
|
+
TYPE postgres,
|
|
421
|
+
HOST '${e(catalog.host)}',
|
|
422
|
+
PORT ${catalog.port},
|
|
423
|
+
DATABASE '${e(catalog.database)}',
|
|
424
|
+
USER '${e(catalog.user)}',
|
|
425
|
+
PASSWORD '${e(catalog.password)}'
|
|
426
|
+
);`;
|
|
427
|
+
case common_1.DucklakeCatalogType.SQLITE:
|
|
428
|
+
case common_1.DucklakeCatalogType.DUCKDB:
|
|
429
|
+
return null;
|
|
430
|
+
default:
|
|
431
|
+
return null;
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
static buildDucklakeDataSecretSql(ducklake) {
|
|
435
|
+
const e = DuckdbWarehouseClient.escapeDuckdbString;
|
|
436
|
+
const { dataPath } = ducklake;
|
|
437
|
+
switch (dataPath.type) {
|
|
438
|
+
case common_1.DucklakeDataPathType.S3: {
|
|
439
|
+
const hasStaticCreds = !!(dataPath.accessKeyId && dataPath.secretAccessKey);
|
|
440
|
+
const providerClause = hasStaticCreds
|
|
441
|
+
? ''
|
|
442
|
+
: `PROVIDER credential_chain, REFRESH auto, VALIDATION 'none',`;
|
|
443
|
+
const keyIdClause = dataPath.accessKeyId
|
|
444
|
+
? `KEY_ID '${e(dataPath.accessKeyId)}',`
|
|
445
|
+
: '';
|
|
446
|
+
const secretClause = dataPath.secretAccessKey
|
|
447
|
+
? `SECRET '${e(dataPath.secretAccessKey)}',`
|
|
448
|
+
: '';
|
|
449
|
+
const endpointClause = dataPath.endpoint
|
|
450
|
+
? `ENDPOINT '${e(dataPath.endpoint)}',`
|
|
451
|
+
: '';
|
|
452
|
+
const regionClause = dataPath.region
|
|
453
|
+
? `REGION '${e(dataPath.region)}',`
|
|
454
|
+
: '';
|
|
455
|
+
const urlStyleClause = dataPath.forcePathStyle === undefined
|
|
456
|
+
? ''
|
|
457
|
+
: `URL_STYLE '${dataPath.forcePathStyle ? 'path' : 'vhost'}',`;
|
|
458
|
+
const useSslClause = dataPath.useSsl === undefined
|
|
459
|
+
? ''
|
|
460
|
+
: `USE_SSL ${dataPath.useSsl},`;
|
|
461
|
+
return `CREATE OR REPLACE SECRET ${DuckdbWarehouseClient.DUCKLAKE_DATA_SECRET} (
|
|
462
|
+
TYPE s3,
|
|
463
|
+
${providerClause}
|
|
464
|
+
${keyIdClause}
|
|
465
|
+
${secretClause}
|
|
466
|
+
${endpointClause}
|
|
467
|
+
${regionClause}
|
|
468
|
+
${urlStyleClause}
|
|
469
|
+
${useSslClause}
|
|
470
|
+
SCOPE '${e(dataPath.url)}'
|
|
471
|
+
);`;
|
|
472
|
+
}
|
|
473
|
+
case common_1.DucklakeDataPathType.GCS: {
|
|
474
|
+
const hasStaticCreds = !!(dataPath.hmacKeyId && dataPath.hmacSecret);
|
|
475
|
+
const providerClause = hasStaticCreds
|
|
476
|
+
? ''
|
|
477
|
+
: `PROVIDER credential_chain,`;
|
|
478
|
+
const keyIdClause = dataPath.hmacKeyId
|
|
479
|
+
? `KEY_ID '${e(dataPath.hmacKeyId)}',`
|
|
480
|
+
: '';
|
|
481
|
+
const secretClause = dataPath.hmacSecret
|
|
482
|
+
? `SECRET '${e(dataPath.hmacSecret)}',`
|
|
483
|
+
: '';
|
|
484
|
+
return `CREATE OR REPLACE SECRET ${DuckdbWarehouseClient.DUCKLAKE_DATA_SECRET} (
|
|
485
|
+
TYPE gcs,
|
|
486
|
+
${providerClause}
|
|
487
|
+
${keyIdClause}
|
|
488
|
+
${secretClause}
|
|
489
|
+
SCOPE '${e(dataPath.url)}'
|
|
490
|
+
);`;
|
|
491
|
+
}
|
|
492
|
+
case common_1.DucklakeDataPathType.AZURE: {
|
|
493
|
+
if (dataPath.connectionString) {
|
|
494
|
+
return `CREATE OR REPLACE SECRET ${DuckdbWarehouseClient.DUCKLAKE_DATA_SECRET} (
|
|
495
|
+
TYPE azure,
|
|
496
|
+
CONNECTION_STRING '${e(dataPath.connectionString)}',
|
|
497
|
+
SCOPE '${e(dataPath.url)}'
|
|
498
|
+
);`;
|
|
499
|
+
}
|
|
500
|
+
if (dataPath.accountName && dataPath.accountKey) {
|
|
501
|
+
return `CREATE OR REPLACE SECRET ${DuckdbWarehouseClient.DUCKLAKE_DATA_SECRET} (
|
|
502
|
+
TYPE azure,
|
|
503
|
+
ACCOUNT_NAME '${e(dataPath.accountName)}',
|
|
504
|
+
ACCOUNT_KEY '${e(dataPath.accountKey)}',
|
|
505
|
+
SCOPE '${e(dataPath.url)}'
|
|
506
|
+
);`;
|
|
507
|
+
}
|
|
508
|
+
if (dataPath.accountName) {
|
|
509
|
+
return `CREATE OR REPLACE SECRET ${DuckdbWarehouseClient.DUCKLAKE_DATA_SECRET} (
|
|
510
|
+
TYPE azure,
|
|
511
|
+
PROVIDER credential_chain,
|
|
512
|
+
ACCOUNT_NAME '${e(dataPath.accountName)}',
|
|
513
|
+
SCOPE '${e(dataPath.url)}'
|
|
514
|
+
);`;
|
|
515
|
+
}
|
|
516
|
+
return null;
|
|
517
|
+
}
|
|
518
|
+
case common_1.DucklakeDataPathType.LOCAL:
|
|
519
|
+
return null;
|
|
520
|
+
default:
|
|
521
|
+
return null;
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
static catalogUsesSecret(ducklake) {
|
|
525
|
+
return ducklake.catalog.type === common_1.DucklakeCatalogType.POSTGRES;
|
|
526
|
+
}
|
|
527
|
+
static buildDucklakeSecretSql(ducklake) {
|
|
528
|
+
if (!DuckdbWarehouseClient.catalogUsesSecret(ducklake))
|
|
529
|
+
return null;
|
|
530
|
+
const e = DuckdbWarehouseClient.escapeDuckdbString;
|
|
531
|
+
const dataPathUrl = ducklake.dataPath.type === common_1.DucklakeDataPathType.LOCAL
|
|
532
|
+
? ducklake.dataPath.path
|
|
533
|
+
: ducklake.dataPath.url;
|
|
534
|
+
return `CREATE OR REPLACE SECRET ${DuckdbWarehouseClient.DUCKLAKE_SECRET} (
|
|
535
|
+
TYPE ducklake,
|
|
536
|
+
METADATA_PATH '',
|
|
537
|
+
DATA_PATH '${e(dataPathUrl)}',
|
|
538
|
+
METADATA_PARAMETERS MAP {'TYPE': 'postgres', 'SECRET': '${DuckdbWarehouseClient.DUCKLAKE_CATALOG_SECRET}'}
|
|
539
|
+
);`;
|
|
540
|
+
}
|
|
541
|
+
static buildDucklakeAttachSql(ducklake) {
|
|
542
|
+
const e = DuckdbWarehouseClient.escapeDuckdbString;
|
|
543
|
+
const alias = ducklake.catalogAlias ?? 'ducklake';
|
|
544
|
+
const quotedAlias = DuckdbWarehouseClient.quoteIdent(alias);
|
|
545
|
+
const stmts = [];
|
|
546
|
+
const catalogSecret = DuckdbWarehouseClient.buildDucklakeCatalogSecretSql(ducklake);
|
|
547
|
+
if (catalogSecret)
|
|
548
|
+
stmts.push(catalogSecret);
|
|
549
|
+
const dataSecret = DuckdbWarehouseClient.buildDucklakeDataSecretSql(ducklake);
|
|
550
|
+
if (dataSecret)
|
|
551
|
+
stmts.push(dataSecret);
|
|
552
|
+
if (DuckdbWarehouseClient.catalogUsesSecret(ducklake)) {
|
|
553
|
+
const ducklakeSecret = DuckdbWarehouseClient.buildDucklakeSecretSql(ducklake);
|
|
554
|
+
if (ducklakeSecret)
|
|
555
|
+
stmts.push(ducklakeSecret);
|
|
556
|
+
stmts.push(`ATTACH 'ducklake:${DuckdbWarehouseClient.DUCKLAKE_SECRET}' AS ${quotedAlias} (READ_ONLY);`);
|
|
557
|
+
}
|
|
558
|
+
else {
|
|
559
|
+
const { catalog } = ducklake;
|
|
560
|
+
const catalogTarget = catalog.type === common_1.DucklakeCatalogType.SQLITE
|
|
561
|
+
? `ducklake:sqlite:${e(catalog.path)}`
|
|
562
|
+
: `ducklake:${e(catalog.path)}`;
|
|
563
|
+
const dataPathUrl = ducklake.dataPath.type === common_1.DucklakeDataPathType.LOCAL
|
|
564
|
+
? ducklake.dataPath.path
|
|
565
|
+
: ducklake.dataPath.url;
|
|
566
|
+
stmts.push(`ATTACH '${catalogTarget}' AS ${quotedAlias} (DATA_PATH '${e(dataPathUrl)}', READ_ONLY);`);
|
|
567
|
+
}
|
|
568
|
+
return stmts;
|
|
569
|
+
}
|
|
338
570
|
static buildS3SecretSql(s3Config) {
|
|
339
571
|
const escape = (v) => DuckdbWarehouseClient.sqlBuilder.escapeString(v);
|
|
340
572
|
const usesStaticCredentials = !DuckdbWarehouseClient.usesS3CredentialChain(s3Config);
|
|
@@ -426,11 +658,18 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
426
658
|
}
|
|
427
659
|
/** Bootstrap for isolated instances — no shared locks needed. */
|
|
428
660
|
async bootstrapIsolatedSession(db, tempDir) {
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
661
|
+
if (!this.ducklakeConfig) {
|
|
662
|
+
await db.run('INSTALL httpfs;');
|
|
663
|
+
await db.run('LOAD httpfs;');
|
|
664
|
+
await DuckdbWarehouseClient.loadAwsExtensionForCredentialChain(db, this.s3Config);
|
|
665
|
+
}
|
|
666
|
+
await DuckdbWarehouseClient.hardenInstance(db, {
|
|
667
|
+
allowKnownExtensionAutoload: !!this.ducklakeConfig,
|
|
668
|
+
});
|
|
433
669
|
await db.run(`SET temp_directory = '${tempDir}';`);
|
|
670
|
+
if (this.ducklakeConfig) {
|
|
671
|
+
await db.run('SET pg_connection_limit = 64;');
|
|
672
|
+
}
|
|
434
673
|
if (this.resourceLimits?.memoryLimit) {
|
|
435
674
|
await db.run(`SET memory_limit = '${this.resourceLimits.memoryLimit}';`);
|
|
436
675
|
}
|
|
@@ -440,7 +679,15 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
440
679
|
if (this.s3Config) {
|
|
441
680
|
await db.run(DuckdbWarehouseClient.buildS3SecretSql(this.s3Config));
|
|
442
681
|
}
|
|
443
|
-
|
|
682
|
+
if (this.ducklakeConfig) {
|
|
683
|
+
const stmts = DuckdbWarehouseClient.buildDucklakeAttachSql(this.ducklakeConfig);
|
|
684
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
685
|
+
for (const stmt of stmts) {
|
|
686
|
+
// eslint-disable-next-line no-await-in-loop
|
|
687
|
+
await db.run(stmt);
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
this.logger?.info(`DuckDB isolated bootstrap: memory_limit=${this.resourceLimits?.memoryLimit ?? 'default'} threads=${this.resourceLimits?.threads ?? 'default'} s3=${this.s3Config ? 'configured' : 'none'} ducklake=${this.ducklakeConfig ? 'configured' : 'none'}`);
|
|
444
691
|
}
|
|
445
692
|
/** Ephemeral DuckDB instance with resource limits (e.g. parquet conversion). */
|
|
446
693
|
async withIsolatedSession(callback) {
|
|
@@ -865,4 +1112,7 @@ exports.DuckdbWarehouseClient = DuckdbWarehouseClient;
|
|
|
865
1112
|
DuckdbWarehouseClient.sharedInstances = new Map();
|
|
866
1113
|
DuckdbWarehouseClient.sharedInstanceSemaphores = new Map();
|
|
867
1114
|
DuckdbWarehouseClient.sqlBuilder = new DuckdbSqlBuilder();
|
|
1115
|
+
DuckdbWarehouseClient.DUCKLAKE_CATALOG_SECRET = '__lightdash_ducklake_catalog';
|
|
1116
|
+
DuckdbWarehouseClient.DUCKLAKE_DATA_SECRET = '__lightdash_ducklake_data';
|
|
1117
|
+
DuckdbWarehouseClient.DUCKLAKE_SECRET = '__lightdash_ducklake';
|
|
868
1118
|
DuckdbWarehouseClient.CONNECT_RETRIES_BEFORE_RECREATE = 2;
|
|
@@ -527,6 +527,7 @@ describe('DuckdbWarehouseClient', () => {
|
|
|
527
527
|
name: 'pass token in connection string for MotherDuck',
|
|
528
528
|
credentials: {
|
|
529
529
|
type: common_1.WarehouseTypes.DUCKDB,
|
|
530
|
+
connectionType: common_1.DuckdbConnectionType.MOTHERDUCK,
|
|
530
531
|
database: 'my_database',
|
|
531
532
|
schema: 'main',
|
|
532
533
|
token: 'my_motherduck_token',
|
|
@@ -548,6 +549,7 @@ describe('DuckdbWarehouseClient', () => {
|
|
|
548
549
|
it('should require a MotherDuck token for direct DuckDB connections', () => {
|
|
549
550
|
expect(() => new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
550
551
|
type: common_1.WarehouseTypes.DUCKDB,
|
|
552
|
+
connectionType: common_1.DuckdbConnectionType.MOTHERDUCK,
|
|
551
553
|
database: 'my_database',
|
|
552
554
|
schema: 'main',
|
|
553
555
|
token: '',
|
|
@@ -556,6 +558,7 @@ describe('DuckdbWarehouseClient', () => {
|
|
|
556
558
|
it('should expose the configured start of week', () => {
|
|
557
559
|
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
558
560
|
type: common_1.WarehouseTypes.DUCKDB,
|
|
561
|
+
connectionType: common_1.DuckdbConnectionType.MOTHERDUCK,
|
|
559
562
|
database: 'analytics',
|
|
560
563
|
schema: 'main',
|
|
561
564
|
token: 'motherduck_token',
|
|
@@ -579,6 +582,7 @@ describe('DuckdbWarehouseClient', () => {
|
|
|
579
582
|
createInstanceMock.mockResolvedValue(createMockConnection(jest.fn(), runMock));
|
|
580
583
|
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
581
584
|
type: common_1.WarehouseTypes.DUCKDB,
|
|
585
|
+
connectionType: common_1.DuckdbConnectionType.MOTHERDUCK,
|
|
582
586
|
database: 'analytics',
|
|
583
587
|
schema: 'main',
|
|
584
588
|
token: 'motherduck_token',
|
|
@@ -596,4 +600,138 @@ describe('DuckdbWarehouseClient', () => {
|
|
|
596
600
|
},
|
|
597
601
|
});
|
|
598
602
|
});
|
|
603
|
+
describe('DuckLake bootstrap', () => {
|
|
604
|
+
const captureRunMock = () => jest.fn().mockResolvedValue({
|
|
605
|
+
getRowObjects: async () => [],
|
|
606
|
+
});
|
|
607
|
+
const collectStatements = (runMock) => runMock.mock.calls.map((c) => c[0]);
|
|
608
|
+
it('attaches a postgres-catalog + S3 DuckLake in the correct order', async () => {
|
|
609
|
+
const runMock = captureRunMock();
|
|
610
|
+
const streamMock = jest.fn(async () => getMockStreamResult([[]], []));
|
|
611
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
612
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
613
|
+
type: common_1.WarehouseTypes.DUCKDB,
|
|
614
|
+
connectionType: common_1.DuckdbConnectionType.DUCKLAKE,
|
|
615
|
+
schema: 'main',
|
|
616
|
+
catalogAlias: 'ducklake',
|
|
617
|
+
catalog: {
|
|
618
|
+
type: common_1.DucklakeCatalogType.POSTGRES,
|
|
619
|
+
host: 'pg.example.com',
|
|
620
|
+
port: 5432,
|
|
621
|
+
database: 'catalog',
|
|
622
|
+
user: 'ducklake_user',
|
|
623
|
+
password: 'p@ss',
|
|
624
|
+
},
|
|
625
|
+
dataPath: {
|
|
626
|
+
type: common_1.DucklakeDataPathType.S3,
|
|
627
|
+
url: 's3://my-bucket/path/',
|
|
628
|
+
accessKeyId: 'AKIAEXAMPLE',
|
|
629
|
+
secretAccessKey: 'SECRETEXAMPLE',
|
|
630
|
+
region: 'us-east-1',
|
|
631
|
+
},
|
|
632
|
+
});
|
|
633
|
+
await client.runQuery('SELECT 1');
|
|
634
|
+
const stmts = collectStatements(runMock);
|
|
635
|
+
const joined = stmts.join('\n');
|
|
636
|
+
// No explicit INSTALL/LOAD in DuckLake mode — rely on autoload.
|
|
637
|
+
expect(joined).not.toMatch(/INSTALL httpfs/);
|
|
638
|
+
expect(joined).not.toMatch(/LOAD httpfs/);
|
|
639
|
+
// Hardening flips autoload to TRUE for DuckLake.
|
|
640
|
+
expect(stmts).toEqual(expect.arrayContaining([
|
|
641
|
+
'SET autoinstall_known_extensions = true;',
|
|
642
|
+
'SET autoload_known_extensions = true;',
|
|
643
|
+
'SET allow_community_extensions = false;',
|
|
644
|
+
'SET allow_unredacted_secrets = false;',
|
|
645
|
+
]));
|
|
646
|
+
const catalogIdx = stmts.findIndex((s) => /__lightdash_ducklake_catalog/.test(s));
|
|
647
|
+
const dataIdx = stmts.findIndex((s) => /__lightdash_ducklake_data/.test(s));
|
|
648
|
+
const duckLakeSecretIdx = stmts.findIndex((s) => /SECRET __lightdash_ducklake\s/.test(s));
|
|
649
|
+
const attachIdx = stmts.findIndex((s) => /^ATTACH 'ducklake:__lightdash_ducklake'/.test(s));
|
|
650
|
+
expect(catalogIdx).toBeGreaterThanOrEqual(0);
|
|
651
|
+
expect(dataIdx).toBeGreaterThan(catalogIdx);
|
|
652
|
+
expect(duckLakeSecretIdx).toBeGreaterThan(dataIdx);
|
|
653
|
+
expect(attachIdx).toBeGreaterThan(duckLakeSecretIdx);
|
|
654
|
+
expect(stmts[catalogIdx]).toMatch(/TYPE postgres/);
|
|
655
|
+
expect(stmts[catalogIdx]).toMatch(/HOST 'pg.example.com'/);
|
|
656
|
+
expect(stmts[catalogIdx]).toMatch(/PASSWORD 'p@ss'/);
|
|
657
|
+
expect(stmts[dataIdx]).toMatch(/TYPE s3/);
|
|
658
|
+
expect(stmts[dataIdx]).toMatch(/KEY_ID 'AKIAEXAMPLE'/);
|
|
659
|
+
expect(stmts[dataIdx]).toMatch(/SCOPE 's3:\/\/my-bucket\/path\/'/);
|
|
660
|
+
});
|
|
661
|
+
it('uses inline ATTACH (no ducklake secret) for SQLite catalog + local data path', async () => {
|
|
662
|
+
const runMock = captureRunMock();
|
|
663
|
+
const streamMock = jest.fn(async () => getMockStreamResult([[]], []));
|
|
664
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
665
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
666
|
+
type: common_1.WarehouseTypes.DUCKDB,
|
|
667
|
+
connectionType: common_1.DuckdbConnectionType.DUCKLAKE,
|
|
668
|
+
schema: 'main',
|
|
669
|
+
catalog: {
|
|
670
|
+
type: common_1.DucklakeCatalogType.SQLITE,
|
|
671
|
+
path: '/tmp/ducklake.sqlite',
|
|
672
|
+
},
|
|
673
|
+
dataPath: {
|
|
674
|
+
type: common_1.DucklakeDataPathType.LOCAL,
|
|
675
|
+
path: '/tmp/ducklake-data',
|
|
676
|
+
},
|
|
677
|
+
});
|
|
678
|
+
await client.runQuery('SELECT 1');
|
|
679
|
+
const stmts = collectStatements(runMock);
|
|
680
|
+
const joined = stmts.join('\n');
|
|
681
|
+
expect(joined).not.toMatch(/__lightdash_ducklake_catalog/);
|
|
682
|
+
expect(joined).not.toMatch(/__lightdash_ducklake_data/);
|
|
683
|
+
expect(joined).not.toMatch(/SECRET __lightdash_ducklake\s/);
|
|
684
|
+
expect(stmts.some((s) => /^ATTACH 'ducklake:sqlite:\/tmp\/ducklake\.sqlite' AS "ducklake" \(DATA_PATH '\/tmp\/ducklake-data', READ_ONLY\);/.test(s))).toBe(true);
|
|
685
|
+
});
|
|
686
|
+
it('rejects user SQL that contains ATTACH even in DuckLake mode', async () => {
|
|
687
|
+
const runMock = captureRunMock();
|
|
688
|
+
const streamMock = jest.fn(async () => getMockStreamResult([[]], []));
|
|
689
|
+
const extractStatements = jest.fn(async () => ({
|
|
690
|
+
count: 1,
|
|
691
|
+
prepare: async () => ({
|
|
692
|
+
statementType: 25, // ATTACH
|
|
693
|
+
destroySync: jest.fn(),
|
|
694
|
+
}),
|
|
695
|
+
}));
|
|
696
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock, {
|
|
697
|
+
extractStatements,
|
|
698
|
+
}));
|
|
699
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
700
|
+
type: common_1.WarehouseTypes.DUCKDB,
|
|
701
|
+
connectionType: common_1.DuckdbConnectionType.DUCKLAKE,
|
|
702
|
+
schema: 'main',
|
|
703
|
+
catalog: {
|
|
704
|
+
type: common_1.DucklakeCatalogType.SQLITE,
|
|
705
|
+
path: '/tmp/c.sqlite',
|
|
706
|
+
},
|
|
707
|
+
dataPath: {
|
|
708
|
+
type: common_1.DucklakeDataPathType.LOCAL,
|
|
709
|
+
path: '/tmp/d',
|
|
710
|
+
},
|
|
711
|
+
});
|
|
712
|
+
await expect(client.runQuery("ATTACH 'ducklake:evil' AS bad;")).rejects.toThrow(/only SELECT statements are allowed/);
|
|
713
|
+
});
|
|
714
|
+
it('keeps autoload disabled for non-DuckLake modes', async () => {
|
|
715
|
+
const runMock = captureRunMock();
|
|
716
|
+
const streamMock = jest.fn(async () => getMockStreamResult([[]], []));
|
|
717
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
718
|
+
const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate({
|
|
719
|
+
type: 'duckdb_s3',
|
|
720
|
+
s3Config: {
|
|
721
|
+
endpoint: 'localhost:9000',
|
|
722
|
+
region: 'us-east-1',
|
|
723
|
+
forcePathStyle: true,
|
|
724
|
+
useSsl: false,
|
|
725
|
+
},
|
|
726
|
+
});
|
|
727
|
+
await client.runQuery('SELECT 1');
|
|
728
|
+
const stmts = collectStatements(runMock);
|
|
729
|
+
expect(stmts).toEqual(expect.arrayContaining([
|
|
730
|
+
'SET autoinstall_known_extensions = false;',
|
|
731
|
+
'SET autoload_known_extensions = false;',
|
|
732
|
+
]));
|
|
733
|
+
// Regression: existing modes still explicitly INSTALL/LOAD httpfs.
|
|
734
|
+
expect(stmts).toEqual(expect.arrayContaining(['INSTALL httpfs;', 'LOAD httpfs;']));
|
|
735
|
+
});
|
|
736
|
+
});
|
|
599
737
|
});
|