@lightdash/warehouses 2.228.1 → 2.229.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo.build +1 -0
- package/package.json +4 -4
- package/dist/.tsbuildinfo +0 -1
- package/dist/ssh/sshTunnelFailure.test.d.ts +0 -2
- package/dist/ssh/sshTunnelFailure.test.d.ts.map +0 -1
- package/dist/ssh/sshTunnelFailure.test.js +0 -90
- package/dist/validateWarehouseColumnReferences.test.d.ts +0 -2
- package/dist/validateWarehouseColumnReferences.test.d.ts.map +0 -1
- package/dist/validateWarehouseColumnReferences.test.js +0 -168
- package/dist/warehouseClients/AthenaWarehouseClient.test.d.ts +0 -2
- package/dist/warehouseClients/AthenaWarehouseClient.test.d.ts.map +0 -1
- package/dist/warehouseClients/AthenaWarehouseClient.test.js +0 -261
- package/dist/warehouseClients/BigqueryWarehouseClient.mock.d.ts +0 -83
- package/dist/warehouseClients/BigqueryWarehouseClient.mock.d.ts.map +0 -1
- package/dist/warehouseClients/BigqueryWarehouseClient.mock.js +0 -130
- package/dist/warehouseClients/BigqueryWarehouseClient.test.d.ts +0 -2
- package/dist/warehouseClients/BigqueryWarehouseClient.test.d.ts.map +0 -1
- package/dist/warehouseClients/BigqueryWarehouseClient.test.js +0 -493
- package/dist/warehouseClients/ClickhouseWarehouseClient.test.d.ts +0 -2
- package/dist/warehouseClients/ClickhouseWarehouseClient.test.d.ts.map +0 -1
- package/dist/warehouseClients/ClickhouseWarehouseClient.test.js +0 -115
- package/dist/warehouseClients/DatabricksWarehouseClient.mock.d.ts +0 -17
- package/dist/warehouseClients/DatabricksWarehouseClient.mock.d.ts.map +0 -1
- package/dist/warehouseClients/DatabricksWarehouseClient.mock.js +0 -111
- package/dist/warehouseClients/DatabricksWarehouseClient.test.d.ts +0 -2
- package/dist/warehouseClients/DatabricksWarehouseClient.test.d.ts.map +0 -1
- package/dist/warehouseClients/DatabricksWarehouseClient.test.js +0 -592
- package/dist/warehouseClients/DatabricksWarehouseStartupRetry.test.d.ts +0 -2
- package/dist/warehouseClients/DatabricksWarehouseStartupRetry.test.d.ts.map +0 -1
- package/dist/warehouseClients/DatabricksWarehouseStartupRetry.test.js +0 -129
- package/dist/warehouseClients/DuckdbWarehouseClient.embedded.test.d.ts +0 -2
- package/dist/warehouseClients/DuckdbWarehouseClient.embedded.test.d.ts.map +0 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.embedded.test.js +0 -92
- package/dist/warehouseClients/DuckdbWarehouseClient.test.d.ts +0 -2
- package/dist/warehouseClients/DuckdbWarehouseClient.test.d.ts.map +0 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.test.js +0 -1749
- package/dist/warehouseClients/MotherduckErrorClassifier.test.d.ts +0 -2
- package/dist/warehouseClients/MotherduckErrorClassifier.test.d.ts.map +0 -1
- package/dist/warehouseClients/MotherduckErrorClassifier.test.js +0 -31
- package/dist/warehouseClients/MotherduckInstanceCache.catalog.test.d.ts +0 -2
- package/dist/warehouseClients/MotherduckInstanceCache.catalog.test.d.ts.map +0 -1
- package/dist/warehouseClients/MotherduckInstanceCache.catalog.test.js +0 -52
- package/dist/warehouseClients/MotherduckInstanceCache.integration.test.d.ts +0 -2
- package/dist/warehouseClients/MotherduckInstanceCache.integration.test.d.ts.map +0 -1
- package/dist/warehouseClients/MotherduckInstanceCache.integration.test.js +0 -172
- package/dist/warehouseClients/MotherduckInstanceCache.security.test.d.ts +0 -2
- package/dist/warehouseClients/MotherduckInstanceCache.security.test.d.ts.map +0 -1
- package/dist/warehouseClients/MotherduckInstanceCache.security.test.js +0 -192
- package/dist/warehouseClients/MotherduckInstanceCache.test.d.ts +0 -2
- package/dist/warehouseClients/MotherduckInstanceCache.test.d.ts.map +0 -1
- package/dist/warehouseClients/MotherduckInstanceCache.test.js +0 -302
- package/dist/warehouseClients/PostgresWarehouseClient.mock.d.ts +0 -8
- package/dist/warehouseClients/PostgresWarehouseClient.mock.d.ts.map +0 -1
- package/dist/warehouseClients/PostgresWarehouseClient.mock.js +0 -66
- package/dist/warehouseClients/PostgresWarehouseClient.test.d.ts +0 -2
- package/dist/warehouseClients/PostgresWarehouseClient.test.d.ts.map +0 -1
- package/dist/warehouseClients/PostgresWarehouseClient.test.js +0 -371
- package/dist/warehouseClients/RedshiftWarehouseClient.test.d.ts +0 -2
- package/dist/warehouseClients/RedshiftWarehouseClient.test.d.ts.map +0 -1
- package/dist/warehouseClients/RedshiftWarehouseClient.test.js +0 -126
- package/dist/warehouseClients/SnowflakeWarehouseClient.mock.d.ts +0 -41
- package/dist/warehouseClients/SnowflakeWarehouseClient.mock.d.ts.map +0 -1
- package/dist/warehouseClients/SnowflakeWarehouseClient.mock.js +0 -184
- package/dist/warehouseClients/SnowflakeWarehouseClient.test.d.ts +0 -2
- package/dist/warehouseClients/SnowflakeWarehouseClient.test.d.ts.map +0 -1
- package/dist/warehouseClients/SnowflakeWarehouseClient.test.js +0 -777
- package/dist/warehouseClients/TrinoWarehouseClient.mock.d.ts +0 -23
- package/dist/warehouseClients/TrinoWarehouseClient.mock.d.ts.map +0 -1
- package/dist/warehouseClients/TrinoWarehouseClient.mock.js +0 -102
- package/dist/warehouseClients/TrinoWarehouseClient.test.d.ts +0 -2
- package/dist/warehouseClients/TrinoWarehouseClient.test.d.ts.map +0 -1
- package/dist/warehouseClients/TrinoWarehouseClient.test.js +0 -230
- package/dist/warehouseClients/WarehouseClient.mock.d.ts +0 -13
- package/dist/warehouseClients/WarehouseClient.mock.d.ts.map +0 -1
- package/dist/warehouseClients/WarehouseClient.mock.js +0 -51
- package/dist/warehouseClients/numericKinds.test.d.ts +0 -2
- package/dist/warehouseClients/numericKinds.test.d.ts.map +0 -1
- package/dist/warehouseClients/numericKinds.test.js +0 -168
- package/dist/warehouseClients/pgProtocolGuard.integration.test.d.ts +0 -2
- package/dist/warehouseClients/pgProtocolGuard.integration.test.d.ts.map +0 -1
- package/dist/warehouseClients/pgProtocolGuard.integration.test.js +0 -93
- package/dist/warehouseClients/pgProtocolGuard.test.d.ts +0 -2
- package/dist/warehouseClients/pgProtocolGuard.test.d.ts.map +0 -1
- package/dist/warehouseClients/pgProtocolGuard.test.js +0 -48
- package/dist/warehouseClients/redshiftIamCredentials.test.d.ts +0 -2
- package/dist/warehouseClients/redshiftIamCredentials.test.d.ts.map +0 -1
- package/dist/warehouseClients/redshiftIamCredentials.test.js +0 -210
- package/dist/warehouseClients/timestampDomain.test.d.ts +0 -2
- package/dist/warehouseClients/timestampDomain.test.d.ts.map +0 -1
- package/dist/warehouseClients/timestampDomain.test.js +0 -101
|
@@ -1,1749 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
36
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
|
-
};
|
|
38
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
-
const common_1 = require("@lightdash/common");
|
|
40
|
-
const crypto_1 = require("crypto");
|
|
41
|
-
const promises_1 = __importDefault(require("fs/promises"));
|
|
42
|
-
const DuckdbWarehouseClient_1 = require("./DuckdbWarehouseClient");
|
|
43
|
-
const MotherduckInstanceCache = __importStar(require("./MotherduckInstanceCache"));
|
|
44
|
-
const createInstanceMock = vi.fn();
|
|
45
|
-
const duckdbS3Credentials = {
|
|
46
|
-
type: 'duckdb_s3',
|
|
47
|
-
s3Config: {
|
|
48
|
-
endpoint: 'localhost:9000',
|
|
49
|
-
region: 'us-east-1',
|
|
50
|
-
forcePathStyle: true,
|
|
51
|
-
useSsl: false,
|
|
52
|
-
},
|
|
53
|
-
};
|
|
54
|
-
// Must provide DuckDBTypeId since mapFieldTypeFromTypeId references it at runtime
|
|
55
|
-
const DUCKDB_TYPE_IDS = {
|
|
56
|
-
BOOLEAN: 1,
|
|
57
|
-
TINYINT: 2,
|
|
58
|
-
SMALLINT: 3,
|
|
59
|
-
INTEGER: 4,
|
|
60
|
-
BIGINT: 5,
|
|
61
|
-
UTINYINT: 6,
|
|
62
|
-
USMALLINT: 7,
|
|
63
|
-
UINTEGER: 8,
|
|
64
|
-
UBIGINT: 9,
|
|
65
|
-
FLOAT: 10,
|
|
66
|
-
DOUBLE: 11,
|
|
67
|
-
TIMESTAMP: 12,
|
|
68
|
-
DATE: 13,
|
|
69
|
-
TIME: 14,
|
|
70
|
-
DECIMAL: 19,
|
|
71
|
-
HUGEINT: 25,
|
|
72
|
-
TIMESTAMP_S: 27,
|
|
73
|
-
TIMESTAMP_MS: 28,
|
|
74
|
-
TIMESTAMP_NS: 29,
|
|
75
|
-
TIMESTAMP_TZ: 31,
|
|
76
|
-
TIME_TZ: 32,
|
|
77
|
-
UHUGEINT: 49,
|
|
78
|
-
// Not referenced in switch — used to test default→STRING
|
|
79
|
-
VARCHAR: 17,
|
|
80
|
-
BLOB: 18,
|
|
81
|
-
};
|
|
82
|
-
vi.mock('@duckdb/node-api', () => ({
|
|
83
|
-
DuckDBTypeId: {
|
|
84
|
-
BOOLEAN: 1,
|
|
85
|
-
TINYINT: 2,
|
|
86
|
-
SMALLINT: 3,
|
|
87
|
-
INTEGER: 4,
|
|
88
|
-
BIGINT: 5,
|
|
89
|
-
UTINYINT: 6,
|
|
90
|
-
USMALLINT: 7,
|
|
91
|
-
UINTEGER: 8,
|
|
92
|
-
UBIGINT: 9,
|
|
93
|
-
FLOAT: 10,
|
|
94
|
-
DOUBLE: 11,
|
|
95
|
-
TIMESTAMP: 12,
|
|
96
|
-
DATE: 13,
|
|
97
|
-
TIME: 14,
|
|
98
|
-
DECIMAL: 19,
|
|
99
|
-
HUGEINT: 25,
|
|
100
|
-
TIMESTAMP_S: 27,
|
|
101
|
-
TIMESTAMP_MS: 28,
|
|
102
|
-
TIMESTAMP_NS: 29,
|
|
103
|
-
TIMESTAMP_TZ: 31,
|
|
104
|
-
TIME_TZ: 32,
|
|
105
|
-
UHUGEINT: 49,
|
|
106
|
-
},
|
|
107
|
-
DuckDBInstance: {
|
|
108
|
-
create: (...args) => createInstanceMock(...args),
|
|
109
|
-
},
|
|
110
|
-
version: () => 'v1.5.2',
|
|
111
|
-
}));
|
|
112
|
-
const getMockStreamResult = (chunks, columnTypeIds) => {
|
|
113
|
-
const columnNames = Object.keys(chunks[0]?.[0] || {});
|
|
114
|
-
return {
|
|
115
|
-
columnCount: columnNames.length,
|
|
116
|
-
columnNames: () => columnNames,
|
|
117
|
-
columnType: (i) => ({ typeId: columnTypeIds[i] ?? 0 }),
|
|
118
|
-
// eslint-disable-next-line object-shorthand, func-names, no-restricted-syntax
|
|
119
|
-
yieldRowObjectJson: async function* () {
|
|
120
|
-
// eslint-disable-next-line no-restricted-syntax
|
|
121
|
-
for (const chunk of chunks) {
|
|
122
|
-
yield chunk;
|
|
123
|
-
}
|
|
124
|
-
},
|
|
125
|
-
};
|
|
126
|
-
};
|
|
127
|
-
const createMockExtractStatements = (overrides) => vi.fn(async () => ({
|
|
128
|
-
count: overrides?.count ?? 1,
|
|
129
|
-
prepare: async () => ({
|
|
130
|
-
statementType: overrides?.statementType ?? 1, // SELECT
|
|
131
|
-
destroySync: vi.fn(),
|
|
132
|
-
}),
|
|
133
|
-
}));
|
|
134
|
-
const createMockConnection = (streamMock, runMock = vi.fn(), opts) => ({
|
|
135
|
-
connect: async () => ({
|
|
136
|
-
run: runMock,
|
|
137
|
-
stream: streamMock,
|
|
138
|
-
extractStatements: opts?.extractStatements ?? createMockExtractStatements(),
|
|
139
|
-
interrupt: opts?.interrupt ?? vi.fn(),
|
|
140
|
-
closeSync: vi.fn(),
|
|
141
|
-
disconnectSync: vi.fn(),
|
|
142
|
-
}),
|
|
143
|
-
closeSync: vi.fn(),
|
|
144
|
-
});
|
|
145
|
-
describe('internal Parquet projects', () => {
|
|
146
|
-
const scope = 'https://storage.googleapis.com/example/events/compacted/org_id=org-a/';
|
|
147
|
-
const url = `${scope}stream=query_events/dt=2026-09-07/part.parquet`;
|
|
148
|
-
const source = () => ({
|
|
149
|
-
scope,
|
|
150
|
-
httpAuth: { bearerToken: 'test-token' },
|
|
151
|
-
tables: [{ name: 'query_events', urls: [url] }],
|
|
152
|
-
});
|
|
153
|
-
let run;
|
|
154
|
-
beforeEach(() => {
|
|
155
|
-
vi.clearAllMocks();
|
|
156
|
-
run = vi.fn();
|
|
157
|
-
createInstanceMock.mockResolvedValue(createMockConnection(vi.fn(async () => getMockStreamResult([[{ count: 2 }]], [5])), run));
|
|
158
|
-
});
|
|
159
|
-
it('binds trusted views, restricts external access and refreshes the manifest each session', async () => {
|
|
160
|
-
const resolveSource = vi.fn(async () => source());
|
|
161
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
162
|
-
type: 'duckdb_parquet',
|
|
163
|
-
resolveSource,
|
|
164
|
-
});
|
|
165
|
-
await client.runQuery('SELECT count(*) FROM query_events');
|
|
166
|
-
await client.runQuery('SELECT count(*) FROM query_events');
|
|
167
|
-
expect(resolveSource).toHaveBeenCalledTimes(2);
|
|
168
|
-
expect(createInstanceMock).toHaveBeenCalledTimes(2);
|
|
169
|
-
expect(run).toHaveBeenCalledWith(`SET allowed_paths = ['${url}'];`);
|
|
170
|
-
expect(run).toHaveBeenCalledWith('SET enable_external_access = false;');
|
|
171
|
-
expect(run).toHaveBeenCalledWith("SET temp_directory = '';");
|
|
172
|
-
expect(run).toHaveBeenCalledWith(expect.stringContaining('CREATE VIEW "query_events" AS SELECT * FROM read_parquet'));
|
|
173
|
-
expect(client.credentials).not.toHaveProperty('httpAuth');
|
|
174
|
-
expect(run).toHaveBeenCalledWith("SET memory_limit = '256MB';");
|
|
175
|
-
for (const cache of [
|
|
176
|
-
'enable_http_metadata_cache',
|
|
177
|
-
'enable_external_file_cache',
|
|
178
|
-
'parquet_metadata_cache',
|
|
179
|
-
]) {
|
|
180
|
-
expect(run).toHaveBeenCalledWith(`SET ${cache} = true;`);
|
|
181
|
-
}
|
|
182
|
-
const statements = run.mock.calls.map(([sql]) => sql);
|
|
183
|
-
const bind = statements.findIndex((sql) => sql.startsWith('CREATE VIEW'));
|
|
184
|
-
expect(run).toHaveBeenCalledWith('SET threads = 32;');
|
|
185
|
-
expect(statements[bind]).toContain('union_by_name = true');
|
|
186
|
-
const instance = await createInstanceMock.mock.results[0].value;
|
|
187
|
-
expect(instance.closeSync).toHaveBeenCalledTimes(2);
|
|
188
|
-
});
|
|
189
|
-
it('honors explicit thread limits during metadata binding and execution', async () => {
|
|
190
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({ type: 'duckdb_parquet', resolveSource: async () => source() }, { sharedResourceLimits: { threads: 1, memoryLimit: '128MB' } });
|
|
191
|
-
await client.runQuery('SELECT count(*) FROM query_events');
|
|
192
|
-
expect(run).toHaveBeenCalledWith("SET memory_limit = '128MB';");
|
|
193
|
-
expect(run).toHaveBeenCalledWith('SET threads = 1;');
|
|
194
|
-
expect(run).not.toHaveBeenCalledWith('SET threads = 32;');
|
|
195
|
-
expect(run).not.toHaveBeenCalledWith('SET threads = 2;');
|
|
196
|
-
});
|
|
197
|
-
it('closes the private cache when view binding fails', async () => {
|
|
198
|
-
run.mockImplementation(async (sql) => {
|
|
199
|
-
if (sql.startsWith('CREATE VIEW'))
|
|
200
|
-
throw new Error('binding failed');
|
|
201
|
-
});
|
|
202
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
203
|
-
type: 'duckdb_parquet',
|
|
204
|
-
resolveSource: async () => source(),
|
|
205
|
-
});
|
|
206
|
-
await expect(client.runQuery('SELECT count(*) FROM query_events')).rejects.toThrow(/Internal analytics query failed/);
|
|
207
|
-
const instance = await createInstanceMock.mock.results[0].value;
|
|
208
|
-
expect(instance.closeSync).toHaveBeenCalledTimes(1);
|
|
209
|
-
});
|
|
210
|
-
it.each([
|
|
211
|
-
'https://storage.googleapis.com/example/events/compacted/org_id=org-b/file.parquet',
|
|
212
|
-
`${scope}../org-b/file.parquet`,
|
|
213
|
-
`${scope}%2e%2e/org-b/file.parquet`,
|
|
214
|
-
`${scope}*.parquet`,
|
|
215
|
-
'/tmp/private.parquet',
|
|
216
|
-
])('rejects files outside the exact manifest scope: %s', async (file) => {
|
|
217
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
218
|
-
type: 'duckdb_parquet',
|
|
219
|
-
resolveSource: async () => ({
|
|
220
|
-
...source(),
|
|
221
|
-
tables: [{ name: 'query_events', urls: [file] }],
|
|
222
|
-
}),
|
|
223
|
-
});
|
|
224
|
-
await expect(client.runQuery('SELECT * FROM query_events')).rejects.toThrow();
|
|
225
|
-
expect(run).not.toHaveBeenCalledWith(expect.stringContaining('CREATE SECRET'));
|
|
226
|
-
});
|
|
227
|
-
it.each([
|
|
228
|
-
{ name: 'x"; SELECT 1; --', urls: [url] },
|
|
229
|
-
{ name: 'query_events', urls: [] },
|
|
230
|
-
])('rejects arbitrary table SQL and empty manifests: $name', async (table) => {
|
|
231
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
232
|
-
type: 'duckdb_parquet',
|
|
233
|
-
resolveSource: async () => ({ ...source(), tables: [table] }),
|
|
234
|
-
});
|
|
235
|
-
await expect(client.runQuery('SELECT 1')).rejects.toThrow(/unique names/);
|
|
236
|
-
});
|
|
237
|
-
it('keeps user-supplied read_parquet forbidden', async () => {
|
|
238
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
239
|
-
type: 'duckdb_parquet',
|
|
240
|
-
resolveSource: async () => source(),
|
|
241
|
-
});
|
|
242
|
-
await expect(client.runQuery(`SELECT * FROM read_parquet('${url}')`)).rejects.toThrow(/query permissions/);
|
|
243
|
-
});
|
|
244
|
-
it('allows exact signed GET URLs without configuring bucket credentials', async () => {
|
|
245
|
-
const signedScope = scope.replace('org_id=', 'org_id%3D');
|
|
246
|
-
const signedUrl = `${url.replace(/=/g, '%3D')}?X-Amz-Signature=test&X-Amz-Expires=900`;
|
|
247
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
248
|
-
type: 'duckdb_parquet',
|
|
249
|
-
resolveSource: async () => ({
|
|
250
|
-
scope: signedScope,
|
|
251
|
-
signedUrls: true,
|
|
252
|
-
tables: [{ name: 'query_events', urls: [signedUrl] }],
|
|
253
|
-
}),
|
|
254
|
-
});
|
|
255
|
-
await client.runQuery('SELECT count(*) FROM query_events');
|
|
256
|
-
expect(run).toHaveBeenCalledWith(`SET allowed_paths = ['${signedUrl}'];`);
|
|
257
|
-
expect(run).not.toHaveBeenCalledWith(expect.stringContaining('CREATE SECRET'));
|
|
258
|
-
expect(run).toHaveBeenCalledWith('SET enable_external_file_cache = true;');
|
|
259
|
-
});
|
|
260
|
-
it.each([
|
|
261
|
-
'SELECT sql FROM duckdb_views()',
|
|
262
|
-
'SELECT * FROM duckdb_external_file_cache()',
|
|
263
|
-
'SELECT * FROM "information_schema"."views"',
|
|
264
|
-
'SELECT * FROM sqlite_master',
|
|
265
|
-
"SELECT * FROM pragma_storage_info('query_events')",
|
|
266
|
-
])('blocks metadata queries that could disclose signed URLs: %s', async (sql) => {
|
|
267
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
268
|
-
type: 'duckdb_parquet',
|
|
269
|
-
resolveSource: async () => source(),
|
|
270
|
-
});
|
|
271
|
-
await expect(client.runQuery(sql)).rejects.toThrow(/catalog access/);
|
|
272
|
-
});
|
|
273
|
-
it('redacts signed URLs from native query failures', async () => {
|
|
274
|
-
run.mockRejectedValue(new Error(`${url}?X-Amz-Signature=private`));
|
|
275
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
276
|
-
type: 'duckdb_parquet',
|
|
277
|
-
resolveSource: async () => source(),
|
|
278
|
-
});
|
|
279
|
-
await expect(client.runQuery('SELECT count(*) FROM query_events')).rejects.toThrow(/^Internal analytics query failed\. Check storage access and query permissions\.$/);
|
|
280
|
-
});
|
|
281
|
-
it('cannot create privileged readers from public project credentials or shared instances', () => {
|
|
282
|
-
expect(() => new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
283
|
-
type: common_1.WarehouseTypes.DUCKDB,
|
|
284
|
-
connectionType: common_1.DuckdbConnectionType.ANALYTICS,
|
|
285
|
-
database: 'memory',
|
|
286
|
-
schema: 'main',
|
|
287
|
-
})).toThrow(/internal project service/);
|
|
288
|
-
expect(() => new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
289
|
-
type: 'duckdb_parquet',
|
|
290
|
-
resolveSource: async () => source(),
|
|
291
|
-
}, { instanceCacheKey: 'shared' })).toThrow(/cannot share/);
|
|
292
|
-
});
|
|
293
|
-
});
|
|
294
|
-
describe('mapFieldTypeFromTypeId', () => {
|
|
295
|
-
it('should map date types', () => {
|
|
296
|
-
expect((0, DuckdbWarehouseClient_1.mapFieldTypeFromTypeId)(DUCKDB_TYPE_IDS.DATE)).toBe(common_1.DimensionType.DATE);
|
|
297
|
-
});
|
|
298
|
-
it('should map timestamp types', () => {
|
|
299
|
-
const timestampIds = [
|
|
300
|
-
DUCKDB_TYPE_IDS.TIMESTAMP,
|
|
301
|
-
DUCKDB_TYPE_IDS.TIMESTAMP_S,
|
|
302
|
-
DUCKDB_TYPE_IDS.TIMESTAMP_MS,
|
|
303
|
-
DUCKDB_TYPE_IDS.TIMESTAMP_NS,
|
|
304
|
-
DUCKDB_TYPE_IDS.TIMESTAMP_TZ,
|
|
305
|
-
DUCKDB_TYPE_IDS.TIME,
|
|
306
|
-
DUCKDB_TYPE_IDS.TIME_TZ,
|
|
307
|
-
];
|
|
308
|
-
timestampIds.forEach((id) => {
|
|
309
|
-
expect((0, DuckdbWarehouseClient_1.mapFieldTypeFromTypeId)(id)).toBe(common_1.DimensionType.TIMESTAMP);
|
|
310
|
-
});
|
|
311
|
-
});
|
|
312
|
-
it('should map boolean type', () => {
|
|
313
|
-
expect((0, DuckdbWarehouseClient_1.mapFieldTypeFromTypeId)(DUCKDB_TYPE_IDS.BOOLEAN)).toBe(common_1.DimensionType.BOOLEAN);
|
|
314
|
-
});
|
|
315
|
-
it('should map numeric types', () => {
|
|
316
|
-
const numericIds = [
|
|
317
|
-
DUCKDB_TYPE_IDS.TINYINT,
|
|
318
|
-
DUCKDB_TYPE_IDS.SMALLINT,
|
|
319
|
-
DUCKDB_TYPE_IDS.INTEGER,
|
|
320
|
-
DUCKDB_TYPE_IDS.BIGINT,
|
|
321
|
-
DUCKDB_TYPE_IDS.UTINYINT,
|
|
322
|
-
DUCKDB_TYPE_IDS.USMALLINT,
|
|
323
|
-
DUCKDB_TYPE_IDS.UINTEGER,
|
|
324
|
-
DUCKDB_TYPE_IDS.UBIGINT,
|
|
325
|
-
DUCKDB_TYPE_IDS.HUGEINT,
|
|
326
|
-
DUCKDB_TYPE_IDS.UHUGEINT,
|
|
327
|
-
DUCKDB_TYPE_IDS.FLOAT,
|
|
328
|
-
DUCKDB_TYPE_IDS.DOUBLE,
|
|
329
|
-
DUCKDB_TYPE_IDS.DECIMAL,
|
|
330
|
-
];
|
|
331
|
-
numericIds.forEach((id) => {
|
|
332
|
-
expect((0, DuckdbWarehouseClient_1.mapFieldTypeFromTypeId)(id)).toBe(common_1.DimensionType.NUMBER);
|
|
333
|
-
});
|
|
334
|
-
});
|
|
335
|
-
it('should default to string for unknown types', () => {
|
|
336
|
-
expect((0, DuckdbWarehouseClient_1.mapFieldTypeFromTypeId)(DUCKDB_TYPE_IDS.VARCHAR)).toBe(common_1.DimensionType.STRING);
|
|
337
|
-
expect((0, DuckdbWarehouseClient_1.mapFieldTypeFromTypeId)(DUCKDB_TYPE_IDS.BLOB)).toBe(common_1.DimensionType.STRING);
|
|
338
|
-
expect((0, DuckdbWarehouseClient_1.mapFieldTypeFromTypeId)(9999)).toBe(common_1.DimensionType.STRING);
|
|
339
|
-
});
|
|
340
|
-
});
|
|
341
|
-
describe('DuckdbWarehouseClient', () => {
|
|
342
|
-
beforeEach(() => {
|
|
343
|
-
vi.clearAllMocks();
|
|
344
|
-
DuckdbWarehouseClient_1.DuckdbWarehouseClient.resetSharedDuckdbStateForTesting();
|
|
345
|
-
MotherduckInstanceCache.resetForTesting();
|
|
346
|
-
MotherduckInstanceCache.configure({
|
|
347
|
-
idleTtlMs: 60000,
|
|
348
|
-
maxAgeMs: 60000,
|
|
349
|
-
maxEntries: 8,
|
|
350
|
-
});
|
|
351
|
-
});
|
|
352
|
-
afterEach(() => {
|
|
353
|
-
vi.useRealTimers();
|
|
354
|
-
});
|
|
355
|
-
it('should return query rows and mapped fields', async () => {
|
|
356
|
-
const rows = [
|
|
357
|
-
{
|
|
358
|
-
customer_name: 'Alice',
|
|
359
|
-
order_count: 2,
|
|
360
|
-
last_order_at: '2026-02-22 12:00:00',
|
|
361
|
-
},
|
|
362
|
-
];
|
|
363
|
-
const streamMock = vi.fn(async () => getMockStreamResult([rows], [
|
|
364
|
-
DUCKDB_TYPE_IDS.VARCHAR,
|
|
365
|
-
DUCKDB_TYPE_IDS.INTEGER,
|
|
366
|
-
DUCKDB_TYPE_IDS.TIMESTAMP,
|
|
367
|
-
]));
|
|
368
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
|
369
|
-
const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate({
|
|
370
|
-
type: 'duckdb_s3',
|
|
371
|
-
s3Config: {
|
|
372
|
-
endpoint: 'localhost:9000',
|
|
373
|
-
region: 'us-east-1',
|
|
374
|
-
forcePathStyle: true,
|
|
375
|
-
useSsl: false,
|
|
376
|
-
},
|
|
377
|
-
});
|
|
378
|
-
const result = await client.runQuery('SELECT * FROM customers');
|
|
379
|
-
expect(result.rows).toEqual(rows);
|
|
380
|
-
expect(result.fields).toEqual({
|
|
381
|
-
customer_name: { type: common_1.DimensionType.STRING },
|
|
382
|
-
order_count: {
|
|
383
|
-
type: common_1.DimensionType.NUMBER,
|
|
384
|
-
numericKind: { kind: 'integer' },
|
|
385
|
-
},
|
|
386
|
-
last_order_at: { type: common_1.DimensionType.TIMESTAMP },
|
|
387
|
-
});
|
|
388
|
-
});
|
|
389
|
-
it('should stream results in multiple chunks', async () => {
|
|
390
|
-
const chunk1 = [{ id: 1 }, { id: 2 }];
|
|
391
|
-
const chunk2 = [{ id: 3 }];
|
|
392
|
-
const streamMock = vi.fn(async () => getMockStreamResult([chunk1, chunk2], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
393
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
|
394
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
395
|
-
const streamCallback = vi.fn();
|
|
396
|
-
const result = await client.executeAsyncQuery({
|
|
397
|
-
sql: 'SELECT id FROM t',
|
|
398
|
-
tags: { project_uuid: 'proj-1' },
|
|
399
|
-
}, streamCallback);
|
|
400
|
-
expect(streamCallback).toHaveBeenCalledTimes(2);
|
|
401
|
-
const idField = {
|
|
402
|
-
id: {
|
|
403
|
-
type: common_1.DimensionType.NUMBER,
|
|
404
|
-
numericKind: { kind: 'integer' },
|
|
405
|
-
},
|
|
406
|
-
};
|
|
407
|
-
expect(streamCallback).toHaveBeenNthCalledWith(1, chunk1, idField);
|
|
408
|
-
expect(streamCallback).toHaveBeenNthCalledWith(2, chunk2, idField);
|
|
409
|
-
expect(result.totalRows).toBe(3);
|
|
410
|
-
});
|
|
411
|
-
it('should handle empty result set', async () => {
|
|
412
|
-
const streamMock = vi.fn(async () => getMockStreamResult([], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
413
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
|
414
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
415
|
-
const result = await client.runQuery('SELECT id FROM empty_table');
|
|
416
|
-
expect(result.rows).toEqual([]);
|
|
417
|
-
expect(result.fields).toEqual({});
|
|
418
|
-
});
|
|
419
|
-
it('skips MotherDuck timezone and profiling configuration, warns once, and runs every query', async () => {
|
|
420
|
-
const runMock = vi.fn();
|
|
421
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
422
|
-
const logger = { info: vi.fn(), warn: vi.fn() };
|
|
423
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
424
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
425
|
-
type: common_1.WarehouseTypes.DUCKDB,
|
|
426
|
-
connectionType: common_1.DuckdbConnectionType.MOTHERDUCK,
|
|
427
|
-
database: 'analytics',
|
|
428
|
-
schema: 'main',
|
|
429
|
-
token: 'motherduck_token',
|
|
430
|
-
}, { logger, enableQueryProfiling: true });
|
|
431
|
-
await client.runQuery('SELECT 1 AS val', undefined, 'Europe/London');
|
|
432
|
-
await client.runQuery('SELECT 2 AS val', undefined, 'Europe/London');
|
|
433
|
-
expect(streamMock).toHaveBeenCalledTimes(2);
|
|
434
|
-
expect(runMock).not.toHaveBeenCalledWith("SET TimeZone = 'Europe/London';");
|
|
435
|
-
expect(runMock).not.toHaveBeenCalledWith("PRAGMA enable_profiling='json';");
|
|
436
|
-
expect(logger.warn).toHaveBeenCalledExactlyOnceWith(expect.stringMatching(/Europe\/London.*MotherDuck.*saas_mode.*server default zone.*configured zone/));
|
|
437
|
-
expect(logger.info).not.toHaveBeenCalledWith(expect.stringContaining('Requested timezone'));
|
|
438
|
-
});
|
|
439
|
-
it('falls back to info for the MotherDuck timezone warning', async () => {
|
|
440
|
-
const runMock = vi.fn();
|
|
441
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
442
|
-
const logger = { info: vi.fn() };
|
|
443
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
444
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
445
|
-
type: common_1.WarehouseTypes.DUCKDB,
|
|
446
|
-
connectionType: common_1.DuckdbConnectionType.MOTHERDUCK,
|
|
447
|
-
database: 'analytics',
|
|
448
|
-
schema: 'main',
|
|
449
|
-
token: 'motherduck_token',
|
|
450
|
-
}, { logger });
|
|
451
|
-
await client.runQuery('SELECT 1 AS val', undefined, 'Europe/London');
|
|
452
|
-
expect(logger.info).toHaveBeenCalledWith(expect.stringMatching(/Europe\/London.*MotherDuck.*saas_mode.*server default zone.*configured zone/));
|
|
453
|
-
});
|
|
454
|
-
describe('MotherDuck instance caching', () => {
|
|
455
|
-
const credentials = {
|
|
456
|
-
type: common_1.WarehouseTypes.DUCKDB,
|
|
457
|
-
connectionType: common_1.DuckdbConnectionType.MOTHERDUCK,
|
|
458
|
-
database: 'analytics',
|
|
459
|
-
schema: 'main',
|
|
460
|
-
token: 'token-a',
|
|
461
|
-
};
|
|
462
|
-
it('reuses an instance without issuing SET or PRAGMA statements and reports connect timing', async () => {
|
|
463
|
-
const runMock = vi.fn();
|
|
464
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
465
|
-
const onPhaseTiming = vi.fn();
|
|
466
|
-
const events = [];
|
|
467
|
-
MotherduckInstanceCache.setObserver((event) => events.push(event));
|
|
468
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
469
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(credentials, {
|
|
470
|
-
enableInstanceCache: true,
|
|
471
|
-
projectUuid: 'project-a',
|
|
472
|
-
});
|
|
473
|
-
await client.streamQuery('SELECT 1 AS val', vi.fn(), {
|
|
474
|
-
onPhaseTiming,
|
|
475
|
-
});
|
|
476
|
-
await client.streamQuery('SELECT 1 AS val', vi.fn(), {
|
|
477
|
-
onPhaseTiming,
|
|
478
|
-
});
|
|
479
|
-
expect(createInstanceMock).toHaveBeenCalledOnce();
|
|
480
|
-
expect(runMock).not.toHaveBeenCalledWith(expect.stringMatching(/^(?:SET|PRAGMA)\b/i));
|
|
481
|
-
const acquisitions = events.filter((event) => event.type === 'acquire');
|
|
482
|
-
expect(acquisitions).toHaveLength(2);
|
|
483
|
-
acquisitions.forEach((event) => {
|
|
484
|
-
expect(onPhaseTiming).toHaveBeenCalledWith('connect', event.waitMs + event.instanceCreateMs + event.connectMs);
|
|
485
|
-
});
|
|
486
|
-
});
|
|
487
|
-
it('reports instance creation and connection as direct connect timing', async () => {
|
|
488
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
489
|
-
const onPhaseTiming = vi.fn();
|
|
490
|
-
const performanceNow = vi
|
|
491
|
-
.spyOn(performance, 'now')
|
|
492
|
-
.mockReturnValueOnce(0)
|
|
493
|
-
.mockReturnValueOnce(10)
|
|
494
|
-
.mockReturnValueOnce(30)
|
|
495
|
-
.mockReturnValueOnce(40)
|
|
496
|
-
.mockReturnValueOnce(45)
|
|
497
|
-
.mockReturnValue(45);
|
|
498
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
|
499
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(credentials);
|
|
500
|
-
try {
|
|
501
|
-
await client.streamQuery('SELECT 1 AS val', vi.fn(), {
|
|
502
|
-
onPhaseTiming,
|
|
503
|
-
});
|
|
504
|
-
}
|
|
505
|
-
finally {
|
|
506
|
-
performanceNow.mockRestore();
|
|
507
|
-
}
|
|
508
|
-
expect(onPhaseTiming).toHaveBeenCalledWith('connect', 25);
|
|
509
|
-
});
|
|
510
|
-
it('falls back to direct sessions when user credentials are required', async () => {
|
|
511
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
512
|
-
createInstanceMock.mockImplementation(async () => createMockConnection(streamMock));
|
|
513
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({ ...credentials, requireUserCredentials: true }, { enableInstanceCache: true, projectUuid: 'project-a' });
|
|
514
|
-
await client.runQuery('SELECT 1 AS val');
|
|
515
|
-
await client.runQuery('SELECT 1 AS val');
|
|
516
|
-
expect(createInstanceMock).toHaveBeenCalledTimes(2);
|
|
517
|
-
});
|
|
518
|
-
it('retries one stale failure before rows are emitted and replaces only the failed entry', async () => {
|
|
519
|
-
const events = [];
|
|
520
|
-
MotherduckInstanceCache.setObserver((event) => events.push(event));
|
|
521
|
-
const staleError = new Error('Connection Error: Connection has already been closed');
|
|
522
|
-
const firstStream = vi.fn().mockRejectedValue(staleError);
|
|
523
|
-
const recoveredStream = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
524
|
-
const firstInstance = createMockConnection(firstStream);
|
|
525
|
-
const recoveredInstance = createMockConnection(recoveredStream);
|
|
526
|
-
createInstanceMock
|
|
527
|
-
.mockResolvedValueOnce(firstInstance)
|
|
528
|
-
.mockResolvedValueOnce(recoveredInstance);
|
|
529
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(credentials, {
|
|
530
|
-
enableInstanceCache: true,
|
|
531
|
-
projectUuid: 'project-a',
|
|
532
|
-
});
|
|
533
|
-
await expect(client.runQuery('SELECT 1 AS val')).resolves.toEqual(expect.objectContaining({ rows: [{ val: 1 }] }));
|
|
534
|
-
expect(createInstanceMock).toHaveBeenCalledTimes(2);
|
|
535
|
-
expect(firstInstance.closeSync).toHaveBeenCalledOnce();
|
|
536
|
-
expect(events).toContainEqual(expect.objectContaining({
|
|
537
|
-
type: 'evict',
|
|
538
|
-
reason: 'stale',
|
|
539
|
-
}));
|
|
540
|
-
expect(events).toContainEqual(expect.objectContaining({
|
|
541
|
-
type: 'retry',
|
|
542
|
-
outcome: 'recovered',
|
|
543
|
-
}));
|
|
544
|
-
});
|
|
545
|
-
it('reports a failed retry and does not loop on a second stale failure', async () => {
|
|
546
|
-
const events = [];
|
|
547
|
-
MotherduckInstanceCache.setObserver((event) => events.push(event));
|
|
548
|
-
const staleError = new Error('Connection Error: Connection has already been closed');
|
|
549
|
-
const firstInstance = createMockConnection(vi.fn().mockRejectedValue(staleError));
|
|
550
|
-
const secondInstance = createMockConnection(vi.fn().mockRejectedValue(staleError));
|
|
551
|
-
createInstanceMock
|
|
552
|
-
.mockResolvedValueOnce(firstInstance)
|
|
553
|
-
.mockResolvedValueOnce(secondInstance);
|
|
554
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(credentials, {
|
|
555
|
-
enableInstanceCache: true,
|
|
556
|
-
projectUuid: 'project-a',
|
|
557
|
-
});
|
|
558
|
-
await expect(client.runQuery('SELECT 1 AS val')).rejects.toThrow(staleError);
|
|
559
|
-
expect(createInstanceMock).toHaveBeenCalledTimes(2);
|
|
560
|
-
expect(firstInstance.closeSync).toHaveBeenCalledOnce();
|
|
561
|
-
expect(secondInstance.closeSync).toHaveBeenCalledOnce();
|
|
562
|
-
expect(events).toContainEqual(expect.objectContaining({
|
|
563
|
-
type: 'retry',
|
|
564
|
-
outcome: 'failed',
|
|
565
|
-
}));
|
|
566
|
-
});
|
|
567
|
-
it('does not retry a stale failure after rows reached the consumer', async () => {
|
|
568
|
-
const events = [];
|
|
569
|
-
MotherduckInstanceCache.setObserver((event) => events.push(event));
|
|
570
|
-
const staleError = new Error('Invalid Input Error: Cannot execute statement of closed connection');
|
|
571
|
-
const streamMock = vi.fn(async () => {
|
|
572
|
-
let hasYielded = false;
|
|
573
|
-
const iterator = {
|
|
574
|
-
next: async () => {
|
|
575
|
-
if (!hasYielded) {
|
|
576
|
-
hasYielded = true;
|
|
577
|
-
return { done: false, value: [{ val: 1 }] };
|
|
578
|
-
}
|
|
579
|
-
throw staleError;
|
|
580
|
-
},
|
|
581
|
-
[Symbol.asyncIterator]() {
|
|
582
|
-
return this;
|
|
583
|
-
},
|
|
584
|
-
};
|
|
585
|
-
return {
|
|
586
|
-
columnCount: 1,
|
|
587
|
-
columnNames: () => ['val'],
|
|
588
|
-
columnType: () => ({ typeId: DUCKDB_TYPE_IDS.INTEGER }),
|
|
589
|
-
yieldRowObjectJson: () => iterator,
|
|
590
|
-
};
|
|
591
|
-
});
|
|
592
|
-
const instance = createMockConnection(streamMock);
|
|
593
|
-
createInstanceMock.mockResolvedValue(instance);
|
|
594
|
-
const streamCallback = vi.fn();
|
|
595
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(credentials, {
|
|
596
|
-
enableInstanceCache: true,
|
|
597
|
-
projectUuid: 'project-a',
|
|
598
|
-
});
|
|
599
|
-
await expect(client.streamQuery('SELECT 1 AS val', streamCallback)).rejects.toThrow(staleError);
|
|
600
|
-
expect(streamCallback).toHaveBeenCalledOnce();
|
|
601
|
-
expect(createInstanceMock).toHaveBeenCalledOnce();
|
|
602
|
-
expect(instance.closeSync).toHaveBeenCalledOnce();
|
|
603
|
-
expect(events).toContainEqual(expect.objectContaining({
|
|
604
|
-
type: 'evict',
|
|
605
|
-
reason: 'stale',
|
|
606
|
-
}));
|
|
607
|
-
expect(events).not.toContainEqual(expect.objectContaining({ type: 'retry' }));
|
|
608
|
-
});
|
|
609
|
-
it('invalidates authentication failures without retrying', async () => {
|
|
610
|
-
const events = [];
|
|
611
|
-
MotherduckInstanceCache.setObserver((event) => events.push(event));
|
|
612
|
-
const authError = new Error('Invalid Input Error: MD Authentication Error: Invalid token');
|
|
613
|
-
const streamMock = vi.fn().mockRejectedValue(authError);
|
|
614
|
-
const instance = createMockConnection(streamMock);
|
|
615
|
-
createInstanceMock.mockResolvedValue(instance);
|
|
616
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(credentials, {
|
|
617
|
-
enableInstanceCache: true,
|
|
618
|
-
projectUuid: 'project-a',
|
|
619
|
-
});
|
|
620
|
-
await expect(client.runQuery('SELECT 1 AS val')).rejects.toThrow(authError);
|
|
621
|
-
expect(createInstanceMock).toHaveBeenCalledOnce();
|
|
622
|
-
expect(instance.closeSync).toHaveBeenCalledOnce();
|
|
623
|
-
expect(events).toContainEqual(expect.objectContaining({
|
|
624
|
-
type: 'evict',
|
|
625
|
-
reason: 'auth',
|
|
626
|
-
}));
|
|
627
|
-
expect(events).not.toContainEqual(expect.objectContaining({ type: 'retry' }));
|
|
628
|
-
});
|
|
629
|
-
it('never includes the cache digest in cached-instance log lines or metadata', async () => {
|
|
630
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
631
|
-
const logger = { info: vi.fn() };
|
|
632
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
|
633
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(credentials, {
|
|
634
|
-
enableInstanceCache: true,
|
|
635
|
-
projectUuid: 'project-a',
|
|
636
|
-
logger,
|
|
637
|
-
});
|
|
638
|
-
await client.runQuery('SELECT 1 AS val');
|
|
639
|
-
const connectionString = createInstanceMock.mock.calls[0][0];
|
|
640
|
-
if (typeof connectionString !== 'string') {
|
|
641
|
-
throw new Error('Expected a MotherDuck connection string');
|
|
642
|
-
}
|
|
643
|
-
const digest = (0, crypto_1.createHash)('sha256')
|
|
644
|
-
.update(JSON.stringify({ connectionString, v: 1 }))
|
|
645
|
-
.digest('hex');
|
|
646
|
-
expect(JSON.stringify(logger.info.mock.calls)).not.toContain(digest);
|
|
647
|
-
});
|
|
648
|
-
});
|
|
649
|
-
describe('session strategy routing', () => {
|
|
650
|
-
const createSuccessfulInstance = () => createMockConnection(vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER])), vi.fn());
|
|
651
|
-
it('keeps embedded playground databases on direct read-only sessions', async () => {
|
|
652
|
-
createInstanceMock.mockResolvedValue(createSuccessfulInstance());
|
|
653
|
-
const onPhaseTiming = vi.fn();
|
|
654
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
655
|
-
type: common_1.WarehouseTypes.DUCKDB,
|
|
656
|
-
connectionType: common_1.DuckdbConnectionType.EMBEDDED,
|
|
657
|
-
dataset: 'jaffle_shop',
|
|
658
|
-
});
|
|
659
|
-
await client.streamQuery('SELECT 1 AS val', vi.fn(), {
|
|
660
|
-
onPhaseTiming,
|
|
661
|
-
});
|
|
662
|
-
expect(createInstanceMock).toHaveBeenCalledWith(expect.stringContaining('jaffle_shop.duckdb'), expect.objectContaining({ access_mode: 'READ_ONLY' }));
|
|
663
|
-
expect(onPhaseTiming).toHaveBeenCalledWith('connect', expect.any(Number));
|
|
664
|
-
});
|
|
665
|
-
it('routes resource-limited in-memory clients to isolated sessions', async () => {
|
|
666
|
-
createInstanceMock.mockResolvedValue(createSuccessfulInstance());
|
|
667
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(undefined, {
|
|
668
|
-
resourceLimits: { memoryLimit: '64MB', threads: 1 },
|
|
669
|
-
});
|
|
670
|
-
await client.runQuery('SELECT 1 AS val');
|
|
671
|
-
await client.runQuery('SELECT 1 AS val');
|
|
672
|
-
expect(createInstanceMock).toHaveBeenCalledTimes(2);
|
|
673
|
-
expect(createInstanceMock).toHaveBeenNthCalledWith(1, ':memory:');
|
|
674
|
-
expect(createInstanceMock).toHaveBeenNthCalledWith(2, ':memory:');
|
|
675
|
-
});
|
|
676
|
-
it('routes explicit cache keys to shared sessions', async () => {
|
|
677
|
-
createInstanceMock.mockResolvedValue(createSuccessfulInstance());
|
|
678
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(undefined, {
|
|
679
|
-
instanceCacheKey: 'routing-shared-instance',
|
|
680
|
-
});
|
|
681
|
-
await client.runQuery('SELECT 1 AS val');
|
|
682
|
-
await client.runQuery('SELECT 1 AS val');
|
|
683
|
-
expect(createInstanceMock).toHaveBeenCalledExactlyOnceWith(':memory:');
|
|
684
|
-
});
|
|
685
|
-
it('keeps DuckLake on its existing shared session strategy', async () => {
|
|
686
|
-
createInstanceMock.mockResolvedValue(createSuccessfulInstance());
|
|
687
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
688
|
-
type: common_1.WarehouseTypes.DUCKDB,
|
|
689
|
-
connectionType: common_1.DuckdbConnectionType.DUCKLAKE,
|
|
690
|
-
schema: 'main',
|
|
691
|
-
catalogAlias: 'ducklake',
|
|
692
|
-
catalog: {
|
|
693
|
-
type: common_1.DucklakeCatalogType.POSTGRES,
|
|
694
|
-
host: 'pg.example.com',
|
|
695
|
-
port: 5432,
|
|
696
|
-
database: 'catalog',
|
|
697
|
-
user: 'ducklake_user',
|
|
698
|
-
password: 'password',
|
|
699
|
-
},
|
|
700
|
-
dataPath: {
|
|
701
|
-
type: common_1.DucklakeDataPathType.S3,
|
|
702
|
-
url: 's3://bucket/path/',
|
|
703
|
-
region: 'us-east-1',
|
|
704
|
-
},
|
|
705
|
-
});
|
|
706
|
-
await client.runQuery('SELECT 1 AS val');
|
|
707
|
-
await client.runQuery('SELECT 1 AS val');
|
|
708
|
-
expect(createInstanceMock).toHaveBeenCalledExactlyOnceWith(':memory:');
|
|
709
|
-
});
|
|
710
|
-
it('keeps default in-memory clients on ephemeral sessions', async () => {
|
|
711
|
-
createInstanceMock.mockResolvedValue(createSuccessfulInstance());
|
|
712
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
713
|
-
await client.runQuery('SELECT 1 AS val');
|
|
714
|
-
await client.runQuery('SELECT 1 AS val');
|
|
715
|
-
expect(createInstanceMock).toHaveBeenCalledTimes(2);
|
|
716
|
-
expect(createInstanceMock).toHaveBeenNthCalledWith(1, ':memory:');
|
|
717
|
-
expect(createInstanceMock).toHaveBeenNthCalledWith(2, ':memory:');
|
|
718
|
-
});
|
|
719
|
-
});
|
|
720
|
-
it('sets timezone for in-memory and embedded DuckDB queries', async () => {
|
|
721
|
-
const runMock = vi.fn();
|
|
722
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
723
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
724
|
-
const inMemoryClient = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
725
|
-
await inMemoryClient.runQuery('SELECT 1 AS val', undefined, 'Europe/London');
|
|
726
|
-
expect(runMock).toHaveBeenCalledWith("SET TimeZone = 'Europe/London';");
|
|
727
|
-
runMock.mockClear();
|
|
728
|
-
const embeddedClient = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
729
|
-
type: common_1.WarehouseTypes.DUCKDB,
|
|
730
|
-
connectionType: common_1.DuckdbConnectionType.EMBEDDED,
|
|
731
|
-
dataset: 'jaffle_shop',
|
|
732
|
-
});
|
|
733
|
-
await embeddedClient.runQuery('SELECT 2 AS val', undefined, 'America/New_York');
|
|
734
|
-
expect(runMock).toHaveBeenCalledWith("SET TimeZone = 'America/New_York';");
|
|
735
|
-
});
|
|
736
|
-
it('should set timezone, S3 config, and shared resource limits before streaming', async () => {
|
|
737
|
-
const runMock = vi.fn();
|
|
738
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
739
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
740
|
-
const config = {
|
|
741
|
-
type: 'duckdb_s3',
|
|
742
|
-
s3Config: {
|
|
743
|
-
endpoint: 'localhost:9000',
|
|
744
|
-
region: 'us-east-1',
|
|
745
|
-
accessKey: 'key',
|
|
746
|
-
secretKey: 'secret',
|
|
747
|
-
forcePathStyle: true,
|
|
748
|
-
useSsl: false,
|
|
749
|
-
},
|
|
750
|
-
};
|
|
751
|
-
const clientA = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(config, {
|
|
752
|
-
sharedResourceLimits: {
|
|
753
|
-
memoryLimit: '3GB',
|
|
754
|
-
threads: 4,
|
|
755
|
-
},
|
|
756
|
-
instanceCacheKey: 'pre-aggregate-query-instance',
|
|
757
|
-
});
|
|
758
|
-
const clientB = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(config, {
|
|
759
|
-
sharedResourceLimits: {
|
|
760
|
-
memoryLimit: '3GB',
|
|
761
|
-
threads: 4,
|
|
762
|
-
},
|
|
763
|
-
instanceCacheKey: 'pre-aggregate-query-instance',
|
|
764
|
-
});
|
|
765
|
-
await clientA.runQuery('SELECT 1 AS val');
|
|
766
|
-
await clientB.runQuery('SELECT 1 AS val');
|
|
767
|
-
expect(createInstanceMock).toHaveBeenCalledTimes(1);
|
|
768
|
-
expect(runMock.mock.calls.filter(([sql]) => sql.includes('CREATE OR REPLACE SECRET __lightdash_s3'))).toHaveLength(1);
|
|
769
|
-
expect(streamMock).toHaveBeenCalledTimes(2);
|
|
770
|
-
});
|
|
771
|
-
it('should use DuckDB credential chain for S3 config without static credentials', async () => {
|
|
772
|
-
const runMock = vi.fn();
|
|
773
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
774
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
775
|
-
const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate({
|
|
776
|
-
type: 'duckdb_s3',
|
|
777
|
-
s3Config: {
|
|
778
|
-
endpoint: 's3.eu-west-1.amazonaws.com',
|
|
779
|
-
region: 'eu-west-1',
|
|
780
|
-
forcePathStyle: false,
|
|
781
|
-
useSsl: true,
|
|
782
|
-
},
|
|
783
|
-
});
|
|
784
|
-
await client.runQuery('SELECT 1 AS val');
|
|
785
|
-
expect(runMock).toHaveBeenCalledWith('INSTALL aws;');
|
|
786
|
-
expect(runMock).toHaveBeenCalledWith('LOAD aws;');
|
|
787
|
-
const secretSql = runMock.mock.calls
|
|
788
|
-
.map(([sql]) => sql)
|
|
789
|
-
.find((sql) => sql.includes('CREATE OR REPLACE SECRET __lightdash_s3'));
|
|
790
|
-
expect(secretSql).toContain('PROVIDER credential_chain');
|
|
791
|
-
expect(secretSql).toContain('REFRESH auto');
|
|
792
|
-
expect(secretSql).toContain("VALIDATION 'none'");
|
|
793
|
-
expect(secretSql).toContain("REGION 'eu-west-1'");
|
|
794
|
-
expect(secretSql).toContain("ENDPOINT 's3.eu-west-1.amazonaws.com'");
|
|
795
|
-
expect(secretSql).not.toContain('KEY_ID');
|
|
796
|
-
expect(secretSql).not.toContain("SECRET '");
|
|
797
|
-
});
|
|
798
|
-
it('scopes S3 credentials to the trusted external-source object', async () => {
|
|
799
|
-
const runMock = vi.fn();
|
|
800
|
-
createInstanceMock.mockResolvedValue(createMockConnection(vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER])), runMock));
|
|
801
|
-
const scope = [
|
|
802
|
-
's3://bucket/external-sources/project/source/table/v1.parquet',
|
|
803
|
-
's3://bucket/external-sources/project/source/table/v2.parquet',
|
|
804
|
-
];
|
|
805
|
-
const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate({
|
|
806
|
-
type: 'duckdb_s3',
|
|
807
|
-
s3Config: {
|
|
808
|
-
endpoint: 's3.amazonaws.com',
|
|
809
|
-
forcePathStyle: false,
|
|
810
|
-
useSsl: true,
|
|
811
|
-
scope,
|
|
812
|
-
},
|
|
813
|
-
});
|
|
814
|
-
await client.runQuery('SELECT 1');
|
|
815
|
-
const secretSql = runMock.mock.calls
|
|
816
|
-
.map(([sql]) => sql)
|
|
817
|
-
.find((sql) => sql.includes('CREATE OR REPLACE SECRET __lightdash_s3'));
|
|
818
|
-
expect(secretSql).toContain(`SCOPE ('${scope[0]}', '${scope[1]}')`);
|
|
819
|
-
});
|
|
820
|
-
it('caps isolated S3 queries per organization', async () => {
|
|
821
|
-
let resolveStream = () => { };
|
|
822
|
-
const streamMock = vi.fn(() => new Promise((resolve) => {
|
|
823
|
-
resolveStream = resolve;
|
|
824
|
-
}));
|
|
825
|
-
createInstanceMock.mockImplementation(async () => createMockConnection(streamMock));
|
|
826
|
-
const createClient = () => DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate({
|
|
827
|
-
type: 'duckdb_s3',
|
|
828
|
-
s3Config: {
|
|
829
|
-
endpoint: 's3.amazonaws.com',
|
|
830
|
-
forcePathStyle: false,
|
|
831
|
-
useSsl: true,
|
|
832
|
-
scope: ['s3://bucket/external-sources/object.parquet'],
|
|
833
|
-
},
|
|
834
|
-
}, {
|
|
835
|
-
resourceLimits: { memoryLimit: '128MB', threads: 1 },
|
|
836
|
-
organizationConcurrencyLimit: 1,
|
|
837
|
-
});
|
|
838
|
-
const tags = { organization_uuid: 'external-source-org' };
|
|
839
|
-
const first = createClient().runQuery('SELECT 1', tags);
|
|
840
|
-
await vi.waitFor(() => expect(streamMock).toHaveBeenCalledOnce());
|
|
841
|
-
await expect(createClient().runQuery('SELECT 2', tags)).rejects.toThrow('External source query capacity is full');
|
|
842
|
-
resolveStream(getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
843
|
-
await first;
|
|
844
|
-
});
|
|
845
|
-
it('should load bundled extensions without runtime installs', async () => {
|
|
846
|
-
const accessMock = vi.spyOn(promises_1.default, 'access').mockResolvedValue();
|
|
847
|
-
try {
|
|
848
|
-
const runMock = vi.fn();
|
|
849
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
850
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
851
|
-
const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate({
|
|
852
|
-
type: 'duckdb_s3',
|
|
853
|
-
s3Config: {
|
|
854
|
-
endpoint: 's3.eu-west-1.amazonaws.com',
|
|
855
|
-
region: 'eu-west-1',
|
|
856
|
-
forcePathStyle: false,
|
|
857
|
-
useSsl: true,
|
|
858
|
-
},
|
|
859
|
-
});
|
|
860
|
-
await client.runQuery('SELECT 1 AS val');
|
|
861
|
-
expect(runMock).toHaveBeenCalledWith(expect.stringContaining("duckdbExtensions/v1.5.2/httpfs.duckdb_extension';"));
|
|
862
|
-
expect(runMock).toHaveBeenCalledWith(expect.stringContaining("duckdbExtensions/v1.5.2/aws.duckdb_extension';"));
|
|
863
|
-
expect(runMock).not.toHaveBeenCalledWith('INSTALL httpfs;');
|
|
864
|
-
expect(runMock).not.toHaveBeenCalledWith('INSTALL aws;');
|
|
865
|
-
}
|
|
866
|
-
finally {
|
|
867
|
-
accessMock.mockRestore();
|
|
868
|
-
}
|
|
869
|
-
});
|
|
870
|
-
it('should surface bundled extension load failures', async () => {
|
|
871
|
-
const accessMock = vi.spyOn(promises_1.default, 'access').mockResolvedValue();
|
|
872
|
-
try {
|
|
873
|
-
const loadError = new Error('Invalid bundled extension');
|
|
874
|
-
const runMock = vi.fn(async (sql) => {
|
|
875
|
-
if (sql.includes('httpfs.duckdb_extension')) {
|
|
876
|
-
throw loadError;
|
|
877
|
-
}
|
|
878
|
-
});
|
|
879
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
880
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
881
|
-
const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate({
|
|
882
|
-
type: 'duckdb_s3',
|
|
883
|
-
s3Config: {
|
|
884
|
-
endpoint: 'localhost:9000',
|
|
885
|
-
region: 'us-east-1',
|
|
886
|
-
accessKey: 'key',
|
|
887
|
-
secretKey: 'secret',
|
|
888
|
-
forcePathStyle: true,
|
|
889
|
-
useSsl: false,
|
|
890
|
-
},
|
|
891
|
-
});
|
|
892
|
-
await expect(client.runQuery('SELECT 1 AS val')).rejects.toThrow(loadError);
|
|
893
|
-
expect(runMock).not.toHaveBeenCalledWith('INSTALL httpfs;');
|
|
894
|
-
}
|
|
895
|
-
finally {
|
|
896
|
-
accessMock.mockRestore();
|
|
897
|
-
}
|
|
898
|
-
});
|
|
899
|
-
it('should use static DuckDB S3 credentials when configured', async () => {
|
|
900
|
-
const runMock = vi.fn();
|
|
901
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
902
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
903
|
-
const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate({
|
|
904
|
-
type: 'duckdb_s3',
|
|
905
|
-
s3Config: {
|
|
906
|
-
endpoint: 'localhost:9000',
|
|
907
|
-
region: 'us-east-1',
|
|
908
|
-
accessKey: 'key',
|
|
909
|
-
secretKey: 'secret',
|
|
910
|
-
forcePathStyle: true,
|
|
911
|
-
useSsl: false,
|
|
912
|
-
},
|
|
913
|
-
});
|
|
914
|
-
await client.runQuery('SELECT 1 AS val');
|
|
915
|
-
expect(runMock).not.toHaveBeenCalledWith('INSTALL aws;');
|
|
916
|
-
expect(runMock).not.toHaveBeenCalledWith('LOAD aws;');
|
|
917
|
-
const secretSql = runMock.mock.calls
|
|
918
|
-
.map(([sql]) => sql)
|
|
919
|
-
.find((sql) => sql.includes('CREATE OR REPLACE SECRET __lightdash_s3'));
|
|
920
|
-
expect(secretSql).not.toContain('PROVIDER credential_chain');
|
|
921
|
-
expect(secretSql).toContain("KEY_ID 'key'");
|
|
922
|
-
expect(secretSql).toContain("SECRET 'secret'");
|
|
923
|
-
expect(secretSql).toContain("URL_STYLE 'path'");
|
|
924
|
-
expect(secretSql).toContain('USE_SSL false');
|
|
925
|
-
});
|
|
926
|
-
it('should treat instanceCacheKey as the shared instance identity', async () => {
|
|
927
|
-
const runMock = vi.fn();
|
|
928
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
929
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
930
|
-
const clientA = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
931
|
-
type: 'duckdb_s3',
|
|
932
|
-
s3Config: {
|
|
933
|
-
endpoint: 'localhost:9000',
|
|
934
|
-
region: 'us-east-1',
|
|
935
|
-
accessKey: 'key-a',
|
|
936
|
-
secretKey: 'secret-a',
|
|
937
|
-
forcePathStyle: true,
|
|
938
|
-
useSsl: false,
|
|
939
|
-
},
|
|
940
|
-
}, {
|
|
941
|
-
sharedResourceLimits: {
|
|
942
|
-
memoryLimit: '3GB',
|
|
943
|
-
},
|
|
944
|
-
instanceCacheKey: 'pre-aggregate-query-instance',
|
|
945
|
-
});
|
|
946
|
-
const clientB = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
947
|
-
type: 'duckdb_s3',
|
|
948
|
-
s3Config: {
|
|
949
|
-
endpoint: 'localhost:9000',
|
|
950
|
-
region: 'us-east-1',
|
|
951
|
-
accessKey: 'key-b',
|
|
952
|
-
secretKey: 'secret-b',
|
|
953
|
-
forcePathStyle: true,
|
|
954
|
-
useSsl: false,
|
|
955
|
-
},
|
|
956
|
-
}, {
|
|
957
|
-
sharedResourceLimits: {
|
|
958
|
-
memoryLimit: '6GB',
|
|
959
|
-
threads: 8,
|
|
960
|
-
},
|
|
961
|
-
instanceCacheKey: 'pre-aggregate-query-instance',
|
|
962
|
-
});
|
|
963
|
-
await clientA.runQuery('SELECT 1 AS val');
|
|
964
|
-
await clientB.runQuery('SELECT 1 AS val');
|
|
965
|
-
expect(createInstanceMock).toHaveBeenCalledTimes(1);
|
|
966
|
-
expect(runMock.mock.calls.filter(([sql]) => sql.includes('CREATE OR REPLACE SECRET __lightdash_s3'))).toHaveLength(1);
|
|
967
|
-
expect(runMock).not.toHaveBeenCalledWith("SET memory_limit = '6GB';");
|
|
968
|
-
expect(runMock).not.toHaveBeenCalledWith('SET threads = 8;');
|
|
969
|
-
});
|
|
970
|
-
it('hardens each query connection on a shared instance', async () => {
|
|
971
|
-
const bootstrapRunMock = vi.fn();
|
|
972
|
-
const queryRunMock = vi.fn();
|
|
973
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
974
|
-
const connectMock = vi
|
|
975
|
-
.fn()
|
|
976
|
-
.mockResolvedValueOnce({
|
|
977
|
-
run: bootstrapRunMock,
|
|
978
|
-
closeSync: vi.fn(),
|
|
979
|
-
disconnectSync: vi.fn(),
|
|
980
|
-
})
|
|
981
|
-
.mockResolvedValueOnce({
|
|
982
|
-
run: queryRunMock,
|
|
983
|
-
stream: streamMock,
|
|
984
|
-
extractStatements: createMockExtractStatements(),
|
|
985
|
-
closeSync: vi.fn(),
|
|
986
|
-
disconnectSync: vi.fn(),
|
|
987
|
-
});
|
|
988
|
-
createInstanceMock.mockResolvedValue({
|
|
989
|
-
connect: connectMock,
|
|
990
|
-
closeSync: vi.fn(),
|
|
991
|
-
});
|
|
992
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(undefined, {
|
|
993
|
-
instanceCacheKey: 'shared-query-hardening',
|
|
994
|
-
sharedResourceLimits: { memoryLimit: '256MB', threads: 1 },
|
|
995
|
-
});
|
|
996
|
-
await client.runQuery('SELECT 1 AS val');
|
|
997
|
-
expect(queryRunMock.mock.calls.map(([sql]) => sql)).toEqual([
|
|
998
|
-
"SET memory_limit = '256MB';",
|
|
999
|
-
'SET threads = 1;',
|
|
1000
|
-
'SET allow_community_extensions = false;',
|
|
1001
|
-
'SET autoinstall_known_extensions = false;',
|
|
1002
|
-
'SET autoload_known_extensions = false;',
|
|
1003
|
-
'SET allow_unredacted_secrets = false;',
|
|
1004
|
-
]);
|
|
1005
|
-
});
|
|
1006
|
-
it('points httpfs at the CA bundle the session config names', async () => {
|
|
1007
|
-
const bootstrapRunMock = vi.fn();
|
|
1008
|
-
const queryRunMock = vi.fn();
|
|
1009
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
1010
|
-
const connectMock = vi
|
|
1011
|
-
.fn()
|
|
1012
|
-
.mockResolvedValueOnce({
|
|
1013
|
-
run: bootstrapRunMock,
|
|
1014
|
-
closeSync: vi.fn(),
|
|
1015
|
-
disconnectSync: vi.fn(),
|
|
1016
|
-
})
|
|
1017
|
-
.mockResolvedValueOnce({
|
|
1018
|
-
run: queryRunMock,
|
|
1019
|
-
stream: streamMock,
|
|
1020
|
-
extractStatements: createMockExtractStatements(),
|
|
1021
|
-
closeSync: vi.fn(),
|
|
1022
|
-
disconnectSync: vi.fn(),
|
|
1023
|
-
});
|
|
1024
|
-
createInstanceMock.mockResolvedValue({
|
|
1025
|
-
connect: connectMock,
|
|
1026
|
-
closeSync: vi.fn(),
|
|
1027
|
-
});
|
|
1028
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
1029
|
-
type: 'duckdb_s3',
|
|
1030
|
-
s3Config: {
|
|
1031
|
-
endpoint: 'results.example.com',
|
|
1032
|
-
region: 'eu-west-1',
|
|
1033
|
-
accessKey: 'key',
|
|
1034
|
-
secretKey: 'secret',
|
|
1035
|
-
forcePathStyle: false,
|
|
1036
|
-
useSsl: true,
|
|
1037
|
-
caCertFile: '/etc/ssl/certs/ca-certificates.crt',
|
|
1038
|
-
},
|
|
1039
|
-
}, { instanceCacheKey: 'ca-bundle-session' });
|
|
1040
|
-
await client.runQuery('SELECT 1 AS val');
|
|
1041
|
-
expect(bootstrapRunMock).toHaveBeenCalledWith("SET ca_cert_file = '/etc/ssl/certs/ca-certificates.crt';");
|
|
1042
|
-
});
|
|
1043
|
-
it('confines embedded user queries without writing profiles to local files', async () => {
|
|
1044
|
-
const runMock = vi.fn();
|
|
1045
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
1046
|
-
const logger = { info: vi.fn() };
|
|
1047
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
1048
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
1049
|
-
type: common_1.WarehouseTypes.DUCKDB,
|
|
1050
|
-
connectionType: common_1.DuckdbConnectionType.EMBEDDED,
|
|
1051
|
-
dataset: 'jaffle_shop',
|
|
1052
|
-
}, { logger });
|
|
1053
|
-
await client.runQuery('SELECT 1 AS val');
|
|
1054
|
-
expect(createInstanceMock).toHaveBeenCalledWith(expect.stringContaining('jaffle_shop.duckdb'), {
|
|
1055
|
-
access_mode: 'READ_ONLY',
|
|
1056
|
-
memory_limit: '256MB',
|
|
1057
|
-
threads: '1',
|
|
1058
|
-
});
|
|
1059
|
-
expect(runMock).toHaveBeenCalledWith("SET disabled_filesystems = 'LocalFileSystem';");
|
|
1060
|
-
expect(runMock).not.toHaveBeenCalledWith("PRAGMA enable_profiling='json';");
|
|
1061
|
-
});
|
|
1062
|
-
it('runs a queued embedded query once a same-organization holder releases', async () => {
|
|
1063
|
-
const pendingStreams = [];
|
|
1064
|
-
const streamMock = vi.fn((_sql) => new Promise((resolve) => {
|
|
1065
|
-
pendingStreams.push(resolve);
|
|
1066
|
-
}));
|
|
1067
|
-
createInstanceMock.mockImplementation(async () => createMockConnection(streamMock));
|
|
1068
|
-
const createClient = () => new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
1069
|
-
type: common_1.WarehouseTypes.DUCKDB,
|
|
1070
|
-
connectionType: common_1.DuckdbConnectionType.EMBEDDED,
|
|
1071
|
-
dataset: 'jaffle_shop',
|
|
1072
|
-
}, { embeddedQueryTimeoutMs: 60000 });
|
|
1073
|
-
const tags = { organization_uuid: 'organization-1' };
|
|
1074
|
-
const queries = Array.from({ length: 6 }, (_, index) => createClient().runQuery(`SELECT ${index + 1} AS val`, tags));
|
|
1075
|
-
await vi.waitFor(() => expect(streamMock).toHaveBeenCalledTimes(4));
|
|
1076
|
-
expect(createInstanceMock).toHaveBeenCalledTimes(4);
|
|
1077
|
-
pendingStreams[0](getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
1078
|
-
await queries[0];
|
|
1079
|
-
await vi.waitFor(() => expect(streamMock).toHaveBeenCalledTimes(5));
|
|
1080
|
-
expect(createInstanceMock).toHaveBeenCalledTimes(5);
|
|
1081
|
-
expect(streamMock.mock.calls[4]?.[0]).toContain('SELECT 5 AS val');
|
|
1082
|
-
pendingStreams[1](getMockStreamResult([[{ val: 2 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
1083
|
-
await queries[1];
|
|
1084
|
-
await vi.waitFor(() => expect(streamMock).toHaveBeenCalledTimes(6));
|
|
1085
|
-
expect(createInstanceMock).toHaveBeenCalledTimes(6);
|
|
1086
|
-
expect(streamMock.mock.calls[5]?.[0]).toContain('SELECT 6 AS val');
|
|
1087
|
-
pendingStreams.forEach((resolve) => resolve(getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER])));
|
|
1088
|
-
await Promise.all(queries.slice(1));
|
|
1089
|
-
});
|
|
1090
|
-
it('times out a queued embedded query with the existing capacity error', async () => {
|
|
1091
|
-
vi.useFakeTimers();
|
|
1092
|
-
const pendingStreams = [];
|
|
1093
|
-
const streamMock = vi.fn(() => new Promise((resolve) => {
|
|
1094
|
-
pendingStreams.push(resolve);
|
|
1095
|
-
}));
|
|
1096
|
-
createInstanceMock.mockImplementation(async () => createMockConnection(streamMock));
|
|
1097
|
-
const createClient = () => new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
1098
|
-
type: common_1.WarehouseTypes.DUCKDB,
|
|
1099
|
-
connectionType: common_1.DuckdbConnectionType.EMBEDDED,
|
|
1100
|
-
dataset: 'jaffle_shop',
|
|
1101
|
-
}, { embeddedQueryTimeoutMs: 60000 });
|
|
1102
|
-
const tags = { organization_uuid: 'organization-1' };
|
|
1103
|
-
const activeQueries = Array.from({ length: 4 }, (_, index) => createClient().runQuery(`SELECT ${index + 1} AS val`, tags));
|
|
1104
|
-
await vi.waitFor(() => expect(streamMock).toHaveBeenCalledTimes(4));
|
|
1105
|
-
const queuedQuery = createClient().runQuery('SELECT 5 AS val', tags);
|
|
1106
|
-
const capacityError = queuedQuery.catch((error) => error);
|
|
1107
|
-
await vi.advanceTimersByTimeAsync(15000);
|
|
1108
|
-
const error = await capacityError;
|
|
1109
|
-
expect(error).toBeInstanceOf(common_1.WarehouseQueryError);
|
|
1110
|
-
expect(error).toMatchObject({
|
|
1111
|
-
message: 'Playground query capacity is full. Try again shortly.',
|
|
1112
|
-
});
|
|
1113
|
-
expect(createInstanceMock).toHaveBeenCalledTimes(4);
|
|
1114
|
-
pendingStreams.forEach((resolve) => resolve(getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER])));
|
|
1115
|
-
await Promise.all(activeQueries);
|
|
1116
|
-
});
|
|
1117
|
-
it('returns embedded capacity when a query throws', async () => {
|
|
1118
|
-
const pendingStreams = [];
|
|
1119
|
-
const streamMock = vi.fn((sql) => {
|
|
1120
|
-
if (sql.includes('SELECT 4')) {
|
|
1121
|
-
return Promise.reject(new Error('Query failed'));
|
|
1122
|
-
}
|
|
1123
|
-
return new Promise((resolve) => {
|
|
1124
|
-
pendingStreams.push(resolve);
|
|
1125
|
-
});
|
|
1126
|
-
});
|
|
1127
|
-
createInstanceMock.mockImplementation(async () => createMockConnection(streamMock));
|
|
1128
|
-
const createClient = () => new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
1129
|
-
type: common_1.WarehouseTypes.DUCKDB,
|
|
1130
|
-
connectionType: common_1.DuckdbConnectionType.EMBEDDED,
|
|
1131
|
-
dataset: 'jaffle_shop',
|
|
1132
|
-
});
|
|
1133
|
-
const tags = { organization_uuid: 'organization-1' };
|
|
1134
|
-
const activeQueries = Array.from({ length: 3 }, (_, index) => createClient().runQuery(`SELECT ${index + 1} AS val`, tags));
|
|
1135
|
-
const failedQuery = createClient().runQuery('SELECT 4 AS val', tags);
|
|
1136
|
-
await expect(failedQuery).rejects.toThrow('Query failed');
|
|
1137
|
-
const nextQuery = createClient().runQuery('SELECT 5 AS val', tags);
|
|
1138
|
-
await vi.waitFor(() => expect(streamMock).toHaveBeenCalledTimes(5));
|
|
1139
|
-
pendingStreams.forEach((resolve) => resolve(getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER])));
|
|
1140
|
-
await Promise.all([...activeQueries, nextQuery]);
|
|
1141
|
-
});
|
|
1142
|
-
it('returns the global slot when organization capacity times out', async () => {
|
|
1143
|
-
vi.useFakeTimers();
|
|
1144
|
-
const pendingStreams = [];
|
|
1145
|
-
const streamMock = vi.fn(() => new Promise((resolve) => {
|
|
1146
|
-
pendingStreams.push(resolve);
|
|
1147
|
-
}));
|
|
1148
|
-
createInstanceMock.mockImplementation(async () => createMockConnection(streamMock));
|
|
1149
|
-
const createClient = () => new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
1150
|
-
type: common_1.WarehouseTypes.DUCKDB,
|
|
1151
|
-
connectionType: common_1.DuckdbConnectionType.EMBEDDED,
|
|
1152
|
-
dataset: 'jaffle_shop',
|
|
1153
|
-
}, { embeddedQueryTimeoutMs: 60000 });
|
|
1154
|
-
const organizationOneTags = {
|
|
1155
|
-
organization_uuid: 'organization-1',
|
|
1156
|
-
};
|
|
1157
|
-
const organizationTwoTags = {
|
|
1158
|
-
organization_uuid: 'organization-2',
|
|
1159
|
-
};
|
|
1160
|
-
const organizationOneQueries = Array.from({ length: 4 }, (_, index) => createClient().runQuery(`SELECT ${index + 1} AS val`, organizationOneTags));
|
|
1161
|
-
await vi.waitFor(() => expect(streamMock).toHaveBeenCalledTimes(4));
|
|
1162
|
-
const timedOutQuery = createClient().runQuery('SELECT 5 AS val', organizationOneTags);
|
|
1163
|
-
const capacityError = timedOutQuery.catch((error) => error);
|
|
1164
|
-
await vi.advanceTimersByTimeAsync(15000);
|
|
1165
|
-
await expect(capacityError).resolves.toMatchObject({
|
|
1166
|
-
name: 'WarehouseQueryError',
|
|
1167
|
-
message: 'Playground query capacity is full. Try again shortly.',
|
|
1168
|
-
});
|
|
1169
|
-
const organizationTwoQueries = Array.from({ length: 4 }, (_, index) => createClient().runQuery(`SELECT ${index + 6} AS val`, organizationTwoTags));
|
|
1170
|
-
await vi.waitFor(() => expect(streamMock).toHaveBeenCalledTimes(8));
|
|
1171
|
-
pendingStreams.forEach((resolve) => resolve(getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER])));
|
|
1172
|
-
await Promise.all([
|
|
1173
|
-
...organizationOneQueries,
|
|
1174
|
-
...organizationTwoQueries,
|
|
1175
|
-
]);
|
|
1176
|
-
});
|
|
1177
|
-
it('logs structured profiles for non-embedded queries and reports metrics', async () => {
|
|
1178
|
-
const runMock = vi.fn(async (sql) => {
|
|
1179
|
-
const match = sql.match(/^PRAGMA profiling_output='(.+)';$/);
|
|
1180
|
-
if (match) {
|
|
1181
|
-
await promises_1.default.writeFile(match[1], JSON.stringify({
|
|
1182
|
-
latency: 4.747,
|
|
1183
|
-
cpu_time: 4.731,
|
|
1184
|
-
rows_returned: 68,
|
|
1185
|
-
total_bytes_read: 20225287,
|
|
1186
|
-
children: [
|
|
1187
|
-
{
|
|
1188
|
-
operator_name: 'READ_PARQUET ',
|
|
1189
|
-
operator_timing: 4.632,
|
|
1190
|
-
operator_cardinality: 9905024,
|
|
1191
|
-
children: [],
|
|
1192
|
-
},
|
|
1193
|
-
],
|
|
1194
|
-
}));
|
|
1195
|
-
}
|
|
1196
|
-
});
|
|
1197
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
1198
|
-
const logger = { info: vi.fn() };
|
|
1199
|
-
const onQueryProfile = vi.fn();
|
|
1200
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
1201
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(undefined, {
|
|
1202
|
-
logger,
|
|
1203
|
-
enableQueryProfiling: true,
|
|
1204
|
-
onQueryProfile,
|
|
1205
|
-
});
|
|
1206
|
-
await client.runQuery('SELECT 1 AS val', {
|
|
1207
|
-
query_uuid: 'query-123',
|
|
1208
|
-
chart_uuid: 'chart-123',
|
|
1209
|
-
dashboard_uuid: 'dashboard-123',
|
|
1210
|
-
query_context: common_1.QueryExecutionContext.DASHBOARD,
|
|
1211
|
-
});
|
|
1212
|
-
expect(logger.info).toHaveBeenCalledWith(expect.stringContaining('DuckDB query profile: latency=4747ms cpu=4731ms'), expect.objectContaining({
|
|
1213
|
-
query_uuid: 'query-123',
|
|
1214
|
-
chart_uuid: 'chart-123',
|
|
1215
|
-
dashboard_uuid: 'dashboard-123',
|
|
1216
|
-
query_context: common_1.QueryExecutionContext.DASHBOARD,
|
|
1217
|
-
latencyMs: 4747,
|
|
1218
|
-
cpuMs: 4731,
|
|
1219
|
-
waitMs: 16,
|
|
1220
|
-
readParquetMs: 4632,
|
|
1221
|
-
rowsScanned: 9905024,
|
|
1222
|
-
rowsReturned: 68,
|
|
1223
|
-
bytesRead: 20225287,
|
|
1224
|
-
scanAmplification: 9905024 / 68,
|
|
1225
|
-
}));
|
|
1226
|
-
expect(onQueryProfile).toHaveBeenCalledWith(expect.objectContaining({
|
|
1227
|
-
latencyMs: 4747,
|
|
1228
|
-
readParquetMs: 4632,
|
|
1229
|
-
bytesRead: 20225287,
|
|
1230
|
-
}));
|
|
1231
|
-
expect(runMock).not.toHaveBeenCalledWith("SET disabled_filesystems = 'LocalFileSystem';");
|
|
1232
|
-
expect(runMock).toHaveBeenCalledWith("PRAGMA enable_profiling='json';");
|
|
1233
|
-
});
|
|
1234
|
-
it('does not enable query profiling for a logger without the opt-in flag', async () => {
|
|
1235
|
-
const runMock = vi.fn();
|
|
1236
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
1237
|
-
const logger = { info: vi.fn() };
|
|
1238
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
1239
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(undefined, { logger });
|
|
1240
|
-
await client.runQuery('SELECT 1 AS val');
|
|
1241
|
-
expect(runMock).not.toHaveBeenCalledWith("PRAGMA enable_profiling='json';");
|
|
1242
|
-
expect(logger.info).not.toHaveBeenCalledWith(expect.stringContaining('DuckDB query profile:'), expect.anything());
|
|
1243
|
-
});
|
|
1244
|
-
it('logs raw profile timings when DuckDB reports cpu above latency', async () => {
|
|
1245
|
-
const runMock = vi.fn(async (sql) => {
|
|
1246
|
-
const match = sql.match(/^PRAGMA profiling_output='(.+)';$/);
|
|
1247
|
-
if (match) {
|
|
1248
|
-
await promises_1.default.writeFile(match[1], JSON.stringify({
|
|
1249
|
-
latency: 0.002,
|
|
1250
|
-
cpu_time: 0.004,
|
|
1251
|
-
rows_returned: 123,
|
|
1252
|
-
total_bytes_read: 0,
|
|
1253
|
-
children: [],
|
|
1254
|
-
}));
|
|
1255
|
-
}
|
|
1256
|
-
});
|
|
1257
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
1258
|
-
const logger = { info: vi.fn() };
|
|
1259
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
1260
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(undefined, {
|
|
1261
|
-
logger,
|
|
1262
|
-
enableQueryProfiling: true,
|
|
1263
|
-
});
|
|
1264
|
-
await client.runQuery('SELECT 1 AS val');
|
|
1265
|
-
expect(logger.info).toHaveBeenCalledWith(expect.stringContaining('DuckDB query profile: latency=2ms cpu=4ms wait=-2ms'), expect.objectContaining({
|
|
1266
|
-
latencyMs: 2,
|
|
1267
|
-
cpuMs: 4,
|
|
1268
|
-
waitMs: -2,
|
|
1269
|
-
}));
|
|
1270
|
-
});
|
|
1271
|
-
describe('SQL security validation', () => {
|
|
1272
|
-
it.each([
|
|
1273
|
-
{
|
|
1274
|
-
name: 'SET statements',
|
|
1275
|
-
sql: "SET s3_endpoint = 'attacker.com'",
|
|
1276
|
-
statementType: 20,
|
|
1277
|
-
},
|
|
1278
|
-
{
|
|
1279
|
-
name: 'COPY statements',
|
|
1280
|
-
sql: "COPY t TO '/tmp/data.csv'",
|
|
1281
|
-
statementType: 11,
|
|
1282
|
-
},
|
|
1283
|
-
{
|
|
1284
|
-
name: 'ATTACH statements',
|
|
1285
|
-
sql: "ATTACH DATABASE 'file.db'",
|
|
1286
|
-
statementType: 25,
|
|
1287
|
-
},
|
|
1288
|
-
{
|
|
1289
|
-
name: 'EXPLAIN statements',
|
|
1290
|
-
sql: 'EXPLAIN SELECT * FROM users',
|
|
1291
|
-
statementType: 4,
|
|
1292
|
-
},
|
|
1293
|
-
])('should reject $name', async ({ sql, statementType }) => {
|
|
1294
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
1295
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, vi.fn(), {
|
|
1296
|
-
extractStatements: createMockExtractStatements({
|
|
1297
|
-
statementType,
|
|
1298
|
-
}),
|
|
1299
|
-
}));
|
|
1300
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
1301
|
-
await expect(client.runQuery(sql)).rejects.toThrow('SQL validation error: only SELECT statements are allowed');
|
|
1302
|
-
expect(streamMock).not.toHaveBeenCalled();
|
|
1303
|
-
});
|
|
1304
|
-
it('should reject multiple statements', async () => {
|
|
1305
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
1306
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, vi.fn(), {
|
|
1307
|
-
extractStatements: createMockExtractStatements({
|
|
1308
|
-
count: 2,
|
|
1309
|
-
}),
|
|
1310
|
-
}));
|
|
1311
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
1312
|
-
await expect(client.runQuery('SELECT 1; DROP TABLE users')).rejects.toThrow('SQL validation error: multiple SQL statements are not allowed');
|
|
1313
|
-
});
|
|
1314
|
-
it.each([
|
|
1315
|
-
'current_setting',
|
|
1316
|
-
'duckdb_settings',
|
|
1317
|
-
'duckdb_secrets',
|
|
1318
|
-
'query',
|
|
1319
|
-
'query_table',
|
|
1320
|
-
])('should reject queries with %s()', async (blockedFunction) => {
|
|
1321
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
1322
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
|
1323
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
1324
|
-
await expect(client.runQuery(`SELECT * FROM ${blockedFunction}()`)).rejects.toThrow(`SQL validation error: function '${blockedFunction}' is not allowed`);
|
|
1325
|
-
});
|
|
1326
|
-
it.each([
|
|
1327
|
-
'read_csv',
|
|
1328
|
-
'read_csv_auto',
|
|
1329
|
-
'read_json',
|
|
1330
|
-
'read_json_auto',
|
|
1331
|
-
'read_json_objects',
|
|
1332
|
-
'read_json_objects_auto',
|
|
1333
|
-
'read_ndjson',
|
|
1334
|
-
'read_ndjson_auto',
|
|
1335
|
-
'read_ndjson_objects',
|
|
1336
|
-
'read_ndjson_objects_auto',
|
|
1337
|
-
'read_parquet',
|
|
1338
|
-
'read_text',
|
|
1339
|
-
'read_blob',
|
|
1340
|
-
'read_xlsx',
|
|
1341
|
-
'parquet_scan',
|
|
1342
|
-
'parquet_metadata',
|
|
1343
|
-
'parquet_schema',
|
|
1344
|
-
'parquet_file_metadata',
|
|
1345
|
-
'parquet_kv_metadata',
|
|
1346
|
-
'parquet_bloom_probe',
|
|
1347
|
-
'sniff_csv',
|
|
1348
|
-
'glob',
|
|
1349
|
-
'sqlite_scan',
|
|
1350
|
-
'postgres_scan',
|
|
1351
|
-
'mysql_scan',
|
|
1352
|
-
])('should reject user queries with %s()', async (blockedFunction) => {
|
|
1353
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
1354
|
-
const extractStatementsMock = createMockExtractStatements();
|
|
1355
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, vi.fn(), {
|
|
1356
|
-
extractStatements: extractStatementsMock,
|
|
1357
|
-
}));
|
|
1358
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
1359
|
-
await expect(client.runQuery(`SELECT * FROM ${blockedFunction}('/tmp/a')`)).rejects.toThrow(`SQL validation error: function '${blockedFunction}' is not allowed`);
|
|
1360
|
-
expect(extractStatementsMock).not.toHaveBeenCalled();
|
|
1361
|
-
expect(streamMock).not.toHaveBeenCalled();
|
|
1362
|
-
});
|
|
1363
|
-
it('should keep file readers blocked when only S3 is configured', async () => {
|
|
1364
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
1365
|
-
const extractStatementsMock = createMockExtractStatements();
|
|
1366
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, vi.fn(), {
|
|
1367
|
-
extractStatements: extractStatementsMock,
|
|
1368
|
-
}));
|
|
1369
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(duckdbS3Credentials);
|
|
1370
|
-
await expect(client.runQuery("SELECT * FROM read_parquet('s3://bucket/data.parquet')")).rejects.toThrow("SQL validation error: function 'read_parquet' is not allowed");
|
|
1371
|
-
expect(extractStatementsMock).not.toHaveBeenCalled();
|
|
1372
|
-
expect(streamMock).not.toHaveBeenCalled();
|
|
1373
|
-
});
|
|
1374
|
-
it.each([
|
|
1375
|
-
{
|
|
1376
|
-
format: 'Parquet',
|
|
1377
|
-
sql: "SELECT * FROM read_parquet('s3://bucket/data.parquet')",
|
|
1378
|
-
},
|
|
1379
|
-
{
|
|
1380
|
-
format: 'inferred JSONL',
|
|
1381
|
-
sql: "SELECT * FROM read_json_auto('s3://bucket/data.jsonl')",
|
|
1382
|
-
},
|
|
1383
|
-
{
|
|
1384
|
-
format: 'typed JSONL',
|
|
1385
|
-
sql: "SELECT * FROM read_json('s3://bucket/data.jsonl', format='newline_delimited')",
|
|
1386
|
-
},
|
|
1387
|
-
])('should allow pre-aggregate clients to read S3 $format files', async ({ sql }) => {
|
|
1388
|
-
const rows = [{ val: 1 }];
|
|
1389
|
-
const streamMock = vi.fn(async () => getMockStreamResult([rows], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
1390
|
-
const extractStatementsMock = createMockExtractStatements();
|
|
1391
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, vi.fn(), {
|
|
1392
|
-
extractStatements: extractStatementsMock,
|
|
1393
|
-
}));
|
|
1394
|
-
const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate(duckdbS3Credentials);
|
|
1395
|
-
const result = await client.runQuery(sql);
|
|
1396
|
-
expect(result.rows).toEqual(rows);
|
|
1397
|
-
expect(extractStatementsMock).toHaveBeenCalledTimes(1);
|
|
1398
|
-
expect(streamMock).toHaveBeenCalledTimes(1);
|
|
1399
|
-
});
|
|
1400
|
-
it('should keep pre-aggregate queries read-only', async () => {
|
|
1401
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
1402
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, vi.fn(), {
|
|
1403
|
-
extractStatements: createMockExtractStatements({
|
|
1404
|
-
statementType: 11, // COPY
|
|
1405
|
-
}),
|
|
1406
|
-
}));
|
|
1407
|
-
const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate(duckdbS3Credentials);
|
|
1408
|
-
await expect(client.runQuery("COPY t TO 's3://bucket/data.parquet'")).rejects.toThrow('SQL validation error: only SELECT statements are allowed');
|
|
1409
|
-
expect(streamMock).not.toHaveBeenCalled();
|
|
1410
|
-
});
|
|
1411
|
-
it('should keep secret introspection blocked for pre-aggregate queries', async () => {
|
|
1412
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
1413
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
|
1414
|
-
const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate(duckdbS3Credentials);
|
|
1415
|
-
await expect(client.runQuery("SELECT current_setting('s3_secret_access_key')")).rejects.toThrow("SQL validation error: function 'current_setting' is not allowed");
|
|
1416
|
-
expect(streamMock).not.toHaveBeenCalled();
|
|
1417
|
-
});
|
|
1418
|
-
it.each(['read_parquet', 'read_json_auto'])('should reject quoted-identifier calls to "%s"()', async (blockedFunction) => {
|
|
1419
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
1420
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
|
1421
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
1422
|
-
await expect(client.runQuery(`SELECT * FROM "${blockedFunction}"('/tmp/a')`)).rejects.toThrow(`SQL validation error: function '${blockedFunction}' is not allowed`);
|
|
1423
|
-
expect(streamMock).not.toHaveBeenCalled();
|
|
1424
|
-
});
|
|
1425
|
-
it('should reject schema-qualified calls to file functions', async () => {
|
|
1426
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
1427
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
|
1428
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
1429
|
-
await expect(client.runQuery("SELECT * FROM main.read_json_auto('/tmp/a.json')")).rejects.toThrow("SQL validation error: function 'read_json_auto' is not allowed");
|
|
1430
|
-
expect(streamMock).not.toHaveBeenCalled();
|
|
1431
|
-
});
|
|
1432
|
-
it('should reject user queries that use file table paths', async () => {
|
|
1433
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
1434
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
|
1435
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
1436
|
-
await expect(client.runQuery("SELECT * FROM '/tmp/data.parquet'")).rejects.toThrow('SQL validation error: file table paths are not allowed');
|
|
1437
|
-
expect(streamMock).not.toHaveBeenCalled();
|
|
1438
|
-
});
|
|
1439
|
-
it('should ignore blocked functions inside SQL comments', async () => {
|
|
1440
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
1441
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
|
1442
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
1443
|
-
// Should NOT throw because current_setting is in a comment
|
|
1444
|
-
const result = await client.runQuery("SELECT 1 -- current_setting('s3_secret_access_key')");
|
|
1445
|
-
expect(result.rows).toEqual([{ val: 1 }]);
|
|
1446
|
-
});
|
|
1447
|
-
it('should ignore blocked file readers inside SQL comments', async () => {
|
|
1448
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
1449
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
|
1450
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
1451
|
-
const result = await client.runQuery("SELECT 1 -- SELECT * FROM read_parquet('/tmp/data.parquet')");
|
|
1452
|
-
expect(result.rows).toEqual([{ val: 1 }]);
|
|
1453
|
-
});
|
|
1454
|
-
it('should allow COPY statements in runSql', async () => {
|
|
1455
|
-
const streamMock = vi.fn();
|
|
1456
|
-
const runMock = vi.fn();
|
|
1457
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock, {
|
|
1458
|
-
extractStatements: createMockExtractStatements({
|
|
1459
|
-
statementType: 10, // COPY
|
|
1460
|
-
}),
|
|
1461
|
-
}));
|
|
1462
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
1463
|
-
await client.runSql("COPY table TO 's3://bucket/data.parquet' (FORMAT PARQUET)");
|
|
1464
|
-
expect(runMock).toHaveBeenCalledWith("COPY table TO 's3://bucket/data.parquet' (FORMAT PARQUET)");
|
|
1465
|
-
});
|
|
1466
|
-
it('should allow internal SQL to read staged files', async () => {
|
|
1467
|
-
const streamMock = vi.fn();
|
|
1468
|
-
const runMock = vi.fn();
|
|
1469
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
1470
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
1471
|
-
await client.runSql("CREATE TABLE staged AS SELECT * FROM read_parquet('s3://bucket/data.parquet')");
|
|
1472
|
-
expect(runMock).toHaveBeenCalledWith("CREATE TABLE staged AS SELECT * FROM read_parquet('s3://bucket/data.parquet')");
|
|
1473
|
-
});
|
|
1474
|
-
it.each([
|
|
1475
|
-
{
|
|
1476
|
-
name: 'SET statements',
|
|
1477
|
-
sql: "SET s3_endpoint = 'attacker.com'",
|
|
1478
|
-
statementType: 20,
|
|
1479
|
-
},
|
|
1480
|
-
{
|
|
1481
|
-
name: 'ATTACH statements',
|
|
1482
|
-
sql: "ATTACH DATABASE 'file.db'",
|
|
1483
|
-
statementType: 25,
|
|
1484
|
-
},
|
|
1485
|
-
{
|
|
1486
|
-
name: 'LOAD statements',
|
|
1487
|
-
sql: "LOAD 'malicious_extension'",
|
|
1488
|
-
statementType: 21,
|
|
1489
|
-
},
|
|
1490
|
-
])('should reject $name in runSql', async ({ sql, statementType }) => {
|
|
1491
|
-
const streamMock = vi.fn();
|
|
1492
|
-
const runMock = vi.fn();
|
|
1493
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock, {
|
|
1494
|
-
extractStatements: createMockExtractStatements({
|
|
1495
|
-
statementType,
|
|
1496
|
-
}),
|
|
1497
|
-
}));
|
|
1498
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
1499
|
-
await expect(client.runSql(sql)).rejects.toThrow(`SQL validation error: statement type ${statementType} is not allowed in internal SQL`);
|
|
1500
|
-
expect(runMock).not.toHaveBeenCalledWith(sql);
|
|
1501
|
-
});
|
|
1502
|
-
it('should reject introspection functions in runSql', async () => {
|
|
1503
|
-
const streamMock = vi.fn();
|
|
1504
|
-
const runMock = vi.fn();
|
|
1505
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
1506
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
1507
|
-
await expect(client.runSql("COPY (SELECT current_setting('s3_secret_access_key')) TO '/tmp/out.csv'")).rejects.toThrow("SQL validation error: function 'current_setting' is not allowed");
|
|
1508
|
-
expect(runMock).not.toHaveBeenCalledWith("COPY (SELECT current_setting('s3_secret_access_key')) TO '/tmp/out.csv'");
|
|
1509
|
-
});
|
|
1510
|
-
it('should allow valid SELECT queries', async () => {
|
|
1511
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ id: 1, name: 'test' }]], [DUCKDB_TYPE_IDS.INTEGER, DUCKDB_TYPE_IDS.VARCHAR]));
|
|
1512
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock));
|
|
1513
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient();
|
|
1514
|
-
const result = await client.runQuery('SELECT id, name FROM users WHERE id = 1');
|
|
1515
|
-
expect(result.rows).toEqual([{ id: 1, name: 'test' }]);
|
|
1516
|
-
});
|
|
1517
|
-
});
|
|
1518
|
-
it.each([
|
|
1519
|
-
{
|
|
1520
|
-
name: 'pass encoded token in SaaS-mode connection string for MotherDuck',
|
|
1521
|
-
credentials: {
|
|
1522
|
-
type: common_1.WarehouseTypes.DUCKDB,
|
|
1523
|
-
connectionType: common_1.DuckdbConnectionType.MOTHERDUCK,
|
|
1524
|
-
database: 'my_database',
|
|
1525
|
-
schema: 'main',
|
|
1526
|
-
token: 'my_motherduck_token+with=symbols',
|
|
1527
|
-
},
|
|
1528
|
-
expectedPath: 'md:my_database?motherduck_token=my_motherduck_token%2Bwith%3Dsymbols&saas_mode=true',
|
|
1529
|
-
},
|
|
1530
|
-
{
|
|
1531
|
-
name: 'encode MotherDuck database names before adding connection parameters',
|
|
1532
|
-
credentials: {
|
|
1533
|
-
type: common_1.WarehouseTypes.DUCKDB,
|
|
1534
|
-
connectionType: common_1.DuckdbConnectionType.MOTHERDUCK,
|
|
1535
|
-
database: 'analytics?motherduck_token=other&saas_mode=false',
|
|
1536
|
-
schema: 'main',
|
|
1537
|
-
token: 'my_motherduck_token',
|
|
1538
|
-
},
|
|
1539
|
-
expectedPath: 'md:analytics%3Fmotherduck_token%3Dother%26saas_mode%3Dfalse?motherduck_token=my_motherduck_token&saas_mode=true',
|
|
1540
|
-
},
|
|
1541
|
-
])('should $name', async ({ credentials, expectedPath }) => {
|
|
1542
|
-
const runMock = vi.fn();
|
|
1543
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[{ id: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
1544
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
1545
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(credentials);
|
|
1546
|
-
await client.runQuery('SELECT 1 AS id');
|
|
1547
|
-
expect(createInstanceMock).toHaveBeenCalledWith(expectedPath);
|
|
1548
|
-
expect(runMock).not.toHaveBeenCalledWith(expect.stringContaining('allow_community_extensions'));
|
|
1549
|
-
expect(runMock).not.toHaveBeenCalledWith(expect.stringContaining('autoinstall_known_extensions'));
|
|
1550
|
-
expect(runMock).not.toHaveBeenCalledWith(expect.stringContaining('autoload_known_extensions'));
|
|
1551
|
-
expect(runMock).not.toHaveBeenCalledWith(expect.stringContaining('allow_unredacted_secrets'));
|
|
1552
|
-
});
|
|
1553
|
-
it('should require a MotherDuck token for direct DuckDB connections', () => {
|
|
1554
|
-
expect(() => new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
1555
|
-
type: common_1.WarehouseTypes.DUCKDB,
|
|
1556
|
-
connectionType: common_1.DuckdbConnectionType.MOTHERDUCK,
|
|
1557
|
-
database: 'my_database',
|
|
1558
|
-
schema: 'main',
|
|
1559
|
-
token: '',
|
|
1560
|
-
})).toThrow('MotherDuck token is required for DuckDB warehouse connections');
|
|
1561
|
-
});
|
|
1562
|
-
it('should expose the configured start of week', () => {
|
|
1563
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
1564
|
-
type: common_1.WarehouseTypes.DUCKDB,
|
|
1565
|
-
connectionType: common_1.DuckdbConnectionType.MOTHERDUCK,
|
|
1566
|
-
database: 'analytics',
|
|
1567
|
-
schema: 'main',
|
|
1568
|
-
token: 'motherduck_token',
|
|
1569
|
-
startOfWeek: common_1.WeekDay.SUNDAY,
|
|
1570
|
-
});
|
|
1571
|
-
expect(client.getStartOfWeek()).toBe(common_1.WeekDay.SUNDAY);
|
|
1572
|
-
});
|
|
1573
|
-
it('should default getFields database to the configured DuckDB database', async () => {
|
|
1574
|
-
const runMock = vi.fn(async () => ({
|
|
1575
|
-
getRowObjects: async () => [
|
|
1576
|
-
{
|
|
1577
|
-
column_name: 'order_id',
|
|
1578
|
-
data_type: 'INTEGER',
|
|
1579
|
-
},
|
|
1580
|
-
{
|
|
1581
|
-
column_name: 'created_at',
|
|
1582
|
-
data_type: 'TIMESTAMP',
|
|
1583
|
-
},
|
|
1584
|
-
],
|
|
1585
|
-
}));
|
|
1586
|
-
createInstanceMock.mockResolvedValue(createMockConnection(vi.fn(), runMock));
|
|
1587
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
1588
|
-
type: common_1.WarehouseTypes.DUCKDB,
|
|
1589
|
-
connectionType: common_1.DuckdbConnectionType.MOTHERDUCK,
|
|
1590
|
-
database: 'analytics',
|
|
1591
|
-
schema: 'main',
|
|
1592
|
-
token: 'motherduck_token',
|
|
1593
|
-
});
|
|
1594
|
-
const result = await client.getFields('orders');
|
|
1595
|
-
expect(runMock).toHaveBeenCalledWith(expect.stringContaining("WHERE table_catalog = 'analytics'"));
|
|
1596
|
-
expect(result).toEqual({
|
|
1597
|
-
analytics: {
|
|
1598
|
-
main: {
|
|
1599
|
-
orders: {
|
|
1600
|
-
order_id: common_1.DimensionType.NUMBER,
|
|
1601
|
-
created_at: common_1.DimensionType.TIMESTAMP,
|
|
1602
|
-
},
|
|
1603
|
-
},
|
|
1604
|
-
},
|
|
1605
|
-
__lightdashTimestampDomains: {
|
|
1606
|
-
analytics: {
|
|
1607
|
-
main: { orders: { created_at: 'naive' } },
|
|
1608
|
-
},
|
|
1609
|
-
},
|
|
1610
|
-
});
|
|
1611
|
-
});
|
|
1612
|
-
describe('DuckLake bootstrap', () => {
|
|
1613
|
-
const captureRunMock = () => vi.fn().mockResolvedValue({
|
|
1614
|
-
getRowObjects: async () => [],
|
|
1615
|
-
});
|
|
1616
|
-
const collectStatements = (runMock) => runMock.mock.calls.map((c) => c[0]);
|
|
1617
|
-
it('attaches a postgres-catalog + S3 DuckLake in the correct order', async () => {
|
|
1618
|
-
const runMock = captureRunMock();
|
|
1619
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[]], []));
|
|
1620
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
1621
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
1622
|
-
type: common_1.WarehouseTypes.DUCKDB,
|
|
1623
|
-
connectionType: common_1.DuckdbConnectionType.DUCKLAKE,
|
|
1624
|
-
schema: 'main',
|
|
1625
|
-
catalogAlias: 'ducklake',
|
|
1626
|
-
catalog: {
|
|
1627
|
-
type: common_1.DucklakeCatalogType.POSTGRES,
|
|
1628
|
-
host: 'pg.example.com',
|
|
1629
|
-
port: 5432,
|
|
1630
|
-
database: 'catalog',
|
|
1631
|
-
user: 'ducklake_user',
|
|
1632
|
-
password: 'p@ss',
|
|
1633
|
-
},
|
|
1634
|
-
dataPath: {
|
|
1635
|
-
type: common_1.DucklakeDataPathType.S3,
|
|
1636
|
-
url: 's3://my-bucket/path/',
|
|
1637
|
-
accessKeyId: 'AKIAEXAMPLE',
|
|
1638
|
-
secretAccessKey: 'SECRETEXAMPLE',
|
|
1639
|
-
region: 'us-east-1',
|
|
1640
|
-
},
|
|
1641
|
-
});
|
|
1642
|
-
await client.runQuery('SELECT 1');
|
|
1643
|
-
const stmts = collectStatements(runMock);
|
|
1644
|
-
const joined = stmts.join('\n');
|
|
1645
|
-
// No explicit INSTALL/LOAD in DuckLake mode — rely on autoload.
|
|
1646
|
-
expect(joined).not.toMatch(/INSTALL httpfs/);
|
|
1647
|
-
expect(joined).not.toMatch(/LOAD httpfs/);
|
|
1648
|
-
expect(stmts).toEqual(expect.arrayContaining([
|
|
1649
|
-
'SET autoinstall_known_extensions = true;',
|
|
1650
|
-
'SET autoload_known_extensions = true;',
|
|
1651
|
-
'SET autoinstall_known_extensions = false;',
|
|
1652
|
-
'SET autoload_known_extensions = false;',
|
|
1653
|
-
'SET allow_community_extensions = false;',
|
|
1654
|
-
'SET allow_unredacted_secrets = false;',
|
|
1655
|
-
]));
|
|
1656
|
-
const catalogIdx = stmts.findIndex((s) => /__lightdash_ducklake_catalog/.test(s));
|
|
1657
|
-
const dataIdx = stmts.findIndex((s) => /__lightdash_ducklake_data/.test(s));
|
|
1658
|
-
const duckLakeSecretIdx = stmts.findIndex((s) => /SECRET __lightdash_ducklake\s/.test(s));
|
|
1659
|
-
const attachIdx = stmts.findIndex((s) => s.startsWith("ATTACH 'ducklake:__lightdash_ducklake'"));
|
|
1660
|
-
expect(catalogIdx).toBeGreaterThanOrEqual(0);
|
|
1661
|
-
expect(dataIdx).toBeGreaterThan(catalogIdx);
|
|
1662
|
-
expect(duckLakeSecretIdx).toBeGreaterThan(dataIdx);
|
|
1663
|
-
expect(attachIdx).toBeGreaterThan(duckLakeSecretIdx);
|
|
1664
|
-
expect(stmts.lastIndexOf('SET autoinstall_known_extensions = false;')).toBeGreaterThan(attachIdx);
|
|
1665
|
-
expect(stmts.lastIndexOf('SET autoload_known_extensions = false;')).toBeGreaterThan(attachIdx);
|
|
1666
|
-
expect(stmts[catalogIdx]).toMatch(/TYPE postgres/);
|
|
1667
|
-
expect(stmts[catalogIdx]).toMatch(/HOST 'pg.example.com'/);
|
|
1668
|
-
expect(stmts[catalogIdx]).toMatch(/PASSWORD 'p@ss'/);
|
|
1669
|
-
expect(stmts[dataIdx]).toMatch(/TYPE s3/);
|
|
1670
|
-
expect(stmts[dataIdx]).toMatch(/KEY_ID 'AKIAEXAMPLE'/);
|
|
1671
|
-
expect(stmts[dataIdx]).toMatch(/SCOPE 's3:\/\/my-bucket\/path\/'/);
|
|
1672
|
-
});
|
|
1673
|
-
it('uses inline ATTACH (no ducklake secret) for SQLite catalog + local data path', async () => {
|
|
1674
|
-
const runMock = captureRunMock();
|
|
1675
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[]], []));
|
|
1676
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
1677
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
1678
|
-
type: common_1.WarehouseTypes.DUCKDB,
|
|
1679
|
-
connectionType: common_1.DuckdbConnectionType.DUCKLAKE,
|
|
1680
|
-
schema: 'main',
|
|
1681
|
-
catalog: {
|
|
1682
|
-
type: common_1.DucklakeCatalogType.SQLITE,
|
|
1683
|
-
path: '/tmp/ducklake.sqlite',
|
|
1684
|
-
},
|
|
1685
|
-
dataPath: {
|
|
1686
|
-
type: common_1.DucklakeDataPathType.LOCAL,
|
|
1687
|
-
path: '/tmp/ducklake-data',
|
|
1688
|
-
},
|
|
1689
|
-
});
|
|
1690
|
-
await client.runQuery('SELECT 1');
|
|
1691
|
-
const stmts = collectStatements(runMock);
|
|
1692
|
-
const joined = stmts.join('\n');
|
|
1693
|
-
expect(joined).not.toMatch(/__lightdash_ducklake_catalog/);
|
|
1694
|
-
expect(joined).not.toMatch(/__lightdash_ducklake_data/);
|
|
1695
|
-
expect(joined).not.toMatch(/SECRET __lightdash_ducklake\s/);
|
|
1696
|
-
expect(stmts.some((s) => s.startsWith("ATTACH 'ducklake:sqlite:/tmp/ducklake.sqlite' AS \"ducklake\" (DATA_PATH '/tmp/ducklake-data', READ_ONLY);"))).toBe(true);
|
|
1697
|
-
});
|
|
1698
|
-
it('rejects user SQL that contains ATTACH even in DuckLake mode', async () => {
|
|
1699
|
-
const runMock = captureRunMock();
|
|
1700
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[]], []));
|
|
1701
|
-
const extractStatements = vi.fn(async () => ({
|
|
1702
|
-
count: 1,
|
|
1703
|
-
prepare: async () => ({
|
|
1704
|
-
statementType: 25, // ATTACH
|
|
1705
|
-
destroySync: vi.fn(),
|
|
1706
|
-
}),
|
|
1707
|
-
}));
|
|
1708
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock, {
|
|
1709
|
-
extractStatements,
|
|
1710
|
-
}));
|
|
1711
|
-
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({
|
|
1712
|
-
type: common_1.WarehouseTypes.DUCKDB,
|
|
1713
|
-
connectionType: common_1.DuckdbConnectionType.DUCKLAKE,
|
|
1714
|
-
schema: 'main',
|
|
1715
|
-
catalog: {
|
|
1716
|
-
type: common_1.DucklakeCatalogType.SQLITE,
|
|
1717
|
-
path: '/tmp/c.sqlite',
|
|
1718
|
-
},
|
|
1719
|
-
dataPath: {
|
|
1720
|
-
type: common_1.DucklakeDataPathType.LOCAL,
|
|
1721
|
-
path: '/tmp/d',
|
|
1722
|
-
},
|
|
1723
|
-
});
|
|
1724
|
-
await expect(client.runQuery("ATTACH 'ducklake:evil' AS bad;")).rejects.toThrow(/only SELECT statements are allowed/);
|
|
1725
|
-
});
|
|
1726
|
-
it('keeps autoload disabled for non-DuckLake modes', async () => {
|
|
1727
|
-
const runMock = captureRunMock();
|
|
1728
|
-
const streamMock = vi.fn(async () => getMockStreamResult([[]], []));
|
|
1729
|
-
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
1730
|
-
const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate({
|
|
1731
|
-
type: 'duckdb_s3',
|
|
1732
|
-
s3Config: {
|
|
1733
|
-
endpoint: 'localhost:9000',
|
|
1734
|
-
region: 'us-east-1',
|
|
1735
|
-
forcePathStyle: true,
|
|
1736
|
-
useSsl: false,
|
|
1737
|
-
},
|
|
1738
|
-
});
|
|
1739
|
-
await client.runQuery('SELECT 1');
|
|
1740
|
-
const stmts = collectStatements(runMock);
|
|
1741
|
-
expect(stmts).toEqual(expect.arrayContaining([
|
|
1742
|
-
'SET autoinstall_known_extensions = false;',
|
|
1743
|
-
'SET autoload_known_extensions = false;',
|
|
1744
|
-
]));
|
|
1745
|
-
// Regression: existing modes still explicitly INSTALL/LOAD httpfs.
|
|
1746
|
-
expect(stmts).toEqual(expect.arrayContaining(['INSTALL httpfs;', 'LOAD httpfs;']));
|
|
1747
|
-
});
|
|
1748
|
-
});
|
|
1749
|
-
});
|