@lightdash/warehouses 1.79.1 → 1.80.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/warehouseClientFromCredentials.d.ts +2 -1
- package/dist/warehouseClientFromCredentials.d.ts.map +1 -1
- package/dist/warehouseClientFromCredentials.js +2 -2
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts +6 -7
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.js +128 -14
- package/dist/warehouseClients/DuckdbWarehouseClient.test.js +307 -0
- package/dist/warehouseClients/MotherduckErrorClassifier.d.ts +3 -0
- package/dist/warehouseClients/MotherduckErrorClassifier.d.ts.map +1 -0
- package/dist/warehouseClients/MotherduckErrorClassifier.js +34 -0
- package/dist/warehouseClients/MotherduckErrorClassifier.test.d.ts +2 -0
- package/dist/warehouseClients/MotherduckErrorClassifier.test.d.ts.map +1 -0
- package/dist/warehouseClients/MotherduckErrorClassifier.test.js +31 -0
- package/dist/warehouseClients/MotherduckInstanceCache.catalog.test.d.ts +2 -0
- package/dist/warehouseClients/MotherduckInstanceCache.catalog.test.d.ts.map +1 -0
- package/dist/warehouseClients/MotherduckInstanceCache.catalog.test.js +52 -0
- package/dist/warehouseClients/MotherduckInstanceCache.d.ts +58 -0
- package/dist/warehouseClients/MotherduckInstanceCache.d.ts.map +1 -0
- package/dist/warehouseClients/MotherduckInstanceCache.integration.test.d.ts +2 -0
- package/dist/warehouseClients/MotherduckInstanceCache.integration.test.d.ts.map +1 -0
- package/dist/warehouseClients/MotherduckInstanceCache.integration.test.js +172 -0
- package/dist/warehouseClients/MotherduckInstanceCache.js +316 -0
- package/dist/warehouseClients/MotherduckInstanceCache.security.test.d.ts +2 -0
- package/dist/warehouseClients/MotherduckInstanceCache.security.test.d.ts.map +1 -0
- package/dist/warehouseClients/MotherduckInstanceCache.security.test.js +192 -0
- package/dist/warehouseClients/MotherduckInstanceCache.test.d.ts +2 -0
- package/dist/warehouseClients/MotherduckInstanceCache.test.d.ts.map +1 -0
- package/dist/warehouseClients/MotherduckInstanceCache.test.js +302 -0
- package/package.json +2 -2
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Run: set -a; source .motherduck.env; set +a; pnpm -F warehouses exec vitest run src/warehouseClients/MotherduckInstanceCache.integration.test.ts
|
|
3
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
4
|
+
if (k2 === undefined) k2 = k;
|
|
5
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
6
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
7
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
8
|
+
}
|
|
9
|
+
Object.defineProperty(o, k2, desc);
|
|
10
|
+
}) : (function(o, m, k, k2) {
|
|
11
|
+
if (k2 === undefined) k2 = k;
|
|
12
|
+
o[k2] = m[k];
|
|
13
|
+
}));
|
|
14
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
15
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
16
|
+
}) : function(o, v) {
|
|
17
|
+
o["default"] = v;
|
|
18
|
+
});
|
|
19
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
20
|
+
var ownKeys = function(o) {
|
|
21
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
22
|
+
var ar = [];
|
|
23
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
24
|
+
return ar;
|
|
25
|
+
};
|
|
26
|
+
return ownKeys(o);
|
|
27
|
+
};
|
|
28
|
+
return function (mod) {
|
|
29
|
+
if (mod && mod.__esModule) return mod;
|
|
30
|
+
var result = {};
|
|
31
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
32
|
+
__setModuleDefault(result, mod);
|
|
33
|
+
return result;
|
|
34
|
+
};
|
|
35
|
+
})();
|
|
36
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
37
|
+
const node_api_1 = require("@duckdb/node-api");
|
|
38
|
+
const crypto_1 = require("crypto");
|
|
39
|
+
const MotherduckInstanceCache = __importStar(require("./MotherduckInstanceCache"));
|
|
40
|
+
const motherduckToken = process.env.MOTHERDUCK_TOKEN?.trim();
|
|
41
|
+
const database = 'jaffle_shop';
|
|
42
|
+
const query = 'SELECT * FROM jaffle_shop.jaffle.orders LIMIT 10';
|
|
43
|
+
const tablePrefix = 'ld_spk822_catalog_freshness_';
|
|
44
|
+
const ownedTablePattern = /^ld_spk822_catalog_freshness_[0-9a-f]{8}_[0-9a-f]{4}_[0-9a-f]{4}_[0-9a-f]{4}_[0-9a-f]{12}$/;
|
|
45
|
+
const getConnectionString = () => {
|
|
46
|
+
if (!motherduckToken) {
|
|
47
|
+
throw new Error('MOTHERDUCK_TOKEN is required');
|
|
48
|
+
}
|
|
49
|
+
return `md:${database}?${new URLSearchParams({
|
|
50
|
+
motherduck_token: motherduckToken,
|
|
51
|
+
saas_mode: 'true',
|
|
52
|
+
}).toString()}`;
|
|
53
|
+
};
|
|
54
|
+
const closeConnection = (connection) => {
|
|
55
|
+
connection.closeSync?.();
|
|
56
|
+
connection.disconnectSync?.();
|
|
57
|
+
};
|
|
58
|
+
const consumeQuery = async (connection, sql) => {
|
|
59
|
+
const result = await connection.stream(sql);
|
|
60
|
+
const rows = [];
|
|
61
|
+
for await (const chunk of result.yieldRowObjectJson()) {
|
|
62
|
+
rows.push(...chunk);
|
|
63
|
+
}
|
|
64
|
+
return rows;
|
|
65
|
+
};
|
|
66
|
+
const withFreshInstance = async (callback) => {
|
|
67
|
+
const instance = await node_api_1.DuckDBInstance.create(getConnectionString());
|
|
68
|
+
const connection = await instance.connect();
|
|
69
|
+
try {
|
|
70
|
+
return await callback(connection);
|
|
71
|
+
}
|
|
72
|
+
finally {
|
|
73
|
+
closeConnection(connection);
|
|
74
|
+
instance.closeSync?.();
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
const measureFreshInstance = async () => {
|
|
78
|
+
const startedAt = performance.now();
|
|
79
|
+
await withFreshInstance(async (connection) => consumeQuery(connection, query));
|
|
80
|
+
return performance.now() - startedAt;
|
|
81
|
+
};
|
|
82
|
+
const measureHeldInstance = async (instance) => {
|
|
83
|
+
const startedAt = performance.now();
|
|
84
|
+
const connection = await instance.connect();
|
|
85
|
+
try {
|
|
86
|
+
await consumeQuery(connection, query);
|
|
87
|
+
return performance.now() - startedAt;
|
|
88
|
+
}
|
|
89
|
+
finally {
|
|
90
|
+
closeConnection(connection);
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
const median = (values) => {
|
|
94
|
+
const sorted = [...values].sort((left, right) => left - right);
|
|
95
|
+
const middle = sorted[Math.floor(sorted.length / 2)];
|
|
96
|
+
if (middle === undefined) {
|
|
97
|
+
throw new Error('Cannot calculate the median of an empty set');
|
|
98
|
+
}
|
|
99
|
+
return middle;
|
|
100
|
+
};
|
|
101
|
+
describe.skipIf(!motherduckToken)('MotherDuck instance cache live integration', () => {
|
|
102
|
+
let createdTableName;
|
|
103
|
+
beforeEach(() => {
|
|
104
|
+
MotherduckInstanceCache.resetForTesting();
|
|
105
|
+
MotherduckInstanceCache.configure({
|
|
106
|
+
idleTtlMs: 60000,
|
|
107
|
+
maxAgeMs: 60000,
|
|
108
|
+
maxEntries: 1,
|
|
109
|
+
});
|
|
110
|
+
});
|
|
111
|
+
afterEach(async () => {
|
|
112
|
+
await MotherduckInstanceCache.closeAll('shutdown');
|
|
113
|
+
if (createdTableName) {
|
|
114
|
+
if (!ownedTablePattern.test(createdTableName)) {
|
|
115
|
+
throw new Error('Refusing to clean up an unowned table');
|
|
116
|
+
}
|
|
117
|
+
await withFreshInstance(async (connection) => {
|
|
118
|
+
await connection.run(`DROP TABLE main."${createdTableName}"`);
|
|
119
|
+
});
|
|
120
|
+
createdTableName = undefined;
|
|
121
|
+
}
|
|
122
|
+
});
|
|
123
|
+
it('keeps a held instance fresh for a table and column created after warmup', async () => {
|
|
124
|
+
const tableName = `${tablePrefix}${(0, crypto_1.randomUUID)().replaceAll('-', '_')}`;
|
|
125
|
+
await MotherduckInstanceCache.withInstance(getConnectionString(), {}, async (instance) => {
|
|
126
|
+
const warmConnection = await instance.connect();
|
|
127
|
+
try {
|
|
128
|
+
await consumeQuery(warmConnection, 'SELECT * FROM information_schema.columns LIMIT 1');
|
|
129
|
+
await withFreshInstance(async (connection) => {
|
|
130
|
+
await connection.run(`CREATE TABLE main."${tableName}" (initial_column INTEGER)`);
|
|
131
|
+
createdTableName = tableName;
|
|
132
|
+
await connection.run(`ALTER TABLE main."${tableName}" ADD COLUMN added_column VARCHAR`);
|
|
133
|
+
});
|
|
134
|
+
const columnsSql = `SELECT column_name FROM information_schema.columns WHERE table_catalog = '${database}' AND table_schema = 'main' AND table_name = '${tableName}' ORDER BY ordinal_position`;
|
|
135
|
+
const warmRows = await consumeQuery(warmConnection, columnsSql);
|
|
136
|
+
const freshConnection = await instance.connect();
|
|
137
|
+
try {
|
|
138
|
+
const freshRows = await consumeQuery(freshConnection, columnsSql);
|
|
139
|
+
expect(warmRows).toEqual([
|
|
140
|
+
{ column_name: 'initial_column' },
|
|
141
|
+
{ column_name: 'added_column' },
|
|
142
|
+
]);
|
|
143
|
+
expect(freshRows).toEqual(warmRows);
|
|
144
|
+
}
|
|
145
|
+
finally {
|
|
146
|
+
closeConnection(freshConnection);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
finally {
|
|
150
|
+
closeConnection(warmConnection);
|
|
151
|
+
}
|
|
152
|
+
});
|
|
153
|
+
}, 60000);
|
|
154
|
+
it('keeps held-instance query cost several times below fresh-instance cost', async () => {
|
|
155
|
+
const freshInstanceMs = [];
|
|
156
|
+
const heldInstanceMs = [];
|
|
157
|
+
await MotherduckInstanceCache.withInstance(getConnectionString(), {}, async (instance) => {
|
|
158
|
+
await measureHeldInstance(instance);
|
|
159
|
+
const collectMeasurements = async (remaining) => {
|
|
160
|
+
if (remaining === 0) {
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
freshInstanceMs.push(await measureFreshInstance());
|
|
164
|
+
heldInstanceMs.push(await measureHeldInstance(instance));
|
|
165
|
+
await collectMeasurements(remaining - 1);
|
|
166
|
+
};
|
|
167
|
+
await collectMeasurements(3);
|
|
168
|
+
});
|
|
169
|
+
// A 3x threshold catches loss of instance reuse while leaving headroom for observed cross-run variance.
|
|
170
|
+
expect(median(freshInstanceMs) / median(heldInstanceMs)).toBeGreaterThanOrEqual(3);
|
|
171
|
+
}, 60000);
|
|
172
|
+
});
|
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MotherduckInstanceCache = exports.resetForTesting = exports.recordRetry = exports.closeAll = exports.invalidateByConnectionString = exports.invalidate = exports.withInstance = exports.setObserver = exports.configure = void 0;
|
|
4
|
+
const node_api_1 = require("@duckdb/node-api");
|
|
5
|
+
const crypto_1 = require("crypto");
|
|
6
|
+
const holdEntry = (entry) => {
|
|
7
|
+
const heldEntry = entry;
|
|
8
|
+
heldEntry.refCount += 1;
|
|
9
|
+
heldEntry.lastUsedAt = performance.now();
|
|
10
|
+
return heldEntry;
|
|
11
|
+
};
|
|
12
|
+
const DEFAULT_OPTIONS = {
|
|
13
|
+
idleTtlMs: 10 * 60000,
|
|
14
|
+
maxAgeMs: 60 * 60000,
|
|
15
|
+
maxEntries: 8,
|
|
16
|
+
maxConsecutiveFailures: 3,
|
|
17
|
+
};
|
|
18
|
+
let options = DEFAULT_OPTIONS;
|
|
19
|
+
let observer = () => undefined;
|
|
20
|
+
let sweepTimer;
|
|
21
|
+
const entries = new Map();
|
|
22
|
+
const pendingCreations = new Map();
|
|
23
|
+
const creationGenerations = new Map();
|
|
24
|
+
const emit = (event) => {
|
|
25
|
+
try {
|
|
26
|
+
observer(event);
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
const cacheKeyFor = (connectionString) => (0, crypto_1.createHash)('sha256')
|
|
33
|
+
.update(JSON.stringify({ connectionString, v: 1 }))
|
|
34
|
+
.digest('hex');
|
|
35
|
+
const closeInstance = (instance) => {
|
|
36
|
+
try {
|
|
37
|
+
instance.closeSync?.();
|
|
38
|
+
return true;
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
};
|
|
44
|
+
const closeEntry = (entry) => {
|
|
45
|
+
if (entry.closed || entry.refCount > 0) {
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
const closingEntry = entry;
|
|
49
|
+
closingEntry.closed = true;
|
|
50
|
+
closeInstance(closingEntry.instance);
|
|
51
|
+
closingEntry.resolveClose();
|
|
52
|
+
};
|
|
53
|
+
const unlinkEntry = (key, entry, reason) => {
|
|
54
|
+
if (entries.get(key) !== entry) {
|
|
55
|
+
return entry.closePromise;
|
|
56
|
+
}
|
|
57
|
+
entries.delete(key);
|
|
58
|
+
const drainingEntry = entry;
|
|
59
|
+
drainingEntry.draining = true;
|
|
60
|
+
emit({
|
|
61
|
+
type: 'evict',
|
|
62
|
+
entryId: entry.entryId,
|
|
63
|
+
projectUuid: entry.projectUuid,
|
|
64
|
+
reason,
|
|
65
|
+
ageMs: performance.now() - entry.createdAt,
|
|
66
|
+
});
|
|
67
|
+
emit({ type: 'size', entries: entries.size });
|
|
68
|
+
closeEntry(entry);
|
|
69
|
+
return entry.closePromise;
|
|
70
|
+
};
|
|
71
|
+
const sweep = () => {
|
|
72
|
+
const now = performance.now();
|
|
73
|
+
entries.forEach((entry, key) => {
|
|
74
|
+
const ageMs = now - entry.createdAt;
|
|
75
|
+
const idleMs = now - entry.lastUsedAt;
|
|
76
|
+
if (ageMs >= options.maxAgeMs) {
|
|
77
|
+
void unlinkEntry(key, entry, 'max_age');
|
|
78
|
+
}
|
|
79
|
+
else if (entry.refCount === 0 && idleMs >= options.idleTtlMs) {
|
|
80
|
+
void unlinkEntry(key, entry, 'idle_ttl');
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
};
|
|
84
|
+
const scheduleSweep = () => {
|
|
85
|
+
if (sweepTimer) {
|
|
86
|
+
clearInterval(sweepTimer);
|
|
87
|
+
}
|
|
88
|
+
const intervalMs = Math.max(1, Math.min(options.idleTtlMs, options.maxAgeMs, 60000));
|
|
89
|
+
sweepTimer = setInterval(() => {
|
|
90
|
+
sweep();
|
|
91
|
+
}, intervalMs);
|
|
92
|
+
sweepTimer.unref();
|
|
93
|
+
};
|
|
94
|
+
const createEntry = async (key, connectionString, generation, projectUuid) => {
|
|
95
|
+
const createStart = performance.now();
|
|
96
|
+
// Deliberately bypass fromCache(): mixing its unbounded singleton with this cache would create conflicting lifecycles.
|
|
97
|
+
// Revisit on @duckdb/node-api upgrades if the binding cache gains eviction bounds.
|
|
98
|
+
const instance = await node_api_1.DuckDBInstance.create(connectionString);
|
|
99
|
+
const instanceCreateMs = performance.now() - createStart;
|
|
100
|
+
if ((creationGenerations.get(key) ?? 0) !== generation) {
|
|
101
|
+
closeInstance(instance);
|
|
102
|
+
return undefined;
|
|
103
|
+
}
|
|
104
|
+
let resolveClose = () => undefined;
|
|
105
|
+
const closePromise = new Promise((resolve) => {
|
|
106
|
+
resolveClose = resolve;
|
|
107
|
+
});
|
|
108
|
+
const now = performance.now();
|
|
109
|
+
const entry = {
|
|
110
|
+
instance,
|
|
111
|
+
entryId: (0, crypto_1.randomUUID)(),
|
|
112
|
+
projectUuid,
|
|
113
|
+
createdAt: now,
|
|
114
|
+
lastUsedAt: now,
|
|
115
|
+
refCount: 1,
|
|
116
|
+
consecutiveFailures: 0,
|
|
117
|
+
draining: false,
|
|
118
|
+
closed: false,
|
|
119
|
+
closePromise,
|
|
120
|
+
resolveClose,
|
|
121
|
+
};
|
|
122
|
+
entries.set(key, entry);
|
|
123
|
+
emit({ type: 'size', entries: entries.size });
|
|
124
|
+
if (entries.size > options.maxEntries) {
|
|
125
|
+
const lru = [...entries.entries()]
|
|
126
|
+
.filter(([, candidate]) => candidate !== entry)
|
|
127
|
+
.sort(([, left], [, right]) => left.lastUsedAt - right.lastUsedAt)
|
|
128
|
+
.at(0);
|
|
129
|
+
if (lru) {
|
|
130
|
+
void unlinkEntry(lru[0], lru[1], 'lru');
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return { entry, instanceCreateMs };
|
|
134
|
+
};
|
|
135
|
+
const resolveCreation = async (key, pending, result, getWaitMs, retry) => {
|
|
136
|
+
let created;
|
|
137
|
+
try {
|
|
138
|
+
created = await pending.promise;
|
|
139
|
+
}
|
|
140
|
+
finally {
|
|
141
|
+
if (pendingCreations.get(key) === pending) {
|
|
142
|
+
pendingCreations.delete(key);
|
|
143
|
+
if ((creationGenerations.get(key) ?? 0) === pending.generation) {
|
|
144
|
+
creationGenerations.delete(key);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
if (!created || pending.invalidated) {
|
|
149
|
+
if (created && result === 'miss') {
|
|
150
|
+
created.entry.refCount -= 1;
|
|
151
|
+
closeEntry(created.entry);
|
|
152
|
+
}
|
|
153
|
+
return retry();
|
|
154
|
+
}
|
|
155
|
+
return {
|
|
156
|
+
entry: result === 'hit' ? holdEntry(created.entry) : created.entry,
|
|
157
|
+
result,
|
|
158
|
+
waitMs: getWaitMs(),
|
|
159
|
+
instanceCreateMs: result === 'miss' ? created.instanceCreateMs : 0,
|
|
160
|
+
};
|
|
161
|
+
};
|
|
162
|
+
const acquire = async (connectionString, projectUuid) => {
|
|
163
|
+
const waitStart = performance.now();
|
|
164
|
+
sweep();
|
|
165
|
+
const key = cacheKeyFor(connectionString);
|
|
166
|
+
const existing = entries.get(key);
|
|
167
|
+
if (existing) {
|
|
168
|
+
return {
|
|
169
|
+
entry: holdEntry(existing),
|
|
170
|
+
result: 'hit',
|
|
171
|
+
waitMs: performance.now() - waitStart,
|
|
172
|
+
instanceCreateMs: 0,
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
const pending = pendingCreations.get(key);
|
|
176
|
+
if (pending) {
|
|
177
|
+
return resolveCreation(key, pending, 'hit', () => performance.now() - waitStart, () => acquire(connectionString, projectUuid));
|
|
178
|
+
}
|
|
179
|
+
const generation = creationGenerations.get(key) ?? 0;
|
|
180
|
+
const waitMs = performance.now() - waitStart;
|
|
181
|
+
const creation = {
|
|
182
|
+
generation,
|
|
183
|
+
invalidated: false,
|
|
184
|
+
promise: createEntry(key, connectionString, generation, projectUuid),
|
|
185
|
+
};
|
|
186
|
+
pendingCreations.set(key, creation);
|
|
187
|
+
return resolveCreation(key, creation, 'miss', () => waitMs, () => acquire(connectionString, projectUuid));
|
|
188
|
+
};
|
|
189
|
+
const configure = (nextOptions) => {
|
|
190
|
+
options = {
|
|
191
|
+
...nextOptions,
|
|
192
|
+
maxConsecutiveFailures: nextOptions.maxConsecutiveFailures ??
|
|
193
|
+
DEFAULT_OPTIONS.maxConsecutiveFailures,
|
|
194
|
+
};
|
|
195
|
+
scheduleSweep();
|
|
196
|
+
};
|
|
197
|
+
exports.configure = configure;
|
|
198
|
+
const setObserver = (nextObserver) => {
|
|
199
|
+
observer = nextObserver;
|
|
200
|
+
emit({ type: 'size', entries: entries.size });
|
|
201
|
+
};
|
|
202
|
+
exports.setObserver = setObserver;
|
|
203
|
+
const withInstance = async (connectionString, ctx, fn) => {
|
|
204
|
+
const acquisition = await acquire(connectionString, ctx.projectUuid);
|
|
205
|
+
const { entry } = acquisition;
|
|
206
|
+
const timing = {
|
|
207
|
+
waitMs: acquisition.waitMs,
|
|
208
|
+
instanceCreateMs: acquisition.instanceCreateMs,
|
|
209
|
+
connectMs: 0,
|
|
210
|
+
};
|
|
211
|
+
const observedInstance = new Proxy(entry.instance, {
|
|
212
|
+
get: (target, property) => {
|
|
213
|
+
if (property === 'connect') {
|
|
214
|
+
return async () => {
|
|
215
|
+
const connectStart = performance.now();
|
|
216
|
+
try {
|
|
217
|
+
return await target.connect();
|
|
218
|
+
}
|
|
219
|
+
finally {
|
|
220
|
+
timing.connectMs += performance.now() - connectStart;
|
|
221
|
+
}
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
const value = Reflect.get(target, property, target);
|
|
225
|
+
return typeof value === 'function' ? value.bind(target) : value;
|
|
226
|
+
},
|
|
227
|
+
});
|
|
228
|
+
try {
|
|
229
|
+
const result = await fn(observedInstance, entry.entryId, timing);
|
|
230
|
+
entry.consecutiveFailures = 0;
|
|
231
|
+
return result;
|
|
232
|
+
}
|
|
233
|
+
catch (error) {
|
|
234
|
+
entry.consecutiveFailures += 1;
|
|
235
|
+
if (entry.consecutiveFailures >= options.maxConsecutiveFailures &&
|
|
236
|
+
!entry.draining) {
|
|
237
|
+
void unlinkEntry(cacheKeyFor(connectionString), entry, 'failures');
|
|
238
|
+
}
|
|
239
|
+
throw error;
|
|
240
|
+
}
|
|
241
|
+
finally {
|
|
242
|
+
entry.refCount -= 1;
|
|
243
|
+
entry.lastUsedAt = performance.now();
|
|
244
|
+
emit({
|
|
245
|
+
type: 'acquire',
|
|
246
|
+
result: acquisition.result,
|
|
247
|
+
entryId: entry.entryId,
|
|
248
|
+
projectUuid: ctx.projectUuid,
|
|
249
|
+
waitMs: timing.waitMs,
|
|
250
|
+
instanceCreateMs: timing.instanceCreateMs,
|
|
251
|
+
connectMs: timing.connectMs,
|
|
252
|
+
});
|
|
253
|
+
if (entry.draining) {
|
|
254
|
+
closeEntry(entry);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
};
|
|
258
|
+
exports.withInstance = withInstance;
|
|
259
|
+
const invalidate = async (entryId, reason) => {
|
|
260
|
+
const match = [...entries.entries()].find(([, entry]) => entry.entryId === entryId);
|
|
261
|
+
if (match) {
|
|
262
|
+
await unlinkEntry(match[0], match[1], reason);
|
|
263
|
+
}
|
|
264
|
+
};
|
|
265
|
+
exports.invalidate = invalidate;
|
|
266
|
+
const invalidateByConnectionString = (connectionString, reason) => {
|
|
267
|
+
const key = cacheKeyFor(connectionString);
|
|
268
|
+
const pending = pendingCreations.get(key);
|
|
269
|
+
if (pending) {
|
|
270
|
+
pending.invalidated = true;
|
|
271
|
+
creationGenerations.set(key, (creationGenerations.get(key) ?? 0) + 1);
|
|
272
|
+
}
|
|
273
|
+
const entry = entries.get(key);
|
|
274
|
+
if (entry) {
|
|
275
|
+
void unlinkEntry(key, entry, reason);
|
|
276
|
+
}
|
|
277
|
+
};
|
|
278
|
+
exports.invalidateByConnectionString = invalidateByConnectionString;
|
|
279
|
+
const closeAll = async (reason) => {
|
|
280
|
+
await Promise.allSettled([...pendingCreations.values()].map(({ promise }) => promise));
|
|
281
|
+
const draining = [...entries.entries()].map(([key, entry]) => unlinkEntry(key, entry, reason));
|
|
282
|
+
await Promise.all(draining);
|
|
283
|
+
};
|
|
284
|
+
exports.closeAll = closeAll;
|
|
285
|
+
const recordRetry = (entryId, outcome) => {
|
|
286
|
+
emit({ type: 'retry', entryId, outcome });
|
|
287
|
+
};
|
|
288
|
+
exports.recordRetry = recordRetry;
|
|
289
|
+
const resetForTesting = () => {
|
|
290
|
+
if (sweepTimer) {
|
|
291
|
+
clearInterval(sweepTimer);
|
|
292
|
+
}
|
|
293
|
+
entries.forEach((entry) => {
|
|
294
|
+
const drainingEntry = entry;
|
|
295
|
+
drainingEntry.draining = true;
|
|
296
|
+
closeEntry(drainingEntry);
|
|
297
|
+
});
|
|
298
|
+
entries.clear();
|
|
299
|
+
pendingCreations.clear();
|
|
300
|
+
creationGenerations.clear();
|
|
301
|
+
options = DEFAULT_OPTIONS;
|
|
302
|
+
observer = () => undefined;
|
|
303
|
+
scheduleSweep();
|
|
304
|
+
};
|
|
305
|
+
exports.resetForTesting = resetForTesting;
|
|
306
|
+
exports.MotherduckInstanceCache = {
|
|
307
|
+
configure: exports.configure,
|
|
308
|
+
setObserver: exports.setObserver,
|
|
309
|
+
withInstance: exports.withInstance,
|
|
310
|
+
invalidate: exports.invalidate,
|
|
311
|
+
invalidateByConnectionString: exports.invalidateByConnectionString,
|
|
312
|
+
recordRetry: exports.recordRetry,
|
|
313
|
+
closeAll: exports.closeAll,
|
|
314
|
+
resetForTesting: exports.resetForTesting,
|
|
315
|
+
};
|
|
316
|
+
scheduleSweep();
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MotherduckInstanceCache.security.test.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/MotherduckInstanceCache.security.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
const common_1 = require("@lightdash/common");
|
|
37
|
+
const DuckdbWarehouseClient_1 = require("./DuckdbWarehouseClient");
|
|
38
|
+
const MotherduckInstanceCache = __importStar(require("./MotherduckInstanceCache"));
|
|
39
|
+
const createInstanceMock = vi.fn();
|
|
40
|
+
vi.mock('@duckdb/node-api', () => ({
|
|
41
|
+
DuckDBTypeId: { INTEGER: 4, VARCHAR: 17 },
|
|
42
|
+
DuckDBInstance: {
|
|
43
|
+
create: (...args) => createInstanceMock(...args),
|
|
44
|
+
},
|
|
45
|
+
version: () => 'v1.5.2',
|
|
46
|
+
}));
|
|
47
|
+
const getMockStreamResult = (marker) => ({
|
|
48
|
+
columnCount: 1,
|
|
49
|
+
columnNames: () => ['marker'],
|
|
50
|
+
columnTypeId: () => 17,
|
|
51
|
+
yieldRowObjectJson: async function* yieldRows() {
|
|
52
|
+
yield [{ marker }];
|
|
53
|
+
},
|
|
54
|
+
});
|
|
55
|
+
const createMockInstance = (streamMock) => ({
|
|
56
|
+
connect: async () => ({
|
|
57
|
+
run: vi.fn(),
|
|
58
|
+
stream: streamMock,
|
|
59
|
+
extractStatements: vi.fn(async () => ({
|
|
60
|
+
count: 1,
|
|
61
|
+
prepare: async () => ({
|
|
62
|
+
statementType: 1,
|
|
63
|
+
destroySync: vi.fn(),
|
|
64
|
+
}),
|
|
65
|
+
})),
|
|
66
|
+
interrupt: vi.fn(),
|
|
67
|
+
closeSync: vi.fn(),
|
|
68
|
+
disconnectSync: vi.fn(),
|
|
69
|
+
}),
|
|
70
|
+
closeSync: vi.fn(),
|
|
71
|
+
});
|
|
72
|
+
const motherduckCredentials = (database, token) => ({
|
|
73
|
+
type: common_1.WarehouseTypes.DUCKDB,
|
|
74
|
+
connectionType: common_1.DuckdbConnectionType.MOTHERDUCK,
|
|
75
|
+
database,
|
|
76
|
+
schema: 'main',
|
|
77
|
+
token,
|
|
78
|
+
});
|
|
79
|
+
const cachedOptions = (projectUuid) => ({
|
|
80
|
+
enableInstanceCache: true,
|
|
81
|
+
projectUuid,
|
|
82
|
+
});
|
|
83
|
+
const successStream = (marker) => vi.fn(async () => getMockStreamResult(marker));
|
|
84
|
+
describe('MotherDuck instance cache security boundaries', () => {
|
|
85
|
+
beforeEach(() => {
|
|
86
|
+
vi.useFakeTimers();
|
|
87
|
+
vi.clearAllMocks();
|
|
88
|
+
MotherduckInstanceCache.resetForTesting();
|
|
89
|
+
MotherduckInstanceCache.configure({
|
|
90
|
+
idleTtlMs: 60000,
|
|
91
|
+
maxAgeMs: 3600000,
|
|
92
|
+
maxEntries: 64,
|
|
93
|
+
});
|
|
94
|
+
DuckdbWarehouseClient_1.DuckdbWarehouseClient.resetSharedDuckdbStateForTesting();
|
|
95
|
+
});
|
|
96
|
+
afterEach(async () => {
|
|
97
|
+
await MotherduckInstanceCache.closeAll('shutdown');
|
|
98
|
+
vi.useRealTimers();
|
|
99
|
+
});
|
|
100
|
+
it('isolates concurrent clients with different tokens for one database', async () => {
|
|
101
|
+
const streamA = successStream('token-a-rows');
|
|
102
|
+
const streamB = successStream('token-b-rows');
|
|
103
|
+
const instanceA = createMockInstance(streamA);
|
|
104
|
+
const instanceB = createMockInstance(streamB);
|
|
105
|
+
createInstanceMock.mockImplementation(async (connectionString) => String(connectionString).includes('token-a')
|
|
106
|
+
? instanceA
|
|
107
|
+
: instanceB);
|
|
108
|
+
const clientA = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(motherduckCredentials('analytics', 'token-a'), cachedOptions('project-a'));
|
|
109
|
+
const clientB = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(motherduckCredentials('analytics', 'token-b'), cachedOptions('project-b'));
|
|
110
|
+
const [resultA, resultB] = await Promise.all([
|
|
111
|
+
clientA.runQuery('SELECT 1'),
|
|
112
|
+
clientB.runQuery('SELECT 1'),
|
|
113
|
+
]);
|
|
114
|
+
expect(createInstanceMock).toHaveBeenCalledTimes(2);
|
|
115
|
+
expect(resultA.rows).toEqual([{ marker: 'token-a-rows' }]);
|
|
116
|
+
expect(resultB.rows).toEqual([{ marker: 'token-b-rows' }]);
|
|
117
|
+
});
|
|
118
|
+
it('isolates clients with one token across different databases', async () => {
|
|
119
|
+
createInstanceMock.mockImplementation(async (connectionString) => createMockInstance(successStream(String(connectionString).startsWith('md:first')
|
|
120
|
+
? 'first-rows'
|
|
121
|
+
: 'second-rows')));
|
|
122
|
+
const firstClient = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(motherduckCredentials('first', 'shared-token'), cachedOptions('project-a'));
|
|
123
|
+
const secondClient = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(motherduckCredentials('second', 'shared-token'), cachedOptions('project-b'));
|
|
124
|
+
const [firstResult, secondResult] = await Promise.all([
|
|
125
|
+
firstClient.runQuery('SELECT 1'),
|
|
126
|
+
secondClient.runQuery('SELECT 1'),
|
|
127
|
+
]);
|
|
128
|
+
expect(createInstanceMock).toHaveBeenCalledTimes(2);
|
|
129
|
+
expect(firstResult.rows).toEqual([{ marker: 'first-rows' }]);
|
|
130
|
+
expect(secondResult.rows).toEqual([{ marker: 'second-rows' }]);
|
|
131
|
+
});
|
|
132
|
+
it('never maps adversarial database names onto another credential entry', async () => {
|
|
133
|
+
const victimConnectionString = `md:analytics?${new URLSearchParams({
|
|
134
|
+
motherduck_token: 'victim-token',
|
|
135
|
+
saas_mode: 'true',
|
|
136
|
+
}).toString()}`;
|
|
137
|
+
const adversarialNames = [
|
|
138
|
+
'analytics?motherduck_token=victim-token',
|
|
139
|
+
'analytics?motherduck_token=victim-token&saas_mode=true',
|
|
140
|
+
'analytics&saas_mode=false',
|
|
141
|
+
'analytics%3Fmotherduck_token%3Dvictim-token',
|
|
142
|
+
'analytics/../analytics',
|
|
143
|
+
'analytics#',
|
|
144
|
+
'analytics/',
|
|
145
|
+
'analytics ',
|
|
146
|
+
'ANALYTICS',
|
|
147
|
+
'café',
|
|
148
|
+
'café',
|
|
149
|
+
'analytics%00',
|
|
150
|
+
'',
|
|
151
|
+
'�',
|
|
152
|
+
];
|
|
153
|
+
createInstanceMock.mockImplementation(async () => createMockInstance(successStream('rows')));
|
|
154
|
+
await Promise.all(adversarialNames.map((database, index) => {
|
|
155
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(motherduckCredentials(database, 'attacker-token'), cachedOptions(`attacker-project-${index}`));
|
|
156
|
+
return client.runQuery('SELECT 1');
|
|
157
|
+
}));
|
|
158
|
+
const adversarialConnectionStrings = createInstanceMock.mock.calls.map(([connectionString]) => String(connectionString));
|
|
159
|
+
expect(adversarialConnectionStrings).not.toContain(victimConnectionString);
|
|
160
|
+
expect(new Set(adversarialConnectionStrings).size).toBe(adversarialNames.length);
|
|
161
|
+
const victimClient = new DuckdbWarehouseClient_1.DuckdbWarehouseClient(motherduckCredentials('analytics', 'victim-token'), cachedOptions('victim-project'));
|
|
162
|
+
await victimClient.runQuery('SELECT 1');
|
|
163
|
+
expect(createInstanceMock).toHaveBeenLastCalledWith(victimConnectionString);
|
|
164
|
+
});
|
|
165
|
+
it('does not close an in-flight instance when max age evicts it', async () => {
|
|
166
|
+
MotherduckInstanceCache.configure({
|
|
167
|
+
idleTtlMs: 60000,
|
|
168
|
+
maxAgeMs: 1000,
|
|
169
|
+
maxEntries: 8,
|
|
170
|
+
});
|
|
171
|
+
const instance = createMockInstance(successStream('rows'));
|
|
172
|
+
createInstanceMock
|
|
173
|
+
.mockResolvedValueOnce(instance)
|
|
174
|
+
.mockResolvedValue(createMockInstance(successStream('other')));
|
|
175
|
+
let releaseQuery = () => undefined;
|
|
176
|
+
const queryGate = new Promise((resolve) => {
|
|
177
|
+
releaseQuery = resolve;
|
|
178
|
+
});
|
|
179
|
+
let entryId = '';
|
|
180
|
+
const inFlight = MotherduckInstanceCache.withInstance('md:analytics?motherduck_token=token-a', {}, async (_instance, acquiredEntryId) => {
|
|
181
|
+
entryId = acquiredEntryId;
|
|
182
|
+
await queryGate;
|
|
183
|
+
});
|
|
184
|
+
await vi.waitFor(() => expect(entryId).not.toBe(''));
|
|
185
|
+
await vi.advanceTimersByTimeAsync(1500);
|
|
186
|
+
await MotherduckInstanceCache.withInstance('md:other?motherduck_token=token-b', {}, async () => undefined);
|
|
187
|
+
expect(instance.closeSync).not.toHaveBeenCalled();
|
|
188
|
+
releaseQuery();
|
|
189
|
+
await inFlight;
|
|
190
|
+
expect(instance.closeSync).toHaveBeenCalledOnce();
|
|
191
|
+
});
|
|
192
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MotherduckInstanceCache.test.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/MotherduckInstanceCache.test.ts"],"names":[],"mappings":""}
|