@makoai/app-sdk 2.11.1 → 2.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +106 -3
- package/data-core.js +62 -0
- package/index.d.ts +58 -3
- package/index.js +1071 -90
- package/package.json +1 -1
package/index.js
CHANGED
|
@@ -20,8 +20,12 @@ import {
|
|
|
20
20
|
missingBinding,
|
|
21
21
|
mentionedBindings,
|
|
22
22
|
quoteIdent,
|
|
23
|
+
isReadOnlySql,
|
|
23
24
|
readsCatalog,
|
|
24
25
|
quoteString,
|
|
26
|
+
crashedEngineError,
|
|
27
|
+
isOutOfMemoryError,
|
|
28
|
+
retryDelayMs,
|
|
25
29
|
toPlainRows,
|
|
26
30
|
parseBindingSnapshot,
|
|
27
31
|
snapshotUrl,
|
|
@@ -32,7 +36,7 @@ import {
|
|
|
32
36
|
* in a page (or this export) says which SDK a deployed app actually runs.
|
|
33
37
|
* Kept in step with package.json by snapshot.test.mjs.
|
|
34
38
|
*/
|
|
35
|
-
export const SDK_VERSION = "2.
|
|
39
|
+
export const SDK_VERSION = "2.13.0";
|
|
36
40
|
if (typeof window !== "undefined") {
|
|
37
41
|
try {
|
|
38
42
|
window.__MAKO_APP_SDK__ = SDK_VERSION;
|
|
@@ -42,6 +46,9 @@ if (typeof window !== "undefined") {
|
|
|
42
46
|
}
|
|
43
47
|
|
|
44
48
|
const DEFAULT_ROW_LIMIT = 500000;
|
|
49
|
+
// A double-read UNION ALL of 55 to 60 bindings overflows the WASM stack and
|
|
50
|
+
// corrupts the engine for every panel (measured). 50 held.
|
|
51
|
+
const MAX_BINDINGS_IN_QUERY = 50;
|
|
45
52
|
const DUCKDB_ESM =
|
|
46
53
|
"https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.29.0/+esm";
|
|
47
54
|
|
|
@@ -53,12 +60,30 @@ const DUCKDB_ESM =
|
|
|
53
60
|
// parquet download instead of waiting for it.
|
|
54
61
|
// ---------------------------------------------------------------------------
|
|
55
62
|
let dbPromise = null;
|
|
63
|
+
let liveEngine = null; // the resolved dbPromise: errors from an older one are history
|
|
64
|
+
// createWorker() makes a new object URL per call and never revokes it, so each
|
|
65
|
+
// reboot after a release would leak the worker script (about 742 KiB). One URL
|
|
66
|
+
// per page, also one download fewer in a frame with no HTTP cache. Not through
|
|
67
|
+
// the download gate: the CDN being down says nothing about Mako's server.
|
|
68
|
+
let workerUrl = null;
|
|
69
|
+
function startWorker(duckdb, script) {
|
|
70
|
+
if (typeof Worker === "undefined") return duckdb.createWorker(script);
|
|
71
|
+
workerUrl ??= fetch(script).then(async res => {
|
|
72
|
+
if (!res.ok) throw new Error("The DuckDB worker could not be loaded (HTTP " + res.status + ")");
|
|
73
|
+
return URL.createObjectURL(await res.blob());
|
|
74
|
+
});
|
|
75
|
+
workerUrl.catch(() => {
|
|
76
|
+
workerUrl = null;
|
|
77
|
+
});
|
|
78
|
+
return workerUrl.then(url => new Worker(url));
|
|
79
|
+
}
|
|
80
|
+
|
|
56
81
|
function getDb() {
|
|
57
82
|
if (!dbPromise) {
|
|
58
83
|
dbPromise = (async () => {
|
|
59
84
|
const duckdb = await import(/* @vite-ignore */ DUCKDB_ESM);
|
|
60
85
|
const bundle = await duckdb.selectBundle(duckdb.getJsDelivrBundles());
|
|
61
|
-
const worker = await duckdb
|
|
86
|
+
const worker = await startWorker(duckdb, bundle.mainWorker);
|
|
62
87
|
const db = new duckdb.AsyncDuckDB(new duckdb.VoidLogger(), worker);
|
|
63
88
|
await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
|
|
64
89
|
// read_parquet lives in an extension DuckDB fetches on first use
|
|
@@ -72,6 +97,7 @@ function getDb() {
|
|
|
72
97
|
} finally {
|
|
73
98
|
await conn.close();
|
|
74
99
|
}
|
|
100
|
+
liveEngine = db;
|
|
75
101
|
return db;
|
|
76
102
|
})();
|
|
77
103
|
// A failed boot (CDN unreachable) is retried by the next query.
|
|
@@ -104,6 +130,8 @@ async function responseError(res, what) {
|
|
|
104
130
|
const error = new Error(what + " (HTTP " + res.status + ")" + (detail ? ": " + detail : ""));
|
|
105
131
|
error.status = res.status;
|
|
106
132
|
if (body && typeof body.retryAfterMs === "number") error.retryAfterMs = body.retryAfterMs;
|
|
133
|
+
// The one thing a mounted hook may ask again. Any other status said what it meant.
|
|
134
|
+
if (res.status === 429 || res.status === 503) error.retry = "network";
|
|
107
135
|
return error;
|
|
108
136
|
}
|
|
109
137
|
|
|
@@ -168,6 +196,102 @@ export function useBindingProgress(name) {
|
|
|
168
196
|
// under its own file name, so a reload never drops a file another load is
|
|
169
197
|
// still reading.
|
|
170
198
|
|
|
199
|
+
// Cloud Run answers a queue timeout with a 429 and no CORS header, so an opaque
|
|
200
|
+
// app frame sees only "Failed to fetch": 12 of 5,087 published parquet requests
|
|
201
|
+
// on 2026-11-02, in 8 bursts. One page-wide gate keeps a page from making it worse.
|
|
202
|
+
// - At most 12 downloads at once; 1 while the gate is recovering from a failure.
|
|
203
|
+
// - A failure closes the gate for 1, 3, 9, 27, then 60 s, with jitter. Failures
|
|
204
|
+
// in one closed window count once. Any answer that is not a 429 or 503 opens it.
|
|
205
|
+
// - Past 3 failures in a row, a request while it is closed fails at once with no
|
|
206
|
+
// network, so a page down to one probe at a time does not keep panels spinning.
|
|
207
|
+
const FETCH_RETRY_MS = [1000, 3000];
|
|
208
|
+
const RETRY_AFTER_CAP_MS = 30000;
|
|
209
|
+
const DOWNLOAD_SLOTS = 12;
|
|
210
|
+
const GATE_MAX_MS = 60000;
|
|
211
|
+
const GATE_FAIL_FAST_AT = 3;
|
|
212
|
+
let gateStreak = 0;
|
|
213
|
+
let gateUntil = 0;
|
|
214
|
+
let gateError = null;
|
|
215
|
+
let slotsInUse = 0;
|
|
216
|
+
const slotQueue = [];
|
|
217
|
+
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
|
218
|
+
const withJitter = ms => ms * (0.75 + Math.random() / 2);
|
|
219
|
+
|
|
220
|
+
function gateFailed(error) {
|
|
221
|
+
gateError = error;
|
|
222
|
+
if (Date.now() < gateUntil) return;
|
|
223
|
+
gateStreak++;
|
|
224
|
+
gateUntil = Date.now() + withJitter(Math.min(GATE_MAX_MS, 1000 * 3 ** (gateStreak - 1)));
|
|
225
|
+
}
|
|
226
|
+
function gateAnswered() {
|
|
227
|
+
if (gateStreak === 0) return;
|
|
228
|
+
gateStreak = 0;
|
|
229
|
+
gateUntil = 0;
|
|
230
|
+
gateError = null;
|
|
231
|
+
pumpSlots();
|
|
232
|
+
wakeRetries();
|
|
233
|
+
}
|
|
234
|
+
async function throughGate() {
|
|
235
|
+
for (;;) {
|
|
236
|
+
const wait = gateUntil - Date.now();
|
|
237
|
+
if (wait <= 0) return;
|
|
238
|
+
if (gateStreak >= GATE_FAIL_FAST_AT) throw gateError;
|
|
239
|
+
await sleep(wait);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
function pumpSlots() {
|
|
243
|
+
let room = (gateStreak ? 1 : DOWNLOAD_SLOTS) - slotsInUse;
|
|
244
|
+
while (room-- > 0 && slotQueue.length) slotQueue.shift()();
|
|
245
|
+
}
|
|
246
|
+
async function withSlot(fn) {
|
|
247
|
+
while (slotsInUse >= (gateStreak ? 1 : DOWNLOAD_SLOTS)) await new Promise(resolve => slotQueue.push(resolve));
|
|
248
|
+
slotsInUse++;
|
|
249
|
+
try {
|
|
250
|
+
return await fn();
|
|
251
|
+
} finally {
|
|
252
|
+
slotsInUse--;
|
|
253
|
+
pumpSlots();
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* fetch, until headers arrive: 3 attempts on a rejected fetch, a 429 or a 503,
|
|
259
|
+
* through the gate. A response of any other status returns at once, and so does
|
|
260
|
+
* a stream once it starts, so a build is never asked for again from here. Waits
|
|
261
|
+
* out a `retryAfterMs` up to 30 s; past that the caller gets the answer. No
|
|
262
|
+
* headers timeout: the Vite plugin may hold headers for a whole build.
|
|
263
|
+
*/
|
|
264
|
+
async function fetchRetrying(url, init) {
|
|
265
|
+
for (let attempt = 0; ; attempt++) {
|
|
266
|
+
const last = attempt >= FETCH_RETRY_MS.length;
|
|
267
|
+
await throughGate();
|
|
268
|
+
let wait = last ? 0 : withJitter(FETCH_RETRY_MS[attempt]);
|
|
269
|
+
try {
|
|
270
|
+
const res = await fetch(url, init);
|
|
271
|
+
if (res.status !== 429 && res.status !== 503) {
|
|
272
|
+
gateAnswered();
|
|
273
|
+
return res;
|
|
274
|
+
}
|
|
275
|
+
const asked = await res
|
|
276
|
+
.clone()
|
|
277
|
+
.json()
|
|
278
|
+
.then(body => body?.retryAfterMs)
|
|
279
|
+
.catch(() => undefined);
|
|
280
|
+
const busy = new Error("The server is busy (HTTP " + res.status + ")");
|
|
281
|
+
busy.retry = "network";
|
|
282
|
+
gateFailed(busy);
|
|
283
|
+
if (last || asked > RETRY_AFTER_CAP_MS) return res;
|
|
284
|
+
if (typeof asked === "number") wait = asked;
|
|
285
|
+
} catch (error) {
|
|
286
|
+
if (!(error instanceof TypeError)) throw error;
|
|
287
|
+
error.retry = "network";
|
|
288
|
+
gateFailed(error);
|
|
289
|
+
if (last) throw error;
|
|
290
|
+
}
|
|
291
|
+
await sleep(wait);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
171
295
|
/**
|
|
172
296
|
* A binding's parquet bytes, phase-aware: asks for NDJSON phase events (a
|
|
173
297
|
* build reports each step, a stored file resolving → signing and a signed
|
|
@@ -178,67 +302,748 @@ async function downloadBinding(name, bust, report) {
|
|
|
178
302
|
const path = "__data/" + encodeURIComponent(name) + ".parquet";
|
|
179
303
|
const what = 'Data for binding "' + name + '" could not be loaded';
|
|
180
304
|
report({ phase: "requesting", rowsWritten: null, bytes: null, bytesTotal: null });
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
305
|
+
// A slot holds the connection, not the wait for a build: one slow warehouse
|
|
306
|
+
// query must not hold up twelve bindings that are ready.
|
|
307
|
+
let res = await withSlot(() =>
|
|
308
|
+
fetchRetrying(path + (bust ? "?refresh=" + Date.now() : ""), {
|
|
309
|
+
headers: { accept: PROGRESS_ACCEPT },
|
|
310
|
+
}),
|
|
311
|
+
);
|
|
312
|
+
let signed = null;
|
|
184
313
|
if (isProgressResponse(res)) {
|
|
185
|
-
|
|
314
|
+
let final;
|
|
315
|
+
try {
|
|
316
|
+
final = await readProgressStream(res, e => report(progressPatch(e)));
|
|
317
|
+
} catch (error) {
|
|
318
|
+
// A cut build stream may have cost a warehouse query. Asking again costs another.
|
|
319
|
+
throw new Error(what + ": the build's stream was cut (" + (error instanceof Error ? error.message : String(error)) + ")");
|
|
320
|
+
}
|
|
186
321
|
if (final.phase === "failed") throw progressError(final, what);
|
|
187
322
|
report({ phase: "downloading", rowsWritten: null, bytes: 0 });
|
|
188
323
|
// No url: the server streams the bytes itself — the same path, asked
|
|
189
324
|
// plainly, serves them (the build already staged or cached them).
|
|
190
|
-
|
|
325
|
+
signed = final.url || path;
|
|
326
|
+
res = null;
|
|
191
327
|
}
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
328
|
+
return withSlot(async () => {
|
|
329
|
+
res ??= await fetchRetrying(signed);
|
|
330
|
+
if (!res.ok) throw await responseError(res, what);
|
|
331
|
+
try {
|
|
332
|
+
return await readBytes(res, (bytes, bytesTotal) =>
|
|
333
|
+
report({ phase: "downloading", rowsWritten: null, bytes, bytesTotal }),
|
|
334
|
+
);
|
|
335
|
+
} catch (error) {
|
|
336
|
+
if (error instanceof TypeError) error.retry = "network"; // the body cut off: no build behind it
|
|
337
|
+
throw error;
|
|
338
|
+
}
|
|
339
|
+
});
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
// ---------------------------------------------------------------------------
|
|
343
|
+
// Memory (2.12). Every table lives in this frame's WASM heap, where DuckDB's
|
|
344
|
+
// memory_limit is 3.1 GiB. 2.8 to 2.11 never dropped one, so an app with many
|
|
345
|
+
// bindings (Web Performance: 106 shards) browsed its way to "Out of Memory".
|
|
346
|
+
//
|
|
347
|
+
// - Every read bumps its binding. Least recently used goes first.
|
|
348
|
+
// - The budget counts what the SDK placed: tables and the parquet copies views
|
|
349
|
+
// read. An app's own tables do not shrink it. The real total still decides
|
|
350
|
+
// whether a table can fit in the heap at all.
|
|
351
|
+
// - A table lands if the parquet's own metadata says it fits, after evicting what
|
|
352
|
+
// no one holds. Measured tables can pass the estimate, so the budget can
|
|
353
|
+
// overshoot (measured: 1,700 MiB against 1,536). With a mostly empty heap, one
|
|
354
|
+
// table may try anyway: an ordinary app with one large parquet stays fast.
|
|
355
|
+
// - What does not fit lands as a view over its parquet in the heap
|
|
356
|
+
// (registerFileBuffer): 7x to 14x smaller than its table. A later read, or the
|
|
357
|
+
// end of pressure, promotes it if room has appeared.
|
|
358
|
+
// - Never a BROWSER_FILEREADER handle over a Blob: duckdb-wasm 1.29.0's openFile
|
|
359
|
+
// writes the file size to HEAPF64[(ptr >> 3)], and `>>` turns a pointer past
|
|
360
|
+
// 2 GiB negative. Reads then fail with "too small to be a Parquet file" (4 of
|
|
361
|
+
// 8 reads with 3.0 GiB of tables, 0 of 8 with 1.6 GiB; heap buffers 0 of 8).
|
|
362
|
+
// - File copies have their own ceiling: 2.5 GiB, and 3.6 GiB with DuckDB's own
|
|
363
|
+
// bytes. registerFileBuffer never checks its malloc: at 4.16 GiB of copies the
|
|
364
|
+
// call never settled and hung the page. Over it the least recently read view
|
|
365
|
+
// goes, pinned too, and downloads again on its next read.
|
|
366
|
+
// - The parquet stays in JS memory as a Blob only so a table can become a view
|
|
367
|
+
// without a download. A Blob can stop being readable (NotReadableError on the
|
|
368
|
+
// third 885 MiB in incognito). Its table then goes instead, and downloads again.
|
|
369
|
+
// Past the Blob budget, the least recently used view is dropped outright.
|
|
370
|
+
// - A binding a mounted hook reads, or a query in flight, is pinned: it goes last.
|
|
371
|
+
// Only loads in flight are never touched.
|
|
372
|
+
// - "Out of Memory" on a query demotes unpinned tables, then the pinned tables
|
|
373
|
+
// that query reads, and retries until it answers or none is left. Other
|
|
374
|
+
// panels' tables stay. For a minute (pressure) tables stay down, then pinned
|
|
375
|
+
// views promote. Only DuckDB's own error counts (see markEngineError).
|
|
376
|
+
// - "Out of Memory" on a load demotes unpinned tables and retries once. A load
|
|
377
|
+
// that still does not fit lands as a view.
|
|
378
|
+
//
|
|
379
|
+
// All catalog changes run one at a time (catalog()), so the bookkeeping here
|
|
380
|
+
// always matches DuckDB's catalog. The worker runs one statement at a time
|
|
381
|
+
// anyway, so this costs no parallelism.
|
|
382
|
+
// ---------------------------------------------------------------------------
|
|
383
|
+
const MIB = 1024 * 1024;
|
|
384
|
+
export const DEFAULT_BINDING_MEMORY_BUDGET = 1536 * MIB;
|
|
385
|
+
export const DEFAULT_BINDING_BLOB_BUDGET = 512 * MIB;
|
|
386
|
+
const DUCKDB_LIMIT = 3174 * MIB; // duckdb-wasm's default memory_limit: 3.1 GiB
|
|
387
|
+
const FILE_CEILING = 2560 * MIB;
|
|
388
|
+
const HEAP_CEILING = 3686 * MIB;
|
|
389
|
+
let memoryBudget = DEFAULT_BINDING_MEMORY_BUDGET;
|
|
390
|
+
let blobBudget = DEFAULT_BINDING_BLOB_BUDGET;
|
|
391
|
+
|
|
392
|
+
// name -> { storage: "table" | "view", blob: Blob | null, file: string | null,
|
|
393
|
+
// fileBytes: number, tableBytes: number | null, demoted: boolean }. `file` is the
|
|
394
|
+
// heap copy of the parquet a view reads, of `fileBytes`. `tableBytes`: what its
|
|
395
|
+
// table cost, measured. `demoted`: a view only because of memory: try to promote.
|
|
396
|
+
const entries = new Map();
|
|
397
|
+
const lastUse = new Map(); // name -> sequence number of its latest read
|
|
398
|
+
let useSeq = 0;
|
|
399
|
+
const pins = new Map(); // name -> how many hooks or queries hold it
|
|
400
|
+
const loading = new Set(); // names with a load in flight
|
|
401
|
+
// Loaded through ensureBinding(): app code may query them by name at any
|
|
402
|
+
// time, so they are demoted (still readable) but never dropped.
|
|
403
|
+
const externalBindings = new Set();
|
|
404
|
+
// Every binding this page ever loaded, for reading one the SDK evicted when the
|
|
405
|
+
// server's index does not list it (index [] apps name their bindings in useQuery).
|
|
406
|
+
const knownBindings = new Set();
|
|
407
|
+
let measuredBytes = 0;
|
|
408
|
+
// After an Out of Memory, tables stay at or under what relief left, for a minute.
|
|
409
|
+
const PRESSURE_MS = 60000;
|
|
410
|
+
let pressureCeiling = Infinity;
|
|
411
|
+
let pressureUntil = 0;
|
|
412
|
+
let pressureTimer = null;
|
|
413
|
+
|
|
414
|
+
function tableCeiling() {
|
|
415
|
+
return Date.now() < pressureUntil ? Math.min(memoryBudget, pressureCeiling) : memoryBudget;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
function touch(name) {
|
|
419
|
+
lastUse.set(name, ++useSeq);
|
|
420
|
+
}
|
|
421
|
+
function isPinned(name) {
|
|
422
|
+
return (pins.get(name) ?? 0) > 0;
|
|
423
|
+
}
|
|
424
|
+
/** Hold `name` against eviction; the returned function lets go (once). */
|
|
425
|
+
function pin(name) {
|
|
426
|
+
knownBindings.add(name);
|
|
427
|
+
pins.set(name, (pins.get(name) ?? 0) + 1);
|
|
428
|
+
let held = true;
|
|
429
|
+
return () => {
|
|
430
|
+
if (!held) return;
|
|
431
|
+
held = false;
|
|
432
|
+
const n = (pins.get(name) ?? 1) - 1;
|
|
433
|
+
if (n > 0) pins.set(name, n);
|
|
434
|
+
else pins.delete(name);
|
|
435
|
+
};
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
let catalogQueue = Promise.resolve();
|
|
439
|
+
function catalog(fn) {
|
|
440
|
+
const run = catalogQueue.then(fn);
|
|
441
|
+
catalogQueue = run.catch(() => {});
|
|
442
|
+
return run;
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
// The statements in one transaction, so another connection never sees the
|
|
446
|
+
// name missing between a DROP and a CREATE.
|
|
447
|
+
async function exec(engine, statements) {
|
|
448
|
+
const conn = await engine.connect();
|
|
449
|
+
try {
|
|
450
|
+
if (statements.length === 1) {
|
|
451
|
+
await conn.query(statements[0]);
|
|
452
|
+
return;
|
|
453
|
+
}
|
|
454
|
+
await conn.query("BEGIN TRANSACTION");
|
|
455
|
+
try {
|
|
456
|
+
for (const sql of statements) await conn.query(sql);
|
|
457
|
+
await conn.query("COMMIT");
|
|
458
|
+
} catch (error) {
|
|
459
|
+
await conn.query("ROLLBACK").catch(() => {});
|
|
460
|
+
throw error;
|
|
461
|
+
}
|
|
462
|
+
} catch (error) {
|
|
463
|
+
throw markEngineError(error, engine);
|
|
464
|
+
} finally {
|
|
465
|
+
await conn.close();
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
/**
|
|
470
|
+
* An error from this engine's own conn.query. Only these can be Out of Memory:
|
|
471
|
+
* a download or a server's build may quote the same words. A crash is noted here
|
|
472
|
+
* and the engine restarts (restartEngine).
|
|
473
|
+
*/
|
|
474
|
+
function markEngineError(error, engine) {
|
|
475
|
+
if (!(error instanceof Error)) return error;
|
|
476
|
+
error.fromEngine = true;
|
|
477
|
+
if (isOutOfMemoryError(error)) error.retry = "memory";
|
|
478
|
+
if (crashedEngineError(error)) {
|
|
479
|
+
error.message = "The data engine crashed (" + error.message + "). It restarts by itself; reload the page if this stays.";
|
|
480
|
+
noteEngineCrash(engine);
|
|
481
|
+
}
|
|
482
|
+
return error;
|
|
483
|
+
}
|
|
484
|
+
const isEngineOom = error => Boolean(error?.fromEngine) && isOutOfMemoryError(error);
|
|
485
|
+
|
|
486
|
+
function dropSql(storage, name) {
|
|
487
|
+
return "DROP " + (storage === "view" ? "VIEW" : "TABLE") + " IF EXISTS " + quoteIdent(name);
|
|
196
488
|
}
|
|
197
489
|
|
|
198
490
|
let fileSeq = 0;
|
|
491
|
+
// Every load registers its bytes under its own file name, so a reload never
|
|
492
|
+
// drops a file another load is still reading.
|
|
493
|
+
function nextFile() {
|
|
494
|
+
return "mako-" + ++fileSeq + ".parquet";
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
/** The table for a registered parquet `file`, in place of whatever `name` was. */
|
|
498
|
+
async function tableFromFile(engine, name, file) {
|
|
499
|
+
const prev = entries.get(name);
|
|
500
|
+
const before = await duckdbBytes(engine);
|
|
501
|
+
const create =
|
|
502
|
+
"CREATE OR REPLACE TABLE " + quoteIdent(name) +
|
|
503
|
+
" AS SELECT * FROM read_parquet(" + quoteString(file) + ")";
|
|
504
|
+
await exec(engine, prev && prev.storage === "view" ? [dropSql("view", name), create] : [create]);
|
|
505
|
+
const after = await duckdbBytes(engine);
|
|
506
|
+
if (before === null || after === null) return prev?.tableBytes ?? null;
|
|
507
|
+
const grown = after - before + (prev?.storage === "table" ? (prev.tableBytes ?? 0) : 0);
|
|
508
|
+
return grown > 0 ? grown : (prev?.tableBytes ?? null);
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
/** The view over a registered parquet `file`, in place of whatever `name` was. */
|
|
512
|
+
async function viewFromFile(engine, name, file) {
|
|
513
|
+
const prev = entries.get(name);
|
|
514
|
+
const create =
|
|
515
|
+
"CREATE OR REPLACE VIEW " + quoteIdent(name) +
|
|
516
|
+
" AS SELECT * FROM read_parquet(" + quoteString(file) + ")";
|
|
517
|
+
await exec(engine, prev && prev.storage === "table" ? [dropSql("table", name), create] : [create]);
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
/**
|
|
521
|
+
* What a registered parquet's table will cost, from its metadata: 16 bytes a cell,
|
|
522
|
+
* or its uncompressed pages if long strings make that more. Measured on 6 parquets
|
|
523
|
+
* from 2 to 402 MiB: the real table was 0.84x to 1.01x of the 16-byte figure.
|
|
524
|
+
*/
|
|
525
|
+
async function estimateTable(engine, file, parquetBytes) {
|
|
526
|
+
const conn = await engine.connect();
|
|
527
|
+
try {
|
|
528
|
+
const f = quoteString(file);
|
|
529
|
+
const result = await conn.query(
|
|
530
|
+
"SELECT greatest(sum(total_uncompressed_size), " +
|
|
531
|
+
"count(DISTINCT path_in_schema) * (SELECT num_rows FROM parquet_file_metadata(" + f + ")) * 16)::DOUBLE " +
|
|
532
|
+
"FROM parquet_metadata(" + f + ")",
|
|
533
|
+
);
|
|
534
|
+
const value = Number(result.getChildAt(0)?.get(0));
|
|
535
|
+
if (Number.isFinite(value) && value > 0) return value;
|
|
536
|
+
} catch {
|
|
537
|
+
/* no metadata to read: the file's own error surfaces when it is read */
|
|
538
|
+
} finally {
|
|
539
|
+
await conn.close();
|
|
540
|
+
}
|
|
541
|
+
return parquetBytes * 16;
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
/** What the SDK placed: tables, and the parquet copies views read. */
|
|
545
|
+
function ownedBytes() {
|
|
546
|
+
let bytes = 0;
|
|
547
|
+
for (const entry of entries.values()) {
|
|
548
|
+
bytes += entry.storage === "table" ? (entry.tableBytes ?? (entry.blob?.size ?? 0) * 16) : entry.fileBytes;
|
|
549
|
+
}
|
|
550
|
+
return bytes;
|
|
551
|
+
}
|
|
552
|
+
function fileBytes() {
|
|
553
|
+
let bytes = 0;
|
|
554
|
+
for (const entry of entries.values()) if (entry.storage === "view") bytes += entry.fileBytes;
|
|
555
|
+
return bytes;
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
/**
|
|
559
|
+
* What this frame's heap holds right now, in bytes: DuckDB's figure, with app
|
|
560
|
+
* tables and every binding table, plus the parquet copies views read, which sit
|
|
561
|
+
* outside its accounting. Query scratch never shows: DuckDB returns it before the
|
|
562
|
+
* next statement runs (measured, also after a failed query). The last figure if
|
|
563
|
+
* it cannot say.
|
|
564
|
+
*/
|
|
565
|
+
async function measureNow(engine) {
|
|
566
|
+
const value = await duckdbBytes(engine);
|
|
567
|
+
if (value !== null) measuredBytes = value + fileBytes();
|
|
568
|
+
reportMemory();
|
|
569
|
+
return measuredBytes;
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
/** duckdb_memory() alone, or null on an engine without it. */
|
|
573
|
+
async function duckdbBytes(engine) {
|
|
574
|
+
const conn = await engine.connect();
|
|
575
|
+
try {
|
|
576
|
+
const result = await conn.query(
|
|
577
|
+
"SELECT coalesce(sum(memory_usage_bytes), 0)::DOUBLE AS bytes FROM duckdb_memory()",
|
|
578
|
+
);
|
|
579
|
+
const value = Number(result.getChildAt(0)?.get(0));
|
|
580
|
+
return Number.isFinite(value) ? value : null;
|
|
581
|
+
} catch (error) {
|
|
582
|
+
markEngineError(error, engine);
|
|
583
|
+
return null; /* the OOM retry still applies */
|
|
584
|
+
} finally {
|
|
585
|
+
await conn.close();
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
// ---------------------------------------------------------------------------
|
|
590
|
+
// Across apps (2.13). Each app frame has its own DuckDB and its own WASM heap,
|
|
591
|
+
// so evicting inside one frame frees nothing for the next app opened. The SDK
|
|
592
|
+
// tells Mako's shell what its heap holds; the shell keeps one budget across its
|
|
593
|
+
// app frames and asks the least recently shown ones to release. A WASM heap
|
|
594
|
+
// never shrinks: demoting a table frees DuckDB's accounting and leaves the bytes
|
|
595
|
+
// reserved. So release means ending the worker, and the report is the most this
|
|
596
|
+
// worker has held. It is measuredBytes, not bindingBytes: an app's own tables
|
|
597
|
+
// sit in the same renderer. Parquet stays here as Blobs ("parked"): the next read
|
|
598
|
+
// boots a new engine and decodes from them, with no download and no gate.
|
|
599
|
+
//
|
|
600
|
+
// A worker can end only when nothing lives in it but bindings. App code that may
|
|
601
|
+
// hold the engine or keep its own state there keeps it for the page's life
|
|
602
|
+
// (keepEngine), says so in its reports, and the shell asks another frame.
|
|
603
|
+
//
|
|
604
|
+
// A release is not a crash: it never counts against MAX_RESTARTS, never reruns a
|
|
605
|
+
// hook, and leaves the gate and the download slots alone. They describe the
|
|
606
|
+
// server. Nothing in flight also means no load holds a slot.
|
|
607
|
+
// ---------------------------------------------------------------------------
|
|
608
|
+
const MEMORY_MESSAGE = "mako-app:duckdb-memory";
|
|
609
|
+
const RELEASE_MESSAGE = "mako-host:duckdb-release";
|
|
610
|
+
const inFrame = typeof window !== "undefined" && window.parent != null && window.parent !== window;
|
|
611
|
+
const parked = new Map(); // name -> Blob of a binding whose engine was released
|
|
612
|
+
let engineKept = false;
|
|
613
|
+
let heapBytes = 0; // the most this worker has held: its heap never gives it back
|
|
614
|
+
let busy = 0; // hooks, withConnection calls and queries in flight
|
|
615
|
+
let reportedBytes = null;
|
|
616
|
+
let reportedKept = false;
|
|
617
|
+
|
|
618
|
+
function reportMemory() {
|
|
619
|
+
heapBytes = Math.max(heapBytes, measuredBytes);
|
|
620
|
+
if (!inFrame || (heapBytes === reportedBytes && engineKept === reportedKept)) return;
|
|
621
|
+
reportedBytes = heapBytes;
|
|
622
|
+
reportedKept = engineKept;
|
|
623
|
+
try {
|
|
624
|
+
window.parent.postMessage({ type: MEMORY_MESSAGE, bytes: heapBytes, canRelease: !engineKept }, "*");
|
|
625
|
+
} catch {
|
|
626
|
+
/* a host that is not Mako: nothing to tell */
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
/** Something only this worker has may exist from now on: never end it. */
|
|
631
|
+
function keepEngine() {
|
|
632
|
+
if (engineKept) return;
|
|
633
|
+
engineKept = true;
|
|
634
|
+
reportMemory();
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
/**
|
|
638
|
+
* The host's ask: give the memory back. Nothing in flight: park every Blob and
|
|
639
|
+
* end the worker. Rows already on screen stay (they are plain JS); the next
|
|
640
|
+
* query reboots. Busy: skipped, and the host asks again. A kept engine stays. So
|
|
641
|
+
* does one with a crash restart due: that ends it and reruns the panels that saw
|
|
642
|
+
* the crash, which a release would not.
|
|
643
|
+
*/
|
|
644
|
+
async function releaseEngine() {
|
|
645
|
+
const idle = () => !engineKept && !restartTimer && busy === 0 && loading.size === 0 && refreshes.size === 0;
|
|
646
|
+
const current = dbPromise;
|
|
647
|
+
if (!current || !idle()) return;
|
|
648
|
+
const engine = await current;
|
|
649
|
+
await catalog(async () => {
|
|
650
|
+
if (dbPromise !== current || !idle()) return;
|
|
651
|
+
// From here to dbPromise = null runs without a yield: no query can grab
|
|
652
|
+
// the old engine in between. liveEngine goes too: a crash report from this
|
|
653
|
+
// engine must not restart the next one.
|
|
654
|
+
for (const [name, entry] of entries) if (entry.blob) parked.set(name, entry.blob);
|
|
655
|
+
entries.clear();
|
|
656
|
+
registered.clear();
|
|
657
|
+
loadedBindings.clear();
|
|
658
|
+
liveEngine = null;
|
|
659
|
+
dbPromise = null;
|
|
660
|
+
measuredBytes = 0;
|
|
661
|
+
heapBytes = 0;
|
|
662
|
+
pressureUntil = 0; // the next engine starts with an empty heap
|
|
663
|
+
clearTimeout(pressureTimer);
|
|
664
|
+
reportMemory();
|
|
665
|
+
await engine.terminate().catch(() => {});
|
|
666
|
+
});
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
if (inFrame) {
|
|
670
|
+
window.addEventListener("message", event => {
|
|
671
|
+
// Only the frame's own parent may ask. The ask can do no harm beyond a
|
|
672
|
+
// slower next query, so the parent's origin (Mako's, whatever host it
|
|
673
|
+
// runs on) is not pinned here.
|
|
674
|
+
if (event.source !== window.parent) return;
|
|
675
|
+
const data = event.data;
|
|
676
|
+
if (!data || data.type !== RELEASE_MESSAGE) return;
|
|
677
|
+
releaseEngine().catch(() => {
|
|
678
|
+
/* the next ask, or the next load's trim, tries again */
|
|
679
|
+
});
|
|
680
|
+
});
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
/** The least recently read binding that passes `test` and nothing holds. */
|
|
684
|
+
function leastRecent(test, pinnedToo = false) {
|
|
685
|
+
let best = null;
|
|
686
|
+
let bestSeq = Infinity;
|
|
687
|
+
for (const [name, entry] of entries) {
|
|
688
|
+
if ((isPinned(name) && !pinnedToo) || loading.has(name) || !test(name, entry)) continue;
|
|
689
|
+
const seq = lastUse.get(name) ?? 0;
|
|
690
|
+
if (seq < bestSeq) {
|
|
691
|
+
best = name;
|
|
692
|
+
bestSeq = seq;
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
return best;
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
/** Forget a binding entirely: the next read downloads it again. */
|
|
699
|
+
async function dropNow(engine, name) {
|
|
700
|
+
const entry = entries.get(name);
|
|
701
|
+
if (!entry) return;
|
|
702
|
+
await exec(engine, [dropSql(entry.storage, name)]);
|
|
703
|
+
if (entry.file) await engine.dropFile(entry.file).catch(() => {});
|
|
704
|
+
entries.delete(name);
|
|
705
|
+
loadedBindings.delete(name);
|
|
706
|
+
registered.delete(name);
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
/**
|
|
710
|
+
* Room in the heap for `incoming` bytes of parquet copy. registerFileBuffer does
|
|
711
|
+
* not check its malloc, so past this point nothing would report the failure. Goes
|
|
712
|
+
* through views least recently read first, pinned too: their next read downloads.
|
|
713
|
+
*/
|
|
714
|
+
async function roomForFile(engine, incoming, except) {
|
|
715
|
+
for (;;) {
|
|
716
|
+
const duckdb = (await duckdbBytes(engine)) ?? 0;
|
|
717
|
+
const files = fileBytes();
|
|
718
|
+
if (files + incoming <= FILE_CEILING && duckdb + files + incoming <= HEAP_CEILING) return;
|
|
719
|
+
const victim = leastRecent(
|
|
720
|
+
(name, entry) => name !== except && entry.storage === "view" && !externalBindings.has(name),
|
|
721
|
+
true,
|
|
722
|
+
);
|
|
723
|
+
if (!victim) {
|
|
724
|
+
const mib = Math.round(incoming / MIB);
|
|
725
|
+
throw new Error("A " + mib + " MiB parquet does not fit in this page's DuckDB heap, which holds " + Math.round((duckdb + files) / MIB) + " MiB.");
|
|
726
|
+
}
|
|
727
|
+
await dropNow(engine, victim);
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
/** One step down: a table becomes a view over a copy of its Blob, or goes. */
|
|
732
|
+
async function evictNow(engine, name) {
|
|
733
|
+
const entry = entries.get(name);
|
|
734
|
+
let file = null;
|
|
735
|
+
if (entry.blob) {
|
|
736
|
+
try {
|
|
737
|
+
await roomForFile(engine, entry.blob.size, name);
|
|
738
|
+
const bytes = new Uint8Array(await entry.blob.arrayBuffer());
|
|
739
|
+
file = nextFile();
|
|
740
|
+
await engine.registerFileBuffer(file, bytes);
|
|
741
|
+
} catch (error) {
|
|
742
|
+
if (error?.fromEngine) throw error;
|
|
743
|
+
file = null; // an unreadable Blob, or no room for its copy
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
if (file) {
|
|
747
|
+
try {
|
|
748
|
+
await viewFromFile(engine, name, file);
|
|
749
|
+
} catch (error) {
|
|
750
|
+
await engine.dropFile(file).catch(() => {});
|
|
751
|
+
throw error;
|
|
752
|
+
}
|
|
753
|
+
if (entry.file) await engine.dropFile(entry.file).catch(() => {});
|
|
754
|
+
entries.set(name, { ...entry, storage: "view", file, fileBytes: entry.blob.size, demoted: true });
|
|
755
|
+
return;
|
|
756
|
+
}
|
|
757
|
+
// App code may read this one by name at any time: it stays a table, and no
|
|
758
|
+
// eviction asks again (evictableTable).
|
|
759
|
+
if (externalBindings.has(name)) {
|
|
760
|
+
entries.set(name, { ...entry, blob: null });
|
|
761
|
+
return;
|
|
762
|
+
}
|
|
763
|
+
await dropNow(engine, name);
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
const evictableTable = except => (name, entry) =>
|
|
767
|
+
name !== except &&
|
|
768
|
+
entry.storage === "table" &&
|
|
769
|
+
(entry.blob !== null || !externalBindings.has(name));
|
|
770
|
+
|
|
771
|
+
/** Evict least recently used tables until the bindings hold at most `target`. */
|
|
772
|
+
async function trimNow(engine, except, target = memoryBudget) {
|
|
773
|
+
for (;;) {
|
|
774
|
+
if (ownedBytes() <= target) break;
|
|
775
|
+
const victim = leastRecent(evictableTable(except));
|
|
776
|
+
if (!victim) break;
|
|
777
|
+
await evictNow(engine, victim);
|
|
778
|
+
}
|
|
779
|
+
for (;;) {
|
|
780
|
+
let blobs = 0;
|
|
781
|
+
for (const entry of entries.values()) if (entry.blob) blobs += entry.blob.size;
|
|
782
|
+
if (blobs <= blobBudget) break;
|
|
783
|
+
const victim = leastRecent(
|
|
784
|
+
(name, entry) =>
|
|
785
|
+
name !== except && entry.blob !== null && !(entry.storage === "view" && externalBindings.has(name)),
|
|
786
|
+
);
|
|
787
|
+
if (!victim) break;
|
|
788
|
+
const entry = entries.get(victim);
|
|
789
|
+
if (entry.storage === "view") await dropNow(engine, victim);
|
|
790
|
+
// A table keeps its rows; evicting it later drops it outright.
|
|
791
|
+
else entries.set(victim, { ...entry, blob: null });
|
|
792
|
+
}
|
|
793
|
+
await measureNow(engine);
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
/**
|
|
797
|
+
* After an Out of Memory error: demote tables, least recently used first, until
|
|
798
|
+
* the bindings hold half of what they did (or half the budget). A load stops at
|
|
799
|
+
* unpinned tables. A query goes on to pinned tables it reads itself (`reads`): a
|
|
800
|
+
* query that can never fit must not take healthy panels down with it. Returns how
|
|
801
|
+
* many it freed; 0: a retry would fail the same way.
|
|
802
|
+
*/
|
|
803
|
+
async function relieveNow(engine, except, reads = null) {
|
|
804
|
+
const target = Math.min(memoryBudget, ownedBytes()) / 2;
|
|
805
|
+
const evictable = evictableTable(except);
|
|
806
|
+
let freed = 0;
|
|
807
|
+
for (;;) {
|
|
808
|
+
const victim =
|
|
809
|
+
leastRecent(evictable) ??
|
|
810
|
+
(reads ? leastRecent((name, entry) => reads.has(name) && entry.blob !== null && evictable(name, entry), true) : null);
|
|
811
|
+
if (!victim) break;
|
|
812
|
+
await evictNow(engine, victim);
|
|
813
|
+
freed++;
|
|
814
|
+
if (ownedBytes() <= target) break;
|
|
815
|
+
}
|
|
816
|
+
if (freed) startPressure();
|
|
817
|
+
await measureNow(engine);
|
|
818
|
+
return freed;
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
function startPressure() {
|
|
822
|
+
pressureCeiling = ownedBytes();
|
|
823
|
+
pressureUntil = Date.now() + PRESSURE_MS;
|
|
824
|
+
clearTimeout(pressureTimer);
|
|
825
|
+
pressureTimer = setTimeout(endPressure, PRESSURE_MS);
|
|
826
|
+
pressureTimer.unref?.();
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
/** Pinned views come back on their own: a mounted panel may not read again soon. */
|
|
830
|
+
async function endPressure() {
|
|
831
|
+
const waiting = [...entries]
|
|
832
|
+
.filter(([name, entry]) => entry.demoted && isPinned(name) && !loading.has(name))
|
|
833
|
+
.sort(([a], [b]) => (lastUse.get(b) ?? 0) - (lastUse.get(a) ?? 0));
|
|
834
|
+
for (const [name] of waiting) {
|
|
835
|
+
await promote(name);
|
|
836
|
+
if (entries.get(name)?.storage !== "table") break; // full: the rest would not fit either
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
/**
|
|
841
|
+
* Whether a table costing `estimate` fits, evicting what no one holds to make
|
|
842
|
+
* room. Checked before decoding: a table placed past the ceiling could only be
|
|
843
|
+
* demoted again.
|
|
844
|
+
*/
|
|
845
|
+
async function roomForTable(engine, name, estimate) {
|
|
846
|
+
const prev = entries.get(name);
|
|
847
|
+
const replaced = !prev ? 0 : prev.storage === "table" ? (prev.tableBytes ?? estimate) : prev.fileBytes;
|
|
848
|
+
const fitsHeap = async () => ((await duckdbBytes(engine)) ?? 0) + estimate <= DUCKDB_LIMIT;
|
|
849
|
+
// A single table over the ceiling: never evict for it. With a mostly empty
|
|
850
|
+
// heap, let it try: an OOM lands it as a view, and queries keep what is left.
|
|
851
|
+
if (estimate > tableCeiling()) {
|
|
852
|
+
return (
|
|
853
|
+
estimate <= memoryBudget * 2 &&
|
|
854
|
+
Date.now() >= pressureUntil &&
|
|
855
|
+
ownedBytes() - replaced <= memoryBudget / 4 &&
|
|
856
|
+
(await fitsHeap())
|
|
857
|
+
);
|
|
858
|
+
}
|
|
859
|
+
for (;;) {
|
|
860
|
+
if (ownedBytes() - replaced + estimate <= tableCeiling()) return fitsHeap();
|
|
861
|
+
const victim = leastRecent(evictableTable(name));
|
|
862
|
+
if (!victim) return false;
|
|
863
|
+
await evictNow(engine, victim);
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
/**
|
|
868
|
+
* Decode a download into DuckDB: a table when it fits, else a view. Reads `buf`,
|
|
869
|
+
* never the Blob: these bytes are in hand and a Blob can be unreadable.
|
|
870
|
+
*/
|
|
871
|
+
async function placeNow(engine, name, buf, blob) {
|
|
872
|
+
const prev = entries.get(name);
|
|
873
|
+
await roomForFile(engine, buf.length, name);
|
|
874
|
+
const file = nextFile();
|
|
875
|
+
const size = buf.length;
|
|
876
|
+
await engine.registerFileBuffer(file, buf); // transfers `buf` to the worker
|
|
877
|
+
const asView = async () => {
|
|
878
|
+
await viewFromFile(engine, name, file);
|
|
879
|
+
if (prev?.file) await engine.dropFile(prev.file).catch(() => {});
|
|
880
|
+
return { storage: "view", file, fileBytes: size, demoted: true };
|
|
881
|
+
};
|
|
882
|
+
const asTable = async () => {
|
|
883
|
+
const tableBytes = await tableFromFile(engine, name, file);
|
|
884
|
+
await engine.dropFile(file).catch(() => {});
|
|
885
|
+
if (prev?.file) await engine.dropFile(prev.file).catch(() => {});
|
|
886
|
+
return { storage: "table", file: null, fileBytes: 0, demoted: false, tableBytes };
|
|
887
|
+
};
|
|
888
|
+
try {
|
|
889
|
+
const estimate = prev?.tableBytes ?? (await estimateTable(engine, file, blob.size));
|
|
890
|
+
if (!(await roomForTable(engine, name, estimate))) return await asView();
|
|
891
|
+
try {
|
|
892
|
+
return await asTable();
|
|
893
|
+
} catch (error) {
|
|
894
|
+
if (!isEngineOom(error)) throw error;
|
|
895
|
+
}
|
|
896
|
+
if (await relieveNow(engine, name)) {
|
|
897
|
+
try {
|
|
898
|
+
return await asTable();
|
|
899
|
+
} catch (error) {
|
|
900
|
+
if (!isEngineOom(error)) throw error;
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
return await asView();
|
|
904
|
+
} catch (error) {
|
|
905
|
+
await engine.dropFile(file).catch(() => {});
|
|
906
|
+
throw error;
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
|
|
199
910
|
async function loadBinding(name, bust) {
|
|
200
911
|
const db = getDb(); // boot in parallel with the download
|
|
201
912
|
db.catch(() => {});
|
|
913
|
+
loading.add(name);
|
|
202
914
|
try {
|
|
203
|
-
|
|
915
|
+
// Parked by a release: the same bytes, no download (unless a refresh made
|
|
916
|
+
// them stale). A Blob can stop being readable: then it downloads.
|
|
917
|
+
const kept = parked.get(name);
|
|
918
|
+
parked.delete(name);
|
|
919
|
+
const parkedBytes = kept && !bust ? await kept.arrayBuffer().catch(() => null) : null;
|
|
920
|
+
const buf = parkedBytes
|
|
921
|
+
? new Uint8Array(parkedBytes)
|
|
922
|
+
: await downloadBinding(name, bust, patch => setProgress(name, patch));
|
|
204
923
|
setProgress(name, { phase: "loading_duckdb" });
|
|
205
924
|
const engine = await db;
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
const
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
}
|
|
218
|
-
} finally {
|
|
219
|
-
await engine.dropFile(file).catch(() => {});
|
|
220
|
-
}
|
|
221
|
-
loadedBindings.add(name);
|
|
925
|
+
// Made before the buffer goes to the worker: the copy a demotion reads.
|
|
926
|
+
const blob = new Blob([buf]);
|
|
927
|
+
await catalog(async () => {
|
|
928
|
+
const placed = await placeNow(engine, name, buf, blob);
|
|
929
|
+
const tableBytes = placed.tableBytes ?? entries.get(name)?.tableBytes ?? null;
|
|
930
|
+
entries.set(name, { ...placed, blob, tableBytes });
|
|
931
|
+
knownBindings.add(name);
|
|
932
|
+
loadedBindings.add(name);
|
|
933
|
+
loading.delete(name);
|
|
934
|
+
await trimNow(engine, name);
|
|
935
|
+
});
|
|
222
936
|
} finally {
|
|
937
|
+
loading.delete(name);
|
|
223
938
|
// A refresh owns the record until its reload is done (refreshBinding).
|
|
224
939
|
if (!refreshes.has(name)) setProgress(name, null);
|
|
225
940
|
}
|
|
226
941
|
}
|
|
227
942
|
|
|
943
|
+
/**
|
|
944
|
+
* A demoted view read again: back to a table from its heap copy, no download, if
|
|
945
|
+
* it fits. Otherwise it stays a view, and the next read asks again: a view that
|
|
946
|
+
* promoted itself into a full heap would only be demoted again.
|
|
947
|
+
*/
|
|
948
|
+
async function promote(name) {
|
|
949
|
+
const engine = await getDb();
|
|
950
|
+
await catalog(async () => {
|
|
951
|
+
const entry = entries.get(name);
|
|
952
|
+
if (!entry || entry.storage !== "view" || !entry.file) return;
|
|
953
|
+
if (!(await roomForTable(engine, name, entry.tableBytes ?? (await estimateTable(engine, entry.file, entry.fileBytes))))) {
|
|
954
|
+
entry.demoted = true;
|
|
955
|
+
return;
|
|
956
|
+
}
|
|
957
|
+
const tableBytes = await tableFromFile(engine, name, entry.file);
|
|
958
|
+
await engine.dropFile(entry.file).catch(() => {});
|
|
959
|
+
entries.set(name, { ...entry, storage: "table", file: null, fileBytes: 0, demoted: false, tableBytes });
|
|
960
|
+
await trimNow(engine, name);
|
|
961
|
+
}).catch(() => {
|
|
962
|
+
const entry = entries.get(name);
|
|
963
|
+
if (entry && entry.storage === "view") entry.demoted = true;
|
|
964
|
+
/* it stays a view, which reads the same rows */
|
|
965
|
+
});
|
|
966
|
+
}
|
|
967
|
+
|
|
228
968
|
const registered = new Map(); // name -> Promise<void>
|
|
229
|
-
const loadedBindings = new Set(); // names
|
|
969
|
+
const loadedBindings = new Set(); // names DuckDB can read (table or view)
|
|
230
970
|
// Refreshed on the server while never loaded on this page: the next load
|
|
231
971
|
// must skip any cache between here and the server.
|
|
232
972
|
const staleBindings = new Set();
|
|
233
973
|
function registerBinding(name) {
|
|
234
|
-
|
|
974
|
+
touch(name);
|
|
975
|
+
knownBindings.add(name);
|
|
976
|
+
const known = registered.get(name);
|
|
977
|
+
if (!known) {
|
|
235
978
|
const load = loadBinding(name, staleBindings.delete(name));
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
979
|
+
return trackRegistered(name, load);
|
|
980
|
+
}
|
|
981
|
+
const entry = entries.get(name);
|
|
982
|
+
if (entry && entry.demoted && !loading.has(name)) {
|
|
983
|
+
entry.demoted = false; // one promotion at a time
|
|
984
|
+
// The view reads fine whatever became of the promise before this one.
|
|
985
|
+
return trackRegistered(name, known.catch(() => {}).then(() => promote(name)));
|
|
240
986
|
}
|
|
241
|
-
return
|
|
987
|
+
return known;
|
|
988
|
+
}
|
|
989
|
+
|
|
990
|
+
/** The promise a next read waits on. A rejected one never stays: it would answer for good. */
|
|
991
|
+
function trackRegistered(name, promise) {
|
|
992
|
+
registered.set(name, promise);
|
|
993
|
+
promise.catch(() => {
|
|
994
|
+
if (registered.get(name) === promise) registered.delete(name);
|
|
995
|
+
});
|
|
996
|
+
return promise;
|
|
997
|
+
}
|
|
998
|
+
|
|
999
|
+
/**
|
|
1000
|
+
* Cap what bindings may hold in DuckDB, in bytes (default 1.5 GiB, half of the
|
|
1001
|
+
* frame's 3.1 GiB heap: queries need the rest). Checked with an estimate before
|
|
1002
|
+
* each load, so it can overshoot. Past it, the least recently read tables become
|
|
1003
|
+
* views over a heap copy of their parquet. `opts.blobBytes` caps the parquet kept
|
|
1004
|
+
* in JS memory (default 512 MiB); past it the oldest are dropped and download
|
|
1005
|
+
* again on their next read. `Infinity` turns either off. Resolves once the engine
|
|
1006
|
+
* is back under the new budget, and ends any memory pressure.
|
|
1007
|
+
*/
|
|
1008
|
+
export async function setBindingMemoryBudget(bytes, opts) {
|
|
1009
|
+
const valid = n => typeof n === "number" && n > 0 && !Number.isNaN(n);
|
|
1010
|
+
if (!valid(bytes)) throw new TypeError("setBindingMemoryBudget: a positive number of bytes");
|
|
1011
|
+
memoryBudget = bytes;
|
|
1012
|
+
pressureUntil = 0; // the caller has just said what tables may hold
|
|
1013
|
+
clearTimeout(pressureTimer);
|
|
1014
|
+
if (opts && opts.blobBytes !== undefined) {
|
|
1015
|
+
if (!valid(opts.blobBytes)) throw new TypeError("setBindingMemoryBudget: blobBytes must be a positive number");
|
|
1016
|
+
blobBudget = opts.blobBytes;
|
|
1017
|
+
}
|
|
1018
|
+
wakeRetries();
|
|
1019
|
+
if (!dbPromise) return;
|
|
1020
|
+
const engine = await getDb();
|
|
1021
|
+
await catalog(() => trimNow(engine, null));
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
/**
|
|
1025
|
+
* What the binding tables hold now: DuckDB's own figure, the budgets, and
|
|
1026
|
+
* each loaded binding, least recently read first.
|
|
1027
|
+
*/
|
|
1028
|
+
export async function getBindingMemory() {
|
|
1029
|
+
const engine = await getDb();
|
|
1030
|
+
const usedBytes = await catalog(() => measureNow(engine));
|
|
1031
|
+
const bindings = [...entries]
|
|
1032
|
+
.sort(([a], [b]) => (lastUse.get(a) ?? 0) - (lastUse.get(b) ?? 0))
|
|
1033
|
+
.map(([name, entry]) => ({
|
|
1034
|
+
name,
|
|
1035
|
+
storage: entry.storage,
|
|
1036
|
+
pinned: isPinned(name),
|
|
1037
|
+
parquetBytes: entry.blob ? entry.blob.size : null,
|
|
1038
|
+
}));
|
|
1039
|
+
return {
|
|
1040
|
+
usedBytes,
|
|
1041
|
+
bindingBytes: ownedBytes(),
|
|
1042
|
+
budgetBytes: memoryBudget,
|
|
1043
|
+
blobBudgetBytes: blobBudget,
|
|
1044
|
+
pressure: Date.now() < pressureUntil,
|
|
1045
|
+
bindings,
|
|
1046
|
+
};
|
|
242
1047
|
}
|
|
243
1048
|
|
|
244
1049
|
// ---------------------------------------------------------------------------
|
|
@@ -252,17 +1057,24 @@ function registerBinding(name) {
|
|
|
252
1057
|
* close or terminate it: every hook on the page reads from it.
|
|
253
1058
|
*/
|
|
254
1059
|
export function getDuckDB() {
|
|
1060
|
+
keepEngine();
|
|
255
1061
|
return getDb();
|
|
256
1062
|
}
|
|
257
1063
|
|
|
258
1064
|
/** Run `fn` with a fresh connection to the shared DuckDB; always closed after. */
|
|
259
1065
|
export async function withConnection(fn) {
|
|
260
|
-
|
|
261
|
-
|
|
1066
|
+
keepEngine();
|
|
1067
|
+
busy++;
|
|
262
1068
|
try {
|
|
263
|
-
|
|
1069
|
+
const db = await getDb();
|
|
1070
|
+
const conn = await db.connect();
|
|
1071
|
+
try {
|
|
1072
|
+
return await fn(conn);
|
|
1073
|
+
} finally {
|
|
1074
|
+
await conn.close();
|
|
1075
|
+
}
|
|
264
1076
|
} finally {
|
|
265
|
-
|
|
1077
|
+
busy--;
|
|
266
1078
|
}
|
|
267
1079
|
}
|
|
268
1080
|
|
|
@@ -270,23 +1082,32 @@ export async function withConnection(fn) {
|
|
|
270
1082
|
* Make sure a binding's table exists in the shared DuckDB, loading it once
|
|
271
1083
|
* per page (concurrent callers share the download). Resolves with the table
|
|
272
1084
|
* name — the binding name; quote it in SQL when it is not a bare identifier.
|
|
1085
|
+
* Each call counts as a read. Under memory pressure the table may become a
|
|
1086
|
+
* view over its parquet (same name, same rows, slower); it is never dropped.
|
|
273
1087
|
*/
|
|
274
1088
|
export async function ensureBinding(name) {
|
|
1089
|
+
keepEngine(); // its contract: the name stays readable, by code the SDK cannot see
|
|
1090
|
+
externalBindings.add(name);
|
|
275
1091
|
await registerBinding(name);
|
|
276
1092
|
return name;
|
|
277
1093
|
}
|
|
278
1094
|
|
|
279
1095
|
/**
|
|
280
|
-
* Forget a loaded binding: drops its table so its memory is freed.
|
|
281
|
-
* query or ensureBinding() downloads it again.
|
|
1096
|
+
* Forget a loaded binding: drops its table (or view) so its memory is freed.
|
|
1097
|
+
* The next query or ensureBinding() downloads it again.
|
|
282
1098
|
*/
|
|
283
1099
|
export async function unloadBinding(name) {
|
|
1100
|
+
parked.delete(name);
|
|
1101
|
+
knownBindings.delete(name);
|
|
284
1102
|
const pending = registered.get(name);
|
|
285
1103
|
if (!pending) return;
|
|
286
1104
|
registered.delete(name);
|
|
287
|
-
|
|
1105
|
+
externalBindings.delete(name);
|
|
288
1106
|
await pending.catch(() => {});
|
|
289
|
-
|
|
1107
|
+
if (registered.has(name)) return; // read again meanwhile: it stays
|
|
1108
|
+
const engine = await getDb();
|
|
1109
|
+
await catalog(() => dropNow(engine, name));
|
|
1110
|
+
wakeRetries();
|
|
290
1111
|
}
|
|
291
1112
|
|
|
292
1113
|
/**
|
|
@@ -319,14 +1140,7 @@ async function reloadBinding(name) {
|
|
|
319
1140
|
// Let a load in flight settle first, or its (older) bytes could land after
|
|
320
1141
|
// the fresh ones and win.
|
|
321
1142
|
await previous.catch(() => {});
|
|
322
|
-
|
|
323
|
-
registered.set(name, load);
|
|
324
|
-
try {
|
|
325
|
-
await load;
|
|
326
|
-
} catch (error) {
|
|
327
|
-
if (registered.get(name) === load) registered.delete(name);
|
|
328
|
-
throw error;
|
|
329
|
-
}
|
|
1143
|
+
await trackRegistered(name, loadBinding(name, true));
|
|
330
1144
|
}
|
|
331
1145
|
|
|
332
1146
|
// ---------------------------------------------------------------------------
|
|
@@ -487,33 +1301,107 @@ export function getViewer() {
|
|
|
487
1301
|
/** Names of every staged binding — written by the dev server next to the
|
|
488
1302
|
* parquet files. Absent (older servers, published builds): empty list. */
|
|
489
1303
|
let indexPromise = null;
|
|
1304
|
+
const INDEX_RETRY_MS = 10000;
|
|
490
1305
|
function bindingIndex() {
|
|
491
|
-
indexPromise
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
1306
|
+
if (indexPromise) return indexPromise;
|
|
1307
|
+
// 20 panels asking at once share one answer, a failed one for 10 s too.
|
|
1308
|
+
const forget = () => {
|
|
1309
|
+
const t = setTimeout(() => {
|
|
1310
|
+
if (indexPromise === pending) indexPromise = null;
|
|
1311
|
+
}, INDEX_RETRY_MS);
|
|
1312
|
+
t.unref?.();
|
|
1313
|
+
};
|
|
1314
|
+
const pending = fetchRetrying("__data/index.json").then(async r => {
|
|
1315
|
+
// A 404 is an older server or a published build: empty, for good.
|
|
1316
|
+
if (!r.ok && r.status !== 404) forget();
|
|
1317
|
+
if (r.status === 429 || r.status === 503) throw await responseError(r, "The binding list could not be loaded");
|
|
1318
|
+
if (!r.ok) return [];
|
|
1319
|
+
const list = await r.json().catch(() => []);
|
|
1320
|
+
return Array.isArray(list) ? list : [];
|
|
1321
|
+
});
|
|
1322
|
+
indexPromise = pending;
|
|
1323
|
+
pending.catch(forget);
|
|
1324
|
+
return pending;
|
|
496
1325
|
}
|
|
497
1326
|
|
|
498
|
-
|
|
1327
|
+
// Out of Memory on a query: free tables, pinned ones the query reads last, and
|
|
1328
|
+
// try again. Each pass demotes at least one table, so this ends. With nothing left
|
|
1329
|
+
// to free, DuckDB's error surfaces and the hook decides (useAsyncQuery).
|
|
1330
|
+
async function runSql(sql, rowLimit, format, reads) {
|
|
1331
|
+
for (;;) {
|
|
1332
|
+
try {
|
|
1333
|
+
return await runSqlOnce(sql, rowLimit, format);
|
|
1334
|
+
} catch (error) {
|
|
1335
|
+
if (!isEngineOom(error)) throw error;
|
|
1336
|
+
const engine = await getDb();
|
|
1337
|
+
const freed = await catalog(() => relieveNow(engine, null, reads)).catch(() => 0);
|
|
1338
|
+
if (!freed) throw error;
|
|
1339
|
+
}
|
|
1340
|
+
}
|
|
1341
|
+
}
|
|
1342
|
+
|
|
1343
|
+
async function runSqlOnce(sql, rowLimit, format) {
|
|
499
1344
|
const cap = rowLimit === null ? Infinity : (rowLimit ?? DEFAULT_ROW_LIMIT);
|
|
500
|
-
const
|
|
501
|
-
|
|
1345
|
+
const writes = !isReadOnlySql(sql);
|
|
1346
|
+
if (writes) keepEngine(); // CREATE TABLE journal_t: only this worker has it
|
|
1347
|
+
busy++; // before the await: a release must not end the engine under it
|
|
502
1348
|
try {
|
|
503
|
-
const
|
|
504
|
-
const
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
//
|
|
508
|
-
|
|
1349
|
+
const db = await getDb();
|
|
1350
|
+
const conn = await db.connect();
|
|
1351
|
+
try {
|
|
1352
|
+
const table = await conn.query(sql);
|
|
1353
|
+
// No load follows to measure what the app just made, and the report owes it.
|
|
1354
|
+
if (writes) catalog(() => measureNow(db)).catch(() => {});
|
|
1355
|
+
const rowCount = table.numRows;
|
|
1356
|
+
if (format === "arrow") {
|
|
1357
|
+
// The Arrow Table as DuckDB returned it: no row objects, no
|
|
1358
|
+
// normalization, no row limit.
|
|
1359
|
+
return { rows: table, fields: table.schema.fields.map(f => f.name), rowCount, truncated: false };
|
|
1360
|
+
}
|
|
1361
|
+
const { rows, fields } = toPlainRows(table, cap);
|
|
1362
|
+
return { rows, fields, rowCount, truncated: rowCount > rows.length };
|
|
1363
|
+
} catch (error) {
|
|
1364
|
+
throw markEngineError(error, db);
|
|
1365
|
+
} finally {
|
|
1366
|
+
await conn.close();
|
|
509
1367
|
}
|
|
510
|
-
const { rows, fields } = toPlainRows(table, cap);
|
|
511
|
-
return { rows, fields, rowCount, truncated: rowCount > rows.length };
|
|
512
1368
|
} finally {
|
|
513
|
-
|
|
1369
|
+
busy--;
|
|
514
1370
|
}
|
|
515
1371
|
}
|
|
516
1372
|
|
|
1373
|
+
// A crashed engine stays crashed: the SQL that broke it can answer, then every
|
|
1374
|
+
// later query fails with a raw WASM trap. So the page starts a new engine.
|
|
1375
|
+
// Bindings download again. Pins stay: they belong to mounted hooks.
|
|
1376
|
+
const RESTART_EVERY_MS = 60000;
|
|
1377
|
+
const MAX_RESTARTS = 3;
|
|
1378
|
+
let lastRestart = -Infinity;
|
|
1379
|
+
let restarts = 0;
|
|
1380
|
+
let restartTimer = null;
|
|
1381
|
+
function noteEngineCrash(engine) {
|
|
1382
|
+
if (engine !== liveEngine || restartTimer || restarts >= MAX_RESTARTS) return;
|
|
1383
|
+
restartTimer = setTimeout(restartEngine, Math.max(0, lastRestart + RESTART_EVERY_MS - Date.now()));
|
|
1384
|
+
}
|
|
1385
|
+
function restartEngine() {
|
|
1386
|
+
restartTimer = null;
|
|
1387
|
+
const old = liveEngine;
|
|
1388
|
+
if (!old) return;
|
|
1389
|
+
restarts++;
|
|
1390
|
+
lastRestart = Date.now();
|
|
1391
|
+
liveEngine = null;
|
|
1392
|
+
dbPromise = null;
|
|
1393
|
+
catalogQueue = Promise.resolve();
|
|
1394
|
+
entries.clear();
|
|
1395
|
+
registered.clear();
|
|
1396
|
+
loadedBindings.clear();
|
|
1397
|
+
measuredBytes = 0;
|
|
1398
|
+
heapBytes = 0;
|
|
1399
|
+
pressureUntil = 0;
|
|
1400
|
+
reportMemory();
|
|
1401
|
+
old.terminate().catch(() => {});
|
|
1402
|
+
setBindingState({ ...bindingState, version: bindingState.version + 1 });
|
|
1403
|
+
}
|
|
1404
|
+
|
|
517
1405
|
const EMPTY = {
|
|
518
1406
|
data: null,
|
|
519
1407
|
fields: null,
|
|
@@ -529,33 +1417,104 @@ const EMPTY = {
|
|
|
529
1417
|
// until the new ones arrive — a dashboard must not blink to empty on every
|
|
530
1418
|
// refresh). `isRefreshing` says whether a refresh in flight concerns this
|
|
531
1419
|
// query, so `refreshing` is true from the click, not only from the reload.
|
|
1420
|
+
// `run(hold)` names each binding it reads through `hold`: those stay pinned
|
|
1421
|
+
// against eviction while this hook shows that query (until the deps change
|
|
1422
|
+
// or it unmounts, and never before its run settles).
|
|
1423
|
+
//
|
|
1424
|
+
// A query that failed on memory or the network runs again by itself, a few times
|
|
1425
|
+
// (retryDelayMs). After that it waits for a reason to expect a different answer:
|
|
1426
|
+
// another hook let go of its bindings, the budget changed, the server answered
|
|
1427
|
+
// again, the tab came back, the network came back. The error stays up until a run
|
|
1428
|
+
// answers. A hidden tab waits too. Anything else stays as it is.
|
|
1429
|
+
const retryWaiters = new Set();
|
|
1430
|
+
function wakeRetries() {
|
|
1431
|
+
const waiting = [...retryWaiters];
|
|
1432
|
+
retryWaiters.clear();
|
|
1433
|
+
for (const fire of waiting) fire();
|
|
1434
|
+
}
|
|
1435
|
+
// The user or the network is back: let one probe through a closed gate, too.
|
|
1436
|
+
function tryAgainNow() {
|
|
1437
|
+
gateUntil = 0;
|
|
1438
|
+
wakeRetries();
|
|
1439
|
+
}
|
|
1440
|
+
if (typeof window !== "undefined" && typeof window.addEventListener === "function") {
|
|
1441
|
+
window.addEventListener("online", tryAgainNow);
|
|
1442
|
+
if (typeof document !== "undefined") {
|
|
1443
|
+
document.addEventListener("visibilitychange", () => {
|
|
1444
|
+
if (!document.hidden) tryAgainNow();
|
|
1445
|
+
});
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
|
|
532
1449
|
function useAsyncQuery(run, deps, isRefreshing) {
|
|
533
1450
|
const bindings = useBindingState();
|
|
534
1451
|
const [state, setState] = React.useState(EMPTY);
|
|
1452
|
+
const [attempt, setAttempt] = React.useState(0);
|
|
535
1453
|
const seenVersion = React.useRef(bindings.version);
|
|
1454
|
+
const seenAttempt = React.useRef(attempt);
|
|
1455
|
+
const retries = React.useRef(0);
|
|
1456
|
+
const retrying = React.useRef(false);
|
|
536
1457
|
React.useEffect(() => {
|
|
537
1458
|
let active = true;
|
|
1459
|
+
let timer = null;
|
|
1460
|
+
let waiter = null;
|
|
538
1461
|
const isRefresh = seenVersion.current !== bindings.version;
|
|
1462
|
+
const isRetry = seenAttempt.current !== attempt;
|
|
539
1463
|
seenVersion.current = bindings.version;
|
|
540
|
-
|
|
541
|
-
|
|
1464
|
+
seenAttempt.current = attempt;
|
|
1465
|
+
retrying.current = false;
|
|
1466
|
+
if (!isRetry) {
|
|
1467
|
+
retries.current = 0;
|
|
1468
|
+
setState(s => (isRefresh && s.data !== null ? { ...s, refreshing: true } : EMPTY));
|
|
1469
|
+
}
|
|
1470
|
+
const held = [];
|
|
1471
|
+
// From the first await to the last. Between a hook's load and its query
|
|
1472
|
+
// nothing else is in flight, and a release there would end the engine
|
|
1473
|
+
// holding the table the query is about to read.
|
|
1474
|
+
busy++;
|
|
1475
|
+
const settled = run(name => held.push(pin(name)));
|
|
1476
|
+
settled.catch(() => {}).finally(() => {
|
|
1477
|
+
busy--;
|
|
1478
|
+
});
|
|
1479
|
+
settled.then(
|
|
542
1480
|
result => {
|
|
543
1481
|
if (active) setState({ ...result, error: null, loading: false, refreshing: false });
|
|
544
1482
|
},
|
|
545
1483
|
error => {
|
|
546
|
-
if (active)
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
1484
|
+
if (!active) return;
|
|
1485
|
+
setState({
|
|
1486
|
+
...EMPTY,
|
|
1487
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1488
|
+
loading: false,
|
|
1489
|
+
});
|
|
1490
|
+
const wait = retryDelayMs(error, retries.current);
|
|
1491
|
+
if (wait === null) return;
|
|
1492
|
+
retries.current++;
|
|
1493
|
+
waiter = () => {
|
|
1494
|
+
if (typeof document !== "undefined" && document.hidden) {
|
|
1495
|
+
retryWaiters.add(waiter);
|
|
1496
|
+
return;
|
|
1497
|
+
}
|
|
1498
|
+
retrying.current = true;
|
|
1499
|
+
setAttempt(n => n + 1);
|
|
1500
|
+
};
|
|
1501
|
+
if (wait === Infinity) retryWaiters.add(waiter);
|
|
1502
|
+
else timer = setTimeout(waiter, wait);
|
|
552
1503
|
},
|
|
553
1504
|
);
|
|
554
1505
|
return () => {
|
|
555
1506
|
active = false;
|
|
1507
|
+
if (timer !== null) clearTimeout(timer);
|
|
1508
|
+
if (waiter) retryWaiters.delete(waiter);
|
|
1509
|
+
const letsGo = !retrying.current;
|
|
1510
|
+
settled.catch(() => {}).finally(() => {
|
|
1511
|
+
for (const release of held) release();
|
|
1512
|
+
// Its own retry frees nothing, or two failing panels would wake each other.
|
|
1513
|
+
if (letsGo && held.length) wakeRetries();
|
|
1514
|
+
});
|
|
556
1515
|
};
|
|
557
1516
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
558
|
-
}, [...deps, bindings.version]);
|
|
1517
|
+
}, [...deps, bindings.version, attempt]);
|
|
559
1518
|
return {
|
|
560
1519
|
...state,
|
|
561
1520
|
refreshing: state.refreshing || isRefreshing(bindings.refreshing),
|
|
@@ -622,9 +1581,10 @@ export function useQuery(name, opts) {
|
|
|
622
1581
|
const withSnapshot = Boolean(opts && opts.snapshot);
|
|
623
1582
|
const preview = useSnapshotPreview(name, withSnapshot, rowLimit);
|
|
624
1583
|
const state = useAsyncQuery(
|
|
625
|
-
async
|
|
1584
|
+
async hold => {
|
|
1585
|
+
hold(name);
|
|
626
1586
|
await registerBinding(name);
|
|
627
|
-
const r = await runSql("SELECT * FROM " + quoteIdent(name), rowLimit);
|
|
1587
|
+
const r = await runSql("SELECT * FROM " + quoteIdent(name), rowLimit, undefined, new Set([name]));
|
|
628
1588
|
return { data: r.rows, fields: r.fields, truncated: r.truncated, rowCount: r.rowCount };
|
|
629
1589
|
},
|
|
630
1590
|
[name, rowLimit],
|
|
@@ -682,24 +1642,41 @@ export function useViewer() {
|
|
|
682
1642
|
* A binding that fails to load must not sink the query silently: when the
|
|
683
1643
|
* SQL then fails, the load failure is what surfaces — it is the cause.
|
|
684
1644
|
*/
|
|
685
|
-
async function runOverBindings(sql, rowLimit, format) {
|
|
686
|
-
|
|
1645
|
+
async function runOverBindings(sql, rowLimit, format, hold) {
|
|
1646
|
+
// No index still answers `select 1`. It matters only if the query then fails.
|
|
1647
|
+
let indexFailure = null;
|
|
1648
|
+
const index = await bindingIndex().catch(error => {
|
|
1649
|
+
indexFailure = error;
|
|
1650
|
+
return [];
|
|
1651
|
+
});
|
|
1652
|
+
const names = [...new Set([...index, ...knownBindings])];
|
|
687
1653
|
const failures = new Map();
|
|
688
1654
|
const attempted = new Set();
|
|
1655
|
+
const reloaded = new Set();
|
|
689
1656
|
const load = n => {
|
|
690
1657
|
attempted.add(n);
|
|
1658
|
+
hold(n);
|
|
691
1659
|
return registerBinding(n).catch(e => {
|
|
692
1660
|
failures.set(n, e);
|
|
693
1661
|
});
|
|
694
1662
|
};
|
|
695
|
-
|
|
1663
|
+
const wanted = readsCatalog(sql) ? names : mentionedBindings(sql, names);
|
|
1664
|
+
if (!readsCatalog(sql) && wanted.length > MAX_BINDINGS_IN_QUERY) {
|
|
1665
|
+
throw new Error(
|
|
1666
|
+
"This query names " + wanted.length + " bindings. One query can read at most " + MAX_BINDINGS_IN_QUERY +
|
|
1667
|
+
": around 60 overflow DuckDB's WASM stack and break the engine for every panel. Read fewer shards, or combine them in the warehouse.",
|
|
1668
|
+
);
|
|
1669
|
+
}
|
|
1670
|
+
await Promise.all(wanted.map(load));
|
|
696
1671
|
for (;;) {
|
|
697
1672
|
try {
|
|
698
|
-
return await runSql(sql, rowLimit, format);
|
|
1673
|
+
return await runSql(sql, rowLimit, format, attempted);
|
|
699
1674
|
} catch (error) {
|
|
700
1675
|
const message = error instanceof Error ? error.message : String(error);
|
|
701
1676
|
const missing = missingBinding(message, names);
|
|
702
|
-
|
|
1677
|
+
// Never asked for, or the SDK evicted it under this query. Once per run.
|
|
1678
|
+
if (missing && !failures.has(missing) && !reloaded.has(missing) && (!attempted.has(missing) || !registered.has(missing))) {
|
|
1679
|
+
reloaded.add(missing);
|
|
703
1680
|
await load(missing);
|
|
704
1681
|
if (failures.has(missing)) throw failures.get(missing);
|
|
705
1682
|
continue;
|
|
@@ -708,11 +1685,15 @@ async function runOverBindings(sql, rowLimit, format) {
|
|
|
708
1685
|
if (message.includes(n)) throw cause;
|
|
709
1686
|
}
|
|
710
1687
|
if (failures.size) {
|
|
711
|
-
const
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
1688
|
+
const causes = [...failures.values()];
|
|
1689
|
+
const details = causes.map(e => (e instanceof Error ? e.message : String(e))).join("; ");
|
|
1690
|
+
const wrapped = new Error(message + " (bindings that failed to load: " + details + ")");
|
|
1691
|
+
const retryable = causes.find(e => e?.retry);
|
|
1692
|
+
wrapped.retry = error?.retry ?? retryable?.retry;
|
|
1693
|
+
wrapped.retryAfterMs = retryable?.retryAfterMs;
|
|
1694
|
+
throw wrapped;
|
|
715
1695
|
}
|
|
1696
|
+
if (indexFailure && !error?.retry) throw indexFailure;
|
|
716
1697
|
throw error;
|
|
717
1698
|
}
|
|
718
1699
|
}
|
|
@@ -722,8 +1703,8 @@ export function useDuckDB(sql, opts) {
|
|
|
722
1703
|
bootEngine();
|
|
723
1704
|
const rowLimit = opts ? opts.rowLimit : undefined;
|
|
724
1705
|
const format = opts && opts.format === "arrow" ? "arrow" : "rows";
|
|
725
|
-
const state = useAsyncQuery(async
|
|
726
|
-
const r = await runOverBindings(sql, rowLimit, format);
|
|
1706
|
+
const state = useAsyncQuery(async hold => {
|
|
1707
|
+
const r = await runOverBindings(sql, rowLimit, format, hold);
|
|
727
1708
|
return { data: r.rows, fields: r.fields, truncated: r.truncated, rowCount: r.rowCount };
|
|
728
1709
|
}, [sql, rowLimit, format], refreshing => refreshing.size > 0);
|
|
729
1710
|
// Every binding the query can read: the combined record says what the
|