@makoai/app-sdk 2.11.0 → 2.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.js CHANGED
@@ -22,6 +22,9 @@ import {
22
22
  quoteIdent,
23
23
  readsCatalog,
24
24
  quoteString,
25
+ crashedEngineError,
26
+ isOutOfMemoryError,
27
+ retryDelayMs,
25
28
  toPlainRows,
26
29
  parseBindingSnapshot,
27
30
  snapshotUrl,
@@ -32,7 +35,7 @@ import {
32
35
  * in a page (or this export) says which SDK a deployed app actually runs.
33
36
  * Kept in step with package.json by snapshot.test.mjs.
34
37
  */
35
- export const SDK_VERSION = "2.11.0";
38
+ export const SDK_VERSION = "2.12.0";
36
39
  if (typeof window !== "undefined") {
37
40
  try {
38
41
  window.__MAKO_APP_SDK__ = SDK_VERSION;
@@ -42,6 +45,9 @@ if (typeof window !== "undefined") {
42
45
  }
43
46
 
44
47
  const DEFAULT_ROW_LIMIT = 500000;
48
+ // A double-read UNION ALL of 55 to 60 bindings overflows the WASM stack and
49
+ // corrupts the engine for every panel (measured). 50 held.
50
+ const MAX_BINDINGS_IN_QUERY = 50;
45
51
  const DUCKDB_ESM =
46
52
  "https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.29.0/+esm";
47
53
 
@@ -53,6 +59,7 @@ const DUCKDB_ESM =
53
59
  // parquet download instead of waiting for it.
54
60
  // ---------------------------------------------------------------------------
55
61
  let dbPromise = null;
62
+ let liveEngine = null; // the resolved dbPromise: errors from an older one are history
56
63
  function getDb() {
57
64
  if (!dbPromise) {
58
65
  dbPromise = (async () => {
@@ -72,6 +79,7 @@ function getDb() {
72
79
  } finally {
73
80
  await conn.close();
74
81
  }
82
+ liveEngine = db;
75
83
  return db;
76
84
  })();
77
85
  // A failed boot (CDN unreachable) is retried by the next query.
@@ -104,6 +112,8 @@ async function responseError(res, what) {
104
112
  const error = new Error(what + " (HTTP " + res.status + ")" + (detail ? ": " + detail : ""));
105
113
  error.status = res.status;
106
114
  if (body && typeof body.retryAfterMs === "number") error.retryAfterMs = body.retryAfterMs;
115
+ // The one thing a mounted hook may ask again. Any other status said what it meant.
116
+ if (res.status === 429 || res.status === 503) error.retry = "network";
107
117
  return error;
108
118
  }
109
119
 
@@ -168,6 +178,102 @@ export function useBindingProgress(name) {
168
178
  // under its own file name, so a reload never drops a file another load is
169
179
  // still reading.
170
180
 
181
+ // Cloud Run answers a queue timeout with a 429 and no CORS header, so an opaque
182
+ // app frame sees only "Failed to fetch": 12 of 5,087 published parquet requests
183
+ // on 2026-11-02, in 8 bursts. One page-wide gate keeps a page from making it worse.
184
+ // - At most 12 downloads at once; 1 while the gate is recovering from a failure.
185
+ // - A failure closes the gate for 1, 3, 9, 27, then 60 s, with jitter. Failures
186
+ // in one closed window count once. Any answer that is not a 429 or 503 opens it.
187
+ // - Past 3 failures in a row, a request while it is closed fails at once with no
188
+ // network, so a page down to one probe at a time does not keep panels spinning.
189
+ const FETCH_RETRY_MS = [1000, 3000];
190
+ const RETRY_AFTER_CAP_MS = 30000;
191
+ const DOWNLOAD_SLOTS = 12;
192
+ const GATE_MAX_MS = 60000;
193
+ const GATE_FAIL_FAST_AT = 3;
194
+ let gateStreak = 0;
195
+ let gateUntil = 0;
196
+ let gateError = null;
197
+ let slotsInUse = 0;
198
+ const slotQueue = [];
199
+ const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
200
+ const withJitter = ms => ms * (0.75 + Math.random() / 2);
201
+
202
+ function gateFailed(error) {
203
+ gateError = error;
204
+ if (Date.now() < gateUntil) return;
205
+ gateStreak++;
206
+ gateUntil = Date.now() + withJitter(Math.min(GATE_MAX_MS, 1000 * 3 ** (gateStreak - 1)));
207
+ }
208
+ function gateAnswered() {
209
+ if (gateStreak === 0) return;
210
+ gateStreak = 0;
211
+ gateUntil = 0;
212
+ gateError = null;
213
+ pumpSlots();
214
+ wakeRetries();
215
+ }
216
+ async function throughGate() {
217
+ for (;;) {
218
+ const wait = gateUntil - Date.now();
219
+ if (wait <= 0) return;
220
+ if (gateStreak >= GATE_FAIL_FAST_AT) throw gateError;
221
+ await sleep(wait);
222
+ }
223
+ }
224
+ function pumpSlots() {
225
+ let room = (gateStreak ? 1 : DOWNLOAD_SLOTS) - slotsInUse;
226
+ while (room-- > 0 && slotQueue.length) slotQueue.shift()();
227
+ }
228
+ async function withSlot(fn) {
229
+ while (slotsInUse >= (gateStreak ? 1 : DOWNLOAD_SLOTS)) await new Promise(resolve => slotQueue.push(resolve));
230
+ slotsInUse++;
231
+ try {
232
+ return await fn();
233
+ } finally {
234
+ slotsInUse--;
235
+ pumpSlots();
236
+ }
237
+ }
238
+
239
+ /**
240
+ * fetch, until headers arrive: 3 attempts on a rejected fetch, a 429 or a 503,
241
+ * through the gate. A response of any other status returns at once, and so does
242
+ * a stream once it starts, so a build is never asked for again from here. Waits
243
+ * out a `retryAfterMs` up to 30 s; past that the caller gets the answer. No
244
+ * headers timeout: the Vite plugin may hold headers for a whole build.
245
+ */
246
+ async function fetchRetrying(url, init) {
247
+ for (let attempt = 0; ; attempt++) {
248
+ const last = attempt >= FETCH_RETRY_MS.length;
249
+ await throughGate();
250
+ let wait = last ? 0 : withJitter(FETCH_RETRY_MS[attempt]);
251
+ try {
252
+ const res = await fetch(url, init);
253
+ if (res.status !== 429 && res.status !== 503) {
254
+ gateAnswered();
255
+ return res;
256
+ }
257
+ const asked = await res
258
+ .clone()
259
+ .json()
260
+ .then(body => body?.retryAfterMs)
261
+ .catch(() => undefined);
262
+ const busy = new Error("The server is busy (HTTP " + res.status + ")");
263
+ busy.retry = "network";
264
+ gateFailed(busy);
265
+ if (last || asked > RETRY_AFTER_CAP_MS) return res;
266
+ if (typeof asked === "number") wait = asked;
267
+ } catch (error) {
268
+ if (!(error instanceof TypeError)) throw error;
269
+ error.retry = "network";
270
+ gateFailed(error);
271
+ if (last) throw error;
272
+ }
273
+ await sleep(wait);
274
+ }
275
+ }
276
+
171
277
  /**
172
278
  * A binding's parquet bytes, phase-aware: asks for NDJSON phase events (a
173
279
  * build reports each step, a stored file resolving → signing and a signed
@@ -178,67 +284,650 @@ async function downloadBinding(name, bust, report) {
178
284
  const path = "__data/" + encodeURIComponent(name) + ".parquet";
179
285
  const what = 'Data for binding "' + name + '" could not be loaded';
180
286
  report({ phase: "requesting", rowsWritten: null, bytes: null, bytesTotal: null });
181
- let res = await fetch(path + (bust ? "?refresh=" + Date.now() : ""), {
182
- headers: { accept: PROGRESS_ACCEPT },
183
- });
287
+ // A slot holds the connection, not the wait for a build: one slow warehouse
288
+ // query must not hold up twelve bindings that are ready.
289
+ let res = await withSlot(() =>
290
+ fetchRetrying(path + (bust ? "?refresh=" + Date.now() : ""), {
291
+ headers: { accept: PROGRESS_ACCEPT },
292
+ }),
293
+ );
294
+ let signed = null;
184
295
  if (isProgressResponse(res)) {
185
- const final = await readProgressStream(res, e => report(progressPatch(e)));
296
+ let final;
297
+ try {
298
+ final = await readProgressStream(res, e => report(progressPatch(e)));
299
+ } catch (error) {
300
+ // A cut build stream may have cost a warehouse query. Asking again costs another.
301
+ throw new Error(what + ": the build's stream was cut (" + (error instanceof Error ? error.message : String(error)) + ")");
302
+ }
186
303
  if (final.phase === "failed") throw progressError(final, what);
187
304
  report({ phase: "downloading", rowsWritten: null, bytes: 0 });
188
305
  // No url: the server streams the bytes itself — the same path, asked
189
306
  // plainly, serves them (the build already staged or cached them).
190
- res = await fetch(final.url || path);
307
+ signed = final.url || path;
308
+ res = null;
191
309
  }
192
- if (!res.ok) throw await responseError(res, what);
193
- return readBytes(res, (bytes, bytesTotal) =>
194
- report({ phase: "downloading", rowsWritten: null, bytes, bytesTotal }),
195
- );
310
+ return withSlot(async () => {
311
+ res ??= await fetchRetrying(signed);
312
+ if (!res.ok) throw await responseError(res, what);
313
+ try {
314
+ return await readBytes(res, (bytes, bytesTotal) =>
315
+ report({ phase: "downloading", rowsWritten: null, bytes, bytesTotal }),
316
+ );
317
+ } catch (error) {
318
+ if (error instanceof TypeError) error.retry = "network"; // the body cut off: no build behind it
319
+ throw error;
320
+ }
321
+ });
322
+ }
323
+
324
+ // ---------------------------------------------------------------------------
325
+ // Memory (2.12). Every table lives in this frame's WASM heap, where DuckDB's
326
+ // memory_limit is 3.1 GiB. 2.8 to 2.11 never dropped one, so an app with many
327
+ // bindings (Web Performance: 106 shards) browsed its way to "Out of Memory".
328
+ //
329
+ // - Every read bumps its binding. Least recently used goes first.
330
+ // - The budget counts what the SDK placed: tables and the parquet copies views
331
+ // read. An app's own tables do not shrink it. The real total still decides
332
+ // whether a table can fit in the heap at all.
333
+ // - A table lands if the parquet's own metadata says it fits, after evicting what
334
+ // no one holds. Measured tables can pass the estimate, so the budget can
335
+ // overshoot (measured: 1,700 MiB against 1,536). With a mostly empty heap, one
336
+ // table may try anyway: an ordinary app with one large parquet stays fast.
337
+ // - What does not fit lands as a view over its parquet in the heap
338
+ // (registerFileBuffer): 7x to 14x smaller than its table. A later read, or the
339
+ // end of pressure, promotes it if room has appeared.
340
+ // - Never a BROWSER_FILEREADER handle over a Blob: duckdb-wasm 1.29.0's openFile
341
+ // writes the file size to HEAPF64[(ptr >> 3)], and `>>` turns a pointer past
342
+ // 2 GiB negative. Reads then fail with "too small to be a Parquet file" (4 of
343
+ // 8 reads with 3.0 GiB of tables, 0 of 8 with 1.6 GiB; heap buffers 0 of 8).
344
+ // - File copies have their own ceiling: 2.5 GiB, and 3.6 GiB with DuckDB's own
345
+ // bytes. registerFileBuffer never checks its malloc: at 4.16 GiB of copies the
346
+ // call never settled and hung the page. Over it the least recently read view
347
+ // goes, pinned too, and downloads again on its next read.
348
+ // - The parquet stays in JS memory as a Blob only so a table can become a view
349
+ // without a download. A Blob can stop being readable (NotReadableError on the
350
+ // third 885 MiB in incognito). Its table then goes instead, and downloads again.
351
+ // Past the Blob budget, the least recently used view is dropped outright.
352
+ // - A binding a mounted hook reads, or a query in flight, is pinned: it goes last.
353
+ // Only loads in flight are never touched.
354
+ // - "Out of Memory" on a query demotes unpinned tables, then the pinned tables
355
+ // that query reads, and retries until it answers or none is left. Other
356
+ // panels' tables stay. For a minute (pressure) tables stay down, then pinned
357
+ // views promote. Only DuckDB's own error counts (see markEngineError).
358
+ // - "Out of Memory" on a load demotes unpinned tables and retries once. A load
359
+ // that still does not fit lands as a view.
360
+ //
361
+ // All catalog changes run one at a time (catalog()), so the bookkeeping here
362
+ // always matches DuckDB's catalog. The worker runs one statement at a time
363
+ // anyway, so this costs no parallelism.
364
+ // ---------------------------------------------------------------------------
365
+ const MIB = 1024 * 1024;
366
+ export const DEFAULT_BINDING_MEMORY_BUDGET = 1536 * MIB;
367
+ export const DEFAULT_BINDING_BLOB_BUDGET = 512 * MIB;
368
+ const DUCKDB_LIMIT = 3174 * MIB; // duckdb-wasm's default memory_limit: 3.1 GiB
369
+ const FILE_CEILING = 2560 * MIB;
370
+ const HEAP_CEILING = 3686 * MIB;
371
+ let memoryBudget = DEFAULT_BINDING_MEMORY_BUDGET;
372
+ let blobBudget = DEFAULT_BINDING_BLOB_BUDGET;
373
+
374
+ // name -> { storage: "table" | "view", blob: Blob | null, file: string | null,
375
+ // fileBytes: number, tableBytes: number | null, demoted: boolean }. `file` is the
376
+ // heap copy of the parquet a view reads, of `fileBytes`. `tableBytes`: what its
377
+ // table cost, measured. `demoted`: a view only because of memory: try to promote.
378
+ const entries = new Map();
379
+ const lastUse = new Map(); // name -> sequence number of its latest read
380
+ let useSeq = 0;
381
+ const pins = new Map(); // name -> how many hooks or queries hold it
382
+ const loading = new Set(); // names with a load in flight
383
+ // Loaded through ensureBinding(): app code may query them by name at any
384
+ // time, so they are demoted (still readable) but never dropped.
385
+ const externalBindings = new Set();
386
+ // Every binding this page ever loaded, for reading one the SDK evicted when the
387
+ // server's index does not list it (index [] apps name their bindings in useQuery).
388
+ const knownBindings = new Set();
389
+ let measuredBytes = 0;
390
+ // After an Out of Memory, tables stay at or under what relief left, for a minute.
391
+ const PRESSURE_MS = 60000;
392
+ let pressureCeiling = Infinity;
393
+ let pressureUntil = 0;
394
+ let pressureTimer = null;
395
+
396
+ function tableCeiling() {
397
+ return Date.now() < pressureUntil ? Math.min(memoryBudget, pressureCeiling) : memoryBudget;
398
+ }
399
+
400
+ function touch(name) {
401
+ lastUse.set(name, ++useSeq);
402
+ }
403
+ function isPinned(name) {
404
+ return (pins.get(name) ?? 0) > 0;
405
+ }
406
+ /** Hold `name` against eviction; the returned function lets go (once). */
407
+ function pin(name) {
408
+ knownBindings.add(name);
409
+ pins.set(name, (pins.get(name) ?? 0) + 1);
410
+ let held = true;
411
+ return () => {
412
+ if (!held) return;
413
+ held = false;
414
+ const n = (pins.get(name) ?? 1) - 1;
415
+ if (n > 0) pins.set(name, n);
416
+ else pins.delete(name);
417
+ };
418
+ }
419
+
420
+ let catalogQueue = Promise.resolve();
421
+ function catalog(fn) {
422
+ const run = catalogQueue.then(fn);
423
+ catalogQueue = run.catch(() => {});
424
+ return run;
425
+ }
426
+
427
+ // The statements in one transaction, so another connection never sees the
428
+ // name missing between a DROP and a CREATE.
429
+ async function exec(engine, statements) {
430
+ const conn = await engine.connect();
431
+ try {
432
+ if (statements.length === 1) {
433
+ await conn.query(statements[0]);
434
+ return;
435
+ }
436
+ await conn.query("BEGIN TRANSACTION");
437
+ try {
438
+ for (const sql of statements) await conn.query(sql);
439
+ await conn.query("COMMIT");
440
+ } catch (error) {
441
+ await conn.query("ROLLBACK").catch(() => {});
442
+ throw error;
443
+ }
444
+ } catch (error) {
445
+ throw markEngineError(error, engine);
446
+ } finally {
447
+ await conn.close();
448
+ }
449
+ }
450
+
451
+ /**
452
+ * An error from this engine's own conn.query. Only these can be Out of Memory:
453
+ * a download or a server's build may quote the same words. A crash is noted here
454
+ * and the engine restarts (restartEngine).
455
+ */
456
+ function markEngineError(error, engine) {
457
+ if (!(error instanceof Error)) return error;
458
+ error.fromEngine = true;
459
+ if (isOutOfMemoryError(error)) error.retry = "memory";
460
+ if (crashedEngineError(error)) {
461
+ error.message = "The data engine crashed (" + error.message + "). It restarts by itself; reload the page if this stays.";
462
+ noteEngineCrash(engine);
463
+ }
464
+ return error;
465
+ }
466
+ const isEngineOom = error => Boolean(error?.fromEngine) && isOutOfMemoryError(error);
467
+
468
+ function dropSql(storage, name) {
469
+ return "DROP " + (storage === "view" ? "VIEW" : "TABLE") + " IF EXISTS " + quoteIdent(name);
196
470
  }
197
471
 
198
472
  let fileSeq = 0;
473
+ // Every load registers its bytes under its own file name, so a reload never
474
+ // drops a file another load is still reading.
475
+ function nextFile() {
476
+ return "mako-" + ++fileSeq + ".parquet";
477
+ }
478
+
479
+ /** The table for a registered parquet `file`, in place of whatever `name` was. */
480
+ async function tableFromFile(engine, name, file) {
481
+ const prev = entries.get(name);
482
+ const before = await duckdbBytes(engine);
483
+ const create =
484
+ "CREATE OR REPLACE TABLE " + quoteIdent(name) +
485
+ " AS SELECT * FROM read_parquet(" + quoteString(file) + ")";
486
+ await exec(engine, prev && prev.storage === "view" ? [dropSql("view", name), create] : [create]);
487
+ const after = await duckdbBytes(engine);
488
+ if (before === null || after === null) return prev?.tableBytes ?? null;
489
+ const grown = after - before + (prev?.storage === "table" ? (prev.tableBytes ?? 0) : 0);
490
+ return grown > 0 ? grown : (prev?.tableBytes ?? null);
491
+ }
492
+
493
+ /** The view over a registered parquet `file`, in place of whatever `name` was. */
494
+ async function viewFromFile(engine, name, file) {
495
+ const prev = entries.get(name);
496
+ const create =
497
+ "CREATE OR REPLACE VIEW " + quoteIdent(name) +
498
+ " AS SELECT * FROM read_parquet(" + quoteString(file) + ")";
499
+ await exec(engine, prev && prev.storage === "table" ? [dropSql("table", name), create] : [create]);
500
+ }
501
+
502
+ /**
503
+ * What a registered parquet's table will cost, from its metadata: 16 bytes a cell,
504
+ * or its uncompressed pages if long strings make that more. Measured on 6 parquets
505
+ * from 2 to 402 MiB: the real table was 0.84x to 1.01x of the 16-byte figure.
506
+ */
507
+ async function estimateTable(engine, file, parquetBytes) {
508
+ const conn = await engine.connect();
509
+ try {
510
+ const f = quoteString(file);
511
+ const result = await conn.query(
512
+ "SELECT greatest(sum(total_uncompressed_size), " +
513
+ "count(DISTINCT path_in_schema) * (SELECT num_rows FROM parquet_file_metadata(" + f + ")) * 16)::DOUBLE " +
514
+ "FROM parquet_metadata(" + f + ")",
515
+ );
516
+ const value = Number(result.getChildAt(0)?.get(0));
517
+ if (Number.isFinite(value) && value > 0) return value;
518
+ } catch {
519
+ /* no metadata to read: the file's own error surfaces when it is read */
520
+ } finally {
521
+ await conn.close();
522
+ }
523
+ return parquetBytes * 16;
524
+ }
525
+
526
+ /** What the SDK placed: tables, and the parquet copies views read. */
527
+ function ownedBytes() {
528
+ let bytes = 0;
529
+ for (const entry of entries.values()) {
530
+ bytes += entry.storage === "table" ? (entry.tableBytes ?? (entry.blob?.size ?? 0) * 16) : entry.fileBytes;
531
+ }
532
+ return bytes;
533
+ }
534
+ function fileBytes() {
535
+ let bytes = 0;
536
+ for (const entry of entries.values()) if (entry.storage === "view") bytes += entry.fileBytes;
537
+ return bytes;
538
+ }
539
+
540
+ /**
541
+ * What this frame's heap holds right now, in bytes: DuckDB's figure, with app
542
+ * tables and every binding table, plus the parquet copies views read, which sit
543
+ * outside its accounting. Query scratch never shows: DuckDB returns it before the
544
+ * next statement runs (measured, also after a failed query). The last figure if
545
+ * it cannot say.
546
+ */
547
+ async function measureNow(engine) {
548
+ const value = await duckdbBytes(engine);
549
+ if (value !== null) measuredBytes = value + fileBytes();
550
+ reportMemory();
551
+ return measuredBytes;
552
+ }
553
+
554
+ /** duckdb_memory() alone, or null on an engine without it. */
555
+ async function duckdbBytes(engine) {
556
+ const conn = await engine.connect();
557
+ try {
558
+ const result = await conn.query(
559
+ "SELECT coalesce(sum(memory_usage_bytes), 0)::DOUBLE AS bytes FROM duckdb_memory()",
560
+ );
561
+ const value = Number(result.getChildAt(0)?.get(0));
562
+ return Number.isFinite(value) ? value : null;
563
+ } catch (error) {
564
+ markEngineError(error, engine);
565
+ return null; /* the OOM retry still applies */
566
+ } finally {
567
+ await conn.close();
568
+ }
569
+ }
570
+
571
+ // Hook for whoever wants the figure (the host frame, in a later step).
572
+ function reportMemory() {}
573
+
574
+ /** The least recently read binding that passes `test` and nothing holds. */
575
+ function leastRecent(test, pinnedToo = false) {
576
+ let best = null;
577
+ let bestSeq = Infinity;
578
+ for (const [name, entry] of entries) {
579
+ if ((isPinned(name) && !pinnedToo) || loading.has(name) || !test(name, entry)) continue;
580
+ const seq = lastUse.get(name) ?? 0;
581
+ if (seq < bestSeq) {
582
+ best = name;
583
+ bestSeq = seq;
584
+ }
585
+ }
586
+ return best;
587
+ }
588
+
589
+ /** Forget a binding entirely: the next read downloads it again. */
590
+ async function dropNow(engine, name) {
591
+ const entry = entries.get(name);
592
+ if (!entry) return;
593
+ await exec(engine, [dropSql(entry.storage, name)]);
594
+ if (entry.file) await engine.dropFile(entry.file).catch(() => {});
595
+ entries.delete(name);
596
+ loadedBindings.delete(name);
597
+ registered.delete(name);
598
+ }
599
+
600
+ /**
601
+ * Room in the heap for `incoming` bytes of parquet copy. registerFileBuffer does
602
+ * not check its malloc, so past this point nothing would report the failure. Goes
603
+ * through views least recently read first, pinned too: their next read downloads.
604
+ */
605
+ async function roomForFile(engine, incoming, except) {
606
+ for (;;) {
607
+ const duckdb = (await duckdbBytes(engine)) ?? 0;
608
+ const files = fileBytes();
609
+ if (files + incoming <= FILE_CEILING && duckdb + files + incoming <= HEAP_CEILING) return;
610
+ const victim = leastRecent(
611
+ (name, entry) => name !== except && entry.storage === "view" && !externalBindings.has(name),
612
+ true,
613
+ );
614
+ if (!victim) {
615
+ const mib = Math.round(incoming / MIB);
616
+ throw new Error("A " + mib + " MiB parquet does not fit in this page's DuckDB heap, which holds " + Math.round((duckdb + files) / MIB) + " MiB.");
617
+ }
618
+ await dropNow(engine, victim);
619
+ }
620
+ }
621
+
622
+ /** One step down: a table becomes a view over a copy of its Blob, or goes. */
623
+ async function evictNow(engine, name) {
624
+ const entry = entries.get(name);
625
+ let file = null;
626
+ if (entry.blob) {
627
+ try {
628
+ await roomForFile(engine, entry.blob.size, name);
629
+ const bytes = new Uint8Array(await entry.blob.arrayBuffer());
630
+ file = nextFile();
631
+ await engine.registerFileBuffer(file, bytes);
632
+ } catch (error) {
633
+ if (error?.fromEngine) throw error;
634
+ file = null; // an unreadable Blob, or no room for its copy
635
+ }
636
+ }
637
+ if (file) {
638
+ try {
639
+ await viewFromFile(engine, name, file);
640
+ } catch (error) {
641
+ await engine.dropFile(file).catch(() => {});
642
+ throw error;
643
+ }
644
+ if (entry.file) await engine.dropFile(entry.file).catch(() => {});
645
+ entries.set(name, { ...entry, storage: "view", file, fileBytes: entry.blob.size, demoted: true });
646
+ return;
647
+ }
648
+ // App code may read this one by name at any time: it stays a table, and no
649
+ // eviction asks again (evictableTable).
650
+ if (externalBindings.has(name)) {
651
+ entries.set(name, { ...entry, blob: null });
652
+ return;
653
+ }
654
+ await dropNow(engine, name);
655
+ }
656
+
657
+ const evictableTable = except => (name, entry) =>
658
+ name !== except &&
659
+ entry.storage === "table" &&
660
+ (entry.blob !== null || !externalBindings.has(name));
661
+
662
+ /** Evict least recently used tables until the bindings hold at most `target`. */
663
+ async function trimNow(engine, except, target = memoryBudget) {
664
+ for (;;) {
665
+ if (ownedBytes() <= target) break;
666
+ const victim = leastRecent(evictableTable(except));
667
+ if (!victim) break;
668
+ await evictNow(engine, victim);
669
+ }
670
+ for (;;) {
671
+ let blobs = 0;
672
+ for (const entry of entries.values()) if (entry.blob) blobs += entry.blob.size;
673
+ if (blobs <= blobBudget) break;
674
+ const victim = leastRecent(
675
+ (name, entry) =>
676
+ name !== except && entry.blob !== null && !(entry.storage === "view" && externalBindings.has(name)),
677
+ );
678
+ if (!victim) break;
679
+ const entry = entries.get(victim);
680
+ if (entry.storage === "view") await dropNow(engine, victim);
681
+ // A table keeps its rows; evicting it later drops it outright.
682
+ else entries.set(victim, { ...entry, blob: null });
683
+ }
684
+ await measureNow(engine);
685
+ }
686
+
687
+ /**
688
+ * After an Out of Memory error: demote tables, least recently used first, until
689
+ * the bindings hold half of what they did (or half the budget). A load stops at
690
+ * unpinned tables. A query goes on to pinned tables it reads itself (`reads`): a
691
+ * query that can never fit must not take healthy panels down with it. Returns how
692
+ * many it freed; 0: a retry would fail the same way.
693
+ */
694
+ async function relieveNow(engine, except, reads = null) {
695
+ const target = Math.min(memoryBudget, ownedBytes()) / 2;
696
+ const evictable = evictableTable(except);
697
+ let freed = 0;
698
+ for (;;) {
699
+ const victim =
700
+ leastRecent(evictable) ??
701
+ (reads ? leastRecent((name, entry) => reads.has(name) && entry.blob !== null && evictable(name, entry), true) : null);
702
+ if (!victim) break;
703
+ await evictNow(engine, victim);
704
+ freed++;
705
+ if (ownedBytes() <= target) break;
706
+ }
707
+ if (freed) startPressure();
708
+ await measureNow(engine);
709
+ return freed;
710
+ }
711
+
712
+ function startPressure() {
713
+ pressureCeiling = ownedBytes();
714
+ pressureUntil = Date.now() + PRESSURE_MS;
715
+ clearTimeout(pressureTimer);
716
+ pressureTimer = setTimeout(endPressure, PRESSURE_MS);
717
+ pressureTimer.unref?.();
718
+ }
719
+
720
+ /** Pinned views come back on their own: a mounted panel may not read again soon. */
721
+ async function endPressure() {
722
+ const waiting = [...entries]
723
+ .filter(([name, entry]) => entry.demoted && isPinned(name) && !loading.has(name))
724
+ .sort(([a], [b]) => (lastUse.get(b) ?? 0) - (lastUse.get(a) ?? 0));
725
+ for (const [name] of waiting) {
726
+ await promote(name);
727
+ if (entries.get(name)?.storage !== "table") break; // full: the rest would not fit either
728
+ }
729
+ }
730
+
731
+ /**
732
+ * Whether a table costing `estimate` fits, evicting what no one holds to make
733
+ * room. Checked before decoding: a table placed past the ceiling could only be
734
+ * demoted again.
735
+ */
736
+ async function roomForTable(engine, name, estimate) {
737
+ const prev = entries.get(name);
738
+ const replaced = !prev ? 0 : prev.storage === "table" ? (prev.tableBytes ?? estimate) : prev.fileBytes;
739
+ const fitsHeap = async () => ((await duckdbBytes(engine)) ?? 0) + estimate <= DUCKDB_LIMIT;
740
+ // A single table over the ceiling: never evict for it. With a mostly empty
741
+ // heap, let it try: an OOM lands it as a view, and queries keep what is left.
742
+ if (estimate > tableCeiling()) {
743
+ return (
744
+ estimate <= memoryBudget * 2 &&
745
+ Date.now() >= pressureUntil &&
746
+ ownedBytes() - replaced <= memoryBudget / 4 &&
747
+ (await fitsHeap())
748
+ );
749
+ }
750
+ for (;;) {
751
+ if (ownedBytes() - replaced + estimate <= tableCeiling()) return fitsHeap();
752
+ const victim = leastRecent(evictableTable(name));
753
+ if (!victim) return false;
754
+ await evictNow(engine, victim);
755
+ }
756
+ }
757
+
758
+ /**
759
+ * Decode a download into DuckDB: a table when it fits, else a view. Reads `buf`,
760
+ * never the Blob: these bytes are in hand and a Blob can be unreadable.
761
+ */
762
+ async function placeNow(engine, name, buf, blob) {
763
+ const prev = entries.get(name);
764
+ await roomForFile(engine, buf.length, name);
765
+ const file = nextFile();
766
+ const size = buf.length;
767
+ await engine.registerFileBuffer(file, buf); // transfers `buf` to the worker
768
+ const asView = async () => {
769
+ await viewFromFile(engine, name, file);
770
+ if (prev?.file) await engine.dropFile(prev.file).catch(() => {});
771
+ return { storage: "view", file, fileBytes: size, demoted: true };
772
+ };
773
+ const asTable = async () => {
774
+ const tableBytes = await tableFromFile(engine, name, file);
775
+ await engine.dropFile(file).catch(() => {});
776
+ if (prev?.file) await engine.dropFile(prev.file).catch(() => {});
777
+ return { storage: "table", file: null, fileBytes: 0, demoted: false, tableBytes };
778
+ };
779
+ try {
780
+ const estimate = prev?.tableBytes ?? (await estimateTable(engine, file, blob.size));
781
+ if (!(await roomForTable(engine, name, estimate))) return await asView();
782
+ try {
783
+ return await asTable();
784
+ } catch (error) {
785
+ if (!isEngineOom(error)) throw error;
786
+ }
787
+ if (await relieveNow(engine, name)) {
788
+ try {
789
+ return await asTable();
790
+ } catch (error) {
791
+ if (!isEngineOom(error)) throw error;
792
+ }
793
+ }
794
+ return await asView();
795
+ } catch (error) {
796
+ await engine.dropFile(file).catch(() => {});
797
+ throw error;
798
+ }
799
+ }
800
+
199
801
  async function loadBinding(name, bust) {
200
802
  const db = getDb(); // boot in parallel with the download
201
803
  db.catch(() => {});
804
+ loading.add(name);
202
805
  try {
203
806
  const buf = await downloadBinding(name, bust, patch => setProgress(name, patch));
204
807
  setProgress(name, { phase: "loading_duckdb" });
205
808
  const engine = await db;
206
- const file = "mako-" + ++fileSeq + ".parquet";
207
- await engine.registerFileBuffer(file, buf);
208
- try {
209
- const conn = await engine.connect();
210
- try {
211
- await conn.query(
212
- "CREATE OR REPLACE TABLE " + quoteIdent(name) +
213
- " AS SELECT * FROM read_parquet(" + quoteString(file) + ")",
214
- );
215
- } finally {
216
- await conn.close();
217
- }
218
- } finally {
219
- await engine.dropFile(file).catch(() => {});
220
- }
221
- loadedBindings.add(name);
809
+ // Made before the buffer goes to the worker: the copy a demotion reads.
810
+ const blob = new Blob([buf]);
811
+ await catalog(async () => {
812
+ const placed = await placeNow(engine, name, buf, blob);
813
+ const tableBytes = placed.tableBytes ?? entries.get(name)?.tableBytes ?? null;
814
+ entries.set(name, { ...placed, blob, tableBytes });
815
+ knownBindings.add(name);
816
+ loadedBindings.add(name);
817
+ loading.delete(name);
818
+ await trimNow(engine, name);
819
+ });
222
820
  } finally {
821
+ loading.delete(name);
223
822
  // A refresh owns the record until its reload is done (refreshBinding).
224
823
  if (!refreshes.has(name)) setProgress(name, null);
225
824
  }
226
825
  }
227
826
 
827
+ /**
828
+ * A demoted view read again: back to a table from its heap copy, no download, if
829
+ * it fits. Otherwise it stays a view, and the next read asks again: a view that
830
+ * promoted itself into a full heap would only be demoted again.
831
+ */
832
+ async function promote(name) {
833
+ const engine = await getDb();
834
+ await catalog(async () => {
835
+ const entry = entries.get(name);
836
+ if (!entry || entry.storage !== "view" || !entry.file) return;
837
+ if (!(await roomForTable(engine, name, entry.tableBytes ?? (await estimateTable(engine, entry.file, entry.fileBytes))))) {
838
+ entry.demoted = true;
839
+ return;
840
+ }
841
+ const tableBytes = await tableFromFile(engine, name, entry.file);
842
+ await engine.dropFile(entry.file).catch(() => {});
843
+ entries.set(name, { ...entry, storage: "table", file: null, fileBytes: 0, demoted: false, tableBytes });
844
+ await trimNow(engine, name);
845
+ }).catch(() => {
846
+ const entry = entries.get(name);
847
+ if (entry && entry.storage === "view") entry.demoted = true;
848
+ /* it stays a view, which reads the same rows */
849
+ });
850
+ }
851
+
228
852
  const registered = new Map(); // name -> Promise<void>
229
- const loadedBindings = new Set(); // names whose table is in DuckDB
853
+ const loadedBindings = new Set(); // names DuckDB can read (table or view)
230
854
  // Refreshed on the server while never loaded on this page: the next load
231
855
  // must skip any cache between here and the server.
232
856
  const staleBindings = new Set();
233
857
  function registerBinding(name) {
234
- if (!registered.has(name)) {
858
+ touch(name);
859
+ knownBindings.add(name);
860
+ const known = registered.get(name);
861
+ if (!known) {
235
862
  const load = loadBinding(name, staleBindings.delete(name));
236
- registered.set(name, load);
237
- load.catch(() => {
238
- if (registered.get(name) === load) registered.delete(name);
239
- });
863
+ return trackRegistered(name, load);
864
+ }
865
+ const entry = entries.get(name);
866
+ if (entry && entry.demoted && !loading.has(name)) {
867
+ entry.demoted = false; // one promotion at a time
868
+ // The view reads fine whatever became of the promise before this one.
869
+ return trackRegistered(name, known.catch(() => {}).then(() => promote(name)));
870
+ }
871
+ return known;
872
+ }
873
+
874
+ /** The promise a next read waits on. A rejected one never stays: it would answer for good. */
875
+ function trackRegistered(name, promise) {
876
+ registered.set(name, promise);
877
+ promise.catch(() => {
878
+ if (registered.get(name) === promise) registered.delete(name);
879
+ });
880
+ return promise;
881
+ }
882
+
883
+ /**
884
+ * Cap what bindings may hold in DuckDB, in bytes (default 1.5 GiB, half of the
885
+ * frame's 3.1 GiB heap: queries need the rest). Checked with an estimate before
886
+ * each load, so it can overshoot. Past it, the least recently read tables become
887
+ * views over a heap copy of their parquet. `opts.blobBytes` caps the parquet kept
888
+ * in JS memory (default 512 MiB); past it the oldest are dropped and download
889
+ * again on their next read. `Infinity` turns either off. Resolves once the engine
890
+ * is back under the new budget, and ends any memory pressure.
891
+ */
892
+ export async function setBindingMemoryBudget(bytes, opts) {
893
+ const valid = n => typeof n === "number" && n > 0 && !Number.isNaN(n);
894
+ if (!valid(bytes)) throw new TypeError("setBindingMemoryBudget: a positive number of bytes");
895
+ memoryBudget = bytes;
896
+ pressureUntil = 0; // the caller has just said what tables may hold
897
+ clearTimeout(pressureTimer);
898
+ if (opts && opts.blobBytes !== undefined) {
899
+ if (!valid(opts.blobBytes)) throw new TypeError("setBindingMemoryBudget: blobBytes must be a positive number");
900
+ blobBudget = opts.blobBytes;
240
901
  }
241
- return registered.get(name);
902
+ wakeRetries();
903
+ if (!dbPromise) return;
904
+ const engine = await getDb();
905
+ await catalog(() => trimNow(engine, null));
906
+ }
907
+
908
+ /**
909
+ * What the binding tables hold now: DuckDB's own figure, the budgets, and
910
+ * each loaded binding, least recently read first.
911
+ */
912
+ export async function getBindingMemory() {
913
+ const engine = await getDb();
914
+ const usedBytes = await catalog(() => measureNow(engine));
915
+ const bindings = [...entries]
916
+ .sort(([a], [b]) => (lastUse.get(a) ?? 0) - (lastUse.get(b) ?? 0))
917
+ .map(([name, entry]) => ({
918
+ name,
919
+ storage: entry.storage,
920
+ pinned: isPinned(name),
921
+ parquetBytes: entry.blob ? entry.blob.size : null,
922
+ }));
923
+ return {
924
+ usedBytes,
925
+ bindingBytes: ownedBytes(),
926
+ budgetBytes: memoryBudget,
927
+ blobBudgetBytes: blobBudget,
928
+ pressure: Date.now() < pressureUntil,
929
+ bindings,
930
+ };
242
931
  }
243
932
 
244
933
  // ---------------------------------------------------------------------------
@@ -270,23 +959,30 @@ export async function withConnection(fn) {
270
959
  * Make sure a binding's table exists in the shared DuckDB, loading it once
271
960
  * per page (concurrent callers share the download). Resolves with the table
272
961
  * name — the binding name; quote it in SQL when it is not a bare identifier.
962
+ * Each call counts as a read. Under memory pressure the table may become a
963
+ * view over its parquet (same name, same rows, slower); it is never dropped.
273
964
  */
274
965
  export async function ensureBinding(name) {
966
+ externalBindings.add(name);
275
967
  await registerBinding(name);
276
968
  return name;
277
969
  }
278
970
 
279
971
  /**
280
- * Forget a loaded binding: drops its table so its memory is freed. The next
281
- * query or ensureBinding() downloads it again.
972
+ * Forget a loaded binding: drops its table (or view) so its memory is freed.
973
+ * The next query or ensureBinding() downloads it again.
282
974
  */
283
975
  export async function unloadBinding(name) {
284
976
  const pending = registered.get(name);
285
977
  if (!pending) return;
286
978
  registered.delete(name);
287
- loadedBindings.delete(name);
979
+ externalBindings.delete(name);
980
+ knownBindings.delete(name);
288
981
  await pending.catch(() => {});
289
- await withConnection(conn => conn.query("DROP TABLE IF EXISTS " + quoteIdent(name)));
982
+ if (registered.has(name)) return; // read again meanwhile: it stays
983
+ const engine = await getDb();
984
+ await catalog(() => dropNow(engine, name));
985
+ wakeRetries();
290
986
  }
291
987
 
292
988
  /**
@@ -319,14 +1015,7 @@ async function reloadBinding(name) {
319
1015
  // Let a load in flight settle first, or its (older) bytes could land after
320
1016
  // the fresh ones and win.
321
1017
  await previous.catch(() => {});
322
- const load = loadBinding(name, true);
323
- registered.set(name, load);
324
- try {
325
- await load;
326
- } catch (error) {
327
- if (registered.get(name) === load) registered.delete(name);
328
- throw error;
329
- }
1018
+ await trackRegistered(name, loadBinding(name, true));
330
1019
  }
331
1020
 
332
1021
  // ---------------------------------------------------------------------------
@@ -487,15 +1176,46 @@ export function getViewer() {
487
1176
  /** Names of every staged binding — written by the dev server next to the
488
1177
  * parquet files. Absent (older servers, published builds): empty list. */
489
1178
  let indexPromise = null;
1179
+ const INDEX_RETRY_MS = 10000;
490
1180
  function bindingIndex() {
491
- indexPromise ??= fetch("__data/index.json")
492
- .then(r => (r.ok ? r.json() : []))
493
- .then(list => (Array.isArray(list) ? list : []))
494
- .catch(() => []);
495
- return indexPromise;
1181
+ if (indexPromise) return indexPromise;
1182
+ // 20 panels asking at once share one answer, a failed one for 10 s too.
1183
+ const forget = () => {
1184
+ const t = setTimeout(() => {
1185
+ if (indexPromise === pending) indexPromise = null;
1186
+ }, INDEX_RETRY_MS);
1187
+ t.unref?.();
1188
+ };
1189
+ const pending = fetchRetrying("__data/index.json").then(async r => {
1190
+ // A 404 is an older server or a published build: empty, for good.
1191
+ if (!r.ok && r.status !== 404) forget();
1192
+ if (r.status === 429 || r.status === 503) throw await responseError(r, "The binding list could not be loaded");
1193
+ if (!r.ok) return [];
1194
+ const list = await r.json().catch(() => []);
1195
+ return Array.isArray(list) ? list : [];
1196
+ });
1197
+ indexPromise = pending;
1198
+ pending.catch(forget);
1199
+ return pending;
496
1200
  }
497
1201
 
498
- async function runSql(sql, rowLimit, format) {
1202
+ // Out of Memory on a query: free tables, pinned ones the query reads last, and
1203
+ // try again. Each pass demotes at least one table, so this ends. With nothing left
1204
+ // to free, DuckDB's error surfaces and the hook decides (useAsyncQuery).
1205
+ async function runSql(sql, rowLimit, format, reads) {
1206
+ for (;;) {
1207
+ try {
1208
+ return await runSqlOnce(sql, rowLimit, format);
1209
+ } catch (error) {
1210
+ if (!isEngineOom(error)) throw error;
1211
+ const engine = await getDb();
1212
+ const freed = await catalog(() => relieveNow(engine, null, reads)).catch(() => 0);
1213
+ if (!freed) throw error;
1214
+ }
1215
+ }
1216
+ }
1217
+
1218
+ async function runSqlOnce(sql, rowLimit, format) {
499
1219
  const cap = rowLimit === null ? Infinity : (rowLimit ?? DEFAULT_ROW_LIMIT);
500
1220
  const db = await getDb();
501
1221
  const conn = await db.connect();
@@ -509,11 +1229,43 @@ async function runSql(sql, rowLimit, format) {
509
1229
  }
510
1230
  const { rows, fields } = toPlainRows(table, cap);
511
1231
  return { rows, fields, rowCount, truncated: rowCount > rows.length };
1232
+ } catch (error) {
1233
+ throw markEngineError(error, db);
512
1234
  } finally {
513
1235
  await conn.close();
514
1236
  }
515
1237
  }
516
1238
 
1239
+ // A crashed engine stays crashed: the SQL that broke it can answer, then every
1240
+ // later query fails with a raw WASM trap. So the page starts a new engine.
1241
+ // Bindings download again. Pins stay: they belong to mounted hooks.
1242
+ const RESTART_EVERY_MS = 60000;
1243
+ const MAX_RESTARTS = 3;
1244
+ let lastRestart = -Infinity;
1245
+ let restarts = 0;
1246
+ let restartTimer = null;
1247
+ function noteEngineCrash(engine) {
1248
+ if (engine !== liveEngine || restartTimer || restarts >= MAX_RESTARTS) return;
1249
+ restartTimer = setTimeout(restartEngine, Math.max(0, lastRestart + RESTART_EVERY_MS - Date.now()));
1250
+ }
1251
+ function restartEngine() {
1252
+ restartTimer = null;
1253
+ const old = liveEngine;
1254
+ if (!old) return;
1255
+ restarts++;
1256
+ lastRestart = Date.now();
1257
+ liveEngine = null;
1258
+ dbPromise = null;
1259
+ catalogQueue = Promise.resolve();
1260
+ entries.clear();
1261
+ registered.clear();
1262
+ loadedBindings.clear();
1263
+ measuredBytes = 0;
1264
+ pressureUntil = 0;
1265
+ old.terminate().catch(() => {});
1266
+ setBindingState({ ...bindingState, version: bindingState.version + 1 });
1267
+ }
1268
+
517
1269
  const EMPTY = {
518
1270
  data: null,
519
1271
  fields: null,
@@ -529,33 +1281,97 @@ const EMPTY = {
529
1281
  // until the new ones arrive — a dashboard must not blink to empty on every
530
1282
  // refresh). `isRefreshing` says whether a refresh in flight concerns this
531
1283
  // query, so `refreshing` is true from the click, not only from the reload.
1284
+ // `run(hold)` names each binding it reads through `hold`: those stay pinned
1285
+ // against eviction while this hook shows that query (until the deps change
1286
+ // or it unmounts, and never before its run settles).
1287
+ //
1288
+ // A query that failed on memory or the network runs again by itself, a few times
1289
+ // (retryDelayMs). After that it waits for a reason to expect a different answer:
1290
+ // another hook let go of its bindings, the budget changed, the server answered
1291
+ // again, the tab came back, the network came back. The error stays up until a run
1292
+ // answers. A hidden tab waits too. Anything else stays as it is.
1293
+ const retryWaiters = new Set();
1294
+ function wakeRetries() {
1295
+ const waiting = [...retryWaiters];
1296
+ retryWaiters.clear();
1297
+ for (const fire of waiting) fire();
1298
+ }
1299
+ // The user or the network is back: let one probe through a closed gate, too.
1300
+ function tryAgainNow() {
1301
+ gateUntil = 0;
1302
+ wakeRetries();
1303
+ }
1304
+ if (typeof window !== "undefined" && typeof window.addEventListener === "function") {
1305
+ window.addEventListener("online", tryAgainNow);
1306
+ if (typeof document !== "undefined") {
1307
+ document.addEventListener("visibilitychange", () => {
1308
+ if (!document.hidden) tryAgainNow();
1309
+ });
1310
+ }
1311
+ }
1312
+
532
1313
  function useAsyncQuery(run, deps, isRefreshing) {
533
1314
  const bindings = useBindingState();
534
1315
  const [state, setState] = React.useState(EMPTY);
1316
+ const [attempt, setAttempt] = React.useState(0);
535
1317
  const seenVersion = React.useRef(bindings.version);
1318
+ const seenAttempt = React.useRef(attempt);
1319
+ const retries = React.useRef(0);
1320
+ const retrying = React.useRef(false);
536
1321
  React.useEffect(() => {
537
1322
  let active = true;
1323
+ let timer = null;
1324
+ let waiter = null;
538
1325
  const isRefresh = seenVersion.current !== bindings.version;
1326
+ const isRetry = seenAttempt.current !== attempt;
539
1327
  seenVersion.current = bindings.version;
540
- setState(s => (isRefresh && s.data !== null ? { ...s, refreshing: true } : EMPTY));
541
- run().then(
1328
+ seenAttempt.current = attempt;
1329
+ retrying.current = false;
1330
+ if (!isRetry) {
1331
+ retries.current = 0;
1332
+ setState(s => (isRefresh && s.data !== null ? { ...s, refreshing: true } : EMPTY));
1333
+ }
1334
+ const held = [];
1335
+ const settled = run(name => held.push(pin(name)));
1336
+ settled.then(
542
1337
  result => {
543
1338
  if (active) setState({ ...result, error: null, loading: false, refreshing: false });
544
1339
  },
545
1340
  error => {
546
- if (active)
547
- setState({
548
- ...EMPTY,
549
- error: error instanceof Error ? error.message : String(error),
550
- loading: false,
551
- });
1341
+ if (!active) return;
1342
+ setState({
1343
+ ...EMPTY,
1344
+ error: error instanceof Error ? error.message : String(error),
1345
+ loading: false,
1346
+ });
1347
+ const wait = retryDelayMs(error, retries.current);
1348
+ if (wait === null) return;
1349
+ retries.current++;
1350
+ waiter = () => {
1351
+ if (typeof document !== "undefined" && document.hidden) {
1352
+ retryWaiters.add(waiter);
1353
+ return;
1354
+ }
1355
+ retrying.current = true;
1356
+ setAttempt(n => n + 1);
1357
+ };
1358
+ if (wait === Infinity) retryWaiters.add(waiter);
1359
+ else timer = setTimeout(waiter, wait);
552
1360
  },
553
1361
  );
554
1362
  return () => {
555
1363
  active = false;
1364
+ if (timer !== null) clearTimeout(timer);
1365
+ if (waiter) retryWaiters.delete(waiter);
1366
+ const letsGo = !retrying.current;
1367
+ settled.catch(() => {}).finally(() => {
1368
+ for (const release of held) release();
1369
+ // Its own retry frees nothing, or two failing panels would wake each other.
1370
+ if (letsGo && held.length) wakeRetries();
1371
+ });
556
1372
  };
557
1373
  // eslint-disable-next-line react-hooks/exhaustive-deps
558
- }, [...deps, bindings.version]);
1374
+ }, [...deps, bindings.version, attempt]);
559
1375
  return {
560
1376
  ...state,
561
1377
  refreshing: state.refreshing || isRefreshing(bindings.refreshing),
@@ -622,9 +1438,10 @@ export function useQuery(name, opts) {
622
1438
  const withSnapshot = Boolean(opts && opts.snapshot);
623
1439
  const preview = useSnapshotPreview(name, withSnapshot, rowLimit);
624
1440
  const state = useAsyncQuery(
625
- async () => {
1441
+ async hold => {
1442
+ hold(name);
626
1443
  await registerBinding(name);
627
- const r = await runSql("SELECT * FROM " + quoteIdent(name), rowLimit);
1444
+ const r = await runSql("SELECT * FROM " + quoteIdent(name), rowLimit, undefined, new Set([name]));
628
1445
  return { data: r.rows, fields: r.fields, truncated: r.truncated, rowCount: r.rowCount };
629
1446
  },
630
1447
  [name, rowLimit],
@@ -682,24 +1499,41 @@ export function useViewer() {
682
1499
  * A binding that fails to load must not sink the query silently: when the
683
1500
  * SQL then fails, the load failure is what surfaces — it is the cause.
684
1501
  */
685
- async function runOverBindings(sql, rowLimit, format) {
686
- const names = await bindingIndex();
1502
+ async function runOverBindings(sql, rowLimit, format, hold) {
1503
+ // No index still answers `select 1`. It matters only if the query then fails.
1504
+ let indexFailure = null;
1505
+ const index = await bindingIndex().catch(error => {
1506
+ indexFailure = error;
1507
+ return [];
1508
+ });
1509
+ const names = [...new Set([...index, ...knownBindings])];
687
1510
  const failures = new Map();
688
1511
  const attempted = new Set();
1512
+ const reloaded = new Set();
689
1513
  const load = n => {
690
1514
  attempted.add(n);
1515
+ hold(n);
691
1516
  return registerBinding(n).catch(e => {
692
1517
  failures.set(n, e);
693
1518
  });
694
1519
  };
695
- await Promise.all((readsCatalog(sql) ? names : mentionedBindings(sql, names)).map(load));
1520
+ const wanted = readsCatalog(sql) ? names : mentionedBindings(sql, names);
1521
+ if (!readsCatalog(sql) && wanted.length > MAX_BINDINGS_IN_QUERY) {
1522
+ throw new Error(
1523
+ "This query names " + wanted.length + " bindings. One query can read at most " + MAX_BINDINGS_IN_QUERY +
1524
+ ": around 60 overflow DuckDB's WASM stack and break the engine for every panel. Read fewer shards, or combine them in the warehouse.",
1525
+ );
1526
+ }
1527
+ await Promise.all(wanted.map(load));
696
1528
  for (;;) {
697
1529
  try {
698
- return await runSql(sql, rowLimit, format);
1530
+ return await runSql(sql, rowLimit, format, attempted);
699
1531
  } catch (error) {
700
1532
  const message = error instanceof Error ? error.message : String(error);
701
1533
  const missing = missingBinding(message, names);
702
- if (missing && !attempted.has(missing)) {
1534
+ // Never asked for, or the SDK evicted it under this query. Once per run.
1535
+ if (missing && !failures.has(missing) && !reloaded.has(missing) && (!attempted.has(missing) || !registered.has(missing))) {
1536
+ reloaded.add(missing);
703
1537
  await load(missing);
704
1538
  if (failures.has(missing)) throw failures.get(missing);
705
1539
  continue;
@@ -708,11 +1542,15 @@ async function runOverBindings(sql, rowLimit, format) {
708
1542
  if (message.includes(n)) throw cause;
709
1543
  }
710
1544
  if (failures.size) {
711
- const details = [...failures.values()]
712
- .map(e => (e instanceof Error ? e.message : String(e)))
713
- .join("; ");
714
- throw new Error(message + " (bindings that failed to load: " + details + ")");
1545
+ const causes = [...failures.values()];
1546
+ const details = causes.map(e => (e instanceof Error ? e.message : String(e))).join("; ");
1547
+ const wrapped = new Error(message + " (bindings that failed to load: " + details + ")");
1548
+ const retryable = causes.find(e => e?.retry);
1549
+ wrapped.retry = error?.retry ?? retryable?.retry;
1550
+ wrapped.retryAfterMs = retryable?.retryAfterMs;
1551
+ throw wrapped;
715
1552
  }
1553
+ if (indexFailure && !error?.retry) throw indexFailure;
716
1554
  throw error;
717
1555
  }
718
1556
  }
@@ -722,8 +1560,8 @@ export function useDuckDB(sql, opts) {
722
1560
  bootEngine();
723
1561
  const rowLimit = opts ? opts.rowLimit : undefined;
724
1562
  const format = opts && opts.format === "arrow" ? "arrow" : "rows";
725
- const state = useAsyncQuery(async () => {
726
- const r = await runOverBindings(sql, rowLimit, format);
1563
+ const state = useAsyncQuery(async hold => {
1564
+ const r = await runOverBindings(sql, rowLimit, format, hold);
727
1565
  return { data: r.rows, fields: r.fields, truncated: r.truncated, rowCount: r.rowCount };
728
1566
  }, [sql, rowLimit, format], refreshing => refreshing.size > 0);
729
1567
  // Every binding the query can read: the combined record says what the