@makoai/app-sdk 2.11.1 → 2.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.js CHANGED
@@ -20,8 +20,12 @@ import {
20
20
  missingBinding,
21
21
  mentionedBindings,
22
22
  quoteIdent,
23
+ isReadOnlySql,
23
24
  readsCatalog,
24
25
  quoteString,
26
+ crashedEngineError,
27
+ isOutOfMemoryError,
28
+ retryDelayMs,
25
29
  toPlainRows,
26
30
  parseBindingSnapshot,
27
31
  snapshotUrl,
@@ -32,7 +36,7 @@ import {
32
36
  * in a page (or this export) says which SDK a deployed app actually runs.
33
37
  * Kept in step with package.json by snapshot.test.mjs.
34
38
  */
35
- export const SDK_VERSION = "2.11.1";
39
+ export const SDK_VERSION = "2.13.0";
36
40
  if (typeof window !== "undefined") {
37
41
  try {
38
42
  window.__MAKO_APP_SDK__ = SDK_VERSION;
@@ -42,6 +46,9 @@ if (typeof window !== "undefined") {
42
46
  }
43
47
 
44
48
  const DEFAULT_ROW_LIMIT = 500000;
49
+ // A double-read UNION ALL of 55 to 60 bindings overflows the WASM stack and
50
+ // corrupts the engine for every panel (measured). 50 held.
51
+ const MAX_BINDINGS_IN_QUERY = 50;
45
52
  const DUCKDB_ESM =
46
53
  "https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.29.0/+esm";
47
54
 
@@ -53,12 +60,30 @@ const DUCKDB_ESM =
53
60
  // parquet download instead of waiting for it.
54
61
  // ---------------------------------------------------------------------------
55
62
  let dbPromise = null;
63
+ let liveEngine = null; // the resolved dbPromise: errors from an older one are history
64
+ // createWorker() makes a new object URL per call and never revokes it, so each
65
+ // reboot after a release would leak the worker script (about 742 KiB). One URL
66
+ // per page, also one download fewer in a frame with no HTTP cache. Not through
67
+ // the download gate: the CDN being down says nothing about Mako's server.
68
+ let workerUrl = null;
69
+ function startWorker(duckdb, script) {
70
+ if (typeof Worker === "undefined") return duckdb.createWorker(script);
71
+ workerUrl ??= fetch(script).then(async res => {
72
+ if (!res.ok) throw new Error("The DuckDB worker could not be loaded (HTTP " + res.status + ")");
73
+ return URL.createObjectURL(await res.blob());
74
+ });
75
+ workerUrl.catch(() => {
76
+ workerUrl = null;
77
+ });
78
+ return workerUrl.then(url => new Worker(url));
79
+ }
80
+
56
81
  function getDb() {
57
82
  if (!dbPromise) {
58
83
  dbPromise = (async () => {
59
84
  const duckdb = await import(/* @vite-ignore */ DUCKDB_ESM);
60
85
  const bundle = await duckdb.selectBundle(duckdb.getJsDelivrBundles());
61
- const worker = await duckdb.createWorker(bundle.mainWorker);
86
+ const worker = await startWorker(duckdb, bundle.mainWorker);
62
87
  const db = new duckdb.AsyncDuckDB(new duckdb.VoidLogger(), worker);
63
88
  await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
64
89
  // read_parquet lives in an extension DuckDB fetches on first use
@@ -72,6 +97,7 @@ function getDb() {
72
97
  } finally {
73
98
  await conn.close();
74
99
  }
100
+ liveEngine = db;
75
101
  return db;
76
102
  })();
77
103
  // A failed boot (CDN unreachable) is retried by the next query.
@@ -104,6 +130,8 @@ async function responseError(res, what) {
104
130
  const error = new Error(what + " (HTTP " + res.status + ")" + (detail ? ": " + detail : ""));
105
131
  error.status = res.status;
106
132
  if (body && typeof body.retryAfterMs === "number") error.retryAfterMs = body.retryAfterMs;
133
+ // The one thing a mounted hook may ask again. Any other status said what it meant.
134
+ if (res.status === 429 || res.status === 503) error.retry = "network";
107
135
  return error;
108
136
  }
109
137
 
@@ -168,6 +196,102 @@ export function useBindingProgress(name) {
168
196
  // under its own file name, so a reload never drops a file another load is
169
197
  // still reading.
170
198
 
199
+ // Cloud Run answers a queue timeout with a 429 and no CORS header, so an opaque
200
+ // app frame sees only "Failed to fetch": 12 of 5,087 published parquet requests
201
+ // on 2026-11-02, in 8 bursts. One page-wide gate keeps a page from making it worse.
202
+ // - At most 12 downloads at once; 1 while the gate is recovering from a failure.
203
+ // - A failure closes the gate for 1, 3, 9, 27, then 60 s, with jitter. Failures
204
+ // in one closed window count once. Any answer that is not a 429 or 503 opens it.
205
+ // - Past 3 failures in a row, a request while it is closed fails at once with no
206
+ // network, so a page down to one probe at a time does not keep panels spinning.
207
+ const FETCH_RETRY_MS = [1000, 3000];
208
+ const RETRY_AFTER_CAP_MS = 30000;
209
+ const DOWNLOAD_SLOTS = 12;
210
+ const GATE_MAX_MS = 60000;
211
+ const GATE_FAIL_FAST_AT = 3;
212
+ let gateStreak = 0;
213
+ let gateUntil = 0;
214
+ let gateError = null;
215
+ let slotsInUse = 0;
216
+ const slotQueue = [];
217
+ const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
218
+ const withJitter = ms => ms * (0.75 + Math.random() / 2);
219
+
220
+ function gateFailed(error) {
221
+ gateError = error;
222
+ if (Date.now() < gateUntil) return;
223
+ gateStreak++;
224
+ gateUntil = Date.now() + withJitter(Math.min(GATE_MAX_MS, 1000 * 3 ** (gateStreak - 1)));
225
+ }
226
+ function gateAnswered() {
227
+ if (gateStreak === 0) return;
228
+ gateStreak = 0;
229
+ gateUntil = 0;
230
+ gateError = null;
231
+ pumpSlots();
232
+ wakeRetries();
233
+ }
234
+ async function throughGate() {
235
+ for (;;) {
236
+ const wait = gateUntil - Date.now();
237
+ if (wait <= 0) return;
238
+ if (gateStreak >= GATE_FAIL_FAST_AT) throw gateError;
239
+ await sleep(wait);
240
+ }
241
+ }
242
+ function pumpSlots() {
243
+ let room = (gateStreak ? 1 : DOWNLOAD_SLOTS) - slotsInUse;
244
+ while (room-- > 0 && slotQueue.length) slotQueue.shift()();
245
+ }
246
+ async function withSlot(fn) {
247
+ while (slotsInUse >= (gateStreak ? 1 : DOWNLOAD_SLOTS)) await new Promise(resolve => slotQueue.push(resolve));
248
+ slotsInUse++;
249
+ try {
250
+ return await fn();
251
+ } finally {
252
+ slotsInUse--;
253
+ pumpSlots();
254
+ }
255
+ }
256
+
257
+ /**
258
+ * fetch, until headers arrive: 3 attempts on a rejected fetch, a 429 or a 503,
259
+ * through the gate. A response of any other status returns at once, and so does
260
+ * a stream once it starts, so a build is never asked for again from here. Waits
261
+ * out a `retryAfterMs` up to 30 s; past that the caller gets the answer. No
262
+ * headers timeout: the Vite plugin may hold headers for a whole build.
263
+ */
264
+ async function fetchRetrying(url, init) {
265
+ for (let attempt = 0; ; attempt++) {
266
+ const last = attempt >= FETCH_RETRY_MS.length;
267
+ await throughGate();
268
+ let wait = last ? 0 : withJitter(FETCH_RETRY_MS[attempt]);
269
+ try {
270
+ const res = await fetch(url, init);
271
+ if (res.status !== 429 && res.status !== 503) {
272
+ gateAnswered();
273
+ return res;
274
+ }
275
+ const asked = await res
276
+ .clone()
277
+ .json()
278
+ .then(body => body?.retryAfterMs)
279
+ .catch(() => undefined);
280
+ const busy = new Error("The server is busy (HTTP " + res.status + ")");
281
+ busy.retry = "network";
282
+ gateFailed(busy);
283
+ if (last || asked > RETRY_AFTER_CAP_MS) return res;
284
+ if (typeof asked === "number") wait = asked;
285
+ } catch (error) {
286
+ if (!(error instanceof TypeError)) throw error;
287
+ error.retry = "network";
288
+ gateFailed(error);
289
+ if (last) throw error;
290
+ }
291
+ await sleep(wait);
292
+ }
293
+ }
294
+
171
295
  /**
172
296
  * A binding's parquet bytes, phase-aware: asks for NDJSON phase events (a
173
297
  * build reports each step, a stored file resolving → signing and a signed
@@ -178,67 +302,748 @@ async function downloadBinding(name, bust, report) {
178
302
  const path = "__data/" + encodeURIComponent(name) + ".parquet";
179
303
  const what = 'Data for binding "' + name + '" could not be loaded';
180
304
  report({ phase: "requesting", rowsWritten: null, bytes: null, bytesTotal: null });
181
- let res = await fetch(path + (bust ? "?refresh=" + Date.now() : ""), {
182
- headers: { accept: PROGRESS_ACCEPT },
183
- });
305
+ // A slot holds the connection, not the wait for a build: one slow warehouse
306
+ // query must not hold up twelve bindings that are ready.
307
+ let res = await withSlot(() =>
308
+ fetchRetrying(path + (bust ? "?refresh=" + Date.now() : ""), {
309
+ headers: { accept: PROGRESS_ACCEPT },
310
+ }),
311
+ );
312
+ let signed = null;
184
313
  if (isProgressResponse(res)) {
185
- const final = await readProgressStream(res, e => report(progressPatch(e)));
314
+ let final;
315
+ try {
316
+ final = await readProgressStream(res, e => report(progressPatch(e)));
317
+ } catch (error) {
318
+ // A cut build stream may have cost a warehouse query. Asking again costs another.
319
+ throw new Error(what + ": the build's stream was cut (" + (error instanceof Error ? error.message : String(error)) + ")");
320
+ }
186
321
  if (final.phase === "failed") throw progressError(final, what);
187
322
  report({ phase: "downloading", rowsWritten: null, bytes: 0 });
188
323
  // No url: the server streams the bytes itself — the same path, asked
189
324
  // plainly, serves them (the build already staged or cached them).
190
- res = await fetch(final.url || path);
325
+ signed = final.url || path;
326
+ res = null;
191
327
  }
192
- if (!res.ok) throw await responseError(res, what);
193
- return readBytes(res, (bytes, bytesTotal) =>
194
- report({ phase: "downloading", rowsWritten: null, bytes, bytesTotal }),
195
- );
328
+ return withSlot(async () => {
329
+ res ??= await fetchRetrying(signed);
330
+ if (!res.ok) throw await responseError(res, what);
331
+ try {
332
+ return await readBytes(res, (bytes, bytesTotal) =>
333
+ report({ phase: "downloading", rowsWritten: null, bytes, bytesTotal }),
334
+ );
335
+ } catch (error) {
336
+ if (error instanceof TypeError) error.retry = "network"; // the body cut off: no build behind it
337
+ throw error;
338
+ }
339
+ });
340
+ }
341
+
342
+ // ---------------------------------------------------------------------------
343
+ // Memory (2.12). Every table lives in this frame's WASM heap, where DuckDB's
344
+ // memory_limit is 3.1 GiB. 2.8 to 2.11 never dropped one, so an app with many
345
+ // bindings (Web Performance: 106 shards) browsed its way to "Out of Memory".
346
+ //
347
+ // - Every read bumps its binding. Least recently used goes first.
348
+ // - The budget counts what the SDK placed: tables and the parquet copies views
349
+ // read. An app's own tables do not shrink it. The real total still decides
350
+ // whether a table can fit in the heap at all.
351
+ // - A table lands if the parquet's own metadata says it fits, after evicting what
352
+ // no one holds. Measured tables can pass the estimate, so the budget can
353
+ // overshoot (measured: 1,700 MiB against 1,536). With a mostly empty heap, one
354
+ // table may try anyway: an ordinary app with one large parquet stays fast.
355
+ // - What does not fit lands as a view over its parquet in the heap
356
+ // (registerFileBuffer): 7x to 14x smaller than its table. A later read, or the
357
+ // end of pressure, promotes it if room has appeared.
358
+ // - Never a BROWSER_FILEREADER handle over a Blob: duckdb-wasm 1.29.0's openFile
359
+ // writes the file size to HEAPF64[(ptr >> 3)], and `>>` turns a pointer past
360
+ // 2 GiB negative. Reads then fail with "too small to be a Parquet file" (4 of
361
+ // 8 reads with 3.0 GiB of tables, 0 of 8 with 1.6 GiB; heap buffers 0 of 8).
362
+ // - File copies have their own ceiling: 2.5 GiB, and 3.6 GiB with DuckDB's own
363
+ // bytes. registerFileBuffer never checks its malloc: at 4.16 GiB of copies the
364
+ // call never settled and hung the page. Over it the least recently read view
365
+ // goes, pinned too, and downloads again on its next read.
366
+ // - The parquet stays in JS memory as a Blob only so a table can become a view
367
+ // without a download. A Blob can stop being readable (NotReadableError on the
368
+ // third 885 MiB in incognito). Its table then goes instead, and downloads again.
369
+ // Past the Blob budget, the least recently used view is dropped outright.
370
+ // - A binding a mounted hook reads, or a query in flight, is pinned: it goes last.
371
+ // Only loads in flight are never touched.
372
+ // - "Out of Memory" on a query demotes unpinned tables, then the pinned tables
373
+ // that query reads, and retries until it answers or none is left. Other
374
+ // panels' tables stay. For a minute (pressure) tables stay down, then pinned
375
+ // views promote. Only DuckDB's own error counts (see markEngineError).
376
+ // - "Out of Memory" on a load demotes unpinned tables and retries once. A load
377
+ // that still does not fit lands as a view.
378
+ //
379
+ // All catalog changes run one at a time (catalog()), so the bookkeeping here
380
+ // always matches DuckDB's catalog. The worker runs one statement at a time
381
+ // anyway, so this costs no parallelism.
382
+ // ---------------------------------------------------------------------------
383
+ const MIB = 1024 * 1024;
384
+ export const DEFAULT_BINDING_MEMORY_BUDGET = 1536 * MIB;
385
+ export const DEFAULT_BINDING_BLOB_BUDGET = 512 * MIB;
386
+ const DUCKDB_LIMIT = 3174 * MIB; // duckdb-wasm's default memory_limit: 3.1 GiB
387
+ const FILE_CEILING = 2560 * MIB;
388
+ const HEAP_CEILING = 3686 * MIB;
389
+ let memoryBudget = DEFAULT_BINDING_MEMORY_BUDGET;
390
+ let blobBudget = DEFAULT_BINDING_BLOB_BUDGET;
391
+
392
+ // name -> { storage: "table" | "view", blob: Blob | null, file: string | null,
393
+ // fileBytes: number, tableBytes: number | null, demoted: boolean }. `file` is the
394
+ // heap copy of the parquet a view reads, of `fileBytes`. `tableBytes`: what its
395
+ // table cost, measured. `demoted`: a view only because of memory: try to promote.
396
+ const entries = new Map();
397
+ const lastUse = new Map(); // name -> sequence number of its latest read
398
+ let useSeq = 0;
399
+ const pins = new Map(); // name -> how many hooks or queries hold it
400
+ const loading = new Set(); // names with a load in flight
401
+ // Loaded through ensureBinding(): app code may query them by name at any
402
+ // time, so they are demoted (still readable) but never dropped.
403
+ const externalBindings = new Set();
404
+ // Every binding this page ever loaded, for reading one the SDK evicted when the
405
+ // server's index does not list it (index [] apps name their bindings in useQuery).
406
+ const knownBindings = new Set();
407
+ let measuredBytes = 0;
408
+ // After an Out of Memory, tables stay at or under what relief left, for a minute.
409
+ const PRESSURE_MS = 60000;
410
+ let pressureCeiling = Infinity;
411
+ let pressureUntil = 0;
412
+ let pressureTimer = null;
413
+
414
+ function tableCeiling() {
415
+ return Date.now() < pressureUntil ? Math.min(memoryBudget, pressureCeiling) : memoryBudget;
416
+ }
417
+
418
+ function touch(name) {
419
+ lastUse.set(name, ++useSeq);
420
+ }
421
+ function isPinned(name) {
422
+ return (pins.get(name) ?? 0) > 0;
423
+ }
424
+ /** Hold `name` against eviction; the returned function lets go (once). */
425
+ function pin(name) {
426
+ knownBindings.add(name);
427
+ pins.set(name, (pins.get(name) ?? 0) + 1);
428
+ let held = true;
429
+ return () => {
430
+ if (!held) return;
431
+ held = false;
432
+ const n = (pins.get(name) ?? 1) - 1;
433
+ if (n > 0) pins.set(name, n);
434
+ else pins.delete(name);
435
+ };
436
+ }
437
+
438
+ let catalogQueue = Promise.resolve();
439
+ function catalog(fn) {
440
+ const run = catalogQueue.then(fn);
441
+ catalogQueue = run.catch(() => {});
442
+ return run;
443
+ }
444
+
445
+ // The statements in one transaction, so another connection never sees the
446
+ // name missing between a DROP and a CREATE.
447
+ async function exec(engine, statements) {
448
+ const conn = await engine.connect();
449
+ try {
450
+ if (statements.length === 1) {
451
+ await conn.query(statements[0]);
452
+ return;
453
+ }
454
+ await conn.query("BEGIN TRANSACTION");
455
+ try {
456
+ for (const sql of statements) await conn.query(sql);
457
+ await conn.query("COMMIT");
458
+ } catch (error) {
459
+ await conn.query("ROLLBACK").catch(() => {});
460
+ throw error;
461
+ }
462
+ } catch (error) {
463
+ throw markEngineError(error, engine);
464
+ } finally {
465
+ await conn.close();
466
+ }
467
+ }
468
+
469
+ /**
470
+ * An error from this engine's own conn.query. Only these can be Out of Memory:
471
+ * a download or a server's build may quote the same words. A crash is noted here
472
+ * and the engine restarts (restartEngine).
473
+ */
474
+ function markEngineError(error, engine) {
475
+ if (!(error instanceof Error)) return error;
476
+ error.fromEngine = true;
477
+ if (isOutOfMemoryError(error)) error.retry = "memory";
478
+ if (crashedEngineError(error)) {
479
+ error.message = "The data engine crashed (" + error.message + "). It restarts by itself; reload the page if this stays.";
480
+ noteEngineCrash(engine);
481
+ }
482
+ return error;
483
+ }
484
+ const isEngineOom = error => Boolean(error?.fromEngine) && isOutOfMemoryError(error);
485
+
486
+ function dropSql(storage, name) {
487
+ return "DROP " + (storage === "view" ? "VIEW" : "TABLE") + " IF EXISTS " + quoteIdent(name);
196
488
  }
197
489
 
198
490
  let fileSeq = 0;
491
+ // Every load registers its bytes under its own file name, so a reload never
492
+ // drops a file another load is still reading.
493
+ function nextFile() {
494
+ return "mako-" + ++fileSeq + ".parquet";
495
+ }
496
+
497
+ /** The table for a registered parquet `file`, in place of whatever `name` was. */
498
+ async function tableFromFile(engine, name, file) {
499
+ const prev = entries.get(name);
500
+ const before = await duckdbBytes(engine);
501
+ const create =
502
+ "CREATE OR REPLACE TABLE " + quoteIdent(name) +
503
+ " AS SELECT * FROM read_parquet(" + quoteString(file) + ")";
504
+ await exec(engine, prev && prev.storage === "view" ? [dropSql("view", name), create] : [create]);
505
+ const after = await duckdbBytes(engine);
506
+ if (before === null || after === null) return prev?.tableBytes ?? null;
507
+ const grown = after - before + (prev?.storage === "table" ? (prev.tableBytes ?? 0) : 0);
508
+ return grown > 0 ? grown : (prev?.tableBytes ?? null);
509
+ }
510
+
511
+ /** The view over a registered parquet `file`, in place of whatever `name` was. */
512
+ async function viewFromFile(engine, name, file) {
513
+ const prev = entries.get(name);
514
+ const create =
515
+ "CREATE OR REPLACE VIEW " + quoteIdent(name) +
516
+ " AS SELECT * FROM read_parquet(" + quoteString(file) + ")";
517
+ await exec(engine, prev && prev.storage === "table" ? [dropSql("table", name), create] : [create]);
518
+ }
519
+
520
+ /**
521
+ * What a registered parquet's table will cost, from its metadata: 16 bytes a cell,
522
+ * or its uncompressed pages if long strings make that more. Measured on 6 parquets
523
+ * from 2 to 402 MiB: the real table was 0.84x to 1.01x of the 16-byte figure.
524
+ */
525
+ async function estimateTable(engine, file, parquetBytes) {
526
+ const conn = await engine.connect();
527
+ try {
528
+ const f = quoteString(file);
529
+ const result = await conn.query(
530
+ "SELECT greatest(sum(total_uncompressed_size), " +
531
+ "count(DISTINCT path_in_schema) * (SELECT num_rows FROM parquet_file_metadata(" + f + ")) * 16)::DOUBLE " +
532
+ "FROM parquet_metadata(" + f + ")",
533
+ );
534
+ const value = Number(result.getChildAt(0)?.get(0));
535
+ if (Number.isFinite(value) && value > 0) return value;
536
+ } catch {
537
+ /* no metadata to read: the file's own error surfaces when it is read */
538
+ } finally {
539
+ await conn.close();
540
+ }
541
+ return parquetBytes * 16;
542
+ }
543
+
544
+ /** What the SDK placed: tables, and the parquet copies views read. */
545
+ function ownedBytes() {
546
+ let bytes = 0;
547
+ for (const entry of entries.values()) {
548
+ bytes += entry.storage === "table" ? (entry.tableBytes ?? (entry.blob?.size ?? 0) * 16) : entry.fileBytes;
549
+ }
550
+ return bytes;
551
+ }
552
+ function fileBytes() {
553
+ let bytes = 0;
554
+ for (const entry of entries.values()) if (entry.storage === "view") bytes += entry.fileBytes;
555
+ return bytes;
556
+ }
557
+
558
+ /**
559
+ * What this frame's heap holds right now, in bytes: DuckDB's figure, with app
560
+ * tables and every binding table, plus the parquet copies views read, which sit
561
+ * outside its accounting. Query scratch never shows: DuckDB returns it before the
562
+ * next statement runs (measured, also after a failed query). The last figure if
563
+ * it cannot say.
564
+ */
565
+ async function measureNow(engine) {
566
+ const value = await duckdbBytes(engine);
567
+ if (value !== null) measuredBytes = value + fileBytes();
568
+ reportMemory();
569
+ return measuredBytes;
570
+ }
571
+
572
+ /** duckdb_memory() alone, or null on an engine without it. */
573
+ async function duckdbBytes(engine) {
574
+ const conn = await engine.connect();
575
+ try {
576
+ const result = await conn.query(
577
+ "SELECT coalesce(sum(memory_usage_bytes), 0)::DOUBLE AS bytes FROM duckdb_memory()",
578
+ );
579
+ const value = Number(result.getChildAt(0)?.get(0));
580
+ return Number.isFinite(value) ? value : null;
581
+ } catch (error) {
582
+ markEngineError(error, engine);
583
+ return null; /* the OOM retry still applies */
584
+ } finally {
585
+ await conn.close();
586
+ }
587
+ }
588
+
589
+ // ---------------------------------------------------------------------------
590
+ // Across apps (2.13). Each app frame has its own DuckDB and its own WASM heap,
591
+ // so evicting inside one frame frees nothing for the next app opened. The SDK
592
+ // tells Mako's shell what its heap holds; the shell keeps one budget across its
593
+ // app frames and asks the least recently shown ones to release. A WASM heap
594
+ // never shrinks: demoting a table frees DuckDB's accounting and leaves the bytes
595
+ // reserved. So release means ending the worker, and the report is the most this
596
+ // worker has held. It is measuredBytes, not bindingBytes: an app's own tables
597
+ // sit in the same renderer. Parquet stays here as Blobs ("parked"): the next read
598
+ // boots a new engine and decodes from them, with no download and no gate.
599
+ //
600
+ // A worker can end only when nothing lives in it but bindings. App code that may
601
+ // hold the engine or keep its own state there keeps it for the page's life
602
+ // (keepEngine), says so in its reports, and the shell asks another frame.
603
+ //
604
+ // A release is not a crash: it never counts against MAX_RESTARTS, never reruns a
605
+ // hook, and leaves the gate and the download slots alone. They describe the
606
+ // server. Nothing in flight also means no load holds a slot.
607
+ // ---------------------------------------------------------------------------
608
+ const MEMORY_MESSAGE = "mako-app:duckdb-memory";
609
+ const RELEASE_MESSAGE = "mako-host:duckdb-release";
610
+ const inFrame = typeof window !== "undefined" && window.parent != null && window.parent !== window;
611
+ const parked = new Map(); // name -> Blob of a binding whose engine was released
612
+ let engineKept = false;
613
+ let heapBytes = 0; // the most this worker has held: its heap never gives it back
614
+ let busy = 0; // hooks, withConnection calls and queries in flight
615
+ let reportedBytes = null;
616
+ let reportedKept = false;
617
+
618
+ function reportMemory() {
619
+ heapBytes = Math.max(heapBytes, measuredBytes);
620
+ if (!inFrame || (heapBytes === reportedBytes && engineKept === reportedKept)) return;
621
+ reportedBytes = heapBytes;
622
+ reportedKept = engineKept;
623
+ try {
624
+ window.parent.postMessage({ type: MEMORY_MESSAGE, bytes: heapBytes, canRelease: !engineKept }, "*");
625
+ } catch {
626
+ /* a host that is not Mako: nothing to tell */
627
+ }
628
+ }
629
+
630
+ /** Something only this worker has may exist from now on: never end it. */
631
+ function keepEngine() {
632
+ if (engineKept) return;
633
+ engineKept = true;
634
+ reportMemory();
635
+ }
636
+
637
+ /**
638
+ * The host's ask: give the memory back. Nothing in flight: park every Blob and
639
+ * end the worker. Rows already on screen stay (they are plain JS); the next
640
+ * query reboots. Busy: skipped, and the host asks again. A kept engine stays. So
641
+ * does one with a crash restart due: that ends it and reruns the panels that saw
642
+ * the crash, which a release would not.
643
+ */
644
+ async function releaseEngine() {
645
+ const idle = () => !engineKept && !restartTimer && busy === 0 && loading.size === 0 && refreshes.size === 0;
646
+ const current = dbPromise;
647
+ if (!current || !idle()) return;
648
+ const engine = await current;
649
+ await catalog(async () => {
650
+ if (dbPromise !== current || !idle()) return;
651
+ // From here to dbPromise = null runs without a yield: no query can grab
652
+ // the old engine in between. liveEngine goes too: a crash report from this
653
+ // engine must not restart the next one.
654
+ for (const [name, entry] of entries) if (entry.blob) parked.set(name, entry.blob);
655
+ entries.clear();
656
+ registered.clear();
657
+ loadedBindings.clear();
658
+ liveEngine = null;
659
+ dbPromise = null;
660
+ measuredBytes = 0;
661
+ heapBytes = 0;
662
+ pressureUntil = 0; // the next engine starts with an empty heap
663
+ clearTimeout(pressureTimer);
664
+ reportMemory();
665
+ await engine.terminate().catch(() => {});
666
+ });
667
+ }
668
+
669
+ if (inFrame) {
670
+ window.addEventListener("message", event => {
671
+ // Only the frame's own parent may ask. The ask can do no harm beyond a
672
+ // slower next query, so the parent's origin (Mako's, whatever host it
673
+ // runs on) is not pinned here.
674
+ if (event.source !== window.parent) return;
675
+ const data = event.data;
676
+ if (!data || data.type !== RELEASE_MESSAGE) return;
677
+ releaseEngine().catch(() => {
678
+ /* the next ask, or the next load's trim, tries again */
679
+ });
680
+ });
681
+ }
682
+
683
+ /** The least recently read binding that passes `test` and nothing holds. */
684
+ function leastRecent(test, pinnedToo = false) {
685
+ let best = null;
686
+ let bestSeq = Infinity;
687
+ for (const [name, entry] of entries) {
688
+ if ((isPinned(name) && !pinnedToo) || loading.has(name) || !test(name, entry)) continue;
689
+ const seq = lastUse.get(name) ?? 0;
690
+ if (seq < bestSeq) {
691
+ best = name;
692
+ bestSeq = seq;
693
+ }
694
+ }
695
+ return best;
696
+ }
697
+
698
+ /** Forget a binding entirely: the next read downloads it again. */
699
+ async function dropNow(engine, name) {
700
+ const entry = entries.get(name);
701
+ if (!entry) return;
702
+ await exec(engine, [dropSql(entry.storage, name)]);
703
+ if (entry.file) await engine.dropFile(entry.file).catch(() => {});
704
+ entries.delete(name);
705
+ loadedBindings.delete(name);
706
+ registered.delete(name);
707
+ }
708
+
709
+ /**
710
+ * Room in the heap for `incoming` bytes of parquet copy. registerFileBuffer does
711
+ * not check its malloc, so past this point nothing would report the failure. Goes
712
+ * through views least recently read first, pinned too: their next read downloads.
713
+ */
714
+ async function roomForFile(engine, incoming, except) {
715
+ for (;;) {
716
+ const duckdb = (await duckdbBytes(engine)) ?? 0;
717
+ const files = fileBytes();
718
+ if (files + incoming <= FILE_CEILING && duckdb + files + incoming <= HEAP_CEILING) return;
719
+ const victim = leastRecent(
720
+ (name, entry) => name !== except && entry.storage === "view" && !externalBindings.has(name),
721
+ true,
722
+ );
723
+ if (!victim) {
724
+ const mib = Math.round(incoming / MIB);
725
+ throw new Error("A " + mib + " MiB parquet does not fit in this page's DuckDB heap, which holds " + Math.round((duckdb + files) / MIB) + " MiB.");
726
+ }
727
+ await dropNow(engine, victim);
728
+ }
729
+ }
730
+
731
+ /** One step down: a table becomes a view over a copy of its Blob, or goes. */
732
+ async function evictNow(engine, name) {
733
+ const entry = entries.get(name);
734
+ let file = null;
735
+ if (entry.blob) {
736
+ try {
737
+ await roomForFile(engine, entry.blob.size, name);
738
+ const bytes = new Uint8Array(await entry.blob.arrayBuffer());
739
+ file = nextFile();
740
+ await engine.registerFileBuffer(file, bytes);
741
+ } catch (error) {
742
+ if (error?.fromEngine) throw error;
743
+ file = null; // an unreadable Blob, or no room for its copy
744
+ }
745
+ }
746
+ if (file) {
747
+ try {
748
+ await viewFromFile(engine, name, file);
749
+ } catch (error) {
750
+ await engine.dropFile(file).catch(() => {});
751
+ throw error;
752
+ }
753
+ if (entry.file) await engine.dropFile(entry.file).catch(() => {});
754
+ entries.set(name, { ...entry, storage: "view", file, fileBytes: entry.blob.size, demoted: true });
755
+ return;
756
+ }
757
+ // App code may read this one by name at any time: it stays a table, and no
758
+ // eviction asks again (evictableTable).
759
+ if (externalBindings.has(name)) {
760
+ entries.set(name, { ...entry, blob: null });
761
+ return;
762
+ }
763
+ await dropNow(engine, name);
764
+ }
765
+
766
+ const evictableTable = except => (name, entry) =>
767
+ name !== except &&
768
+ entry.storage === "table" &&
769
+ (entry.blob !== null || !externalBindings.has(name));
770
+
771
+ /** Evict least recently used tables until the bindings hold at most `target`. */
772
+ async function trimNow(engine, except, target = memoryBudget) {
773
+ for (;;) {
774
+ if (ownedBytes() <= target) break;
775
+ const victim = leastRecent(evictableTable(except));
776
+ if (!victim) break;
777
+ await evictNow(engine, victim);
778
+ }
779
+ for (;;) {
780
+ let blobs = 0;
781
+ for (const entry of entries.values()) if (entry.blob) blobs += entry.blob.size;
782
+ if (blobs <= blobBudget) break;
783
+ const victim = leastRecent(
784
+ (name, entry) =>
785
+ name !== except && entry.blob !== null && !(entry.storage === "view" && externalBindings.has(name)),
786
+ );
787
+ if (!victim) break;
788
+ const entry = entries.get(victim);
789
+ if (entry.storage === "view") await dropNow(engine, victim);
790
+ // A table keeps its rows; evicting it later drops it outright.
791
+ else entries.set(victim, { ...entry, blob: null });
792
+ }
793
+ await measureNow(engine);
794
+ }
795
+
796
+ /**
797
+ * After an Out of Memory error: demote tables, least recently used first, until
798
+ * the bindings hold half of what they did (or half the budget). A load stops at
799
+ * unpinned tables. A query goes on to pinned tables it reads itself (`reads`): a
800
+ * query that can never fit must not take healthy panels down with it. Returns how
801
+ * many it freed; 0: a retry would fail the same way.
802
+ */
803
+ async function relieveNow(engine, except, reads = null) {
804
+ const target = Math.min(memoryBudget, ownedBytes()) / 2;
805
+ const evictable = evictableTable(except);
806
+ let freed = 0;
807
+ for (;;) {
808
+ const victim =
809
+ leastRecent(evictable) ??
810
+ (reads ? leastRecent((name, entry) => reads.has(name) && entry.blob !== null && evictable(name, entry), true) : null);
811
+ if (!victim) break;
812
+ await evictNow(engine, victim);
813
+ freed++;
814
+ if (ownedBytes() <= target) break;
815
+ }
816
+ if (freed) startPressure();
817
+ await measureNow(engine);
818
+ return freed;
819
+ }
820
+
821
+ function startPressure() {
822
+ pressureCeiling = ownedBytes();
823
+ pressureUntil = Date.now() + PRESSURE_MS;
824
+ clearTimeout(pressureTimer);
825
+ pressureTimer = setTimeout(endPressure, PRESSURE_MS);
826
+ pressureTimer.unref?.();
827
+ }
828
+
829
+ /** Pinned views come back on their own: a mounted panel may not read again soon. */
830
+ async function endPressure() {
831
+ const waiting = [...entries]
832
+ .filter(([name, entry]) => entry.demoted && isPinned(name) && !loading.has(name))
833
+ .sort(([a], [b]) => (lastUse.get(b) ?? 0) - (lastUse.get(a) ?? 0));
834
+ for (const [name] of waiting) {
835
+ await promote(name);
836
+ if (entries.get(name)?.storage !== "table") break; // full: the rest would not fit either
837
+ }
838
+ }
839
+
840
+ /**
841
+ * Whether a table costing `estimate` fits, evicting what no one holds to make
842
+ * room. Checked before decoding: a table placed past the ceiling could only be
843
+ * demoted again.
844
+ */
845
+ async function roomForTable(engine, name, estimate) {
846
+ const prev = entries.get(name);
847
+ const replaced = !prev ? 0 : prev.storage === "table" ? (prev.tableBytes ?? estimate) : prev.fileBytes;
848
+ const fitsHeap = async () => ((await duckdbBytes(engine)) ?? 0) + estimate <= DUCKDB_LIMIT;
849
+ // A single table over the ceiling: never evict for it. With a mostly empty
850
+ // heap, let it try: an OOM lands it as a view, and queries keep what is left.
851
+ if (estimate > tableCeiling()) {
852
+ return (
853
+ estimate <= memoryBudget * 2 &&
854
+ Date.now() >= pressureUntil &&
855
+ ownedBytes() - replaced <= memoryBudget / 4 &&
856
+ (await fitsHeap())
857
+ );
858
+ }
859
+ for (;;) {
860
+ if (ownedBytes() - replaced + estimate <= tableCeiling()) return fitsHeap();
861
+ const victim = leastRecent(evictableTable(name));
862
+ if (!victim) return false;
863
+ await evictNow(engine, victim);
864
+ }
865
+ }
866
+
867
+ /**
868
+ * Decode a download into DuckDB: a table when it fits, else a view. Reads `buf`,
869
+ * never the Blob: these bytes are in hand and a Blob can be unreadable.
870
+ */
871
+ async function placeNow(engine, name, buf, blob) {
872
+ const prev = entries.get(name);
873
+ await roomForFile(engine, buf.length, name);
874
+ const file = nextFile();
875
+ const size = buf.length;
876
+ await engine.registerFileBuffer(file, buf); // transfers `buf` to the worker
877
+ const asView = async () => {
878
+ await viewFromFile(engine, name, file);
879
+ if (prev?.file) await engine.dropFile(prev.file).catch(() => {});
880
+ return { storage: "view", file, fileBytes: size, demoted: true };
881
+ };
882
+ const asTable = async () => {
883
+ const tableBytes = await tableFromFile(engine, name, file);
884
+ await engine.dropFile(file).catch(() => {});
885
+ if (prev?.file) await engine.dropFile(prev.file).catch(() => {});
886
+ return { storage: "table", file: null, fileBytes: 0, demoted: false, tableBytes };
887
+ };
888
+ try {
889
+ const estimate = prev?.tableBytes ?? (await estimateTable(engine, file, blob.size));
890
+ if (!(await roomForTable(engine, name, estimate))) return await asView();
891
+ try {
892
+ return await asTable();
893
+ } catch (error) {
894
+ if (!isEngineOom(error)) throw error;
895
+ }
896
+ if (await relieveNow(engine, name)) {
897
+ try {
898
+ return await asTable();
899
+ } catch (error) {
900
+ if (!isEngineOom(error)) throw error;
901
+ }
902
+ }
903
+ return await asView();
904
+ } catch (error) {
905
+ await engine.dropFile(file).catch(() => {});
906
+ throw error;
907
+ }
908
+ }
909
+
199
910
  async function loadBinding(name, bust) {
200
911
  const db = getDb(); // boot in parallel with the download
201
912
  db.catch(() => {});
913
+ loading.add(name);
202
914
  try {
203
- const buf = await downloadBinding(name, bust, patch => setProgress(name, patch));
915
+ // Parked by a release: the same bytes, no download (unless a refresh made
916
+ // them stale). A Blob can stop being readable: then it downloads.
917
+ const kept = parked.get(name);
918
+ parked.delete(name);
919
+ const parkedBytes = kept && !bust ? await kept.arrayBuffer().catch(() => null) : null;
920
+ const buf = parkedBytes
921
+ ? new Uint8Array(parkedBytes)
922
+ : await downloadBinding(name, bust, patch => setProgress(name, patch));
204
923
  setProgress(name, { phase: "loading_duckdb" });
205
924
  const engine = await db;
206
- const file = "mako-" + ++fileSeq + ".parquet";
207
- await engine.registerFileBuffer(file, buf);
208
- try {
209
- const conn = await engine.connect();
210
- try {
211
- await conn.query(
212
- "CREATE OR REPLACE TABLE " + quoteIdent(name) +
213
- " AS SELECT * FROM read_parquet(" + quoteString(file) + ")",
214
- );
215
- } finally {
216
- await conn.close();
217
- }
218
- } finally {
219
- await engine.dropFile(file).catch(() => {});
220
- }
221
- loadedBindings.add(name);
925
+ // Made before the buffer goes to the worker: the copy a demotion reads.
926
+ const blob = new Blob([buf]);
927
+ await catalog(async () => {
928
+ const placed = await placeNow(engine, name, buf, blob);
929
+ const tableBytes = placed.tableBytes ?? entries.get(name)?.tableBytes ?? null;
930
+ entries.set(name, { ...placed, blob, tableBytes });
931
+ knownBindings.add(name);
932
+ loadedBindings.add(name);
933
+ loading.delete(name);
934
+ await trimNow(engine, name);
935
+ });
222
936
  } finally {
937
+ loading.delete(name);
223
938
  // A refresh owns the record until its reload is done (refreshBinding).
224
939
  if (!refreshes.has(name)) setProgress(name, null);
225
940
  }
226
941
  }
227
942
 
943
+ /**
944
+ * A demoted view read again: back to a table from its heap copy, no download, if
945
+ * it fits. Otherwise it stays a view, and the next read asks again: a view that
946
+ * promoted itself into a full heap would only be demoted again.
947
+ */
948
+ async function promote(name) {
949
+ const engine = await getDb();
950
+ await catalog(async () => {
951
+ const entry = entries.get(name);
952
+ if (!entry || entry.storage !== "view" || !entry.file) return;
953
+ if (!(await roomForTable(engine, name, entry.tableBytes ?? (await estimateTable(engine, entry.file, entry.fileBytes))))) {
954
+ entry.demoted = true;
955
+ return;
956
+ }
957
+ const tableBytes = await tableFromFile(engine, name, entry.file);
958
+ await engine.dropFile(entry.file).catch(() => {});
959
+ entries.set(name, { ...entry, storage: "table", file: null, fileBytes: 0, demoted: false, tableBytes });
960
+ await trimNow(engine, name);
961
+ }).catch(() => {
962
+ const entry = entries.get(name);
963
+ if (entry && entry.storage === "view") entry.demoted = true;
964
+ /* it stays a view, which reads the same rows */
965
+ });
966
+ }
967
+
228
968
  const registered = new Map(); // name -> Promise<void>
229
- const loadedBindings = new Set(); // names whose table is in DuckDB
969
+ const loadedBindings = new Set(); // names DuckDB can read (table or view)
230
970
  // Refreshed on the server while never loaded on this page: the next load
231
971
  // must skip any cache between here and the server.
232
972
  const staleBindings = new Set();
233
973
  function registerBinding(name) {
234
- if (!registered.has(name)) {
974
+ touch(name);
975
+ knownBindings.add(name);
976
+ const known = registered.get(name);
977
+ if (!known) {
235
978
  const load = loadBinding(name, staleBindings.delete(name));
236
- registered.set(name, load);
237
- load.catch(() => {
238
- if (registered.get(name) === load) registered.delete(name);
239
- });
979
+ return trackRegistered(name, load);
980
+ }
981
+ const entry = entries.get(name);
982
+ if (entry && entry.demoted && !loading.has(name)) {
983
+ entry.demoted = false; // one promotion at a time
984
+ // The view reads fine whatever became of the promise before this one.
985
+ return trackRegistered(name, known.catch(() => {}).then(() => promote(name)));
240
986
  }
241
- return registered.get(name);
987
+ return known;
988
+ }
989
+
990
+ /** The promise a next read waits on. A rejected one never stays: it would answer for good. */
991
+ function trackRegistered(name, promise) {
992
+ registered.set(name, promise);
993
+ promise.catch(() => {
994
+ if (registered.get(name) === promise) registered.delete(name);
995
+ });
996
+ return promise;
997
+ }
998
+
999
+ /**
1000
+ * Cap what bindings may hold in DuckDB, in bytes (default 1.5 GiB, half of the
1001
+ * frame's 3.1 GiB heap: queries need the rest). Checked with an estimate before
1002
+ * each load, so it can overshoot. Past it, the least recently read tables become
1003
+ * views over a heap copy of their parquet. `opts.blobBytes` caps the parquet kept
1004
+ * in JS memory (default 512 MiB); past it the oldest are dropped and download
1005
+ * again on their next read. `Infinity` turns either off. Resolves once the engine
1006
+ * is back under the new budget, and ends any memory pressure.
1007
+ */
1008
+ export async function setBindingMemoryBudget(bytes, opts) {
1009
+ const valid = n => typeof n === "number" && n > 0 && !Number.isNaN(n);
1010
+ if (!valid(bytes)) throw new TypeError("setBindingMemoryBudget: a positive number of bytes");
1011
+ memoryBudget = bytes;
1012
+ pressureUntil = 0; // the caller has just said what tables may hold
1013
+ clearTimeout(pressureTimer);
1014
+ if (opts && opts.blobBytes !== undefined) {
1015
+ if (!valid(opts.blobBytes)) throw new TypeError("setBindingMemoryBudget: blobBytes must be a positive number");
1016
+ blobBudget = opts.blobBytes;
1017
+ }
1018
+ wakeRetries();
1019
+ if (!dbPromise) return;
1020
+ const engine = await getDb();
1021
+ await catalog(() => trimNow(engine, null));
1022
+ }
1023
+
1024
+ /**
1025
+ * What the binding tables hold now: DuckDB's own figure, the budgets, and
1026
+ * each loaded binding, least recently read first.
1027
+ */
1028
+ export async function getBindingMemory() {
1029
+ const engine = await getDb();
1030
+ const usedBytes = await catalog(() => measureNow(engine));
1031
+ const bindings = [...entries]
1032
+ .sort(([a], [b]) => (lastUse.get(a) ?? 0) - (lastUse.get(b) ?? 0))
1033
+ .map(([name, entry]) => ({
1034
+ name,
1035
+ storage: entry.storage,
1036
+ pinned: isPinned(name),
1037
+ parquetBytes: entry.blob ? entry.blob.size : null,
1038
+ }));
1039
+ return {
1040
+ usedBytes,
1041
+ bindingBytes: ownedBytes(),
1042
+ budgetBytes: memoryBudget,
1043
+ blobBudgetBytes: blobBudget,
1044
+ pressure: Date.now() < pressureUntil,
1045
+ bindings,
1046
+ };
242
1047
  }
243
1048
 
244
1049
  // ---------------------------------------------------------------------------
@@ -252,17 +1057,24 @@ function registerBinding(name) {
252
1057
  * close or terminate it: every hook on the page reads from it.
253
1058
  */
254
1059
  export function getDuckDB() {
1060
+ keepEngine();
255
1061
  return getDb();
256
1062
  }
257
1063
 
258
1064
  /** Run `fn` with a fresh connection to the shared DuckDB; always closed after. */
259
1065
  export async function withConnection(fn) {
260
- const db = await getDb();
261
- const conn = await db.connect();
1066
+ keepEngine();
1067
+ busy++;
262
1068
  try {
263
- return await fn(conn);
1069
+ const db = await getDb();
1070
+ const conn = await db.connect();
1071
+ try {
1072
+ return await fn(conn);
1073
+ } finally {
1074
+ await conn.close();
1075
+ }
264
1076
  } finally {
265
- await conn.close();
1077
+ busy--;
266
1078
  }
267
1079
  }
268
1080
 
@@ -270,23 +1082,32 @@ export async function withConnection(fn) {
270
1082
  * Make sure a binding's table exists in the shared DuckDB, loading it once
271
1083
  * per page (concurrent callers share the download). Resolves with the table
272
1084
  * name — the binding name; quote it in SQL when it is not a bare identifier.
1085
+ * Each call counts as a read. Under memory pressure the table may become a
1086
+ * view over its parquet (same name, same rows, slower); it is never dropped.
273
1087
  */
274
1088
  export async function ensureBinding(name) {
1089
+ keepEngine(); // its contract: the name stays readable, by code the SDK cannot see
1090
+ externalBindings.add(name);
275
1091
  await registerBinding(name);
276
1092
  return name;
277
1093
  }
278
1094
 
279
1095
  /**
280
- * Forget a loaded binding: drops its table so its memory is freed. The next
281
- * query or ensureBinding() downloads it again.
1096
+ * Forget a loaded binding: drops its table (or view) so its memory is freed.
1097
+ * The next query or ensureBinding() downloads it again.
282
1098
  */
283
1099
  export async function unloadBinding(name) {
1100
+ parked.delete(name);
1101
+ knownBindings.delete(name);
284
1102
  const pending = registered.get(name);
285
1103
  if (!pending) return;
286
1104
  registered.delete(name);
287
- loadedBindings.delete(name);
1105
+ externalBindings.delete(name);
288
1106
  await pending.catch(() => {});
289
- await withConnection(conn => conn.query("DROP TABLE IF EXISTS " + quoteIdent(name)));
1107
+ if (registered.has(name)) return; // read again meanwhile: it stays
1108
+ const engine = await getDb();
1109
+ await catalog(() => dropNow(engine, name));
1110
+ wakeRetries();
290
1111
  }
291
1112
 
292
1113
  /**
@@ -319,14 +1140,7 @@ async function reloadBinding(name) {
319
1140
  // Let a load in flight settle first, or its (older) bytes could land after
320
1141
  // the fresh ones and win.
321
1142
  await previous.catch(() => {});
322
- const load = loadBinding(name, true);
323
- registered.set(name, load);
324
- try {
325
- await load;
326
- } catch (error) {
327
- if (registered.get(name) === load) registered.delete(name);
328
- throw error;
329
- }
1143
+ await trackRegistered(name, loadBinding(name, true));
330
1144
  }
331
1145
 
332
1146
  // ---------------------------------------------------------------------------
@@ -487,33 +1301,107 @@ export function getViewer() {
487
1301
  /** Names of every staged binding — written by the dev server next to the
488
1302
  * parquet files. Absent (older servers, published builds): empty list. */
489
1303
  let indexPromise = null;
1304
+ const INDEX_RETRY_MS = 10000;
490
1305
  function bindingIndex() {
491
- indexPromise ??= fetch("__data/index.json")
492
- .then(r => (r.ok ? r.json() : []))
493
- .then(list => (Array.isArray(list) ? list : []))
494
- .catch(() => []);
495
- return indexPromise;
1306
+ if (indexPromise) return indexPromise;
1307
+ // 20 panels asking at once share one answer, a failed one for 10 s too.
1308
+ const forget = () => {
1309
+ const t = setTimeout(() => {
1310
+ if (indexPromise === pending) indexPromise = null;
1311
+ }, INDEX_RETRY_MS);
1312
+ t.unref?.();
1313
+ };
1314
+ const pending = fetchRetrying("__data/index.json").then(async r => {
1315
+ // A 404 is an older server or a published build: empty, for good.
1316
+ if (!r.ok && r.status !== 404) forget();
1317
+ if (r.status === 429 || r.status === 503) throw await responseError(r, "The binding list could not be loaded");
1318
+ if (!r.ok) return [];
1319
+ const list = await r.json().catch(() => []);
1320
+ return Array.isArray(list) ? list : [];
1321
+ });
1322
+ indexPromise = pending;
1323
+ pending.catch(forget);
1324
+ return pending;
496
1325
  }
497
1326
 
498
- async function runSql(sql, rowLimit, format) {
1327
+ // Out of Memory on a query: free tables, pinned ones the query reads last, and
1328
+ // try again. Each pass demotes at least one table, so this ends. With nothing left
1329
+ // to free, DuckDB's error surfaces and the hook decides (useAsyncQuery).
1330
+ async function runSql(sql, rowLimit, format, reads) {
1331
+ for (;;) {
1332
+ try {
1333
+ return await runSqlOnce(sql, rowLimit, format);
1334
+ } catch (error) {
1335
+ if (!isEngineOom(error)) throw error;
1336
+ const engine = await getDb();
1337
+ const freed = await catalog(() => relieveNow(engine, null, reads)).catch(() => 0);
1338
+ if (!freed) throw error;
1339
+ }
1340
+ }
1341
+ }
1342
+
1343
+ async function runSqlOnce(sql, rowLimit, format) {
499
1344
  const cap = rowLimit === null ? Infinity : (rowLimit ?? DEFAULT_ROW_LIMIT);
500
- const db = await getDb();
501
- const conn = await db.connect();
1345
+ const writes = !isReadOnlySql(sql);
1346
+ if (writes) keepEngine(); // CREATE TABLE journal_t: only this worker has it
1347
+ busy++; // before the await: a release must not end the engine under it
502
1348
  try {
503
- const table = await conn.query(sql);
504
- const rowCount = table.numRows;
505
- if (format === "arrow") {
506
- // The Arrow Table as DuckDB returned it: no row objects, no
507
- // normalization, no row limit.
508
- return { rows: table, fields: table.schema.fields.map(f => f.name), rowCount, truncated: false };
1349
+ const db = await getDb();
1350
+ const conn = await db.connect();
1351
+ try {
1352
+ const table = await conn.query(sql);
1353
+ // No load follows to measure what the app just made, and the report owes it.
1354
+ if (writes) catalog(() => measureNow(db)).catch(() => {});
1355
+ const rowCount = table.numRows;
1356
+ if (format === "arrow") {
1357
+ // The Arrow Table as DuckDB returned it: no row objects, no
1358
+ // normalization, no row limit.
1359
+ return { rows: table, fields: table.schema.fields.map(f => f.name), rowCount, truncated: false };
1360
+ }
1361
+ const { rows, fields } = toPlainRows(table, cap);
1362
+ return { rows, fields, rowCount, truncated: rowCount > rows.length };
1363
+ } catch (error) {
1364
+ throw markEngineError(error, db);
1365
+ } finally {
1366
+ await conn.close();
509
1367
  }
510
- const { rows, fields } = toPlainRows(table, cap);
511
- return { rows, fields, rowCount, truncated: rowCount > rows.length };
512
1368
  } finally {
513
- await conn.close();
1369
+ busy--;
514
1370
  }
515
1371
  }
516
1372
 
1373
+ // A crashed engine stays crashed: the SQL that broke it can answer, then every
1374
+ // later query fails with a raw WASM trap. So the page starts a new engine.
1375
+ // Bindings download again. Pins stay: they belong to mounted hooks.
1376
+ const RESTART_EVERY_MS = 60000;
1377
+ const MAX_RESTARTS = 3;
1378
+ let lastRestart = -Infinity;
1379
+ let restarts = 0;
1380
+ let restartTimer = null;
1381
+ function noteEngineCrash(engine) {
1382
+ if (engine !== liveEngine || restartTimer || restarts >= MAX_RESTARTS) return;
1383
+ restartTimer = setTimeout(restartEngine, Math.max(0, lastRestart + RESTART_EVERY_MS - Date.now()));
1384
+ }
1385
+ function restartEngine() {
1386
+ restartTimer = null;
1387
+ const old = liveEngine;
1388
+ if (!old) return;
1389
+ restarts++;
1390
+ lastRestart = Date.now();
1391
+ liveEngine = null;
1392
+ dbPromise = null;
1393
+ catalogQueue = Promise.resolve();
1394
+ entries.clear();
1395
+ registered.clear();
1396
+ loadedBindings.clear();
1397
+ measuredBytes = 0;
1398
+ heapBytes = 0;
1399
+ pressureUntil = 0;
1400
+ reportMemory();
1401
+ old.terminate().catch(() => {});
1402
+ setBindingState({ ...bindingState, version: bindingState.version + 1 });
1403
+ }
1404
+
517
1405
  const EMPTY = {
518
1406
  data: null,
519
1407
  fields: null,
@@ -529,33 +1417,104 @@ const EMPTY = {
529
1417
  // until the new ones arrive — a dashboard must not blink to empty on every
530
1418
  // refresh). `isRefreshing` says whether a refresh in flight concerns this
531
1419
  // query, so `refreshing` is true from the click, not only from the reload.
1420
+ // `run(hold)` names each binding it reads through `hold`: those stay pinned
1421
+ // against eviction while this hook shows that query (until the deps change
1422
+ // or it unmounts, and never before its run settles).
1423
+ //
1424
+ // A query that failed on memory or the network runs again by itself, a few times
1425
+ // (retryDelayMs). After that it waits for a reason to expect a different answer:
1426
+ // another hook let go of its bindings, the budget changed, the server answered
1427
+ // again, the tab came back, the network came back. The error stays up until a run
1428
+ // answers. A hidden tab waits too. Anything else stays as it is.
1429
+ const retryWaiters = new Set();
1430
+ function wakeRetries() {
1431
+ const waiting = [...retryWaiters];
1432
+ retryWaiters.clear();
1433
+ for (const fire of waiting) fire();
1434
+ }
1435
+ // The user or the network is back: let one probe through a closed gate, too.
1436
+ function tryAgainNow() {
1437
+ gateUntil = 0;
1438
+ wakeRetries();
1439
+ }
1440
+ if (typeof window !== "undefined" && typeof window.addEventListener === "function") {
1441
+ window.addEventListener("online", tryAgainNow);
1442
+ if (typeof document !== "undefined") {
1443
+ document.addEventListener("visibilitychange", () => {
1444
+ if (!document.hidden) tryAgainNow();
1445
+ });
1446
+ }
1447
+ }
1448
+
532
1449
  function useAsyncQuery(run, deps, isRefreshing) {
533
1450
  const bindings = useBindingState();
534
1451
  const [state, setState] = React.useState(EMPTY);
1452
+ const [attempt, setAttempt] = React.useState(0);
535
1453
  const seenVersion = React.useRef(bindings.version);
1454
+ const seenAttempt = React.useRef(attempt);
1455
+ const retries = React.useRef(0);
1456
+ const retrying = React.useRef(false);
536
1457
  React.useEffect(() => {
537
1458
  let active = true;
1459
+ let timer = null;
1460
+ let waiter = null;
538
1461
  const isRefresh = seenVersion.current !== bindings.version;
1462
+ const isRetry = seenAttempt.current !== attempt;
539
1463
  seenVersion.current = bindings.version;
540
- setState(s => (isRefresh && s.data !== null ? { ...s, refreshing: true } : EMPTY));
541
- run().then(
1464
+ seenAttempt.current = attempt;
1465
+ retrying.current = false;
1466
+ if (!isRetry) {
1467
+ retries.current = 0;
1468
+ setState(s => (isRefresh && s.data !== null ? { ...s, refreshing: true } : EMPTY));
1469
+ }
1470
+ const held = [];
1471
+ // From the first await to the last. Between a hook's load and its query
1472
+ // nothing else is in flight, and a release there would end the engine
1473
+ // holding the table the query is about to read.
1474
+ busy++;
1475
+ const settled = run(name => held.push(pin(name)));
1476
+ settled.catch(() => {}).finally(() => {
1477
+ busy--;
1478
+ });
1479
+ settled.then(
542
1480
  result => {
543
1481
  if (active) setState({ ...result, error: null, loading: false, refreshing: false });
544
1482
  },
545
1483
  error => {
546
- if (active)
547
- setState({
548
- ...EMPTY,
549
- error: error instanceof Error ? error.message : String(error),
550
- loading: false,
551
- });
1484
+ if (!active) return;
1485
+ setState({
1486
+ ...EMPTY,
1487
+ error: error instanceof Error ? error.message : String(error),
1488
+ loading: false,
1489
+ });
1490
+ const wait = retryDelayMs(error, retries.current);
1491
+ if (wait === null) return;
1492
+ retries.current++;
1493
+ waiter = () => {
1494
+ if (typeof document !== "undefined" && document.hidden) {
1495
+ retryWaiters.add(waiter);
1496
+ return;
1497
+ }
1498
+ retrying.current = true;
1499
+ setAttempt(n => n + 1);
1500
+ };
1501
+ if (wait === Infinity) retryWaiters.add(waiter);
1502
+ else timer = setTimeout(waiter, wait);
552
1503
  },
553
1504
  );
554
1505
  return () => {
555
1506
  active = false;
1507
+ if (timer !== null) clearTimeout(timer);
1508
+ if (waiter) retryWaiters.delete(waiter);
1509
+ const letsGo = !retrying.current;
1510
+ settled.catch(() => {}).finally(() => {
1511
+ for (const release of held) release();
1512
+ // Its own retry frees nothing, or two failing panels would wake each other.
1513
+ if (letsGo && held.length) wakeRetries();
1514
+ });
556
1515
  };
557
1516
  // eslint-disable-next-line react-hooks/exhaustive-deps
558
- }, [...deps, bindings.version]);
1517
+ }, [...deps, bindings.version, attempt]);
559
1518
  return {
560
1519
  ...state,
561
1520
  refreshing: state.refreshing || isRefreshing(bindings.refreshing),
@@ -622,9 +1581,10 @@ export function useQuery(name, opts) {
622
1581
  const withSnapshot = Boolean(opts && opts.snapshot);
623
1582
  const preview = useSnapshotPreview(name, withSnapshot, rowLimit);
624
1583
  const state = useAsyncQuery(
625
- async () => {
1584
+ async hold => {
1585
+ hold(name);
626
1586
  await registerBinding(name);
627
- const r = await runSql("SELECT * FROM " + quoteIdent(name), rowLimit);
1587
+ const r = await runSql("SELECT * FROM " + quoteIdent(name), rowLimit, undefined, new Set([name]));
628
1588
  return { data: r.rows, fields: r.fields, truncated: r.truncated, rowCount: r.rowCount };
629
1589
  },
630
1590
  [name, rowLimit],
@@ -682,24 +1642,41 @@ export function useViewer() {
682
1642
  * A binding that fails to load must not sink the query silently: when the
683
1643
  * SQL then fails, the load failure is what surfaces — it is the cause.
684
1644
  */
685
- async function runOverBindings(sql, rowLimit, format) {
686
- const names = await bindingIndex();
1645
+ async function runOverBindings(sql, rowLimit, format, hold) {
1646
+ // No index still answers `select 1`. It matters only if the query then fails.
1647
+ let indexFailure = null;
1648
+ const index = await bindingIndex().catch(error => {
1649
+ indexFailure = error;
1650
+ return [];
1651
+ });
1652
+ const names = [...new Set([...index, ...knownBindings])];
687
1653
  const failures = new Map();
688
1654
  const attempted = new Set();
1655
+ const reloaded = new Set();
689
1656
  const load = n => {
690
1657
  attempted.add(n);
1658
+ hold(n);
691
1659
  return registerBinding(n).catch(e => {
692
1660
  failures.set(n, e);
693
1661
  });
694
1662
  };
695
- await Promise.all((readsCatalog(sql) ? names : mentionedBindings(sql, names)).map(load));
1663
+ const wanted = readsCatalog(sql) ? names : mentionedBindings(sql, names);
1664
+ if (!readsCatalog(sql) && wanted.length > MAX_BINDINGS_IN_QUERY) {
1665
+ throw new Error(
1666
+ "This query names " + wanted.length + " bindings. One query can read at most " + MAX_BINDINGS_IN_QUERY +
1667
+ ": around 60 overflow DuckDB's WASM stack and break the engine for every panel. Read fewer shards, or combine them in the warehouse.",
1668
+ );
1669
+ }
1670
+ await Promise.all(wanted.map(load));
696
1671
  for (;;) {
697
1672
  try {
698
- return await runSql(sql, rowLimit, format);
1673
+ return await runSql(sql, rowLimit, format, attempted);
699
1674
  } catch (error) {
700
1675
  const message = error instanceof Error ? error.message : String(error);
701
1676
  const missing = missingBinding(message, names);
702
- if (missing && !attempted.has(missing)) {
1677
+ // Never asked for, or the SDK evicted it under this query. Once per run.
1678
+ if (missing && !failures.has(missing) && !reloaded.has(missing) && (!attempted.has(missing) || !registered.has(missing))) {
1679
+ reloaded.add(missing);
703
1680
  await load(missing);
704
1681
  if (failures.has(missing)) throw failures.get(missing);
705
1682
  continue;
@@ -708,11 +1685,15 @@ async function runOverBindings(sql, rowLimit, format) {
708
1685
  if (message.includes(n)) throw cause;
709
1686
  }
710
1687
  if (failures.size) {
711
- const details = [...failures.values()]
712
- .map(e => (e instanceof Error ? e.message : String(e)))
713
- .join("; ");
714
- throw new Error(message + " (bindings that failed to load: " + details + ")");
1688
+ const causes = [...failures.values()];
1689
+ const details = causes.map(e => (e instanceof Error ? e.message : String(e))).join("; ");
1690
+ const wrapped = new Error(message + " (bindings that failed to load: " + details + ")");
1691
+ const retryable = causes.find(e => e?.retry);
1692
+ wrapped.retry = error?.retry ?? retryable?.retry;
1693
+ wrapped.retryAfterMs = retryable?.retryAfterMs;
1694
+ throw wrapped;
715
1695
  }
1696
+ if (indexFailure && !error?.retry) throw indexFailure;
716
1697
  throw error;
717
1698
  }
718
1699
  }
@@ -722,8 +1703,8 @@ export function useDuckDB(sql, opts) {
722
1703
  bootEngine();
723
1704
  const rowLimit = opts ? opts.rowLimit : undefined;
724
1705
  const format = opts && opts.format === "arrow" ? "arrow" : "rows";
725
- const state = useAsyncQuery(async () => {
726
- const r = await runOverBindings(sql, rowLimit, format);
1706
+ const state = useAsyncQuery(async hold => {
1707
+ const r = await runOverBindings(sql, rowLimit, format, hold);
727
1708
  return { data: r.rows, fields: r.fields, truncated: r.truncated, rowCount: r.rowCount };
728
1709
  }, [sql, rowLimit, format], refreshing => refreshing.size > 0);
729
1710
  // Every binding the query can read: the combined record says what the