@makoai/app-sdk 2.11.0 → 2.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +97 -3
- package/data-core.js +45 -0
- package/index.d.ts +50 -1
- package/index.js +911 -73
- package/package.json +1 -1
- package/ui.css +44 -6
- package/ui.d.ts +1 -1
- package/ui.js +3 -1
package/index.js
CHANGED
|
@@ -22,6 +22,9 @@ import {
|
|
|
22
22
|
quoteIdent,
|
|
23
23
|
readsCatalog,
|
|
24
24
|
quoteString,
|
|
25
|
+
crashedEngineError,
|
|
26
|
+
isOutOfMemoryError,
|
|
27
|
+
retryDelayMs,
|
|
25
28
|
toPlainRows,
|
|
26
29
|
parseBindingSnapshot,
|
|
27
30
|
snapshotUrl,
|
|
@@ -32,7 +35,7 @@ import {
|
|
|
32
35
|
* in a page (or this export) says which SDK a deployed app actually runs.
|
|
33
36
|
* Kept in step with package.json by snapshot.test.mjs.
|
|
34
37
|
*/
|
|
35
|
-
export const SDK_VERSION = "2.
|
|
38
|
+
export const SDK_VERSION = "2.12.0";
|
|
36
39
|
if (typeof window !== "undefined") {
|
|
37
40
|
try {
|
|
38
41
|
window.__MAKO_APP_SDK__ = SDK_VERSION;
|
|
@@ -42,6 +45,9 @@ if (typeof window !== "undefined") {
|
|
|
42
45
|
}
|
|
43
46
|
|
|
44
47
|
const DEFAULT_ROW_LIMIT = 500000;
|
|
48
|
+
// A double-read UNION ALL of 55 to 60 bindings overflows the WASM stack and
|
|
49
|
+
// corrupts the engine for every panel (measured). 50 held.
|
|
50
|
+
const MAX_BINDINGS_IN_QUERY = 50;
|
|
45
51
|
const DUCKDB_ESM =
|
|
46
52
|
"https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.29.0/+esm";
|
|
47
53
|
|
|
@@ -53,6 +59,7 @@ const DUCKDB_ESM =
|
|
|
53
59
|
// parquet download instead of waiting for it.
|
|
54
60
|
// ---------------------------------------------------------------------------
|
|
55
61
|
let dbPromise = null;
|
|
62
|
+
let liveEngine = null; // the resolved dbPromise: errors from an older one are history
|
|
56
63
|
function getDb() {
|
|
57
64
|
if (!dbPromise) {
|
|
58
65
|
dbPromise = (async () => {
|
|
@@ -72,6 +79,7 @@ function getDb() {
|
|
|
72
79
|
} finally {
|
|
73
80
|
await conn.close();
|
|
74
81
|
}
|
|
82
|
+
liveEngine = db;
|
|
75
83
|
return db;
|
|
76
84
|
})();
|
|
77
85
|
// A failed boot (CDN unreachable) is retried by the next query.
|
|
@@ -104,6 +112,8 @@ async function responseError(res, what) {
|
|
|
104
112
|
const error = new Error(what + " (HTTP " + res.status + ")" + (detail ? ": " + detail : ""));
|
|
105
113
|
error.status = res.status;
|
|
106
114
|
if (body && typeof body.retryAfterMs === "number") error.retryAfterMs = body.retryAfterMs;
|
|
115
|
+
// The one thing a mounted hook may ask again. Any other status said what it meant.
|
|
116
|
+
if (res.status === 429 || res.status === 503) error.retry = "network";
|
|
107
117
|
return error;
|
|
108
118
|
}
|
|
109
119
|
|
|
@@ -168,6 +178,102 @@ export function useBindingProgress(name) {
|
|
|
168
178
|
// under its own file name, so a reload never drops a file another load is
|
|
169
179
|
// still reading.
|
|
170
180
|
|
|
181
|
+
// Cloud Run answers a queue timeout with a 429 and no CORS header, so an opaque
|
|
182
|
+
// app frame sees only "Failed to fetch": 12 of 5,087 published parquet requests
|
|
183
|
+
// on 2026-11-02, in 8 bursts. One page-wide gate keeps a page from making it worse.
|
|
184
|
+
// - At most 12 downloads at once; 1 while the gate is recovering from a failure.
|
|
185
|
+
// - A failure closes the gate for 1, 3, 9, 27, then 60 s, with jitter. Failures
|
|
186
|
+
// in one closed window count once. Any answer that is not a 429 or 503 opens it.
|
|
187
|
+
// - Past 3 failures in a row, a request while it is closed fails at once with no
|
|
188
|
+
// network, so a page down to one probe at a time does not keep panels spinning.
|
|
189
|
+
const FETCH_RETRY_MS = [1000, 3000];
|
|
190
|
+
const RETRY_AFTER_CAP_MS = 30000;
|
|
191
|
+
const DOWNLOAD_SLOTS = 12;
|
|
192
|
+
const GATE_MAX_MS = 60000;
|
|
193
|
+
const GATE_FAIL_FAST_AT = 3;
|
|
194
|
+
let gateStreak = 0;
|
|
195
|
+
let gateUntil = 0;
|
|
196
|
+
let gateError = null;
|
|
197
|
+
let slotsInUse = 0;
|
|
198
|
+
const slotQueue = [];
|
|
199
|
+
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
|
200
|
+
const withJitter = ms => ms * (0.75 + Math.random() / 2);
|
|
201
|
+
|
|
202
|
+
function gateFailed(error) {
|
|
203
|
+
gateError = error;
|
|
204
|
+
if (Date.now() < gateUntil) return;
|
|
205
|
+
gateStreak++;
|
|
206
|
+
gateUntil = Date.now() + withJitter(Math.min(GATE_MAX_MS, 1000 * 3 ** (gateStreak - 1)));
|
|
207
|
+
}
|
|
208
|
+
function gateAnswered() {
|
|
209
|
+
if (gateStreak === 0) return;
|
|
210
|
+
gateStreak = 0;
|
|
211
|
+
gateUntil = 0;
|
|
212
|
+
gateError = null;
|
|
213
|
+
pumpSlots();
|
|
214
|
+
wakeRetries();
|
|
215
|
+
}
|
|
216
|
+
async function throughGate() {
|
|
217
|
+
for (;;) {
|
|
218
|
+
const wait = gateUntil - Date.now();
|
|
219
|
+
if (wait <= 0) return;
|
|
220
|
+
if (gateStreak >= GATE_FAIL_FAST_AT) throw gateError;
|
|
221
|
+
await sleep(wait);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
function pumpSlots() {
|
|
225
|
+
let room = (gateStreak ? 1 : DOWNLOAD_SLOTS) - slotsInUse;
|
|
226
|
+
while (room-- > 0 && slotQueue.length) slotQueue.shift()();
|
|
227
|
+
}
|
|
228
|
+
async function withSlot(fn) {
|
|
229
|
+
while (slotsInUse >= (gateStreak ? 1 : DOWNLOAD_SLOTS)) await new Promise(resolve => slotQueue.push(resolve));
|
|
230
|
+
slotsInUse++;
|
|
231
|
+
try {
|
|
232
|
+
return await fn();
|
|
233
|
+
} finally {
|
|
234
|
+
slotsInUse--;
|
|
235
|
+
pumpSlots();
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* fetch, until headers arrive: 3 attempts on a rejected fetch, a 429 or a 503,
|
|
241
|
+
* through the gate. A response of any other status returns at once, and so does
|
|
242
|
+
* a stream once it starts, so a build is never asked for again from here. Waits
|
|
243
|
+
* out a `retryAfterMs` up to 30 s; past that the caller gets the answer. No
|
|
244
|
+
* headers timeout: the Vite plugin may hold headers for a whole build.
|
|
245
|
+
*/
|
|
246
|
+
async function fetchRetrying(url, init) {
|
|
247
|
+
for (let attempt = 0; ; attempt++) {
|
|
248
|
+
const last = attempt >= FETCH_RETRY_MS.length;
|
|
249
|
+
await throughGate();
|
|
250
|
+
let wait = last ? 0 : withJitter(FETCH_RETRY_MS[attempt]);
|
|
251
|
+
try {
|
|
252
|
+
const res = await fetch(url, init);
|
|
253
|
+
if (res.status !== 429 && res.status !== 503) {
|
|
254
|
+
gateAnswered();
|
|
255
|
+
return res;
|
|
256
|
+
}
|
|
257
|
+
const asked = await res
|
|
258
|
+
.clone()
|
|
259
|
+
.json()
|
|
260
|
+
.then(body => body?.retryAfterMs)
|
|
261
|
+
.catch(() => undefined);
|
|
262
|
+
const busy = new Error("The server is busy (HTTP " + res.status + ")");
|
|
263
|
+
busy.retry = "network";
|
|
264
|
+
gateFailed(busy);
|
|
265
|
+
if (last || asked > RETRY_AFTER_CAP_MS) return res;
|
|
266
|
+
if (typeof asked === "number") wait = asked;
|
|
267
|
+
} catch (error) {
|
|
268
|
+
if (!(error instanceof TypeError)) throw error;
|
|
269
|
+
error.retry = "network";
|
|
270
|
+
gateFailed(error);
|
|
271
|
+
if (last) throw error;
|
|
272
|
+
}
|
|
273
|
+
await sleep(wait);
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
|
|
171
277
|
/**
|
|
172
278
|
* A binding's parquet bytes, phase-aware: asks for NDJSON phase events (a
|
|
173
279
|
* build reports each step, a stored file resolving → signing and a signed
|
|
@@ -178,67 +284,650 @@ async function downloadBinding(name, bust, report) {
|
|
|
178
284
|
const path = "__data/" + encodeURIComponent(name) + ".parquet";
|
|
179
285
|
const what = 'Data for binding "' + name + '" could not be loaded';
|
|
180
286
|
report({ phase: "requesting", rowsWritten: null, bytes: null, bytesTotal: null });
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
287
|
+
// A slot holds the connection, not the wait for a build: one slow warehouse
|
|
288
|
+
// query must not hold up twelve bindings that are ready.
|
|
289
|
+
let res = await withSlot(() =>
|
|
290
|
+
fetchRetrying(path + (bust ? "?refresh=" + Date.now() : ""), {
|
|
291
|
+
headers: { accept: PROGRESS_ACCEPT },
|
|
292
|
+
}),
|
|
293
|
+
);
|
|
294
|
+
let signed = null;
|
|
184
295
|
if (isProgressResponse(res)) {
|
|
185
|
-
|
|
296
|
+
let final;
|
|
297
|
+
try {
|
|
298
|
+
final = await readProgressStream(res, e => report(progressPatch(e)));
|
|
299
|
+
} catch (error) {
|
|
300
|
+
// A cut build stream may have cost a warehouse query. Asking again costs another.
|
|
301
|
+
throw new Error(what + ": the build's stream was cut (" + (error instanceof Error ? error.message : String(error)) + ")");
|
|
302
|
+
}
|
|
186
303
|
if (final.phase === "failed") throw progressError(final, what);
|
|
187
304
|
report({ phase: "downloading", rowsWritten: null, bytes: 0 });
|
|
188
305
|
// No url: the server streams the bytes itself — the same path, asked
|
|
189
306
|
// plainly, serves them (the build already staged or cached them).
|
|
190
|
-
|
|
307
|
+
signed = final.url || path;
|
|
308
|
+
res = null;
|
|
191
309
|
}
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
310
|
+
return withSlot(async () => {
|
|
311
|
+
res ??= await fetchRetrying(signed);
|
|
312
|
+
if (!res.ok) throw await responseError(res, what);
|
|
313
|
+
try {
|
|
314
|
+
return await readBytes(res, (bytes, bytesTotal) =>
|
|
315
|
+
report({ phase: "downloading", rowsWritten: null, bytes, bytesTotal }),
|
|
316
|
+
);
|
|
317
|
+
} catch (error) {
|
|
318
|
+
if (error instanceof TypeError) error.retry = "network"; // the body cut off: no build behind it
|
|
319
|
+
throw error;
|
|
320
|
+
}
|
|
321
|
+
});
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
// ---------------------------------------------------------------------------
|
|
325
|
+
// Memory (2.12). Every table lives in this frame's WASM heap, where DuckDB's
|
|
326
|
+
// memory_limit is 3.1 GiB. 2.8 to 2.11 never dropped one, so an app with many
|
|
327
|
+
// bindings (Web Performance: 106 shards) browsed its way to "Out of Memory".
|
|
328
|
+
//
|
|
329
|
+
// - Every read bumps its binding. Least recently used goes first.
|
|
330
|
+
// - The budget counts what the SDK placed: tables and the parquet copies views
|
|
331
|
+
// read. An app's own tables do not shrink it. The real total still decides
|
|
332
|
+
// whether a table can fit in the heap at all.
|
|
333
|
+
// - A table lands if the parquet's own metadata says it fits, after evicting what
|
|
334
|
+
// no one holds. Measured tables can pass the estimate, so the budget can
|
|
335
|
+
// overshoot (measured: 1,700 MiB against 1,536). With a mostly empty heap, one
|
|
336
|
+
// table may try anyway: an ordinary app with one large parquet stays fast.
|
|
337
|
+
// - What does not fit lands as a view over its parquet in the heap
|
|
338
|
+
// (registerFileBuffer): 7x to 14x smaller than its table. A later read, or the
|
|
339
|
+
// end of pressure, promotes it if room has appeared.
|
|
340
|
+
// - Never a BROWSER_FILEREADER handle over a Blob: duckdb-wasm 1.29.0's openFile
|
|
341
|
+
// writes the file size to HEAPF64[(ptr >> 3)], and `>>` turns a pointer past
|
|
342
|
+
// 2 GiB negative. Reads then fail with "too small to be a Parquet file" (4 of
|
|
343
|
+
// 8 reads with 3.0 GiB of tables, 0 of 8 with 1.6 GiB; heap buffers 0 of 8).
|
|
344
|
+
// - File copies have their own ceiling: 2.5 GiB, and 3.6 GiB with DuckDB's own
|
|
345
|
+
// bytes. registerFileBuffer never checks its malloc: at 4.16 GiB of copies the
|
|
346
|
+
// call never settled and hung the page. Over it the least recently read view
|
|
347
|
+
// goes, pinned too, and downloads again on its next read.
|
|
348
|
+
// - The parquet stays in JS memory as a Blob only so a table can become a view
|
|
349
|
+
// without a download. A Blob can stop being readable (NotReadableError on the
|
|
350
|
+
// third 885 MiB in incognito). Its table then goes instead, and downloads again.
|
|
351
|
+
// Past the Blob budget, the least recently used view is dropped outright.
|
|
352
|
+
// - A binding a mounted hook reads, or a query in flight, is pinned: it goes last.
|
|
353
|
+
// Only loads in flight are never touched.
|
|
354
|
+
// - "Out of Memory" on a query demotes unpinned tables, then the pinned tables
|
|
355
|
+
// that query reads, and retries until it answers or none is left. Other
|
|
356
|
+
// panels' tables stay. For a minute (pressure) tables stay down, then pinned
|
|
357
|
+
// views promote. Only DuckDB's own error counts (see markEngineError).
|
|
358
|
+
// - "Out of Memory" on a load demotes unpinned tables and retries once. A load
|
|
359
|
+
// that still does not fit lands as a view.
|
|
360
|
+
//
|
|
361
|
+
// All catalog changes run one at a time (catalog()), so the bookkeeping here
|
|
362
|
+
// always matches DuckDB's catalog. The worker runs one statement at a time
|
|
363
|
+
// anyway, so this costs no parallelism.
|
|
364
|
+
// ---------------------------------------------------------------------------
|
|
365
|
+
const MIB = 1024 * 1024;
|
|
366
|
+
export const DEFAULT_BINDING_MEMORY_BUDGET = 1536 * MIB;
|
|
367
|
+
export const DEFAULT_BINDING_BLOB_BUDGET = 512 * MIB;
|
|
368
|
+
const DUCKDB_LIMIT = 3174 * MIB; // duckdb-wasm's default memory_limit: 3.1 GiB
|
|
369
|
+
const FILE_CEILING = 2560 * MIB;
|
|
370
|
+
const HEAP_CEILING = 3686 * MIB;
|
|
371
|
+
let memoryBudget = DEFAULT_BINDING_MEMORY_BUDGET;
|
|
372
|
+
let blobBudget = DEFAULT_BINDING_BLOB_BUDGET;
|
|
373
|
+
|
|
374
|
+
// name -> { storage: "table" | "view", blob: Blob | null, file: string | null,
|
|
375
|
+
// fileBytes: number, tableBytes: number | null, demoted: boolean }. `file` is the
|
|
376
|
+
// heap copy of the parquet a view reads, of `fileBytes`. `tableBytes`: what its
|
|
377
|
+
// table cost, measured. `demoted`: a view only because of memory: try to promote.
|
|
378
|
+
const entries = new Map();
|
|
379
|
+
const lastUse = new Map(); // name -> sequence number of its latest read
|
|
380
|
+
let useSeq = 0;
|
|
381
|
+
const pins = new Map(); // name -> how many hooks or queries hold it
|
|
382
|
+
const loading = new Set(); // names with a load in flight
|
|
383
|
+
// Loaded through ensureBinding(): app code may query them by name at any
|
|
384
|
+
// time, so they are demoted (still readable) but never dropped.
|
|
385
|
+
const externalBindings = new Set();
|
|
386
|
+
// Every binding this page ever loaded, for reading one the SDK evicted when the
|
|
387
|
+
// server's index does not list it (index [] apps name their bindings in useQuery).
|
|
388
|
+
const knownBindings = new Set();
|
|
389
|
+
let measuredBytes = 0;
|
|
390
|
+
// After an Out of Memory, tables stay at or under what relief left, for a minute.
|
|
391
|
+
const PRESSURE_MS = 60000;
|
|
392
|
+
let pressureCeiling = Infinity;
|
|
393
|
+
let pressureUntil = 0;
|
|
394
|
+
let pressureTimer = null;
|
|
395
|
+
|
|
396
|
+
function tableCeiling() {
|
|
397
|
+
return Date.now() < pressureUntil ? Math.min(memoryBudget, pressureCeiling) : memoryBudget;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
function touch(name) {
|
|
401
|
+
lastUse.set(name, ++useSeq);
|
|
402
|
+
}
|
|
403
|
+
function isPinned(name) {
|
|
404
|
+
return (pins.get(name) ?? 0) > 0;
|
|
405
|
+
}
|
|
406
|
+
/** Hold `name` against eviction; the returned function lets go (once). */
|
|
407
|
+
function pin(name) {
|
|
408
|
+
knownBindings.add(name);
|
|
409
|
+
pins.set(name, (pins.get(name) ?? 0) + 1);
|
|
410
|
+
let held = true;
|
|
411
|
+
return () => {
|
|
412
|
+
if (!held) return;
|
|
413
|
+
held = false;
|
|
414
|
+
const n = (pins.get(name) ?? 1) - 1;
|
|
415
|
+
if (n > 0) pins.set(name, n);
|
|
416
|
+
else pins.delete(name);
|
|
417
|
+
};
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
let catalogQueue = Promise.resolve();
|
|
421
|
+
function catalog(fn) {
|
|
422
|
+
const run = catalogQueue.then(fn);
|
|
423
|
+
catalogQueue = run.catch(() => {});
|
|
424
|
+
return run;
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
// The statements in one transaction, so another connection never sees the
|
|
428
|
+
// name missing between a DROP and a CREATE.
|
|
429
|
+
async function exec(engine, statements) {
|
|
430
|
+
const conn = await engine.connect();
|
|
431
|
+
try {
|
|
432
|
+
if (statements.length === 1) {
|
|
433
|
+
await conn.query(statements[0]);
|
|
434
|
+
return;
|
|
435
|
+
}
|
|
436
|
+
await conn.query("BEGIN TRANSACTION");
|
|
437
|
+
try {
|
|
438
|
+
for (const sql of statements) await conn.query(sql);
|
|
439
|
+
await conn.query("COMMIT");
|
|
440
|
+
} catch (error) {
|
|
441
|
+
await conn.query("ROLLBACK").catch(() => {});
|
|
442
|
+
throw error;
|
|
443
|
+
}
|
|
444
|
+
} catch (error) {
|
|
445
|
+
throw markEngineError(error, engine);
|
|
446
|
+
} finally {
|
|
447
|
+
await conn.close();
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
/**
|
|
452
|
+
* An error from this engine's own conn.query. Only these can be Out of Memory:
|
|
453
|
+
* a download or a server's build may quote the same words. A crash is noted here
|
|
454
|
+
* and the engine restarts (restartEngine).
|
|
455
|
+
*/
|
|
456
|
+
function markEngineError(error, engine) {
|
|
457
|
+
if (!(error instanceof Error)) return error;
|
|
458
|
+
error.fromEngine = true;
|
|
459
|
+
if (isOutOfMemoryError(error)) error.retry = "memory";
|
|
460
|
+
if (crashedEngineError(error)) {
|
|
461
|
+
error.message = "The data engine crashed (" + error.message + "). It restarts by itself; reload the page if this stays.";
|
|
462
|
+
noteEngineCrash(engine);
|
|
463
|
+
}
|
|
464
|
+
return error;
|
|
465
|
+
}
|
|
466
|
+
const isEngineOom = error => Boolean(error?.fromEngine) && isOutOfMemoryError(error);
|
|
467
|
+
|
|
468
|
+
function dropSql(storage, name) {
|
|
469
|
+
return "DROP " + (storage === "view" ? "VIEW" : "TABLE") + " IF EXISTS " + quoteIdent(name);
|
|
196
470
|
}
|
|
197
471
|
|
|
198
472
|
let fileSeq = 0;
|
|
473
|
+
// Every load registers its bytes under its own file name, so a reload never
|
|
474
|
+
// drops a file another load is still reading.
|
|
475
|
+
function nextFile() {
|
|
476
|
+
return "mako-" + ++fileSeq + ".parquet";
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
/** The table for a registered parquet `file`, in place of whatever `name` was. */
|
|
480
|
+
async function tableFromFile(engine, name, file) {
|
|
481
|
+
const prev = entries.get(name);
|
|
482
|
+
const before = await duckdbBytes(engine);
|
|
483
|
+
const create =
|
|
484
|
+
"CREATE OR REPLACE TABLE " + quoteIdent(name) +
|
|
485
|
+
" AS SELECT * FROM read_parquet(" + quoteString(file) + ")";
|
|
486
|
+
await exec(engine, prev && prev.storage === "view" ? [dropSql("view", name), create] : [create]);
|
|
487
|
+
const after = await duckdbBytes(engine);
|
|
488
|
+
if (before === null || after === null) return prev?.tableBytes ?? null;
|
|
489
|
+
const grown = after - before + (prev?.storage === "table" ? (prev.tableBytes ?? 0) : 0);
|
|
490
|
+
return grown > 0 ? grown : (prev?.tableBytes ?? null);
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
/** The view over a registered parquet `file`, in place of whatever `name` was. */
|
|
494
|
+
async function viewFromFile(engine, name, file) {
|
|
495
|
+
const prev = entries.get(name);
|
|
496
|
+
const create =
|
|
497
|
+
"CREATE OR REPLACE VIEW " + quoteIdent(name) +
|
|
498
|
+
" AS SELECT * FROM read_parquet(" + quoteString(file) + ")";
|
|
499
|
+
await exec(engine, prev && prev.storage === "table" ? [dropSql("table", name), create] : [create]);
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
/**
|
|
503
|
+
* What a registered parquet's table will cost, from its metadata: 16 bytes a cell,
|
|
504
|
+
* or its uncompressed pages if long strings make that more. Measured on 6 parquets
|
|
505
|
+
* from 2 to 402 MiB: the real table was 0.84x to 1.01x of the 16-byte figure.
|
|
506
|
+
*/
|
|
507
|
+
async function estimateTable(engine, file, parquetBytes) {
|
|
508
|
+
const conn = await engine.connect();
|
|
509
|
+
try {
|
|
510
|
+
const f = quoteString(file);
|
|
511
|
+
const result = await conn.query(
|
|
512
|
+
"SELECT greatest(sum(total_uncompressed_size), " +
|
|
513
|
+
"count(DISTINCT path_in_schema) * (SELECT num_rows FROM parquet_file_metadata(" + f + ")) * 16)::DOUBLE " +
|
|
514
|
+
"FROM parquet_metadata(" + f + ")",
|
|
515
|
+
);
|
|
516
|
+
const value = Number(result.getChildAt(0)?.get(0));
|
|
517
|
+
if (Number.isFinite(value) && value > 0) return value;
|
|
518
|
+
} catch {
|
|
519
|
+
/* no metadata to read: the file's own error surfaces when it is read */
|
|
520
|
+
} finally {
|
|
521
|
+
await conn.close();
|
|
522
|
+
}
|
|
523
|
+
return parquetBytes * 16;
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
/** What the SDK placed: tables, and the parquet copies views read. */
|
|
527
|
+
function ownedBytes() {
|
|
528
|
+
let bytes = 0;
|
|
529
|
+
for (const entry of entries.values()) {
|
|
530
|
+
bytes += entry.storage === "table" ? (entry.tableBytes ?? (entry.blob?.size ?? 0) * 16) : entry.fileBytes;
|
|
531
|
+
}
|
|
532
|
+
return bytes;
|
|
533
|
+
}
|
|
534
|
+
function fileBytes() {
|
|
535
|
+
let bytes = 0;
|
|
536
|
+
for (const entry of entries.values()) if (entry.storage === "view") bytes += entry.fileBytes;
|
|
537
|
+
return bytes;
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
/**
|
|
541
|
+
* What this frame's heap holds right now, in bytes: DuckDB's figure, with app
|
|
542
|
+
* tables and every binding table, plus the parquet copies views read, which sit
|
|
543
|
+
* outside its accounting. Query scratch never shows: DuckDB returns it before the
|
|
544
|
+
* next statement runs (measured, also after a failed query). The last figure if
|
|
545
|
+
* it cannot say.
|
|
546
|
+
*/
|
|
547
|
+
async function measureNow(engine) {
|
|
548
|
+
const value = await duckdbBytes(engine);
|
|
549
|
+
if (value !== null) measuredBytes = value + fileBytes();
|
|
550
|
+
reportMemory();
|
|
551
|
+
return measuredBytes;
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
/** duckdb_memory() alone, or null on an engine without it. */
|
|
555
|
+
async function duckdbBytes(engine) {
|
|
556
|
+
const conn = await engine.connect();
|
|
557
|
+
try {
|
|
558
|
+
const result = await conn.query(
|
|
559
|
+
"SELECT coalesce(sum(memory_usage_bytes), 0)::DOUBLE AS bytes FROM duckdb_memory()",
|
|
560
|
+
);
|
|
561
|
+
const value = Number(result.getChildAt(0)?.get(0));
|
|
562
|
+
return Number.isFinite(value) ? value : null;
|
|
563
|
+
} catch (error) {
|
|
564
|
+
markEngineError(error, engine);
|
|
565
|
+
return null; /* the OOM retry still applies */
|
|
566
|
+
} finally {
|
|
567
|
+
await conn.close();
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
// Hook for whoever wants the figure (the host frame, in a later step).
|
|
572
|
+
function reportMemory() {}
|
|
573
|
+
|
|
574
|
+
/** The least recently read binding that passes `test` and nothing holds. */
|
|
575
|
+
function leastRecent(test, pinnedToo = false) {
|
|
576
|
+
let best = null;
|
|
577
|
+
let bestSeq = Infinity;
|
|
578
|
+
for (const [name, entry] of entries) {
|
|
579
|
+
if ((isPinned(name) && !pinnedToo) || loading.has(name) || !test(name, entry)) continue;
|
|
580
|
+
const seq = lastUse.get(name) ?? 0;
|
|
581
|
+
if (seq < bestSeq) {
|
|
582
|
+
best = name;
|
|
583
|
+
bestSeq = seq;
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
return best;
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
/** Forget a binding entirely: the next read downloads it again. */
|
|
590
|
+
async function dropNow(engine, name) {
|
|
591
|
+
const entry = entries.get(name);
|
|
592
|
+
if (!entry) return;
|
|
593
|
+
await exec(engine, [dropSql(entry.storage, name)]);
|
|
594
|
+
if (entry.file) await engine.dropFile(entry.file).catch(() => {});
|
|
595
|
+
entries.delete(name);
|
|
596
|
+
loadedBindings.delete(name);
|
|
597
|
+
registered.delete(name);
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
/**
|
|
601
|
+
* Room in the heap for `incoming` bytes of parquet copy. registerFileBuffer does
|
|
602
|
+
* not check its malloc, so past this point nothing would report the failure. Goes
|
|
603
|
+
* through views least recently read first, pinned too: their next read downloads.
|
|
604
|
+
*/
|
|
605
|
+
async function roomForFile(engine, incoming, except) {
|
|
606
|
+
for (;;) {
|
|
607
|
+
const duckdb = (await duckdbBytes(engine)) ?? 0;
|
|
608
|
+
const files = fileBytes();
|
|
609
|
+
if (files + incoming <= FILE_CEILING && duckdb + files + incoming <= HEAP_CEILING) return;
|
|
610
|
+
const victim = leastRecent(
|
|
611
|
+
(name, entry) => name !== except && entry.storage === "view" && !externalBindings.has(name),
|
|
612
|
+
true,
|
|
613
|
+
);
|
|
614
|
+
if (!victim) {
|
|
615
|
+
const mib = Math.round(incoming / MIB);
|
|
616
|
+
throw new Error("A " + mib + " MiB parquet does not fit in this page's DuckDB heap, which holds " + Math.round((duckdb + files) / MIB) + " MiB.");
|
|
617
|
+
}
|
|
618
|
+
await dropNow(engine, victim);
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
/** One step down: a table becomes a view over a copy of its Blob, or goes. */
|
|
623
|
+
async function evictNow(engine, name) {
|
|
624
|
+
const entry = entries.get(name);
|
|
625
|
+
let file = null;
|
|
626
|
+
if (entry.blob) {
|
|
627
|
+
try {
|
|
628
|
+
await roomForFile(engine, entry.blob.size, name);
|
|
629
|
+
const bytes = new Uint8Array(await entry.blob.arrayBuffer());
|
|
630
|
+
file = nextFile();
|
|
631
|
+
await engine.registerFileBuffer(file, bytes);
|
|
632
|
+
} catch (error) {
|
|
633
|
+
if (error?.fromEngine) throw error;
|
|
634
|
+
file = null; // an unreadable Blob, or no room for its copy
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
if (file) {
|
|
638
|
+
try {
|
|
639
|
+
await viewFromFile(engine, name, file);
|
|
640
|
+
} catch (error) {
|
|
641
|
+
await engine.dropFile(file).catch(() => {});
|
|
642
|
+
throw error;
|
|
643
|
+
}
|
|
644
|
+
if (entry.file) await engine.dropFile(entry.file).catch(() => {});
|
|
645
|
+
entries.set(name, { ...entry, storage: "view", file, fileBytes: entry.blob.size, demoted: true });
|
|
646
|
+
return;
|
|
647
|
+
}
|
|
648
|
+
// App code may read this one by name at any time: it stays a table, and no
|
|
649
|
+
// eviction asks again (evictableTable).
|
|
650
|
+
if (externalBindings.has(name)) {
|
|
651
|
+
entries.set(name, { ...entry, blob: null });
|
|
652
|
+
return;
|
|
653
|
+
}
|
|
654
|
+
await dropNow(engine, name);
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
const evictableTable = except => (name, entry) =>
|
|
658
|
+
name !== except &&
|
|
659
|
+
entry.storage === "table" &&
|
|
660
|
+
(entry.blob !== null || !externalBindings.has(name));
|
|
661
|
+
|
|
662
|
+
/** Evict least recently used tables until the bindings hold at most `target`. */
|
|
663
|
+
async function trimNow(engine, except, target = memoryBudget) {
|
|
664
|
+
for (;;) {
|
|
665
|
+
if (ownedBytes() <= target) break;
|
|
666
|
+
const victim = leastRecent(evictableTable(except));
|
|
667
|
+
if (!victim) break;
|
|
668
|
+
await evictNow(engine, victim);
|
|
669
|
+
}
|
|
670
|
+
for (;;) {
|
|
671
|
+
let blobs = 0;
|
|
672
|
+
for (const entry of entries.values()) if (entry.blob) blobs += entry.blob.size;
|
|
673
|
+
if (blobs <= blobBudget) break;
|
|
674
|
+
const victim = leastRecent(
|
|
675
|
+
(name, entry) =>
|
|
676
|
+
name !== except && entry.blob !== null && !(entry.storage === "view" && externalBindings.has(name)),
|
|
677
|
+
);
|
|
678
|
+
if (!victim) break;
|
|
679
|
+
const entry = entries.get(victim);
|
|
680
|
+
if (entry.storage === "view") await dropNow(engine, victim);
|
|
681
|
+
// A table keeps its rows; evicting it later drops it outright.
|
|
682
|
+
else entries.set(victim, { ...entry, blob: null });
|
|
683
|
+
}
|
|
684
|
+
await measureNow(engine);
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
/**
|
|
688
|
+
* After an Out of Memory error: demote tables, least recently used first, until
|
|
689
|
+
* the bindings hold half of what they did (or half the budget). A load stops at
|
|
690
|
+
* unpinned tables. A query goes on to pinned tables it reads itself (`reads`): a
|
|
691
|
+
* query that can never fit must not take healthy panels down with it. Returns how
|
|
692
|
+
* many it freed; 0: a retry would fail the same way.
|
|
693
|
+
*/
|
|
694
|
+
async function relieveNow(engine, except, reads = null) {
|
|
695
|
+
const target = Math.min(memoryBudget, ownedBytes()) / 2;
|
|
696
|
+
const evictable = evictableTable(except);
|
|
697
|
+
let freed = 0;
|
|
698
|
+
for (;;) {
|
|
699
|
+
const victim =
|
|
700
|
+
leastRecent(evictable) ??
|
|
701
|
+
(reads ? leastRecent((name, entry) => reads.has(name) && entry.blob !== null && evictable(name, entry), true) : null);
|
|
702
|
+
if (!victim) break;
|
|
703
|
+
await evictNow(engine, victim);
|
|
704
|
+
freed++;
|
|
705
|
+
if (ownedBytes() <= target) break;
|
|
706
|
+
}
|
|
707
|
+
if (freed) startPressure();
|
|
708
|
+
await measureNow(engine);
|
|
709
|
+
return freed;
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
function startPressure() {
|
|
713
|
+
pressureCeiling = ownedBytes();
|
|
714
|
+
pressureUntil = Date.now() + PRESSURE_MS;
|
|
715
|
+
clearTimeout(pressureTimer);
|
|
716
|
+
pressureTimer = setTimeout(endPressure, PRESSURE_MS);
|
|
717
|
+
pressureTimer.unref?.();
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
/** Pinned views come back on their own: a mounted panel may not read again soon. */
|
|
721
|
+
async function endPressure() {
|
|
722
|
+
const waiting = [...entries]
|
|
723
|
+
.filter(([name, entry]) => entry.demoted && isPinned(name) && !loading.has(name))
|
|
724
|
+
.sort(([a], [b]) => (lastUse.get(b) ?? 0) - (lastUse.get(a) ?? 0));
|
|
725
|
+
for (const [name] of waiting) {
|
|
726
|
+
await promote(name);
|
|
727
|
+
if (entries.get(name)?.storage !== "table") break; // full: the rest would not fit either
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
/**
|
|
732
|
+
* Whether a table costing `estimate` fits, evicting what no one holds to make
|
|
733
|
+
* room. Checked before decoding: a table placed past the ceiling could only be
|
|
734
|
+
* demoted again.
|
|
735
|
+
*/
|
|
736
|
+
async function roomForTable(engine, name, estimate) {
|
|
737
|
+
const prev = entries.get(name);
|
|
738
|
+
const replaced = !prev ? 0 : prev.storage === "table" ? (prev.tableBytes ?? estimate) : prev.fileBytes;
|
|
739
|
+
const fitsHeap = async () => ((await duckdbBytes(engine)) ?? 0) + estimate <= DUCKDB_LIMIT;
|
|
740
|
+
// A single table over the ceiling: never evict for it. With a mostly empty
|
|
741
|
+
// heap, let it try: an OOM lands it as a view, and queries keep what is left.
|
|
742
|
+
if (estimate > tableCeiling()) {
|
|
743
|
+
return (
|
|
744
|
+
estimate <= memoryBudget * 2 &&
|
|
745
|
+
Date.now() >= pressureUntil &&
|
|
746
|
+
ownedBytes() - replaced <= memoryBudget / 4 &&
|
|
747
|
+
(await fitsHeap())
|
|
748
|
+
);
|
|
749
|
+
}
|
|
750
|
+
for (;;) {
|
|
751
|
+
if (ownedBytes() - replaced + estimate <= tableCeiling()) return fitsHeap();
|
|
752
|
+
const victim = leastRecent(evictableTable(name));
|
|
753
|
+
if (!victim) return false;
|
|
754
|
+
await evictNow(engine, victim);
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
|
|
758
|
+
/**
|
|
759
|
+
* Decode a download into DuckDB: a table when it fits, else a view. Reads `buf`,
|
|
760
|
+
* never the Blob: these bytes are in hand and a Blob can be unreadable.
|
|
761
|
+
*/
|
|
762
|
+
async function placeNow(engine, name, buf, blob) {
|
|
763
|
+
const prev = entries.get(name);
|
|
764
|
+
await roomForFile(engine, buf.length, name);
|
|
765
|
+
const file = nextFile();
|
|
766
|
+
const size = buf.length;
|
|
767
|
+
await engine.registerFileBuffer(file, buf); // transfers `buf` to the worker
|
|
768
|
+
const asView = async () => {
|
|
769
|
+
await viewFromFile(engine, name, file);
|
|
770
|
+
if (prev?.file) await engine.dropFile(prev.file).catch(() => {});
|
|
771
|
+
return { storage: "view", file, fileBytes: size, demoted: true };
|
|
772
|
+
};
|
|
773
|
+
const asTable = async () => {
|
|
774
|
+
const tableBytes = await tableFromFile(engine, name, file);
|
|
775
|
+
await engine.dropFile(file).catch(() => {});
|
|
776
|
+
if (prev?.file) await engine.dropFile(prev.file).catch(() => {});
|
|
777
|
+
return { storage: "table", file: null, fileBytes: 0, demoted: false, tableBytes };
|
|
778
|
+
};
|
|
779
|
+
try {
|
|
780
|
+
const estimate = prev?.tableBytes ?? (await estimateTable(engine, file, blob.size));
|
|
781
|
+
if (!(await roomForTable(engine, name, estimate))) return await asView();
|
|
782
|
+
try {
|
|
783
|
+
return await asTable();
|
|
784
|
+
} catch (error) {
|
|
785
|
+
if (!isEngineOom(error)) throw error;
|
|
786
|
+
}
|
|
787
|
+
if (await relieveNow(engine, name)) {
|
|
788
|
+
try {
|
|
789
|
+
return await asTable();
|
|
790
|
+
} catch (error) {
|
|
791
|
+
if (!isEngineOom(error)) throw error;
|
|
792
|
+
}
|
|
793
|
+
}
|
|
794
|
+
return await asView();
|
|
795
|
+
} catch (error) {
|
|
796
|
+
await engine.dropFile(file).catch(() => {});
|
|
797
|
+
throw error;
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
|
|
199
801
|
async function loadBinding(name, bust) {
|
|
200
802
|
const db = getDb(); // boot in parallel with the download
|
|
201
803
|
db.catch(() => {});
|
|
804
|
+
loading.add(name);
|
|
202
805
|
try {
|
|
203
806
|
const buf = await downloadBinding(name, bust, patch => setProgress(name, patch));
|
|
204
807
|
setProgress(name, { phase: "loading_duckdb" });
|
|
205
808
|
const engine = await db;
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
const
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
}
|
|
218
|
-
} finally {
|
|
219
|
-
await engine.dropFile(file).catch(() => {});
|
|
220
|
-
}
|
|
221
|
-
loadedBindings.add(name);
|
|
809
|
+
// Made before the buffer goes to the worker: the copy a demotion reads.
|
|
810
|
+
const blob = new Blob([buf]);
|
|
811
|
+
await catalog(async () => {
|
|
812
|
+
const placed = await placeNow(engine, name, buf, blob);
|
|
813
|
+
const tableBytes = placed.tableBytes ?? entries.get(name)?.tableBytes ?? null;
|
|
814
|
+
entries.set(name, { ...placed, blob, tableBytes });
|
|
815
|
+
knownBindings.add(name);
|
|
816
|
+
loadedBindings.add(name);
|
|
817
|
+
loading.delete(name);
|
|
818
|
+
await trimNow(engine, name);
|
|
819
|
+
});
|
|
222
820
|
} finally {
|
|
821
|
+
loading.delete(name);
|
|
223
822
|
// A refresh owns the record until its reload is done (refreshBinding).
|
|
224
823
|
if (!refreshes.has(name)) setProgress(name, null);
|
|
225
824
|
}
|
|
226
825
|
}
|
|
227
826
|
|
|
827
|
+
/**
|
|
828
|
+
* A demoted view read again: back to a table from its heap copy, no download, if
|
|
829
|
+
* it fits. Otherwise it stays a view, and the next read asks again: a view that
|
|
830
|
+
* promoted itself into a full heap would only be demoted again.
|
|
831
|
+
*/
|
|
832
|
+
async function promote(name) {
|
|
833
|
+
const engine = await getDb();
|
|
834
|
+
await catalog(async () => {
|
|
835
|
+
const entry = entries.get(name);
|
|
836
|
+
if (!entry || entry.storage !== "view" || !entry.file) return;
|
|
837
|
+
if (!(await roomForTable(engine, name, entry.tableBytes ?? (await estimateTable(engine, entry.file, entry.fileBytes))))) {
|
|
838
|
+
entry.demoted = true;
|
|
839
|
+
return;
|
|
840
|
+
}
|
|
841
|
+
const tableBytes = await tableFromFile(engine, name, entry.file);
|
|
842
|
+
await engine.dropFile(entry.file).catch(() => {});
|
|
843
|
+
entries.set(name, { ...entry, storage: "table", file: null, fileBytes: 0, demoted: false, tableBytes });
|
|
844
|
+
await trimNow(engine, name);
|
|
845
|
+
}).catch(() => {
|
|
846
|
+
const entry = entries.get(name);
|
|
847
|
+
if (entry && entry.storage === "view") entry.demoted = true;
|
|
848
|
+
/* it stays a view, which reads the same rows */
|
|
849
|
+
});
|
|
850
|
+
}
|
|
851
|
+
|
|
228
852
|
const registered = new Map(); // name -> Promise<void>
|
|
229
|
-
const loadedBindings = new Set(); // names
|
|
853
|
+
const loadedBindings = new Set(); // names DuckDB can read (table or view)
|
|
230
854
|
// Refreshed on the server while never loaded on this page: the next load
|
|
231
855
|
// must skip any cache between here and the server.
|
|
232
856
|
const staleBindings = new Set();
|
|
233
857
|
function registerBinding(name) {
|
|
234
|
-
|
|
858
|
+
touch(name);
|
|
859
|
+
knownBindings.add(name);
|
|
860
|
+
const known = registered.get(name);
|
|
861
|
+
if (!known) {
|
|
235
862
|
const load = loadBinding(name, staleBindings.delete(name));
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
863
|
+
return trackRegistered(name, load);
|
|
864
|
+
}
|
|
865
|
+
const entry = entries.get(name);
|
|
866
|
+
if (entry && entry.demoted && !loading.has(name)) {
|
|
867
|
+
entry.demoted = false; // one promotion at a time
|
|
868
|
+
// The view reads fine whatever became of the promise before this one.
|
|
869
|
+
return trackRegistered(name, known.catch(() => {}).then(() => promote(name)));
|
|
870
|
+
}
|
|
871
|
+
return known;
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
/** The promise a next read waits on. A rejected one never stays: it would answer for good. */
|
|
875
|
+
function trackRegistered(name, promise) {
|
|
876
|
+
registered.set(name, promise);
|
|
877
|
+
promise.catch(() => {
|
|
878
|
+
if (registered.get(name) === promise) registered.delete(name);
|
|
879
|
+
});
|
|
880
|
+
return promise;
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
/**
|
|
884
|
+
* Cap what bindings may hold in DuckDB, in bytes (default 1.5 GiB, half of the
|
|
885
|
+
* frame's 3.1 GiB heap: queries need the rest). Checked with an estimate before
|
|
886
|
+
* each load, so it can overshoot. Past it, the least recently read tables become
|
|
887
|
+
* views over a heap copy of their parquet. `opts.blobBytes` caps the parquet kept
|
|
888
|
+
* in JS memory (default 512 MiB); past it the oldest are dropped and download
|
|
889
|
+
* again on their next read. `Infinity` turns either off. Resolves once the engine
|
|
890
|
+
* is back under the new budget, and ends any memory pressure.
|
|
891
|
+
*/
|
|
892
|
+
export async function setBindingMemoryBudget(bytes, opts) {
|
|
893
|
+
const valid = n => typeof n === "number" && n > 0 && !Number.isNaN(n);
|
|
894
|
+
if (!valid(bytes)) throw new TypeError("setBindingMemoryBudget: a positive number of bytes");
|
|
895
|
+
memoryBudget = bytes;
|
|
896
|
+
pressureUntil = 0; // the caller has just said what tables may hold
|
|
897
|
+
clearTimeout(pressureTimer);
|
|
898
|
+
if (opts && opts.blobBytes !== undefined) {
|
|
899
|
+
if (!valid(opts.blobBytes)) throw new TypeError("setBindingMemoryBudget: blobBytes must be a positive number");
|
|
900
|
+
blobBudget = opts.blobBytes;
|
|
240
901
|
}
|
|
241
|
-
|
|
902
|
+
wakeRetries();
|
|
903
|
+
if (!dbPromise) return;
|
|
904
|
+
const engine = await getDb();
|
|
905
|
+
await catalog(() => trimNow(engine, null));
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
/**
|
|
909
|
+
* What the binding tables hold now: DuckDB's own figure, the budgets, and
|
|
910
|
+
* each loaded binding, least recently read first.
|
|
911
|
+
*/
|
|
912
|
+
export async function getBindingMemory() {
|
|
913
|
+
const engine = await getDb();
|
|
914
|
+
const usedBytes = await catalog(() => measureNow(engine));
|
|
915
|
+
const bindings = [...entries]
|
|
916
|
+
.sort(([a], [b]) => (lastUse.get(a) ?? 0) - (lastUse.get(b) ?? 0))
|
|
917
|
+
.map(([name, entry]) => ({
|
|
918
|
+
name,
|
|
919
|
+
storage: entry.storage,
|
|
920
|
+
pinned: isPinned(name),
|
|
921
|
+
parquetBytes: entry.blob ? entry.blob.size : null,
|
|
922
|
+
}));
|
|
923
|
+
return {
|
|
924
|
+
usedBytes,
|
|
925
|
+
bindingBytes: ownedBytes(),
|
|
926
|
+
budgetBytes: memoryBudget,
|
|
927
|
+
blobBudgetBytes: blobBudget,
|
|
928
|
+
pressure: Date.now() < pressureUntil,
|
|
929
|
+
bindings,
|
|
930
|
+
};
|
|
242
931
|
}
|
|
243
932
|
|
|
244
933
|
// ---------------------------------------------------------------------------
|
|
@@ -270,23 +959,30 @@ export async function withConnection(fn) {
|
|
|
270
959
|
* Make sure a binding's table exists in the shared DuckDB, loading it once
|
|
271
960
|
* per page (concurrent callers share the download). Resolves with the table
|
|
272
961
|
* name — the binding name; quote it in SQL when it is not a bare identifier.
|
|
962
|
+
* Each call counts as a read. Under memory pressure the table may become a
|
|
963
|
+
* view over its parquet (same name, same rows, slower); it is never dropped.
|
|
273
964
|
*/
|
|
274
965
|
export async function ensureBinding(name) {
|
|
966
|
+
externalBindings.add(name);
|
|
275
967
|
await registerBinding(name);
|
|
276
968
|
return name;
|
|
277
969
|
}
|
|
278
970
|
|
|
279
971
|
/**
|
|
280
|
-
* Forget a loaded binding: drops its table so its memory is freed.
|
|
281
|
-
* query or ensureBinding() downloads it again.
|
|
972
|
+
* Forget a loaded binding: drops its table (or view) so its memory is freed.
|
|
973
|
+
* The next query or ensureBinding() downloads it again.
|
|
282
974
|
*/
|
|
283
975
|
export async function unloadBinding(name) {
|
|
284
976
|
const pending = registered.get(name);
|
|
285
977
|
if (!pending) return;
|
|
286
978
|
registered.delete(name);
|
|
287
|
-
|
|
979
|
+
externalBindings.delete(name);
|
|
980
|
+
knownBindings.delete(name);
|
|
288
981
|
await pending.catch(() => {});
|
|
289
|
-
|
|
982
|
+
if (registered.has(name)) return; // read again meanwhile: it stays
|
|
983
|
+
const engine = await getDb();
|
|
984
|
+
await catalog(() => dropNow(engine, name));
|
|
985
|
+
wakeRetries();
|
|
290
986
|
}
|
|
291
987
|
|
|
292
988
|
/**
|
|
@@ -319,14 +1015,7 @@ async function reloadBinding(name) {
|
|
|
319
1015
|
// Let a load in flight settle first, or its (older) bytes could land after
|
|
320
1016
|
// the fresh ones and win.
|
|
321
1017
|
await previous.catch(() => {});
|
|
322
|
-
|
|
323
|
-
registered.set(name, load);
|
|
324
|
-
try {
|
|
325
|
-
await load;
|
|
326
|
-
} catch (error) {
|
|
327
|
-
if (registered.get(name) === load) registered.delete(name);
|
|
328
|
-
throw error;
|
|
329
|
-
}
|
|
1018
|
+
await trackRegistered(name, loadBinding(name, true));
|
|
330
1019
|
}
|
|
331
1020
|
|
|
332
1021
|
// ---------------------------------------------------------------------------
|
|
@@ -487,15 +1176,46 @@ export function getViewer() {
|
|
|
487
1176
|
/** Names of every staged binding — written by the dev server next to the
|
|
488
1177
|
* parquet files. Absent (older servers, published builds): empty list. */
|
|
489
1178
|
let indexPromise = null;
|
|
1179
|
+
const INDEX_RETRY_MS = 10000;
|
|
490
1180
|
function bindingIndex() {
|
|
491
|
-
indexPromise
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
1181
|
+
if (indexPromise) return indexPromise;
|
|
1182
|
+
// 20 panels asking at once share one answer, a failed one for 10 s too.
|
|
1183
|
+
const forget = () => {
|
|
1184
|
+
const t = setTimeout(() => {
|
|
1185
|
+
if (indexPromise === pending) indexPromise = null;
|
|
1186
|
+
}, INDEX_RETRY_MS);
|
|
1187
|
+
t.unref?.();
|
|
1188
|
+
};
|
|
1189
|
+
const pending = fetchRetrying("__data/index.json").then(async r => {
|
|
1190
|
+
// A 404 is an older server or a published build: empty, for good.
|
|
1191
|
+
if (!r.ok && r.status !== 404) forget();
|
|
1192
|
+
if (r.status === 429 || r.status === 503) throw await responseError(r, "The binding list could not be loaded");
|
|
1193
|
+
if (!r.ok) return [];
|
|
1194
|
+
const list = await r.json().catch(() => []);
|
|
1195
|
+
return Array.isArray(list) ? list : [];
|
|
1196
|
+
});
|
|
1197
|
+
indexPromise = pending;
|
|
1198
|
+
pending.catch(forget);
|
|
1199
|
+
return pending;
|
|
496
1200
|
}
|
|
497
1201
|
|
|
498
|
-
|
|
1202
|
+
// Out of Memory on a query: free tables, pinned ones the query reads last, and
|
|
1203
|
+
// try again. Each pass demotes at least one table, so this ends. With nothing left
|
|
1204
|
+
// to free, DuckDB's error surfaces and the hook decides (useAsyncQuery).
|
|
1205
|
+
async function runSql(sql, rowLimit, format, reads) {
|
|
1206
|
+
for (;;) {
|
|
1207
|
+
try {
|
|
1208
|
+
return await runSqlOnce(sql, rowLimit, format);
|
|
1209
|
+
} catch (error) {
|
|
1210
|
+
if (!isEngineOom(error)) throw error;
|
|
1211
|
+
const engine = await getDb();
|
|
1212
|
+
const freed = await catalog(() => relieveNow(engine, null, reads)).catch(() => 0);
|
|
1213
|
+
if (!freed) throw error;
|
|
1214
|
+
}
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
async function runSqlOnce(sql, rowLimit, format) {
|
|
499
1219
|
const cap = rowLimit === null ? Infinity : (rowLimit ?? DEFAULT_ROW_LIMIT);
|
|
500
1220
|
const db = await getDb();
|
|
501
1221
|
const conn = await db.connect();
|
|
@@ -509,11 +1229,43 @@ async function runSql(sql, rowLimit, format) {
|
|
|
509
1229
|
}
|
|
510
1230
|
const { rows, fields } = toPlainRows(table, cap);
|
|
511
1231
|
return { rows, fields, rowCount, truncated: rowCount > rows.length };
|
|
1232
|
+
} catch (error) {
|
|
1233
|
+
throw markEngineError(error, db);
|
|
512
1234
|
} finally {
|
|
513
1235
|
await conn.close();
|
|
514
1236
|
}
|
|
515
1237
|
}
|
|
516
1238
|
|
|
1239
|
+
// A crashed engine stays crashed: the SQL that broke it can answer, then every
|
|
1240
|
+
// later query fails with a raw WASM trap. So the page starts a new engine.
|
|
1241
|
+
// Bindings download again. Pins stay: they belong to mounted hooks.
|
|
1242
|
+
const RESTART_EVERY_MS = 60000;
|
|
1243
|
+
const MAX_RESTARTS = 3;
|
|
1244
|
+
let lastRestart = -Infinity;
|
|
1245
|
+
let restarts = 0;
|
|
1246
|
+
let restartTimer = null;
|
|
1247
|
+
function noteEngineCrash(engine) {
|
|
1248
|
+
if (engine !== liveEngine || restartTimer || restarts >= MAX_RESTARTS) return;
|
|
1249
|
+
restartTimer = setTimeout(restartEngine, Math.max(0, lastRestart + RESTART_EVERY_MS - Date.now()));
|
|
1250
|
+
}
|
|
1251
|
+
function restartEngine() {
|
|
1252
|
+
restartTimer = null;
|
|
1253
|
+
const old = liveEngine;
|
|
1254
|
+
if (!old) return;
|
|
1255
|
+
restarts++;
|
|
1256
|
+
lastRestart = Date.now();
|
|
1257
|
+
liveEngine = null;
|
|
1258
|
+
dbPromise = null;
|
|
1259
|
+
catalogQueue = Promise.resolve();
|
|
1260
|
+
entries.clear();
|
|
1261
|
+
registered.clear();
|
|
1262
|
+
loadedBindings.clear();
|
|
1263
|
+
measuredBytes = 0;
|
|
1264
|
+
pressureUntil = 0;
|
|
1265
|
+
old.terminate().catch(() => {});
|
|
1266
|
+
setBindingState({ ...bindingState, version: bindingState.version + 1 });
|
|
1267
|
+
}
|
|
1268
|
+
|
|
517
1269
|
const EMPTY = {
|
|
518
1270
|
data: null,
|
|
519
1271
|
fields: null,
|
|
@@ -529,33 +1281,97 @@ const EMPTY = {
|
|
|
529
1281
|
// until the new ones arrive — a dashboard must not blink to empty on every
|
|
530
1282
|
// refresh). `isRefreshing` says whether a refresh in flight concerns this
|
|
531
1283
|
// query, so `refreshing` is true from the click, not only from the reload.
|
|
1284
|
+
// `run(hold)` names each binding it reads through `hold`: those stay pinned
|
|
1285
|
+
// against eviction while this hook shows that query (until the deps change
|
|
1286
|
+
// or it unmounts, and never before its run settles).
|
|
1287
|
+
//
|
|
1288
|
+
// A query that failed on memory or the network runs again by itself, a few times
|
|
1289
|
+
// (retryDelayMs). After that it waits for a reason to expect a different answer:
|
|
1290
|
+
// another hook let go of its bindings, the budget changed, the server answered
|
|
1291
|
+
// again, the tab came back, the network came back. The error stays up until a run
|
|
1292
|
+
// answers. A hidden tab waits too. Anything else stays as it is.
|
|
1293
|
+
const retryWaiters = new Set();
|
|
1294
|
+
function wakeRetries() {
|
|
1295
|
+
const waiting = [...retryWaiters];
|
|
1296
|
+
retryWaiters.clear();
|
|
1297
|
+
for (const fire of waiting) fire();
|
|
1298
|
+
}
|
|
1299
|
+
// The user or the network is back: let one probe through a closed gate, too.
|
|
1300
|
+
function tryAgainNow() {
|
|
1301
|
+
gateUntil = 0;
|
|
1302
|
+
wakeRetries();
|
|
1303
|
+
}
|
|
1304
|
+
if (typeof window !== "undefined" && typeof window.addEventListener === "function") {
|
|
1305
|
+
window.addEventListener("online", tryAgainNow);
|
|
1306
|
+
if (typeof document !== "undefined") {
|
|
1307
|
+
document.addEventListener("visibilitychange", () => {
|
|
1308
|
+
if (!document.hidden) tryAgainNow();
|
|
1309
|
+
});
|
|
1310
|
+
}
|
|
1311
|
+
}
|
|
1312
|
+
|
|
532
1313
|
function useAsyncQuery(run, deps, isRefreshing) {
|
|
533
1314
|
const bindings = useBindingState();
|
|
534
1315
|
const [state, setState] = React.useState(EMPTY);
|
|
1316
|
+
const [attempt, setAttempt] = React.useState(0);
|
|
535
1317
|
const seenVersion = React.useRef(bindings.version);
|
|
1318
|
+
const seenAttempt = React.useRef(attempt);
|
|
1319
|
+
const retries = React.useRef(0);
|
|
1320
|
+
const retrying = React.useRef(false);
|
|
536
1321
|
React.useEffect(() => {
|
|
537
1322
|
let active = true;
|
|
1323
|
+
let timer = null;
|
|
1324
|
+
let waiter = null;
|
|
538
1325
|
const isRefresh = seenVersion.current !== bindings.version;
|
|
1326
|
+
const isRetry = seenAttempt.current !== attempt;
|
|
539
1327
|
seenVersion.current = bindings.version;
|
|
540
|
-
|
|
541
|
-
|
|
1328
|
+
seenAttempt.current = attempt;
|
|
1329
|
+
retrying.current = false;
|
|
1330
|
+
if (!isRetry) {
|
|
1331
|
+
retries.current = 0;
|
|
1332
|
+
setState(s => (isRefresh && s.data !== null ? { ...s, refreshing: true } : EMPTY));
|
|
1333
|
+
}
|
|
1334
|
+
const held = [];
|
|
1335
|
+
const settled = run(name => held.push(pin(name)));
|
|
1336
|
+
settled.then(
|
|
542
1337
|
result => {
|
|
543
1338
|
if (active) setState({ ...result, error: null, loading: false, refreshing: false });
|
|
544
1339
|
},
|
|
545
1340
|
error => {
|
|
546
|
-
if (active)
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
1341
|
+
if (!active) return;
|
|
1342
|
+
setState({
|
|
1343
|
+
...EMPTY,
|
|
1344
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1345
|
+
loading: false,
|
|
1346
|
+
});
|
|
1347
|
+
const wait = retryDelayMs(error, retries.current);
|
|
1348
|
+
if (wait === null) return;
|
|
1349
|
+
retries.current++;
|
|
1350
|
+
waiter = () => {
|
|
1351
|
+
if (typeof document !== "undefined" && document.hidden) {
|
|
1352
|
+
retryWaiters.add(waiter);
|
|
1353
|
+
return;
|
|
1354
|
+
}
|
|
1355
|
+
retrying.current = true;
|
|
1356
|
+
setAttempt(n => n + 1);
|
|
1357
|
+
};
|
|
1358
|
+
if (wait === Infinity) retryWaiters.add(waiter);
|
|
1359
|
+
else timer = setTimeout(waiter, wait);
|
|
552
1360
|
},
|
|
553
1361
|
);
|
|
554
1362
|
return () => {
|
|
555
1363
|
active = false;
|
|
1364
|
+
if (timer !== null) clearTimeout(timer);
|
|
1365
|
+
if (waiter) retryWaiters.delete(waiter);
|
|
1366
|
+
const letsGo = !retrying.current;
|
|
1367
|
+
settled.catch(() => {}).finally(() => {
|
|
1368
|
+
for (const release of held) release();
|
|
1369
|
+
// Its own retry frees nothing, or two failing panels would wake each other.
|
|
1370
|
+
if (letsGo && held.length) wakeRetries();
|
|
1371
|
+
});
|
|
556
1372
|
};
|
|
557
1373
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
558
|
-
}, [...deps, bindings.version]);
|
|
1374
|
+
}, [...deps, bindings.version, attempt]);
|
|
559
1375
|
return {
|
|
560
1376
|
...state,
|
|
561
1377
|
refreshing: state.refreshing || isRefreshing(bindings.refreshing),
|
|
@@ -622,9 +1438,10 @@ export function useQuery(name, opts) {
|
|
|
622
1438
|
const withSnapshot = Boolean(opts && opts.snapshot);
|
|
623
1439
|
const preview = useSnapshotPreview(name, withSnapshot, rowLimit);
|
|
624
1440
|
const state = useAsyncQuery(
|
|
625
|
-
async
|
|
1441
|
+
async hold => {
|
|
1442
|
+
hold(name);
|
|
626
1443
|
await registerBinding(name);
|
|
627
|
-
const r = await runSql("SELECT * FROM " + quoteIdent(name), rowLimit);
|
|
1444
|
+
const r = await runSql("SELECT * FROM " + quoteIdent(name), rowLimit, undefined, new Set([name]));
|
|
628
1445
|
return { data: r.rows, fields: r.fields, truncated: r.truncated, rowCount: r.rowCount };
|
|
629
1446
|
},
|
|
630
1447
|
[name, rowLimit],
|
|
@@ -682,24 +1499,41 @@ export function useViewer() {
|
|
|
682
1499
|
* A binding that fails to load must not sink the query silently: when the
|
|
683
1500
|
* SQL then fails, the load failure is what surfaces — it is the cause.
|
|
684
1501
|
*/
|
|
685
|
-
async function runOverBindings(sql, rowLimit, format) {
|
|
686
|
-
|
|
1502
|
+
async function runOverBindings(sql, rowLimit, format, hold) {
|
|
1503
|
+
// No index still answers `select 1`. It matters only if the query then fails.
|
|
1504
|
+
let indexFailure = null;
|
|
1505
|
+
const index = await bindingIndex().catch(error => {
|
|
1506
|
+
indexFailure = error;
|
|
1507
|
+
return [];
|
|
1508
|
+
});
|
|
1509
|
+
const names = [...new Set([...index, ...knownBindings])];
|
|
687
1510
|
const failures = new Map();
|
|
688
1511
|
const attempted = new Set();
|
|
1512
|
+
const reloaded = new Set();
|
|
689
1513
|
const load = n => {
|
|
690
1514
|
attempted.add(n);
|
|
1515
|
+
hold(n);
|
|
691
1516
|
return registerBinding(n).catch(e => {
|
|
692
1517
|
failures.set(n, e);
|
|
693
1518
|
});
|
|
694
1519
|
};
|
|
695
|
-
|
|
1520
|
+
const wanted = readsCatalog(sql) ? names : mentionedBindings(sql, names);
|
|
1521
|
+
if (!readsCatalog(sql) && wanted.length > MAX_BINDINGS_IN_QUERY) {
|
|
1522
|
+
throw new Error(
|
|
1523
|
+
"This query names " + wanted.length + " bindings. One query can read at most " + MAX_BINDINGS_IN_QUERY +
|
|
1524
|
+
": around 60 overflow DuckDB's WASM stack and break the engine for every panel. Read fewer shards, or combine them in the warehouse.",
|
|
1525
|
+
);
|
|
1526
|
+
}
|
|
1527
|
+
await Promise.all(wanted.map(load));
|
|
696
1528
|
for (;;) {
|
|
697
1529
|
try {
|
|
698
|
-
return await runSql(sql, rowLimit, format);
|
|
1530
|
+
return await runSql(sql, rowLimit, format, attempted);
|
|
699
1531
|
} catch (error) {
|
|
700
1532
|
const message = error instanceof Error ? error.message : String(error);
|
|
701
1533
|
const missing = missingBinding(message, names);
|
|
702
|
-
|
|
1534
|
+
// Never asked for, or the SDK evicted it under this query. Once per run.
|
|
1535
|
+
if (missing && !failures.has(missing) && !reloaded.has(missing) && (!attempted.has(missing) || !registered.has(missing))) {
|
|
1536
|
+
reloaded.add(missing);
|
|
703
1537
|
await load(missing);
|
|
704
1538
|
if (failures.has(missing)) throw failures.get(missing);
|
|
705
1539
|
continue;
|
|
@@ -708,11 +1542,15 @@ async function runOverBindings(sql, rowLimit, format) {
|
|
|
708
1542
|
if (message.includes(n)) throw cause;
|
|
709
1543
|
}
|
|
710
1544
|
if (failures.size) {
|
|
711
|
-
const
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
1545
|
+
const causes = [...failures.values()];
|
|
1546
|
+
const details = causes.map(e => (e instanceof Error ? e.message : String(e))).join("; ");
|
|
1547
|
+
const wrapped = new Error(message + " (bindings that failed to load: " + details + ")");
|
|
1548
|
+
const retryable = causes.find(e => e?.retry);
|
|
1549
|
+
wrapped.retry = error?.retry ?? retryable?.retry;
|
|
1550
|
+
wrapped.retryAfterMs = retryable?.retryAfterMs;
|
|
1551
|
+
throw wrapped;
|
|
715
1552
|
}
|
|
1553
|
+
if (indexFailure && !error?.retry) throw indexFailure;
|
|
716
1554
|
throw error;
|
|
717
1555
|
}
|
|
718
1556
|
}
|
|
@@ -722,8 +1560,8 @@ export function useDuckDB(sql, opts) {
|
|
|
722
1560
|
bootEngine();
|
|
723
1561
|
const rowLimit = opts ? opts.rowLimit : undefined;
|
|
724
1562
|
const format = opts && opts.format === "arrow" ? "arrow" : "rows";
|
|
725
|
-
const state = useAsyncQuery(async
|
|
726
|
-
const r = await runOverBindings(sql, rowLimit, format);
|
|
1563
|
+
const state = useAsyncQuery(async hold => {
|
|
1564
|
+
const r = await runOverBindings(sql, rowLimit, format, hold);
|
|
727
1565
|
return { data: r.rows, fields: r.fields, truncated: r.truncated, rowCount: r.rowCount };
|
|
728
1566
|
}, [sql, rowLimit, format], refreshing => refreshing.size > 0);
|
|
729
1567
|
// Every binding the query can read: the combined record says what the
|