@makoai/app-sdk 2.12.0 → 2.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -0
- package/data-core.js +17 -0
- package/index.d.ts +10 -4
- package/index.js +248 -42
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -173,6 +173,58 @@ for the whole heap, and `pressure`.
|
|
|
173
173
|
- A failed `__data/index.json` is kept for 10 s. A query that needs no binding still
|
|
174
174
|
answers. A query that fails then shows the index failure and retries.
|
|
175
175
|
|
|
176
|
+
### Compressed tables on DuckDB 1.4 (2.14)
|
|
177
|
+
|
|
178
|
+
The engine moved from duckdb-wasm 1.29.0 (DuckDB 1.1.1) to 1.32.0 (DuckDB 1.4.3).
|
|
179
|
+
Binding tables live in `ATTACH ':memory:' (COMPRESS)`, and a view with the
|
|
180
|
+
binding's name reads them.
|
|
181
|
+
|
|
182
|
+
| Stored as tables, real parquet | 1.29.0 | 1.32.0 | Saved |
|
|
183
|
+
| --- | --- | --- | --- |
|
|
184
|
+
| Web Performance 7d, 10.9M rows | 3,003 MiB | 176 MiB | 94.1% |
|
|
185
|
+
| Traffic Performance, 2.4M rows | 977 MiB | 75 MiB | 92.4% |
|
|
186
|
+
| Financial Model, 489k rows | 153 MiB | 26 MiB | 83.1% |
|
|
187
|
+
| INTL Sales, 1.9M rows | 601 MiB | 109 MiB | 81.9% |
|
|
188
|
+
| Who's Looking, 198k rows | 33 MiB | 10 MiB | 69.1% |
|
|
189
|
+
|
|
190
|
+
- Very long strings stay uncompressed: `comps_*`, at 720 KB a row, saved 0%.
|
|
191
|
+
- A table you create is uncompressed: it lives in the default database.
|
|
192
|
+
- A binding's name is a view. `INSERT`, `ALTER`, `DROP TABLE` and
|
|
193
|
+
`CREATE OR REPLACE TABLE <binding>` fail: copy it into a table with another name.
|
|
194
|
+
- A CTE read twice is held whole in scratch, and no setting stops that. Write
|
|
195
|
+
`WITH u AS NOT MATERIALIZED (...)` over big bindings: 1.29.0 parses it too.
|
|
196
|
+
- `TIMESTAMPTZ` reads UTC, as it did on 1.29.0: the SDK runs
|
|
197
|
+
`SET GLOBAL TimeZone='UTC'` at boot.
|
|
198
|
+
- Boot also fetches ICU from extensions.duckdb.org. If that fails the boot fails,
|
|
199
|
+
and a hook retries at 5, 20 and 60 s.
|
|
200
|
+
- Row order changes where a query has no `ORDER BY`, and so does `any_value`.
|
|
201
|
+
|
|
202
|
+
### Memory across apps (2.13)
|
|
203
|
+
|
|
204
|
+
Inside Mako, each app frame posts what its heap holds. Past 2 GiB across the
|
|
205
|
+
open app tabs, Mako asks the least recently shown ones to release, and asks a
|
|
206
|
+
busy frame again 10 s later. A WASM heap never shrinks, so a release ends that
|
|
207
|
+
frame's DuckDB worker. Its rows stay on screen and its parquet stays as Blobs.
|
|
208
|
+
The next query there boots a new engine and reloads its parquet with no
|
|
209
|
+
download. A reboot takes 0.6 s on the app's own origin and 0.6 to 2.6 s in an
|
|
210
|
+
opaque frame, which has no HTTP cache and fetches the 35 MiB WASM again.
|
|
211
|
+
|
|
212
|
+
- A frame keeps its engine for the page's life once app code calls
|
|
213
|
+
`getDuckDB()`, `withConnection()` or `ensureBinding()`, or sends `useDuckDB`
|
|
214
|
+
SQL that is not a plain read (`CREATE TABLE`, `SET`, anything the SDK does
|
|
215
|
+
not recognise). Only that worker has what they made. Mako counts its heap and
|
|
216
|
+
asks another frame, so an app that needs to be a good neighbour sticks to
|
|
217
|
+
read-only `useQuery` and `useDuckDB`.
|
|
218
|
+
- A release is not a crash. It does not count toward the 3 restarts a page gets,
|
|
219
|
+
reruns no panel and leaves the download gate as it is. A parked binding reads
|
|
220
|
+
with no network even while the gate is closed.
|
|
221
|
+
- Frames on an older SDK, the dev-mode preview and apps that boot their own
|
|
222
|
+
DuckDB send no report. Mako cannot count or ask them. An app gets 2.13 on its
|
|
223
|
+
next commit.
|
|
224
|
+
- A report covers tables and parquet copies. Query scratch stays in the heap
|
|
225
|
+
unseen, so the real heap is larger (1037 MiB reported, about 1500 MiB in the
|
|
226
|
+
renderer).
|
|
227
|
+
|
|
176
228
|
### Loading cost (2.8)
|
|
177
229
|
|
|
178
230
|
- `useDuckDB` downloads only the bindings its SQL names. A missed name (for
|
package/data-core.js
CHANGED
|
@@ -46,6 +46,23 @@ export function readsCatalog(sql) {
|
|
|
46
46
|
return CATALOG_QUERY.test(String(sql));
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
+
const SQL_NOISE = /'(?:[^']|'')*'|"(?:[^"]|"")*"|--[^\n]*|\/\*[\s\S]*?\*\//g;
|
|
50
|
+
const READ_STATEMENT = /^(?:\(|(?:SELECT|FROM|VALUES|TABLE|DESC|DESCRIBE|SHOW|SUMMARIZE|WITH)\b)/i;
|
|
51
|
+
const WRITE_WORD = /\b(?:INSERT|UPDATE|DELETE|CREATE|DROP|ALTER|MERGE)\b/i;
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Whether `sql` only reads. Anything it does not recognise counts as a write:
|
|
55
|
+
* the cost of a wrong "write" is a frame that keeps its engine, and the cost of
|
|
56
|
+
* a wrong "read" is an app that loses its tables on a release.
|
|
57
|
+
*/
|
|
58
|
+
export function isReadOnlySql(sql) {
|
|
59
|
+
return String(sql)
|
|
60
|
+
.replace(SQL_NOISE, " ")
|
|
61
|
+
.split(";")
|
|
62
|
+
.map(statement => statement.trim())
|
|
63
|
+
.every(s => s === "" || (READ_STATEMENT.test(s) && !(/^WITH\b/i.test(s) && WRITE_WORD.test(s))));
|
|
64
|
+
}
|
|
65
|
+
|
|
49
66
|
/**
|
|
50
67
|
* The binding a DuckDB "Table with name X does not exist" error is about —
|
|
51
68
|
* X itself when the index lists it, else its case-insensitive match (DuckDB
|
package/index.d.ts
CHANGED
|
@@ -263,11 +263,16 @@ export interface RefreshOptions {
|
|
|
263
263
|
/**
|
|
264
264
|
* The page's shared DuckDB, booted on first call. Use it for anything the
|
|
265
265
|
* hooks do not cover instead of importing @duckdb/duckdb-wasm, hyparquet or
|
|
266
|
-
* apache-arrow readers. Never close or terminate it.
|
|
266
|
+
* apache-arrow readers. Never close or terminate it. Calling it keeps the
|
|
267
|
+
* engine for the page's life: Mako cannot give this frame's heap back when the
|
|
268
|
+
* page runs short (see withConnection, ensureBinding and SQL that writes).
|
|
267
269
|
*/
|
|
268
270
|
export function getDuckDB(): Promise<SharedDuckDB>;
|
|
269
271
|
|
|
270
|
-
/**
|
|
272
|
+
/**
|
|
273
|
+
* Run `fn` on a fresh connection to the shared DuckDB; closed afterwards. Keeps
|
|
274
|
+
* the engine for the page's life, like getDuckDB().
|
|
275
|
+
*/
|
|
271
276
|
export function withConnection<T>(fn: (conn: DuckDBConnection) => Promise<T>): Promise<T>;
|
|
272
277
|
|
|
273
278
|
/**
|
|
@@ -275,7 +280,8 @@ export function withConnection<T>(fn: (conn: DuckDBConnection) => Promise<T>): P
|
|
|
275
280
|
* concurrent callers share the download). Resolves with the table name.
|
|
276
281
|
* Each call counts as a read for eviction. Under memory pressure the table
|
|
277
282
|
* may become a view over its parquet (same name and rows, slower to query);
|
|
278
|
-
* a binding loaded this way is never dropped outright.
|
|
283
|
+
* a binding loaded this way is never dropped outright. Keeps the engine for the
|
|
284
|
+
* page's life, like getDuckDB().
|
|
279
285
|
*/
|
|
280
286
|
export function ensureBinding(name: string): Promise<string>;
|
|
281
287
|
|
|
@@ -316,7 +322,7 @@ export interface BindingMemory {
|
|
|
316
322
|
/** Every loaded binding, least recently read first. */
|
|
317
323
|
bindings: {
|
|
318
324
|
name: string;
|
|
319
|
-
/** "table":
|
|
325
|
+
/** "table": held compressed in the heap. "view": decoded from its parquet copy in the heap on each query. */
|
|
320
326
|
storage: "table" | "view";
|
|
321
327
|
/** Read by a mounted hook or a query in flight: evicted last. */
|
|
322
328
|
pinned: boolean;
|
package/index.js
CHANGED
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
missingBinding,
|
|
21
21
|
mentionedBindings,
|
|
22
22
|
quoteIdent,
|
|
23
|
+
isReadOnlySql,
|
|
23
24
|
readsCatalog,
|
|
24
25
|
quoteString,
|
|
25
26
|
crashedEngineError,
|
|
@@ -35,7 +36,7 @@ import {
|
|
|
35
36
|
* in a page (or this export) says which SDK a deployed app actually runs.
|
|
36
37
|
* Kept in step with package.json by snapshot.test.mjs.
|
|
37
38
|
*/
|
|
38
|
-
export const SDK_VERSION = "2.
|
|
39
|
+
export const SDK_VERSION = "2.14.0";
|
|
39
40
|
if (typeof window !== "undefined") {
|
|
40
41
|
try {
|
|
41
42
|
window.__MAKO_APP_SDK__ = SDK_VERSION;
|
|
@@ -49,7 +50,7 @@ const DEFAULT_ROW_LIMIT = 500000;
|
|
|
49
50
|
// corrupts the engine for every panel (measured). 50 held.
|
|
50
51
|
const MAX_BINDINGS_IN_QUERY = 50;
|
|
51
52
|
const DUCKDB_ESM =
|
|
52
|
-
"https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.
|
|
53
|
+
"https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.32.0/+esm";
|
|
53
54
|
|
|
54
55
|
// ---------------------------------------------------------------------------
|
|
55
56
|
// DuckDB: one lazy instance per page, bindings registered on first use. The
|
|
@@ -60,24 +61,64 @@ const DUCKDB_ESM =
|
|
|
60
61
|
// ---------------------------------------------------------------------------
|
|
61
62
|
let dbPromise = null;
|
|
62
63
|
let liveEngine = null; // the resolved dbPromise: errors from an older one are history
|
|
64
|
+
// createWorker() makes a new object URL per call and never revokes it, so each
|
|
65
|
+
// reboot after a release would leak the worker script (about 742 KiB). One URL
|
|
66
|
+
// per page, also one download fewer in a frame with no HTTP cache. Not through
|
|
67
|
+
// the download gate: the CDN being down says nothing about Mako's server.
|
|
68
|
+
let workerUrl = null;
|
|
69
|
+
function startWorker(duckdb, script) {
|
|
70
|
+
if (typeof Worker === "undefined") return duckdb.createWorker(script);
|
|
71
|
+
workerUrl ??= fetch(script).then(async res => {
|
|
72
|
+
if (!res.ok) throw new Error("The DuckDB worker could not be loaded (HTTP " + res.status + ")");
|
|
73
|
+
return URL.createObjectURL(await res.blob());
|
|
74
|
+
});
|
|
75
|
+
workerUrl.catch(() => {
|
|
76
|
+
workerUrl = null;
|
|
77
|
+
});
|
|
78
|
+
return workerUrl.then(url => new Worker(url));
|
|
79
|
+
}
|
|
80
|
+
|
|
63
81
|
function getDb() {
|
|
64
82
|
if (!dbPromise) {
|
|
65
83
|
dbPromise = (async () => {
|
|
66
84
|
const duckdb = await import(/* @vite-ignore */ DUCKDB_ESM);
|
|
67
85
|
const bundle = await duckdb.selectBundle(duckdb.getJsDelivrBundles());
|
|
68
|
-
const worker = await duckdb
|
|
86
|
+
const worker = await startWorker(duckdb, bundle.mainWorker);
|
|
69
87
|
const db = new duckdb.AsyncDuckDB(new duckdb.VoidLogger(), worker);
|
|
70
88
|
await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
|
|
71
89
|
// read_parquet lives in an extension DuckDB fetches on first use
|
|
72
90
|
// (~660 kB from extensions.duckdb.org). Load it now, while the
|
|
73
91
|
// binding bytes are still downloading, not after they arrive.
|
|
74
|
-
const conn = await db.connect();
|
|
75
92
|
try {
|
|
76
|
-
await
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
93
|
+
const conn = await db.connect();
|
|
94
|
+
try {
|
|
95
|
+
try {
|
|
96
|
+
await conn.query("LOAD parquet");
|
|
97
|
+
} catch {
|
|
98
|
+
/* autoload on first read_parquet still applies */
|
|
99
|
+
}
|
|
100
|
+
// 1.32 reads TIMESTAMPTZ in the browser's zone once ICU loads. Apps were
|
|
101
|
+
// written against 1.29, which read UTC. Fetches ICU from
|
|
102
|
+
// extensions.duckdb.org, like parquet above. A failure here fails the
|
|
103
|
+
// boot on purpose: the next query retries, where a skipped SET would
|
|
104
|
+
// shift every timestamp without a word.
|
|
105
|
+
try {
|
|
106
|
+
await conn.query("SET GLOBAL TimeZone='UTC'");
|
|
107
|
+
} catch (error) {
|
|
108
|
+
if (error instanceof Error) error.retry = "network"; // the XHR for ICU
|
|
109
|
+
throw error;
|
|
110
|
+
}
|
|
111
|
+
// Binding tables live here, compressed at CHECKPOINT: 17 bytes a row
|
|
112
|
+
// against 289 on Web Performance's 7d set. Tables an app creates stay in
|
|
113
|
+
// the default database, uncompressed.
|
|
114
|
+
await conn.query("ATTACH ':memory:' AS mako_tables (COMPRESS)");
|
|
115
|
+
} finally {
|
|
116
|
+
await conn.close();
|
|
117
|
+
}
|
|
118
|
+
} catch (error) {
|
|
119
|
+
// A failed boot retries with a new worker: end this one.
|
|
120
|
+
await db.terminate().catch(() => {});
|
|
121
|
+
throw error;
|
|
81
122
|
}
|
|
82
123
|
liveEngine = db;
|
|
83
124
|
return db;
|
|
@@ -172,7 +213,7 @@ export function useBindingProgress(name) {
|
|
|
172
213
|
// Each binding becomes a TABLE, decoded from its parquet once: a view over
|
|
173
214
|
// read_parquet re-decoded the file on every query (SELECT * of a 149k-row
|
|
174
215
|
// binding: 366 ms through a view, 174 ms from a table). The file buffer is
|
|
175
|
-
// dropped as soon as the table exists. CREATE OR REPLACE swaps the new table
|
|
216
|
+
// dropped as soon as the table exists. CREATE OR REPLACE VIEW swaps the new table
|
|
176
217
|
// in with one commit, so a query running during a reload reads the old rows
|
|
177
218
|
// or the new ones, never a missing table. Every load registers its bytes
|
|
178
219
|
// under its own file name, so a reload never drops a file another load is
|
|
@@ -465,10 +506,6 @@ function markEngineError(error, engine) {
|
|
|
465
506
|
}
|
|
466
507
|
const isEngineOom = error => Boolean(error?.fromEngine) && isOutOfMemoryError(error);
|
|
467
508
|
|
|
468
|
-
function dropSql(storage, name) {
|
|
469
|
-
return "DROP " + (storage === "view" ? "VIEW" : "TABLE") + " IF EXISTS " + quoteIdent(name);
|
|
470
|
-
}
|
|
471
|
-
|
|
472
509
|
let fileSeq = 0;
|
|
473
510
|
// Every load registers its bytes under its own file name, so a reload never
|
|
474
511
|
// drops a file another load is still reading.
|
|
@@ -476,14 +513,48 @@ function nextFile() {
|
|
|
476
513
|
return "mako-" + ++fileSeq + ".parquet";
|
|
477
514
|
}
|
|
478
515
|
|
|
479
|
-
|
|
516
|
+
// name -> its table in mako_tables. Each load gets a fresh table, so a view
|
|
517
|
+
// never points at a table whose columns changed under it.
|
|
518
|
+
const stores = new Map();
|
|
519
|
+
let storeSeq = 0;
|
|
520
|
+
function storeSql(store) {
|
|
521
|
+
return "mako_tables.main." + quoteIdent(store);
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
/**
|
|
525
|
+
* Drop the compressed table behind `name`, if any. Runs after the view has moved,
|
|
526
|
+
* so a failure costs memory until the page reloads and never fails the load.
|
|
527
|
+
*/
|
|
528
|
+
async function dropStore(engine, name) {
|
|
529
|
+
const store = stores.get(name);
|
|
530
|
+
if (!store) return;
|
|
531
|
+
stores.delete(name);
|
|
532
|
+
await exec(engine, ["DROP TABLE IF EXISTS " + storeSql(store)]).catch(() => {});
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
/**
|
|
536
|
+
* A registered parquet `file` decoded into a compressed table, and a view named
|
|
537
|
+
* `name` over it. The name is always a view, so CREATE OR REPLACE VIEW swaps it
|
|
538
|
+
* atomically. A transaction cannot write to two databases, so the order keeps it
|
|
539
|
+
* readable: new table, then the view, then the old table goes.
|
|
540
|
+
*/
|
|
480
541
|
async function tableFromFile(engine, name, file) {
|
|
481
542
|
const prev = entries.get(name);
|
|
482
543
|
const before = await duckdbBytes(engine);
|
|
483
|
-
const
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
544
|
+
const store = "t" + ++storeSeq;
|
|
545
|
+
await exec(engine, ["CREATE TABLE " + storeSql(store) + " AS SELECT * FROM read_parquet(" + quoteString(file) + ")"]);
|
|
546
|
+
try {
|
|
547
|
+
// Only a CHECKPOINT compresses. It fails while a transaction is open on
|
|
548
|
+
// another connection: the table stays readable, and the next load's
|
|
549
|
+
// CHECKPOINT compresses it.
|
|
550
|
+
await exec(engine, ["CHECKPOINT mako_tables"]).catch(() => {});
|
|
551
|
+
await exec(engine, ["CREATE OR REPLACE VIEW " + quoteIdent(name) + " AS SELECT * FROM " + storeSql(store)]);
|
|
552
|
+
} catch (error) {
|
|
553
|
+
await exec(engine, ["DROP TABLE IF EXISTS " + storeSql(store)]).catch(() => {});
|
|
554
|
+
throw error;
|
|
555
|
+
}
|
|
556
|
+
await dropStore(engine, name);
|
|
557
|
+
stores.set(name, store);
|
|
487
558
|
const after = await duckdbBytes(engine);
|
|
488
559
|
if (before === null || after === null) return prev?.tableBytes ?? null;
|
|
489
560
|
const grown = after - before + (prev?.storage === "table" ? (prev.tableBytes ?? 0) : 0);
|
|
@@ -496,7 +567,8 @@ async function viewFromFile(engine, name, file) {
|
|
|
496
567
|
const create =
|
|
497
568
|
"CREATE OR REPLACE VIEW " + quoteIdent(name) +
|
|
498
569
|
" AS SELECT * FROM read_parquet(" + quoteString(file) + ")";
|
|
499
|
-
await exec(engine,
|
|
570
|
+
await exec(engine, [create]);
|
|
571
|
+
await dropStore(engine, name);
|
|
500
572
|
}
|
|
501
573
|
|
|
502
574
|
/**
|
|
@@ -568,8 +640,100 @@ async function duckdbBytes(engine) {
|
|
|
568
640
|
}
|
|
569
641
|
}
|
|
570
642
|
|
|
571
|
-
//
|
|
572
|
-
|
|
643
|
+
// ---------------------------------------------------------------------------
|
|
644
|
+
// Across apps (2.13). Each app frame has its own DuckDB and its own WASM heap,
|
|
645
|
+
// so evicting inside one frame frees nothing for the next app opened. The SDK
|
|
646
|
+
// tells Mako's shell what its heap holds; the shell keeps one budget across its
|
|
647
|
+
// app frames and asks the least recently shown ones to release. A WASM heap
|
|
648
|
+
// never shrinks: demoting a table frees DuckDB's accounting and leaves the bytes
|
|
649
|
+
// reserved. So release means ending the worker, and the report is the most this
|
|
650
|
+
// worker has held. It is measuredBytes, not bindingBytes: an app's own tables
|
|
651
|
+
// sit in the same renderer. Parquet stays here as Blobs ("parked"): the next read
|
|
652
|
+
// boots a new engine and decodes from them, with no download and no gate.
|
|
653
|
+
//
|
|
654
|
+
// A worker can end only when nothing lives in it but bindings. App code that may
|
|
655
|
+
// hold the engine or keep its own state there keeps it for the page's life
|
|
656
|
+
// (keepEngine), says so in its reports, and the shell asks another frame.
|
|
657
|
+
//
|
|
658
|
+
// A release is not a crash: it never counts against MAX_RESTARTS, never reruns a
|
|
659
|
+
// hook, and leaves the gate and the download slots alone. They describe the
|
|
660
|
+
// server. Nothing in flight also means no load holds a slot.
|
|
661
|
+
// ---------------------------------------------------------------------------
|
|
662
|
+
const MEMORY_MESSAGE = "mako-app:duckdb-memory";
|
|
663
|
+
const RELEASE_MESSAGE = "mako-host:duckdb-release";
|
|
664
|
+
const inFrame = typeof window !== "undefined" && window.parent != null && window.parent !== window;
|
|
665
|
+
const parked = new Map(); // name -> Blob of a binding whose engine was released
|
|
666
|
+
let engineKept = false;
|
|
667
|
+
let heapBytes = 0; // the most this worker has held: its heap never gives it back
|
|
668
|
+
let busy = 0; // hooks, withConnection calls and queries in flight
|
|
669
|
+
let reportedBytes = null;
|
|
670
|
+
let reportedKept = false;
|
|
671
|
+
|
|
672
|
+
function reportMemory() {
|
|
673
|
+
heapBytes = Math.max(heapBytes, measuredBytes);
|
|
674
|
+
if (!inFrame || (heapBytes === reportedBytes && engineKept === reportedKept)) return;
|
|
675
|
+
reportedBytes = heapBytes;
|
|
676
|
+
reportedKept = engineKept;
|
|
677
|
+
try {
|
|
678
|
+
window.parent.postMessage({ type: MEMORY_MESSAGE, bytes: heapBytes, canRelease: !engineKept }, "*");
|
|
679
|
+
} catch {
|
|
680
|
+
/* a host that is not Mako: nothing to tell */
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
/** Something only this worker has may exist from now on: never end it. */
|
|
685
|
+
function keepEngine() {
|
|
686
|
+
if (engineKept) return;
|
|
687
|
+
engineKept = true;
|
|
688
|
+
reportMemory();
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
/**
|
|
692
|
+
* The host's ask: give the memory back. Nothing in flight: park every Blob and
|
|
693
|
+
* end the worker. Rows already on screen stay (they are plain JS); the next
|
|
694
|
+
* query reboots. Busy: skipped, and the host asks again. A kept engine stays. So
|
|
695
|
+
* does one with a crash restart due: that ends it and reruns the panels that saw
|
|
696
|
+
* the crash, which a release would not.
|
|
697
|
+
*/
|
|
698
|
+
async function releaseEngine() {
|
|
699
|
+
const idle = () => !engineKept && !restartTimer && busy === 0 && loading.size === 0 && refreshes.size === 0;
|
|
700
|
+
const current = dbPromise;
|
|
701
|
+
if (!current || !idle()) return;
|
|
702
|
+
const engine = await current;
|
|
703
|
+
await catalog(async () => {
|
|
704
|
+
if (dbPromise !== current || !idle()) return;
|
|
705
|
+
// From here to dbPromise = null runs without a yield: no query can grab
|
|
706
|
+
// the old engine in between. liveEngine goes too: a crash report from this
|
|
707
|
+
// engine must not restart the next one.
|
|
708
|
+
for (const [name, entry] of entries) if (entry.blob) parked.set(name, entry.blob);
|
|
709
|
+
entries.clear();
|
|
710
|
+
stores.clear();
|
|
711
|
+
registered.clear();
|
|
712
|
+
loadedBindings.clear();
|
|
713
|
+
liveEngine = null;
|
|
714
|
+
dbPromise = null;
|
|
715
|
+
measuredBytes = 0;
|
|
716
|
+
heapBytes = 0;
|
|
717
|
+
pressureUntil = 0; // the next engine starts with an empty heap
|
|
718
|
+
clearTimeout(pressureTimer);
|
|
719
|
+
reportMemory();
|
|
720
|
+
await engine.terminate().catch(() => {});
|
|
721
|
+
});
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
if (inFrame) {
|
|
725
|
+
window.addEventListener("message", event => {
|
|
726
|
+
// Only the frame's own parent may ask. The ask can do no harm beyond a
|
|
727
|
+
// slower next query, so the parent's origin (Mako's, whatever host it
|
|
728
|
+
// runs on) is not pinned here.
|
|
729
|
+
if (event.source !== window.parent) return;
|
|
730
|
+
const data = event.data;
|
|
731
|
+
if (!data || data.type !== RELEASE_MESSAGE) return;
|
|
732
|
+
releaseEngine().catch(() => {
|
|
733
|
+
/* the next ask, or the next load's trim, tries again */
|
|
734
|
+
});
|
|
735
|
+
});
|
|
736
|
+
}
|
|
573
737
|
|
|
574
738
|
/** The least recently read binding that passes `test` and nothing holds. */
|
|
575
739
|
function leastRecent(test, pinnedToo = false) {
|
|
@@ -590,7 +754,8 @@ function leastRecent(test, pinnedToo = false) {
|
|
|
590
754
|
async function dropNow(engine, name) {
|
|
591
755
|
const entry = entries.get(name);
|
|
592
756
|
if (!entry) return;
|
|
593
|
-
await exec(engine, [
|
|
757
|
+
await exec(engine, ["DROP VIEW IF EXISTS " + quoteIdent(name)]);
|
|
758
|
+
await dropStore(engine, name);
|
|
594
759
|
if (entry.file) await engine.dropFile(entry.file).catch(() => {});
|
|
595
760
|
entries.delete(name);
|
|
596
761
|
loadedBindings.delete(name);
|
|
@@ -771,7 +936,11 @@ async function placeNow(engine, name, buf, blob) {
|
|
|
771
936
|
return { storage: "view", file, fileBytes: size, demoted: true };
|
|
772
937
|
};
|
|
773
938
|
const asTable = async () => {
|
|
939
|
+
// The table is decoded in full before it compresses, and a WASM heap never
|
|
940
|
+
// gives that back. The commit compresses, so no later measure can see it.
|
|
941
|
+
const decoded = await estimateTable(engine, file, blob.size);
|
|
774
942
|
const tableBytes = await tableFromFile(engine, name, file);
|
|
943
|
+
heapBytes = Math.max(heapBytes, measuredBytes + decoded);
|
|
775
944
|
await engine.dropFile(file).catch(() => {});
|
|
776
945
|
if (prev?.file) await engine.dropFile(prev.file).catch(() => {});
|
|
777
946
|
return { storage: "table", file: null, fileBytes: 0, demoted: false, tableBytes };
|
|
@@ -803,7 +972,14 @@ async function loadBinding(name, bust) {
|
|
|
803
972
|
db.catch(() => {});
|
|
804
973
|
loading.add(name);
|
|
805
974
|
try {
|
|
806
|
-
|
|
975
|
+
// Parked by a release: the same bytes, no download (unless a refresh made
|
|
976
|
+
// them stale). A Blob can stop being readable: then it downloads.
|
|
977
|
+
const kept = parked.get(name);
|
|
978
|
+
parked.delete(name);
|
|
979
|
+
const parkedBytes = kept && !bust ? await kept.arrayBuffer().catch(() => null) : null;
|
|
980
|
+
const buf = parkedBytes
|
|
981
|
+
? new Uint8Array(parkedBytes)
|
|
982
|
+
: await downloadBinding(name, bust, patch => setProgress(name, patch));
|
|
807
983
|
setProgress(name, { phase: "loading_duckdb" });
|
|
808
984
|
const engine = await db;
|
|
809
985
|
// Made before the buffer goes to the worker: the copy a demotion reads.
|
|
@@ -838,7 +1014,9 @@ async function promote(name) {
|
|
|
838
1014
|
entry.demoted = true;
|
|
839
1015
|
return;
|
|
840
1016
|
}
|
|
1017
|
+
const decoded = await estimateTable(engine, entry.file, entry.fileBytes ?? 0);
|
|
841
1018
|
const tableBytes = await tableFromFile(engine, name, entry.file);
|
|
1019
|
+
heapBytes = Math.max(heapBytes, measuredBytes + decoded); // decoded before it compressed
|
|
842
1020
|
await engine.dropFile(entry.file).catch(() => {});
|
|
843
1021
|
entries.set(name, { ...entry, storage: "table", file: null, fileBytes: 0, demoted: false, tableBytes });
|
|
844
1022
|
await trimNow(engine, name);
|
|
@@ -941,17 +1119,24 @@ export async function getBindingMemory() {
|
|
|
941
1119
|
* close or terminate it: every hook on the page reads from it.
|
|
942
1120
|
*/
|
|
943
1121
|
export function getDuckDB() {
|
|
1122
|
+
keepEngine();
|
|
944
1123
|
return getDb();
|
|
945
1124
|
}
|
|
946
1125
|
|
|
947
1126
|
/** Run `fn` with a fresh connection to the shared DuckDB; always closed after. */
|
|
948
1127
|
export async function withConnection(fn) {
|
|
949
|
-
|
|
950
|
-
|
|
1128
|
+
keepEngine();
|
|
1129
|
+
busy++;
|
|
951
1130
|
try {
|
|
952
|
-
|
|
1131
|
+
const db = await getDb();
|
|
1132
|
+
const conn = await db.connect();
|
|
1133
|
+
try {
|
|
1134
|
+
return await fn(conn);
|
|
1135
|
+
} finally {
|
|
1136
|
+
await conn.close();
|
|
1137
|
+
}
|
|
953
1138
|
} finally {
|
|
954
|
-
|
|
1139
|
+
busy--;
|
|
955
1140
|
}
|
|
956
1141
|
}
|
|
957
1142
|
|
|
@@ -963,6 +1148,7 @@ export async function withConnection(fn) {
|
|
|
963
1148
|
* view over its parquet (same name, same rows, slower); it is never dropped.
|
|
964
1149
|
*/
|
|
965
1150
|
export async function ensureBinding(name) {
|
|
1151
|
+
keepEngine(); // its contract: the name stays readable, by code the SDK cannot see
|
|
966
1152
|
externalBindings.add(name);
|
|
967
1153
|
await registerBinding(name);
|
|
968
1154
|
return name;
|
|
@@ -973,11 +1159,12 @@ export async function ensureBinding(name) {
|
|
|
973
1159
|
* The next query or ensureBinding() downloads it again.
|
|
974
1160
|
*/
|
|
975
1161
|
export async function unloadBinding(name) {
|
|
1162
|
+
parked.delete(name);
|
|
1163
|
+
knownBindings.delete(name);
|
|
976
1164
|
const pending = registered.get(name);
|
|
977
1165
|
if (!pending) return;
|
|
978
1166
|
registered.delete(name);
|
|
979
1167
|
externalBindings.delete(name);
|
|
980
|
-
knownBindings.delete(name);
|
|
981
1168
|
await pending.catch(() => {});
|
|
982
1169
|
if (registered.has(name)) return; // read again meanwhile: it stays
|
|
983
1170
|
const engine = await getDb();
|
|
@@ -1217,22 +1404,31 @@ async function runSql(sql, rowLimit, format, reads) {
|
|
|
1217
1404
|
|
|
1218
1405
|
async function runSqlOnce(sql, rowLimit, format) {
|
|
1219
1406
|
const cap = rowLimit === null ? Infinity : (rowLimit ?? DEFAULT_ROW_LIMIT);
|
|
1220
|
-
const
|
|
1221
|
-
|
|
1407
|
+
const writes = !isReadOnlySql(sql);
|
|
1408
|
+
if (writes) keepEngine(); // CREATE TABLE journal_t: only this worker has it
|
|
1409
|
+
busy++; // before the await: a release must not end the engine under it
|
|
1222
1410
|
try {
|
|
1223
|
-
const
|
|
1224
|
-
const
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
//
|
|
1228
|
-
|
|
1411
|
+
const db = await getDb();
|
|
1412
|
+
const conn = await db.connect();
|
|
1413
|
+
try {
|
|
1414
|
+
const table = await conn.query(sql);
|
|
1415
|
+
// No load follows to measure what the app just made, and the report owes it.
|
|
1416
|
+
if (writes) catalog(() => measureNow(db)).catch(() => {});
|
|
1417
|
+
const rowCount = table.numRows;
|
|
1418
|
+
if (format === "arrow") {
|
|
1419
|
+
// The Arrow Table as DuckDB returned it: no row objects, no
|
|
1420
|
+
// normalization, no row limit.
|
|
1421
|
+
return { rows: table, fields: table.schema.fields.map(f => f.name), rowCount, truncated: false };
|
|
1422
|
+
}
|
|
1423
|
+
const { rows, fields } = toPlainRows(table, cap);
|
|
1424
|
+
return { rows, fields, rowCount, truncated: rowCount > rows.length };
|
|
1425
|
+
} catch (error) {
|
|
1426
|
+
throw markEngineError(error, db);
|
|
1427
|
+
} finally {
|
|
1428
|
+
await conn.close();
|
|
1229
1429
|
}
|
|
1230
|
-
const { rows, fields } = toPlainRows(table, cap);
|
|
1231
|
-
return { rows, fields, rowCount, truncated: rowCount > rows.length };
|
|
1232
|
-
} catch (error) {
|
|
1233
|
-
throw markEngineError(error, db);
|
|
1234
1430
|
} finally {
|
|
1235
|
-
|
|
1431
|
+
busy--;
|
|
1236
1432
|
}
|
|
1237
1433
|
}
|
|
1238
1434
|
|
|
@@ -1258,10 +1454,13 @@ function restartEngine() {
|
|
|
1258
1454
|
dbPromise = null;
|
|
1259
1455
|
catalogQueue = Promise.resolve();
|
|
1260
1456
|
entries.clear();
|
|
1457
|
+
stores.clear();
|
|
1261
1458
|
registered.clear();
|
|
1262
1459
|
loadedBindings.clear();
|
|
1263
1460
|
measuredBytes = 0;
|
|
1461
|
+
heapBytes = 0;
|
|
1264
1462
|
pressureUntil = 0;
|
|
1463
|
+
reportMemory();
|
|
1265
1464
|
old.terminate().catch(() => {});
|
|
1266
1465
|
setBindingState({ ...bindingState, version: bindingState.version + 1 });
|
|
1267
1466
|
}
|
|
@@ -1332,7 +1531,14 @@ function useAsyncQuery(run, deps, isRefreshing) {
|
|
|
1332
1531
|
setState(s => (isRefresh && s.data !== null ? { ...s, refreshing: true } : EMPTY));
|
|
1333
1532
|
}
|
|
1334
1533
|
const held = [];
|
|
1534
|
+
// From the first await to the last. Between a hook's load and its query
|
|
1535
|
+
// nothing else is in flight, and a release there would end the engine
|
|
1536
|
+
// holding the table the query is about to read.
|
|
1537
|
+
busy++;
|
|
1335
1538
|
const settled = run(name => held.push(pin(name)));
|
|
1539
|
+
settled.catch(() => {}).finally(() => {
|
|
1540
|
+
busy--;
|
|
1541
|
+
});
|
|
1336
1542
|
settled.then(
|
|
1337
1543
|
result => {
|
|
1338
1544
|
if (active) setState({ ...result, error: null, loading: false, refreshing: false });
|