@makoai/app-sdk 2.13.0 → 2.14.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/README.md +26 -0
  2. package/index.d.ts +1 -1
  3. package/index.js +149 -49
  4. package/package.json +1 -1
package/README.md CHANGED
@@ -173,6 +173,32 @@ for the whole heap, and `pressure`.
173
173
  - A failed `__data/index.json` is kept for 10 s. A query that needs no binding still
174
174
  answers. A query that fails then shows the index failure and retries.
175
175
 
176
+ ### Compressed tables on DuckDB 1.4 (2.14)
177
+
178
+ The engine moved from duckdb-wasm 1.29.0 (DuckDB 1.1.1) to 1.32.0 (DuckDB 1.4.3).
179
+ Binding tables live in `ATTACH ':memory:' (COMPRESS)`, and a view with the
180
+ binding's name reads them.
181
+
182
+ | Stored as tables, real parquet | 1.29.0 | 1.32.0 | Saved |
183
+ | --- | --- | --- | --- |
184
+ | Web Performance 7d, 10.9M rows | 3,003 MiB | 176 MiB | 94.1% |
185
+ | Traffic Performance, 2.4M rows | 977 MiB | 75 MiB | 92.4% |
186
+ | Financial Model, 489k rows | 153 MiB | 26 MiB | 83.1% |
187
+ | INTL Sales, 1.9M rows | 601 MiB | 109 MiB | 81.9% |
188
+ | Who's Looking, 198k rows | 33 MiB | 10 MiB | 69.1% |
189
+
190
+ - Very long strings stay uncompressed: `comps_*`, at 720 KB a row, saved 0%.
191
+ - A table you create is uncompressed: it lives in the default database.
192
+ - A binding's name is a view. `INSERT`, `ALTER`, `DROP TABLE` and
193
+ `CREATE OR REPLACE TABLE <binding>` fail: copy it into a table with another name.
194
+ - A CTE read twice is held whole in scratch, and no setting stops that. Write
195
+ `WITH u AS NOT MATERIALIZED (...)` over big bindings: 1.29.0 parses it too.
196
+ - `TIMESTAMPTZ` reads UTC, as it did on 1.29.0: the SDK runs
197
+ `SET GLOBAL TimeZone='UTC'` at boot.
198
+ - Boot also fetches ICU from extensions.duckdb.org. If that fails the boot fails,
199
+ and a hook retries at 5, 20 and 60 s.
200
+ - Row order changes where a query has no `ORDER BY`, and so does `any_value`.
201
+
176
202
  ### Memory across apps (2.13)
177
203
 
178
204
  Inside Mako, each app frame posts what its heap holds. Past 2 GiB across the
package/index.d.ts CHANGED
@@ -322,7 +322,7 @@ export interface BindingMemory {
322
322
  /** Every loaded binding, least recently read first. */
323
323
  bindings: {
324
324
  name: string;
325
- /** "table": decoded in the heap. "view": decoded from its parquet copy in the heap on each query. */
325
+ /** "table": held compressed in the heap. "view": decoded from its parquet copy in the heap on each query. */
326
326
  storage: "table" | "view";
327
327
  /** Read by a mounted hook or a query in flight: evicted last. */
328
328
  pinned: boolean;
package/index.js CHANGED
@@ -36,7 +36,7 @@ import {
36
36
  * in a page (or this export) says which SDK a deployed app actually runs.
37
37
  * Kept in step with package.json by snapshot.test.mjs.
38
38
  */
39
- export const SDK_VERSION = "2.13.0";
39
+ export const SDK_VERSION = "2.14.1";
40
40
  if (typeof window !== "undefined") {
41
41
  try {
42
42
  window.__MAKO_APP_SDK__ = SDK_VERSION;
@@ -50,7 +50,7 @@ const DEFAULT_ROW_LIMIT = 500000;
50
50
  // corrupts the engine for every panel (measured). 50 held.
51
51
  const MAX_BINDINGS_IN_QUERY = 50;
52
52
  const DUCKDB_ESM =
53
- "https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.29.0/+esm";
53
+ "https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.32.0/+esm";
54
54
 
55
55
  // ---------------------------------------------------------------------------
56
56
  // DuckDB: one lazy instance per page, bindings registered on first use. The
@@ -89,13 +89,36 @@ function getDb() {
89
89
  // read_parquet lives in an extension DuckDB fetches on first use
90
90
  // (~660 kB from extensions.duckdb.org). Load it now, while the
91
91
  // binding bytes are still downloading, not after they arrive.
92
- const conn = await db.connect();
93
92
  try {
94
- await conn.query("LOAD parquet");
95
- } catch {
96
- /* autoload on first read_parquet still applies */
97
- } finally {
98
- await conn.close();
93
+ const conn = await db.connect();
94
+ try {
95
+ try {
96
+ await conn.query("LOAD parquet");
97
+ } catch {
98
+ /* autoload on first read_parquet still applies */
99
+ }
100
+ // 1.32 reads TIMESTAMPTZ in the browser's zone once ICU loads. Apps were
101
+ // written against 1.29, which read UTC. Fetches ICU from
102
+ // extensions.duckdb.org, like parquet above. A failure here fails the
103
+ // boot on purpose: the next query retries, where a skipped SET would
104
+ // shift every timestamp without a word.
105
+ try {
106
+ await conn.query("SET GLOBAL TimeZone='UTC'");
107
+ } catch (error) {
108
+ if (error instanceof Error) error.retry = "network"; // the XHR for ICU
109
+ throw error;
110
+ }
111
+ // Binding tables live here, compressed at CHECKPOINT: 17 bytes a row
112
+ // against 289 on Web Performance's 7d set. Tables an app creates stay in
113
+ // the default database, uncompressed.
114
+ await conn.query("ATTACH ':memory:' AS mako_tables (COMPRESS)");
115
+ } finally {
116
+ await conn.close().catch(() => {});
117
+ }
118
+ } catch (error) {
119
+ // A failed boot retries with a new worker: end this one.
120
+ await db.terminate().catch(() => {});
121
+ throw error;
99
122
  }
100
123
  liveEngine = db;
101
124
  return db;
@@ -190,7 +213,7 @@ export function useBindingProgress(name) {
190
213
  // Each binding becomes a TABLE, decoded from its parquet once: a view over
191
214
  // read_parquet re-decoded the file on every query (SELECT * of a 149k-row
192
215
  // binding: 366 ms through a view, 174 ms from a table). The file buffer is
193
- // dropped as soon as the table exists. CREATE OR REPLACE swaps the new table
216
+ // dropped as soon as the table exists. CREATE OR REPLACE VIEW swaps the new table
194
217
  // in with one commit, so a query running during a reload reads the old rows
195
218
  // or the new ones, never a missing table. Every load registers its bytes
196
219
  // under its own file name, so a reload never drops a file another load is
@@ -303,30 +326,11 @@ async function downloadBinding(name, bust, report) {
303
326
  const what = 'Data for binding "' + name + '" could not be loaded';
304
327
  report({ phase: "requesting", rowsWritten: null, bytes: null, bytesTotal: null });
305
328
  // A slot holds the connection, not the wait for a build: one slow warehouse
306
- // query must not hold up twelve bindings that are ready.
307
- let res = await withSlot(() =>
308
- fetchRetrying(path + (bust ? "?refresh=" + Date.now() : ""), {
309
- headers: { accept: PROGRESS_ACCEPT },
310
- }),
311
- );
312
- let signed = null;
313
- if (isProgressResponse(res)) {
314
- let final;
315
- try {
316
- final = await readProgressStream(res, e => report(progressPatch(e)));
317
- } catch (error) {
318
- // A cut build stream may have cost a warehouse query. Asking again costs another.
319
- throw new Error(what + ": the build's stream was cut (" + (error instanceof Error ? error.message : String(error)) + ")");
320
- }
321
- if (final.phase === "failed") throw progressError(final, what);
322
- report({ phase: "downloading", rowsWritten: null, bytes: 0 });
323
- // No url: the server streams the bytes itself — the same path, asked
324
- // plainly, serves them (the build already staged or cached them).
325
- signed = final.url || path;
326
- res = null;
327
- }
328
- return withSlot(async () => {
329
- res ??= await fetchRetrying(signed);
329
+ // query must not hold up twelve bindings that are ready. A plain answer is read
330
+ // inside the slot that fetched it. Its body holds a connection, and over HTTP/1.1
331
+ // (6 per host, `vite dev`) queueing that body behind fetches waiting for one
332
+ // stalled Web Performance for good.
333
+ const readPlain = async res => {
330
334
  if (!res.ok) throw await responseError(res, what);
331
335
  try {
332
336
  return await readBytes(res, (bytes, bytesTotal) =>
@@ -336,7 +340,26 @@ async function downloadBinding(name, bust, report) {
336
340
  if (error instanceof TypeError) error.retry = "network"; // the body cut off: no build behind it
337
341
  throw error;
338
342
  }
343
+ };
344
+ const first = await withSlot(async () => {
345
+ const res = await fetchRetrying(path + (bust ? "?refresh=" + Date.now() : ""), {
346
+ headers: { accept: PROGRESS_ACCEPT },
347
+ });
348
+ return isProgressResponse(res) ? { stream: res } : { bytes: await readPlain(res) };
339
349
  });
350
+ if (!first.stream) return first.bytes;
351
+ let final;
352
+ try {
353
+ final = await readProgressStream(first.stream, e => report(progressPatch(e)));
354
+ } catch (error) {
355
+ // A cut build stream may have cost a warehouse query. Asking again costs another.
356
+ throw new Error(what + ": the build's stream was cut (" + (error instanceof Error ? error.message : String(error)) + ")");
357
+ }
358
+ if (final.phase === "failed") throw progressError(final, what);
359
+ report({ phase: "downloading", rowsWritten: null, bytes: 0 });
360
+ // No url: the server streams the bytes itself — the same path, asked
361
+ // plainly, serves them (the build already staged or cached them).
362
+ return withSlot(async () => readPlain(await fetchRetrying(final.url || path)));
340
363
  }
341
364
 
342
365
  // ---------------------------------------------------------------------------
@@ -442,6 +465,24 @@ function catalog(fn) {
442
465
  return run;
443
466
  }
444
467
 
468
+ // A trap in close() must not replace the query's own answer or error: a finally
469
+ // that throws wins over the catch, and the panel showed raw "null function".
470
+ // Nor may a close wait forever: on a trapped worker it can stay pending until
471
+ // restartEngine terminates that worker, and then it never settles. The hook and
472
+ // the query around it stayed counted in `busy` for the page's life, so every
473
+ // later release was skipped.
474
+ const CLOSE_WAIT_MS = 1000;
475
+ async function closeConn(conn, engine) {
476
+ let timer;
477
+ try {
478
+ await Promise.race([conn.close(), new Promise(resolve => (timer = setTimeout(resolve, CLOSE_WAIT_MS)))]);
479
+ } catch (error) {
480
+ markEngineError(error, engine);
481
+ } finally {
482
+ clearTimeout(timer);
483
+ }
484
+ }
485
+
445
486
  // The statements in one transaction, so another connection never sees the
446
487
  // name missing between a DROP and a CREATE.
447
488
  async function exec(engine, statements) {
@@ -462,7 +503,7 @@ async function exec(engine, statements) {
462
503
  } catch (error) {
463
504
  throw markEngineError(error, engine);
464
505
  } finally {
465
- await conn.close();
506
+ await closeConn(conn, engine);
466
507
  }
467
508
  }
468
509
 
@@ -483,10 +524,6 @@ function markEngineError(error, engine) {
483
524
  }
484
525
  const isEngineOom = error => Boolean(error?.fromEngine) && isOutOfMemoryError(error);
485
526
 
486
- function dropSql(storage, name) {
487
- return "DROP " + (storage === "view" ? "VIEW" : "TABLE") + " IF EXISTS " + quoteIdent(name);
488
- }
489
-
490
527
  let fileSeq = 0;
491
528
  // Every load registers its bytes under its own file name, so a reload never
492
529
  // drops a file another load is still reading.
@@ -494,14 +531,48 @@ function nextFile() {
494
531
  return "mako-" + ++fileSeq + ".parquet";
495
532
  }
496
533
 
497
- /** The table for a registered parquet `file`, in place of whatever `name` was. */
534
+ // name -> its table in mako_tables. Each load gets a fresh table, so a view
535
+ // never points at a table whose columns changed under it.
536
+ const stores = new Map();
537
+ let storeSeq = 0;
538
+ function storeSql(store) {
539
+ return "mako_tables.main." + quoteIdent(store);
540
+ }
541
+
542
+ /**
543
+ * Drop the compressed table behind `name`, if any. Runs after the view has moved,
544
+ * so a failure costs memory until the page reloads and never fails the load.
545
+ */
546
+ async function dropStore(engine, name) {
547
+ const store = stores.get(name);
548
+ if (!store) return;
549
+ stores.delete(name);
550
+ await exec(engine, ["DROP TABLE IF EXISTS " + storeSql(store)]).catch(() => {});
551
+ }
552
+
553
+ /**
554
+ * A registered parquet `file` decoded into a compressed table, and a view named
555
+ * `name` over it. The name is always a view, so CREATE OR REPLACE VIEW swaps it
556
+ * atomically. A transaction cannot write to two databases, so the order keeps it
557
+ * readable: new table, then the view, then the old table goes.
558
+ */
498
559
  async function tableFromFile(engine, name, file) {
499
560
  const prev = entries.get(name);
500
561
  const before = await duckdbBytes(engine);
501
- const create =
502
- "CREATE OR REPLACE TABLE " + quoteIdent(name) +
503
- " AS SELECT * FROM read_parquet(" + quoteString(file) + ")";
504
- await exec(engine, prev && prev.storage === "view" ? [dropSql("view", name), create] : [create]);
562
+ const store = "t" + ++storeSeq;
563
+ await exec(engine, ["CREATE TABLE " + storeSql(store) + " AS SELECT * FROM read_parquet(" + quoteString(file) + ")"]);
564
+ try {
565
+ // Only a CHECKPOINT compresses. It fails while a transaction is open on
566
+ // another connection: the table stays readable, and the next load's
567
+ // CHECKPOINT compresses it.
568
+ await exec(engine, ["CHECKPOINT mako_tables"]).catch(() => {});
569
+ await exec(engine, ["CREATE OR REPLACE VIEW " + quoteIdent(name) + " AS SELECT * FROM " + storeSql(store)]);
570
+ } catch (error) {
571
+ await exec(engine, ["DROP TABLE IF EXISTS " + storeSql(store)]).catch(() => {});
572
+ throw error;
573
+ }
574
+ await dropStore(engine, name);
575
+ stores.set(name, store);
505
576
  const after = await duckdbBytes(engine);
506
577
  if (before === null || after === null) return prev?.tableBytes ?? null;
507
578
  const grown = after - before + (prev?.storage === "table" ? (prev.tableBytes ?? 0) : 0);
@@ -514,7 +585,8 @@ async function viewFromFile(engine, name, file) {
514
585
  const create =
515
586
  "CREATE OR REPLACE VIEW " + quoteIdent(name) +
516
587
  " AS SELECT * FROM read_parquet(" + quoteString(file) + ")";
517
- await exec(engine, prev && prev.storage === "table" ? [dropSql("table", name), create] : [create]);
588
+ await exec(engine, [create]);
589
+ await dropStore(engine, name);
518
590
  }
519
591
 
520
592
  /**
@@ -536,7 +608,7 @@ async function estimateTable(engine, file, parquetBytes) {
536
608
  } catch {
537
609
  /* no metadata to read: the file's own error surfaces when it is read */
538
610
  } finally {
539
- await conn.close();
611
+ await closeConn(conn, engine);
540
612
  }
541
613
  return parquetBytes * 16;
542
614
  }
@@ -582,7 +654,7 @@ async function duckdbBytes(engine) {
582
654
  markEngineError(error, engine);
583
655
  return null; /* the OOM retry still applies */
584
656
  } finally {
585
- await conn.close();
657
+ await closeConn(conn, engine);
586
658
  }
587
659
  }
588
660
 
@@ -653,6 +725,7 @@ async function releaseEngine() {
653
725
  // engine must not restart the next one.
654
726
  for (const [name, entry] of entries) if (entry.blob) parked.set(name, entry.blob);
655
727
  entries.clear();
728
+ stores.clear();
656
729
  registered.clear();
657
730
  loadedBindings.clear();
658
731
  liveEngine = null;
@@ -699,7 +772,8 @@ function leastRecent(test, pinnedToo = false) {
699
772
  async function dropNow(engine, name) {
700
773
  const entry = entries.get(name);
701
774
  if (!entry) return;
702
- await exec(engine, [dropSql(entry.storage, name)]);
775
+ await exec(engine, ["DROP VIEW IF EXISTS " + quoteIdent(name)]);
776
+ await dropStore(engine, name);
703
777
  if (entry.file) await engine.dropFile(entry.file).catch(() => {});
704
778
  entries.delete(name);
705
779
  loadedBindings.delete(name);
@@ -880,7 +954,11 @@ async function placeNow(engine, name, buf, blob) {
880
954
  return { storage: "view", file, fileBytes: size, demoted: true };
881
955
  };
882
956
  const asTable = async () => {
957
+ // The table is decoded in full before it compresses, and a WASM heap never
958
+ // gives that back. The commit compresses, so no later measure can see it.
959
+ const decoded = await estimateTable(engine, file, blob.size);
883
960
  const tableBytes = await tableFromFile(engine, name, file);
961
+ heapBytes = Math.max(heapBytes, measuredBytes + decoded);
884
962
  await engine.dropFile(file).catch(() => {});
885
963
  if (prev?.file) await engine.dropFile(prev.file).catch(() => {});
886
964
  return { storage: "table", file: null, fileBytes: 0, demoted: false, tableBytes };
@@ -954,7 +1032,9 @@ async function promote(name) {
954
1032
  entry.demoted = true;
955
1033
  return;
956
1034
  }
1035
+ const decoded = await estimateTable(engine, entry.file, entry.fileBytes ?? 0);
957
1036
  const tableBytes = await tableFromFile(engine, name, entry.file);
1037
+ heapBytes = Math.max(heapBytes, measuredBytes + decoded); // decoded before it compressed
958
1038
  await engine.dropFile(entry.file).catch(() => {});
959
1039
  entries.set(name, { ...entry, storage: "table", file: null, fileBytes: 0, demoted: false, tableBytes });
960
1040
  await trimNow(engine, name);
@@ -1071,7 +1151,7 @@ export async function withConnection(fn) {
1071
1151
  try {
1072
1152
  return await fn(conn);
1073
1153
  } finally {
1074
- await conn.close();
1154
+ await closeConn(conn, db);
1075
1155
  }
1076
1156
  } finally {
1077
1157
  busy--;
@@ -1340,7 +1420,25 @@ async function runSql(sql, rowLimit, format, reads) {
1340
1420
  }
1341
1421
  }
1342
1422
 
1423
+ // The first query to trap an engine is its culprit: the others in flight only
1424
+ // failed because the engine was already gone. Twice first means it traps every
1425
+ // engine it runs on, so it stops running. Without this a mounted panel reruns on
1426
+ // each restart, traps that engine too and takes every other panel down with it,
1427
+ // until the 3 restarts are spent and the page stays dead. A changed query is a
1428
+ // different string, so a new filter still runs.
1429
+ const CULPRIT_LIMIT = 2;
1430
+ const culprits = new Map();
1431
+ let blamedEngine = null;
1432
+ function blameFirstTrap(sql, error, engine) {
1433
+ if (!crashedEngineError(error) || blamedEngine === engine) return;
1434
+ blamedEngine = engine;
1435
+ culprits.set(sql, (culprits.get(sql) ?? 0) + 1);
1436
+ }
1437
+
1343
1438
  async function runSqlOnce(sql, rowLimit, format) {
1439
+ if ((culprits.get(sql) ?? 0) >= CULPRIT_LIMIT) {
1440
+ throw new Error("This query crashed the data engine twice, so it stopped running. Change the filter or reload the page.");
1441
+ }
1344
1442
  const cap = rowLimit === null ? Infinity : (rowLimit ?? DEFAULT_ROW_LIMIT);
1345
1443
  const writes = !isReadOnlySql(sql);
1346
1444
  if (writes) keepEngine(); // CREATE TABLE journal_t: only this worker has it
@@ -1361,9 +1459,10 @@ async function runSqlOnce(sql, rowLimit, format) {
1361
1459
  const { rows, fields } = toPlainRows(table, cap);
1362
1460
  return { rows, fields, rowCount, truncated: rowCount > rows.length };
1363
1461
  } catch (error) {
1462
+ blameFirstTrap(sql, error, db);
1364
1463
  throw markEngineError(error, db);
1365
1464
  } finally {
1366
- await conn.close();
1465
+ await closeConn(conn, db);
1367
1466
  }
1368
1467
  } finally {
1369
1468
  busy--;
@@ -1392,6 +1491,7 @@ function restartEngine() {
1392
1491
  dbPromise = null;
1393
1492
  catalogQueue = Promise.resolve();
1394
1493
  entries.clear();
1494
+ stores.clear();
1395
1495
  registered.clear();
1396
1496
  loadedBindings.clear();
1397
1497
  measuredBytes = 0;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@makoai/app-sdk",
3
- "version": "2.13.0",
3
+ "version": "2.14.1",
4
4
  "description": "Mako app SDK: data bindings (useQuery/useDuckDB), URL state, theme, and the house dashboard kit (/ui).",
5
5
  "license": "MIT",
6
6
  "repository": {