@makoai/app-sdk 2.13.0 → 2.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -0
- package/index.d.ts +1 -1
- package/index.js +83 -20
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -173,6 +173,32 @@ for the whole heap, and `pressure`.
|
|
|
173
173
|
- A failed `__data/index.json` is kept for 10 s. A query that needs no binding still
|
|
174
174
|
answers. A query that fails then shows the index failure and retries.
|
|
175
175
|
|
|
176
|
+
### Compressed tables on DuckDB 1.4 (2.14)
|
|
177
|
+
|
|
178
|
+
The engine moved from duckdb-wasm 1.29.0 (DuckDB 1.1.1) to 1.32.0 (DuckDB 1.4.3).
|
|
179
|
+
Binding tables live in `ATTACH ':memory:' (COMPRESS)`, and a view with the
|
|
180
|
+
binding's name reads them.
|
|
181
|
+
|
|
182
|
+
| Stored as tables, real parquet | 1.29.0 | 1.32.0 | Saved |
|
|
183
|
+
| --- | --- | --- | --- |
|
|
184
|
+
| Web Performance 7d, 10.9M rows | 3,003 MiB | 176 MiB | 94.1% |
|
|
185
|
+
| Traffic Performance, 2.4M rows | 977 MiB | 75 MiB | 92.4% |
|
|
186
|
+
| Financial Model, 489k rows | 153 MiB | 26 MiB | 83.1% |
|
|
187
|
+
| INTL Sales, 1.9M rows | 601 MiB | 109 MiB | 81.9% |
|
|
188
|
+
| Who's Looking, 198k rows | 33 MiB | 10 MiB | 69.1% |
|
|
189
|
+
|
|
190
|
+
- Very long strings stay uncompressed: `comps_*`, at 720 KB a row, saved 0%.
|
|
191
|
+
- A table you create is uncompressed: it lives in the default database.
|
|
192
|
+
- A binding's name is a view. `INSERT`, `ALTER`, `DROP TABLE` and
|
|
193
|
+
`CREATE OR REPLACE TABLE <binding>` fail: copy it into a table with another name.
|
|
194
|
+
- A CTE read twice is held whole in scratch, and no setting stops that. Write
|
|
195
|
+
`WITH u AS NOT MATERIALIZED (...)` over big bindings: 1.29.0 parses it too.
|
|
196
|
+
- `TIMESTAMPTZ` reads UTC, as it did on 1.29.0: the SDK runs
|
|
197
|
+
`SET GLOBAL TimeZone='UTC'` at boot.
|
|
198
|
+
- Boot also fetches ICU from extensions.duckdb.org. If that fails the boot fails,
|
|
199
|
+
and a hook retries at 5, 20 and 60 s.
|
|
200
|
+
- Row order changes where a query has no `ORDER BY`, and so does `any_value`.
|
|
201
|
+
|
|
176
202
|
### Memory across apps (2.13)
|
|
177
203
|
|
|
178
204
|
Inside Mako, each app frame posts what its heap holds. Past 2 GiB across the
|
package/index.d.ts
CHANGED
|
@@ -322,7 +322,7 @@ export interface BindingMemory {
|
|
|
322
322
|
/** Every loaded binding, least recently read first. */
|
|
323
323
|
bindings: {
|
|
324
324
|
name: string;
|
|
325
|
-
/** "table":
|
|
325
|
+
/** "table": held compressed in the heap. "view": decoded from its parquet copy in the heap on each query. */
|
|
326
326
|
storage: "table" | "view";
|
|
327
327
|
/** Read by a mounted hook or a query in flight: evicted last. */
|
|
328
328
|
pinned: boolean;
|
package/index.js
CHANGED
|
@@ -36,7 +36,7 @@ import {
|
|
|
36
36
|
* in a page (or this export) says which SDK a deployed app actually runs.
|
|
37
37
|
* Kept in step with package.json by snapshot.test.mjs.
|
|
38
38
|
*/
|
|
39
|
-
export const SDK_VERSION = "2.
|
|
39
|
+
export const SDK_VERSION = "2.14.0";
|
|
40
40
|
if (typeof window !== "undefined") {
|
|
41
41
|
try {
|
|
42
42
|
window.__MAKO_APP_SDK__ = SDK_VERSION;
|
|
@@ -50,7 +50,7 @@ const DEFAULT_ROW_LIMIT = 500000;
|
|
|
50
50
|
// corrupts the engine for every panel (measured). 50 held.
|
|
51
51
|
const MAX_BINDINGS_IN_QUERY = 50;
|
|
52
52
|
const DUCKDB_ESM =
|
|
53
|
-
"https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.
|
|
53
|
+
"https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.32.0/+esm";
|
|
54
54
|
|
|
55
55
|
// ---------------------------------------------------------------------------
|
|
56
56
|
// DuckDB: one lazy instance per page, bindings registered on first use. The
|
|
@@ -89,13 +89,36 @@ function getDb() {
|
|
|
89
89
|
// read_parquet lives in an extension DuckDB fetches on first use
|
|
90
90
|
// (~660 kB from extensions.duckdb.org). Load it now, while the
|
|
91
91
|
// binding bytes are still downloading, not after they arrive.
|
|
92
|
-
const conn = await db.connect();
|
|
93
92
|
try {
|
|
94
|
-
await
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
93
|
+
const conn = await db.connect();
|
|
94
|
+
try {
|
|
95
|
+
try {
|
|
96
|
+
await conn.query("LOAD parquet");
|
|
97
|
+
} catch {
|
|
98
|
+
/* autoload on first read_parquet still applies */
|
|
99
|
+
}
|
|
100
|
+
// 1.32 reads TIMESTAMPTZ in the browser's zone once ICU loads. Apps were
|
|
101
|
+
// written against 1.29, which read UTC. Fetches ICU from
|
|
102
|
+
// extensions.duckdb.org, like parquet above. A failure here fails the
|
|
103
|
+
// boot on purpose: the next query retries, where a skipped SET would
|
|
104
|
+
// shift every timestamp without a word.
|
|
105
|
+
try {
|
|
106
|
+
await conn.query("SET GLOBAL TimeZone='UTC'");
|
|
107
|
+
} catch (error) {
|
|
108
|
+
if (error instanceof Error) error.retry = "network"; // the XHR for ICU
|
|
109
|
+
throw error;
|
|
110
|
+
}
|
|
111
|
+
// Binding tables live here, compressed at CHECKPOINT: 17 bytes a row
|
|
112
|
+
// against 289 on Web Performance's 7d set. Tables an app creates stay in
|
|
113
|
+
// the default database, uncompressed.
|
|
114
|
+
await conn.query("ATTACH ':memory:' AS mako_tables (COMPRESS)");
|
|
115
|
+
} finally {
|
|
116
|
+
await conn.close();
|
|
117
|
+
}
|
|
118
|
+
} catch (error) {
|
|
119
|
+
// A failed boot retries with a new worker: end this one.
|
|
120
|
+
await db.terminate().catch(() => {});
|
|
121
|
+
throw error;
|
|
99
122
|
}
|
|
100
123
|
liveEngine = db;
|
|
101
124
|
return db;
|
|
@@ -190,7 +213,7 @@ export function useBindingProgress(name) {
|
|
|
190
213
|
// Each binding becomes a TABLE, decoded from its parquet once: a view over
|
|
191
214
|
// read_parquet re-decoded the file on every query (SELECT * of a 149k-row
|
|
192
215
|
// binding: 366 ms through a view, 174 ms from a table). The file buffer is
|
|
193
|
-
// dropped as soon as the table exists. CREATE OR REPLACE swaps the new table
|
|
216
|
+
// dropped as soon as the table exists. CREATE OR REPLACE VIEW swaps the new table
|
|
194
217
|
// in with one commit, so a query running during a reload reads the old rows
|
|
195
218
|
// or the new ones, never a missing table. Every load registers its bytes
|
|
196
219
|
// under its own file name, so a reload never drops a file another load is
|
|
@@ -483,10 +506,6 @@ function markEngineError(error, engine) {
|
|
|
483
506
|
}
|
|
484
507
|
const isEngineOom = error => Boolean(error?.fromEngine) && isOutOfMemoryError(error);
|
|
485
508
|
|
|
486
|
-
function dropSql(storage, name) {
|
|
487
|
-
return "DROP " + (storage === "view" ? "VIEW" : "TABLE") + " IF EXISTS " + quoteIdent(name);
|
|
488
|
-
}
|
|
489
|
-
|
|
490
509
|
let fileSeq = 0;
|
|
491
510
|
// Every load registers its bytes under its own file name, so a reload never
|
|
492
511
|
// drops a file another load is still reading.
|
|
@@ -494,14 +513,48 @@ function nextFile() {
|
|
|
494
513
|
return "mako-" + ++fileSeq + ".parquet";
|
|
495
514
|
}
|
|
496
515
|
|
|
497
|
-
|
|
516
|
+
// name -> its table in mako_tables. Each load gets a fresh table, so a view
|
|
517
|
+
// never points at a table whose columns changed under it.
|
|
518
|
+
const stores = new Map();
|
|
519
|
+
let storeSeq = 0;
|
|
520
|
+
function storeSql(store) {
|
|
521
|
+
return "mako_tables.main." + quoteIdent(store);
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
/**
|
|
525
|
+
* Drop the compressed table behind `name`, if any. Runs after the view has moved,
|
|
526
|
+
* so a failure costs memory until the page reloads and never fails the load.
|
|
527
|
+
*/
|
|
528
|
+
async function dropStore(engine, name) {
|
|
529
|
+
const store = stores.get(name);
|
|
530
|
+
if (!store) return;
|
|
531
|
+
stores.delete(name);
|
|
532
|
+
await exec(engine, ["DROP TABLE IF EXISTS " + storeSql(store)]).catch(() => {});
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
/**
|
|
536
|
+
* A registered parquet `file` decoded into a compressed table, and a view named
|
|
537
|
+
* `name` over it. The name is always a view, so CREATE OR REPLACE VIEW swaps it
|
|
538
|
+
* atomically. A transaction cannot write to two databases, so the order keeps it
|
|
539
|
+
* readable: new table, then the view, then the old table goes.
|
|
540
|
+
*/
|
|
498
541
|
async function tableFromFile(engine, name, file) {
|
|
499
542
|
const prev = entries.get(name);
|
|
500
543
|
const before = await duckdbBytes(engine);
|
|
501
|
-
const
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
544
|
+
const store = "t" + ++storeSeq;
|
|
545
|
+
await exec(engine, ["CREATE TABLE " + storeSql(store) + " AS SELECT * FROM read_parquet(" + quoteString(file) + ")"]);
|
|
546
|
+
try {
|
|
547
|
+
// Only a CHECKPOINT compresses. It fails while a transaction is open on
|
|
548
|
+
// another connection: the table stays readable, and the next load's
|
|
549
|
+
// CHECKPOINT compresses it.
|
|
550
|
+
await exec(engine, ["CHECKPOINT mako_tables"]).catch(() => {});
|
|
551
|
+
await exec(engine, ["CREATE OR REPLACE VIEW " + quoteIdent(name) + " AS SELECT * FROM " + storeSql(store)]);
|
|
552
|
+
} catch (error) {
|
|
553
|
+
await exec(engine, ["DROP TABLE IF EXISTS " + storeSql(store)]).catch(() => {});
|
|
554
|
+
throw error;
|
|
555
|
+
}
|
|
556
|
+
await dropStore(engine, name);
|
|
557
|
+
stores.set(name, store);
|
|
505
558
|
const after = await duckdbBytes(engine);
|
|
506
559
|
if (before === null || after === null) return prev?.tableBytes ?? null;
|
|
507
560
|
const grown = after - before + (prev?.storage === "table" ? (prev.tableBytes ?? 0) : 0);
|
|
@@ -514,7 +567,8 @@ async function viewFromFile(engine, name, file) {
|
|
|
514
567
|
const create =
|
|
515
568
|
"CREATE OR REPLACE VIEW " + quoteIdent(name) +
|
|
516
569
|
" AS SELECT * FROM read_parquet(" + quoteString(file) + ")";
|
|
517
|
-
await exec(engine,
|
|
570
|
+
await exec(engine, [create]);
|
|
571
|
+
await dropStore(engine, name);
|
|
518
572
|
}
|
|
519
573
|
|
|
520
574
|
/**
|
|
@@ -653,6 +707,7 @@ async function releaseEngine() {
|
|
|
653
707
|
// engine must not restart the next one.
|
|
654
708
|
for (const [name, entry] of entries) if (entry.blob) parked.set(name, entry.blob);
|
|
655
709
|
entries.clear();
|
|
710
|
+
stores.clear();
|
|
656
711
|
registered.clear();
|
|
657
712
|
loadedBindings.clear();
|
|
658
713
|
liveEngine = null;
|
|
@@ -699,7 +754,8 @@ function leastRecent(test, pinnedToo = false) {
|
|
|
699
754
|
async function dropNow(engine, name) {
|
|
700
755
|
const entry = entries.get(name);
|
|
701
756
|
if (!entry) return;
|
|
702
|
-
await exec(engine, [
|
|
757
|
+
await exec(engine, ["DROP VIEW IF EXISTS " + quoteIdent(name)]);
|
|
758
|
+
await dropStore(engine, name);
|
|
703
759
|
if (entry.file) await engine.dropFile(entry.file).catch(() => {});
|
|
704
760
|
entries.delete(name);
|
|
705
761
|
loadedBindings.delete(name);
|
|
@@ -880,7 +936,11 @@ async function placeNow(engine, name, buf, blob) {
|
|
|
880
936
|
return { storage: "view", file, fileBytes: size, demoted: true };
|
|
881
937
|
};
|
|
882
938
|
const asTable = async () => {
|
|
939
|
+
// The table is decoded in full before it compresses, and a WASM heap never
|
|
940
|
+
// gives that back. The commit compresses, so no later measure can see it.
|
|
941
|
+
const decoded = await estimateTable(engine, file, blob.size);
|
|
883
942
|
const tableBytes = await tableFromFile(engine, name, file);
|
|
943
|
+
heapBytes = Math.max(heapBytes, measuredBytes + decoded);
|
|
884
944
|
await engine.dropFile(file).catch(() => {});
|
|
885
945
|
if (prev?.file) await engine.dropFile(prev.file).catch(() => {});
|
|
886
946
|
return { storage: "table", file: null, fileBytes: 0, demoted: false, tableBytes };
|
|
@@ -954,7 +1014,9 @@ async function promote(name) {
|
|
|
954
1014
|
entry.demoted = true;
|
|
955
1015
|
return;
|
|
956
1016
|
}
|
|
1017
|
+
const decoded = await estimateTable(engine, entry.file, entry.fileBytes ?? 0);
|
|
957
1018
|
const tableBytes = await tableFromFile(engine, name, entry.file);
|
|
1019
|
+
heapBytes = Math.max(heapBytes, measuredBytes + decoded); // decoded before it compressed
|
|
958
1020
|
await engine.dropFile(entry.file).catch(() => {});
|
|
959
1021
|
entries.set(name, { ...entry, storage: "table", file: null, fileBytes: 0, demoted: false, tableBytes });
|
|
960
1022
|
await trimNow(engine, name);
|
|
@@ -1392,6 +1454,7 @@ function restartEngine() {
|
|
|
1392
1454
|
dbPromise = null;
|
|
1393
1455
|
catalogQueue = Promise.resolve();
|
|
1394
1456
|
entries.clear();
|
|
1457
|
+
stores.clear();
|
|
1395
1458
|
registered.clear();
|
|
1396
1459
|
loadedBindings.clear();
|
|
1397
1460
|
measuredBytes = 0;
|