@makoai/app-sdk 2.7.4 → 2.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -29,8 +29,8 @@ import { useQuery, useDuckDB, useSearchParams, useTheme } from "@makoai/app-sdk"
29
29
  // Rows of bindings/<name>.sql, materialized to parquet by Mako.
30
30
  const { data, loading, error } = useQuery("latest_sales");
31
31
 
32
- // Analytical SQL over every binding (DuckDB-WASM in the browser; table
33
- // names are binding names).
32
+ // Analytical SQL over the bindings (DuckDB-WASM in the browser; table
33
+ // names are binding names). Only the bindings the SQL names download.
34
34
  const totals = useDuckDB("select country, sum(revenue) r from latest_sales group by 1");
35
35
 
36
36
  // Rematerialize on demand: re-runs the binding's query on the warehouse,
@@ -55,6 +55,38 @@ or 502 with the query's error.
55
55
  `useTheme` follows the OS preference. Theme tokens (`--background`,
56
56
  `--chart-1`, …) match the ones the scaffold's `styles.css` declares.
57
57
 
58
+ ### Custom data work: one engine (2.8)
59
+
60
+ Never import `@duckdb/duckdb-wasm`, `hyparquet` or `apache-arrow` parquet
61
+ readers in an app. Use the page's engine through the SDK:
62
+
63
+ ```ts
64
+ import { getDuckDB, withConnection, ensureBinding, unloadBinding, fetchBindingBytes, useDuckDB } from "@makoai/app-sdk";
65
+
66
+ const big = useDuckDB("select lat, lng from listings", { format: "arrow" }); // data: Arrow Table
67
+ const table = await ensureBinding("valuations"); // loaded once, returns "valuations"
68
+ const n = await withConnection(c => c.query(`select count(*) from ${table}`));
69
+ await unloadBinding("valuations"); // free its memory
70
+ const bytes = await fetchBindingBytes("valuations", { onProgress: p => p && console.log(p.phase, p.bytes, p.bytesTotal) });
71
+ const db = await getDuckDB(); // shared: never close() or terminate()
72
+ await db.registerFileBuffer("valuations.parquet", bytes); // read_parquet with your own options
73
+ ```
74
+
75
+ ### Loading cost (2.8)
76
+
77
+ - `useDuckDB` downloads only the bindings its SQL names. A missed name (for
78
+ example `query_table('x')`) loads when DuckDB reports the table missing,
79
+ and the query re-runs. `SHOW TABLES` / `information_schema` load all.
80
+ - Each binding is decoded once into a DuckDB table and its file buffer is
81
+ freed. A refresh swaps the table in one commit, so a running query never
82
+ sees it missing. A table costs about 3x the parquet in memory (10 MB of
83
+ parquet, 149k rows: 33 MB).
84
+ - The DuckDB engine and its parquet extension start loading on the first
85
+ data hook render, in parallel with the first binding download.
86
+ - Rows decode column by column: 149k rows x 16 columns in 190 ms, down
87
+ from 740 ms. Values are unchanged (DATE as `YYYY-MM-DD`, TIMESTAMP as
88
+ ISO 8601, BigInt as Number when safe).
89
+
58
90
  Data arrives from `__data/<name>.parquet`, relative to the page — the same
59
91
  path in Mako's sandbox, in a published app, and on a laptop.
60
92
 
@@ -231,9 +263,11 @@ once, and polls it until the parquet is ready — so a query that runs for
231
263
  minutes is waited for instead of failing when a proxy cuts the request at
232
264
  100 s. `pollIntervalMs` / `buildTimeoutMs` tune the wait (1 s, 30 min).
233
265
 
234
- Results are cached under `node_modules/.mako-data/` for five minutes
235
- (`revalidateMs`; `?refresh` bypasses), next to a fingerprint of the text they
236
- were built from: after an edit the cache is never served, however long
266
+ Results are cached under `node_modules/.mako-data/` next to a fingerprint of
267
+ the text they were built from. A cached copy of the current text is served
268
+ at once. Once it is older than `revalidateMs` (five minutes), it is also
269
+ rebuilt in the background for the next load. `?refresh` waits for a
270
+ build. A copy from other text is never served: after an edit the cache is never served, however long
237
271
  `revalidateMs` is, and while the API is unreachable a stale copy is served
238
272
  only if it was built from the same text. It is `apply: "serve"` only —
239
273
  production builds never load it.
package/data-core.js ADDED
@@ -0,0 +1,212 @@
1
+ // Pure helpers behind @makoai/app-sdk's data hooks — no React and no
2
+ // DuckDB, so `node --test` can cover them without installing anything.
3
+
4
+ /** A SQL identifier, double-quoted (DuckDB folds case either way). */
5
+ export function quoteIdent(name) {
6
+ return '"' + String(name).replace(/"/g, '""') + '"';
7
+ }
8
+
9
+ /** A SQL string literal. */
10
+ export function quoteString(value) {
11
+ return "'" + String(value).replace(/'/g, "''") + "'";
12
+ }
13
+
14
+ // String literals and comments first (skipped), then quoted and bare
15
+ // identifiers (kept).
16
+ const SQL_TOKENS = /'(?:[^']|'')*'|--[^\n]*|\/\*[\s\S]*?\*\/|"((?:[^"]|"")+)"|([A-Za-z_][A-Za-z0-9_$]*)/g;
17
+
18
+ /**
19
+ * The staged bindings whose names appear as identifiers in `sql`. A hint,
20
+ * not a parse: a column that shares a binding's name costs one extra
21
+ * download; a reference it misses is caught by the retry on DuckDB's
22
+ * "Table with name … does not exist" error.
23
+ */
24
+ export function mentionedBindings(sql, names) {
25
+ if (!names.length) return [];
26
+ const byLower = new Map();
27
+ for (const n of names) {
28
+ const key = n.toLowerCase();
29
+ if (!byLower.has(key)) byLower.set(key, n);
30
+ }
31
+ const found = new Set();
32
+ for (const m of String(sql).matchAll(SQL_TOKENS)) {
33
+ const ident = m[1] != null ? m[1].replace(/""/g, '"') : m[2];
34
+ if (ident == null) continue;
35
+ const hit = byLower.get(ident.toLowerCase());
36
+ if (hit) found.add(hit);
37
+ }
38
+ return [...found];
39
+ }
40
+
41
+ // Queries that list tables instead of naming them (SHOW TABLES,
42
+ // information_schema, duckdb_tables() …) see every binding, as they did when
43
+ // all of them were registered up front.
44
+ const CATALOG_QUERY = /\b(?:information_schema|duckdb_(?:tables|columns|views|constraints|indexes)|sqlite_master|pg_catalog|show_tables)\b|\bshow\s+(?:all\s+)?tables\b/i;
45
+ export function readsCatalog(sql) {
46
+ return CATALOG_QUERY.test(String(sql));
47
+ }
48
+
49
+ /**
50
+ * The binding a DuckDB "Table with name X does not exist" error is about —
51
+ * X itself when the index lists it, else its case-insensitive match (DuckDB
52
+ * folds identifier case). null for any other error, or a name no binding has.
53
+ */
54
+ export function missingBinding(message, names) {
55
+ const m = /Table with name (.+?) does not exist/.exec(String(message));
56
+ if (!m) return null;
57
+ const table = m[1];
58
+ if (names.includes(table)) return table;
59
+ const lower = table.toLowerCase();
60
+ return names.find(n => n.toLowerCase() === lower) ?? null;
61
+ }
62
+
63
+ // Arrow → plain JS, with the types apps actually want. DuckDB hands DATE and
64
+ // TIMESTAMP columns to JS as epoch milliseconds; v1 apps (JSON over the wire)
65
+ // always saw strings, and a migrated formatDate() that did
66
+ // new Date(`${d}T00:00:00`) on a number threw and unmounted the whole app
67
+ // (apps.md §15.4). So: DATE → "YYYY-MM-DD", TIMESTAMP → ISO 8601, BigInt →
68
+ // Number when it fits. Everything else passes through untouched.
69
+ const DATE = 1;
70
+ const TIMESTAMP = 2;
71
+ const OTHER = 0;
72
+ function columnKind(field) {
73
+ const type = String(field.type);
74
+ if (type.startsWith("Date")) return DATE;
75
+ if (type.startsWith("Timestamp")) return TIMESTAMP;
76
+ return OTHER;
77
+ }
78
+
79
+ // Fast paths for the two column types that dominate decode time. Each
80
+ // reads Arrow's buffers directly and yields exactly what Arrow's getter
81
+ // would; a chunk it does not fully understand returns null and the column
82
+ // takes the general path (Arrow's own iterator).
83
+ const UTF8 = 5; // arrow Type.Utf8
84
+ const TIMESTAMP_TYPE = 10; // arrow Type.Timestamp
85
+ const MAX_SAFE = Number.MAX_SAFE_INTEGER;
86
+ let textDecoder = null;
87
+ function plainChunks(source, typeId) {
88
+ const chunks = source.data;
89
+ if (!Array.isArray(chunks) || !chunks.length) return null;
90
+ for (const d of chunks) {
91
+ if (!d || !d.type || d.type.typeId !== typeId || d.offset !== 0) return null;
92
+ if (d.nullCount > 0 && typeof d.getValid !== "function") return null;
93
+ }
94
+ return chunks;
95
+ }
96
+
97
+ // Utf8: Arrow runs a TextDecoder per cell. A chunk whose bytes are all
98
+ // ASCII decodes in one call and slices by offset — the same strings.
99
+ function asciiColumn(source, n) {
100
+ const chunks = plainChunks(source, UTF8);
101
+ if (!chunks) return null;
102
+ textDecoder ??= new TextDecoder();
103
+ const out = new Array(n);
104
+ let r = 0;
105
+ for (const d of chunks) {
106
+ const { values, valueOffsets, length } = d;
107
+ if (!(values instanceof Uint8Array) || !(valueOffsets instanceof Int32Array)) return null;
108
+ const start = valueOffsets[0];
109
+ const end = valueOffsets[length];
110
+ for (let i = start; i < end; i++) if (values[i] > 0x7f) return null;
111
+ const text = textDecoder.decode(values.subarray(start, end));
112
+ const nulls = d.nullCount > 0;
113
+ for (let i = 0; i < length && r < n; i++) {
114
+ out[r++] = nulls && !d.getValid(i)
115
+ ? null
116
+ : text.slice(valueOffsets[i] - start, valueOffsets[i + 1] - start);
117
+ }
118
+ }
119
+ return out;
120
+ }
121
+
122
+ // Timestamp: Arrow divides BigInts per cell (getTimestampMicrosecond is
123
+ // `Number(v / 1000n) + Number(v % 1000n) / 1000`). Within ±2^53 the same
124
+ // quotient and remainder are exact in doubles, so the same two IEEE
125
+ // operations give the same millisecond value. A value outside that range
126
+ // sends the column to the general path.
127
+ function timestampColumn(source, n) {
128
+ const chunks = plainChunks(source, TIMESTAMP_TYPE);
129
+ if (!chunks) return null;
130
+ const out = new Array(n);
131
+ let r = 0;
132
+ for (const d of chunks) {
133
+ const { values, length } = d;
134
+ if (!(values instanceof BigInt64Array)) return null;
135
+ const unit = d.type.unit; // 0 s, 1 ms, 2 µs, 3 ns
136
+ if (unit !== 0 && unit !== 1 && unit !== 2 && unit !== 3) return null;
137
+ const nulls = d.nullCount > 0;
138
+ for (let i = 0; i < length && r < n; i++) {
139
+ if (nulls && !d.getValid(i)) {
140
+ out[r++] = null;
141
+ continue;
142
+ }
143
+ const v = Number(values[i]);
144
+ if (v > MAX_SAFE || v < -MAX_SAFE) return null;
145
+ let ms;
146
+ if (unit === 0) ms = 1000 * v;
147
+ else if (unit === 1) ms = v;
148
+ else {
149
+ const div = unit === 2 ? 1000 : 1000000;
150
+ const rem = v % div;
151
+ ms = (v - rem) / div + rem / div;
152
+ }
153
+ out[r++] = new Date(ms).toISOString();
154
+ }
155
+ }
156
+ return out;
157
+ }
158
+
159
+ /**
160
+ * An Arrow table as `{ rows, fields }`: one plain object per row, keyed by
161
+ * column name, values normalized as above, at most `cap` rows.
162
+ *
163
+ * Column by column: each column's values come from Arrow's own vector
164
+ * iterator (the same getter a row proxy's `row[name]` calls), so the output
165
+ * is identical to walking `for (const row of table)` — without a Proxy trap
166
+ * and a field-name lookup per cell, and with each distinct DATE formatted
167
+ * once. 149k rows × 16 columns: 740 ms before, 190 ms after (Chromium).
168
+ */
169
+ export function toPlainRows(table, cap) {
170
+ const schemaFields = table.schema.fields;
171
+ const fields = schemaFields.map(f => f.name);
172
+ const limit = cap == null ? Infinity : cap;
173
+ const n = Math.max(0, Math.min(table.numRows, limit));
174
+ const rows = new Array(n);
175
+ for (let r = 0; r < n; r++) rows[r] = {};
176
+ for (let c = 0; c < fields.length; c++) {
177
+ const name = fields[c];
178
+ // A row proxy resolves a name to its FIRST column; a duplicated name
179
+ // reads that column's values (decoded with this column's type).
180
+ const source = table.getChildAt(fields.indexOf(name));
181
+ const kind = columnKind(schemaFields[c]);
182
+ let r = 0;
183
+ if (n === 0 || !source) {
184
+ for (; r < n; r++) rows[r][name] = undefined;
185
+ continue;
186
+ }
187
+ const fast =
188
+ kind === OTHER ? asciiColumn(source, n) : kind === TIMESTAMP ? timestampColumn(source, n) : null;
189
+ if (fast) {
190
+ for (; r < n; r++) rows[r][name] = fast[r];
191
+ continue;
192
+ }
193
+ const dates = kind === DATE ? new Map() : null;
194
+ for (const v of source) {
195
+ if (r >= n) break;
196
+ let out = v;
197
+ if (v != null) {
198
+ if (kind === DATE) {
199
+ out = dates.get(v);
200
+ if (out === undefined) {
201
+ out = new Date(Number(v)).toISOString().slice(0, 10);
202
+ dates.set(v, out);
203
+ }
204
+ } else if (kind === TIMESTAMP) out = new Date(Number(v)).toISOString();
205
+ else if (typeof v === "bigint" && v >= Number.MIN_SAFE_INTEGER && v <= Number.MAX_SAFE_INTEGER)
206
+ out = Number(v);
207
+ }
208
+ rows[r++][name] = out;
209
+ }
210
+ }
211
+ return { rows, fields };
212
+ }
package/index.d.ts CHANGED
@@ -135,6 +135,31 @@ export interface DuckDBState<Row = Record<string, unknown>>
135
135
  export interface QueryOptions {
136
136
  rowLimit?: number | null;
137
137
  }
138
+ export interface DuckDBOptions extends QueryOptions {
139
+ /**
140
+ * "arrow": `data` is DuckDB's Arrow Table as returned (no row objects, no
141
+ * date/BigInt normalization, `rowLimit` ignored). Default "rows".
142
+ */
143
+ format?: "rows" | "arrow";
144
+ }
145
+
146
+ /**
147
+ * The page's shared DuckDB-WASM engine (`AsyncDuckDB` from
148
+ * @duckdb/duckdb-wasm 1.29), typed loosely so the SDK needs no dependency.
149
+ * Never close() or terminate() it: every hook on the page reads from it.
150
+ */
151
+ export interface SharedDuckDB {
152
+ connect(): Promise<DuckDBConnection>;
153
+ registerFileBuffer(name: string, buffer: Uint8Array): Promise<void>;
154
+ dropFile(name: string): Promise<void>;
155
+ [key: string]: unknown;
156
+ }
157
+ export interface DuckDBConnection {
158
+ /** Resolves with an Arrow Table. */
159
+ query(sql: string): Promise<any>;
160
+ close(): Promise<void>;
161
+ [key: string]: unknown;
162
+ }
138
163
  export interface MakoLocation {
139
164
  pathname: string;
140
165
  search: string;
@@ -153,10 +178,16 @@ export function useQuery<Row = Record<string, unknown>>(
153
178
  progress: BindingProgress | null;
154
179
  };
155
180
 
156
- /** Analytical SQL over the app's bindings; table names are binding names. */
181
+ /**
182
+ * Analytical SQL over the app's bindings; table names are binding names.
183
+ * Since 2.8 only the bindings the SQL names are downloaded (each decoded
184
+ * once into a DuckDB table); one it misses loads on DuckDB's "table does
185
+ * not exist" and the query re-runs. Catalog queries (SHOW TABLES,
186
+ * information_schema) load every binding.
187
+ */
157
188
  export function useDuckDB<Row = Record<string, unknown>>(
158
189
  sql: string,
159
- opts?: QueryOptions,
190
+ opts?: QueryOptions & { format?: "rows" },
160
191
  ): DuckDBState<Row> & {
161
192
  refresh: () => Promise<RefreshResult[]>;
162
193
  /** The slowest pending binding's wait, with sums; null when none. */
@@ -164,6 +195,18 @@ export function useDuckDB<Row = Record<string, unknown>>(
164
195
  /** Every pending binding's progress, by name. */
165
196
  bindings: Record<string, BindingProgress>;
166
197
  };
198
+ export function useDuckDB(
199
+ sql: string,
200
+ opts: DuckDBOptions & { format: "arrow" },
201
+ ): Omit<DuckDBState, "data"> & {
202
+ /** The Arrow Table (apache-arrow 17) DuckDB returned. */
203
+ data: any | null;
204
+ refresh: () => Promise<RefreshResult[]>;
205
+ /** The slowest pending binding's wait, with sums; null when none. */
206
+ progress: CombinedBindingProgress | null;
207
+ /** Every pending binding's progress, by name. */
208
+ bindings: Record<string, BindingProgress>;
209
+ };
167
210
 
168
211
  export interface RefreshOptions {
169
212
  /**
@@ -173,6 +216,42 @@ export interface RefreshOptions {
173
216
  onProgress?: (progress: BindingProgress | null) => void;
174
217
  }
175
218
 
219
+ /**
220
+ * The page's shared DuckDB, booted on first call. Use it for anything the
221
+ * hooks do not cover instead of importing @duckdb/duckdb-wasm, hyparquet or
222
+ * apache-arrow readers. Never close or terminate it.
223
+ */
224
+ export function getDuckDB(): Promise<SharedDuckDB>;
225
+
226
+ /** Run `fn` on a fresh connection to the shared DuckDB; closed afterwards. */
227
+ export function withConnection<T>(fn: (conn: DuckDBConnection) => Promise<T>): Promise<T>;
228
+
229
+ /**
230
+ * Load a binding into the shared DuckDB as a table (once per page;
231
+ * concurrent callers share the download). Resolves with the table name.
232
+ */
233
+ export function ensureBinding(name: string): Promise<string>;
234
+
235
+ /** Drop a loaded binding's table to free memory; the next read reloads it. */
236
+ export function unloadBinding(name: string): Promise<void>;
237
+
238
+ /**
239
+ * The binding's raw parquet bytes, not registered anywhere — for an app with
240
+ * its own reader (register them via getDuckDB().registerFileBuffer to use
241
+ * read_parquet options). Rejects like a failed load (`status`, message).
242
+ */
243
+ export function fetchBindingBytes(
244
+ name: string,
245
+ opts?: {
246
+ fresh?: boolean;
247
+ /**
248
+ * The same phases a hook reports (requesting → resolving/signing or a
249
+ * build → downloading with `bytes` / `bytesTotal`), then `null`.
250
+ */
251
+ onProgress?: (progress: BindingProgress | null) => void;
252
+ },
253
+ ): Promise<Uint8Array>;
254
+
176
255
  /**
177
256
  * Rematerialize one binding on demand and reload it everywhere it is used.
178
257
  * Concurrent calls for the same binding share one request.
package/index.js CHANGED
@@ -16,6 +16,15 @@ export {
16
16
  bindingPhaseStage,
17
17
  } from "./progress.js";
18
18
 
19
+ import {
20
+ missingBinding,
21
+ mentionedBindings,
22
+ quoteIdent,
23
+ readsCatalog,
24
+ quoteString,
25
+ toPlainRows,
26
+ } from "./data-core.js";
27
+
19
28
  const DEFAULT_ROW_LIMIT = 500000;
20
29
  const DUCKDB_ESM =
21
30
  "https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.29.0/+esm";
@@ -23,21 +32,48 @@ const DUCKDB_ESM =
23
32
  // ---------------------------------------------------------------------------
24
33
  // DuckDB: one lazy instance per page, bindings registered on first use. The
25
34
  // engine arrives from the CDN at runtime, so this package needs no install
26
- // step and no dependencies of its own.
35
+ // step and no dependencies of its own. The first data hook to render starts
36
+ // the boot (WASM download + instantiate, ~1 s) so it overlaps the first
37
+ // parquet download instead of waiting for it.
27
38
  // ---------------------------------------------------------------------------
28
39
  let dbPromise = null;
29
40
  function getDb() {
30
- dbPromise ??= (async () => {
31
- const duckdb = await import(/* @vite-ignore */ DUCKDB_ESM);
32
- const bundle = await duckdb.selectBundle(duckdb.getJsDelivrBundles());
33
- const worker = await duckdb.createWorker(bundle.mainWorker);
34
- const db = new duckdb.AsyncDuckDB(new duckdb.VoidLogger(), worker);
35
- await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
36
- return db;
37
- })();
41
+ if (!dbPromise) {
42
+ dbPromise = (async () => {
43
+ const duckdb = await import(/* @vite-ignore */ DUCKDB_ESM);
44
+ const bundle = await duckdb.selectBundle(duckdb.getJsDelivrBundles());
45
+ const worker = await duckdb.createWorker(bundle.mainWorker);
46
+ const db = new duckdb.AsyncDuckDB(new duckdb.VoidLogger(), worker);
47
+ await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
48
+ // read_parquet lives in an extension DuckDB fetches on first use
49
+ // (~660 kB from extensions.duckdb.org). Load it now, while the
50
+ // binding bytes are still downloading, not after they arrive.
51
+ const conn = await db.connect();
52
+ try {
53
+ await conn.query("LOAD parquet");
54
+ } catch {
55
+ /* autoload on first read_parquet still applies */
56
+ } finally {
57
+ await conn.close();
58
+ }
59
+ return db;
60
+ })();
61
+ // A failed boot (CDN unreachable) is retried by the next query.
62
+ dbPromise.catch(() => {
63
+ dbPromise = null;
64
+ });
65
+ }
38
66
  return dbPromise;
39
67
  }
40
68
 
69
+ /** Start the engine early; a no-op outside a browser and once started. */
70
+ function bootEngine() {
71
+ if (typeof window === "undefined" || typeof Worker === "undefined") return;
72
+ getDb().catch(() => {
73
+ /* the query that needs it reports the failure */
74
+ });
75
+ }
76
+
41
77
  // The server answers failures with JSON { error, hint, retryAfterMs } —
42
78
  // show that, not a bare status code (and never let it reach DuckDB, where it
43
79
  // would surface as a baffling Catalog Error).
@@ -107,42 +143,64 @@ export function useBindingProgress(name) {
107
143
  return name === undefined ? all : (all[name] ?? null);
108
144
  }
109
145
 
110
- async function loadBinding(name, bust) {
146
+ // Each binding becomes a TABLE, decoded from its parquet once: a view over
147
+ // read_parquet re-decoded the file on every query (SELECT * of a 149k-row
148
+ // binding: 366 ms through a view, 174 ms from a table). The file buffer is
149
+ // dropped as soon as the table exists. CREATE OR REPLACE swaps the new table
150
+ // in with one commit, so a query running during a reload reads the old rows
151
+ // or the new ones, never a missing table. Every load registers its bytes
152
+ // under its own file name, so a reload never drops a file another load is
153
+ // still reading.
154
+
155
+ /**
156
+ * A binding's parquet bytes, phase-aware: asks for NDJSON phase events (a
157
+ * build reports each step, a stored file resolving → signing and a signed
158
+ * URL) and reports them through `report(patch)`. A server that predates the
159
+ * events answers with the bytes, as it always did.
160
+ */
161
+ async function downloadBinding(name, bust, report) {
111
162
  const path = "__data/" + encodeURIComponent(name) + ".parquet";
112
163
  const what = 'Data for binding "' + name + '" could not be loaded';
113
- setProgress(name, { phase: "requesting", rowsWritten: null, bytes: null, bytesTotal: null });
164
+ report({ phase: "requesting", rowsWritten: null, bytes: null, bytesTotal: null });
165
+ let res = await fetch(path + (bust ? "?refresh=" + Date.now() : ""), {
166
+ headers: { accept: PROGRESS_ACCEPT },
167
+ });
168
+ if (isProgressResponse(res)) {
169
+ const final = await readProgressStream(res, e => report(progressPatch(e)));
170
+ if (final.phase === "failed") throw progressError(final, what);
171
+ report({ phase: "downloading", rowsWritten: null, bytes: 0 });
172
+ // No url: the server streams the bytes itself — the same path, asked
173
+ // plainly, serves them (the build already staged or cached them).
174
+ res = await fetch(final.url || path);
175
+ }
176
+ if (!res.ok) throw await responseError(res, what);
177
+ return readBytes(res, (bytes, bytesTotal) =>
178
+ report({ phase: "downloading", rowsWritten: null, bytes, bytesTotal }),
179
+ );
180
+ }
181
+
182
+ let fileSeq = 0;
183
+ async function loadBinding(name, bust) {
184
+ const db = getDb(); // boot in parallel with the download
185
+ db.catch(() => {});
114
186
  try {
115
- // Ask for phase events: a build (laptop, sandbox live binding) reports
116
- // each step, a stored file reports resolving → signing and hands back
117
- // the signed URL. A server that predates them answers as it always did.
118
- let res = await fetch(path + (bust ? "?refresh=" + Date.now() : ""), {
119
- headers: { accept: PROGRESS_ACCEPT },
120
- });
121
- if (isProgressResponse(res)) {
122
- const final = await readProgressStream(res, e => setProgress(name, progressPatch(e)));
123
- if (final.phase === "failed") throw progressError(final, what);
124
- setProgress(name, { phase: "downloading", rowsWritten: null, bytes: 0 });
125
- // No url: the server streams the bytes itself — the same path, asked
126
- // plainly, serves them (the build already staged or cached them).
127
- res = await fetch(final.url || path);
128
- }
129
- if (!res.ok) throw await responseError(res, what);
130
- const buf = await readBytes(res, (bytes, bytesTotal) =>
131
- setProgress(name, { phase: "downloading", rowsWritten: null, bytes, bytesTotal }),
132
- );
187
+ const buf = await downloadBinding(name, bust, patch => setProgress(name, patch));
133
188
  setProgress(name, { phase: "loading_duckdb" });
134
- const db = await getDb();
135
- if (bust) await db.dropFile(name + ".parquet").catch(() => {});
136
- await db.registerFileBuffer(name + ".parquet", buf);
137
- const conn = await db.connect();
189
+ const engine = await db;
190
+ const file = "mako-" + ++fileSeq + ".parquet";
191
+ await engine.registerFileBuffer(file, buf);
138
192
  try {
139
- // A view per binding so SQL can name tables by binding name.
140
- await conn.query(
141
- 'CREATE OR REPLACE VIEW "' + name.replace(/"/g, '""') +
142
- "\" AS SELECT * FROM read_parquet('" + name + ".parquet')",
143
- );
193
+ const conn = await engine.connect();
194
+ try {
195
+ await conn.query(
196
+ "CREATE OR REPLACE TABLE " + quoteIdent(name) +
197
+ " AS SELECT * FROM read_parquet(" + quoteString(file) + ")",
198
+ );
199
+ } finally {
200
+ await conn.close();
201
+ }
144
202
  } finally {
145
- await conn.close();
203
+ await engine.dropFile(file).catch(() => {});
146
204
  }
147
205
  } finally {
148
206
  // A refresh owns the record until its reload is done (refreshBinding).
@@ -151,27 +209,103 @@ async function loadBinding(name, bust) {
151
209
  }
152
210
 
153
211
  const registered = new Map(); // name -> Promise<void>
212
+ // Refreshed on the server while never loaded on this page: the next load
213
+ // must skip any cache between here and the server.
214
+ const staleBindings = new Set();
154
215
  function registerBinding(name) {
155
216
  if (!registered.has(name)) {
156
- const load = loadBinding(name, false);
217
+ const load = loadBinding(name, staleBindings.delete(name));
157
218
  registered.set(name, load);
158
- load.catch(() => registered.delete(name));
219
+ load.catch(() => {
220
+ if (registered.get(name) === load) registered.delete(name);
221
+ });
159
222
  }
160
223
  return registered.get(name);
161
224
  }
162
225
 
226
+ // ---------------------------------------------------------------------------
227
+ // Escape hatches — the same engine and the same loader the hooks use, for
228
+ // apps that need more than useQuery/useDuckDB. One DuckDB per page: an app
229
+ // that boots its own pays a second ~1 s boot and a second copy of the data.
230
+ // ---------------------------------------------------------------------------
231
+
232
+ /**
233
+ * The page's shared DuckDB (an AsyncDuckDB), booted on first call. Never
234
+ * close or terminate it: every hook on the page reads from it.
235
+ */
236
+ export function getDuckDB() {
237
+ return getDb();
238
+ }
239
+
240
+ /** Run `fn` with a fresh connection to the shared DuckDB; always closed after. */
241
+ export async function withConnection(fn) {
242
+ const db = await getDb();
243
+ const conn = await db.connect();
244
+ try {
245
+ return await fn(conn);
246
+ } finally {
247
+ await conn.close();
248
+ }
249
+ }
250
+
251
+ /**
252
+ * Make sure a binding's table exists in the shared DuckDB, loading it once
253
+ * per page (concurrent callers share the download). Resolves with the table
254
+ * name — the binding name; quote it in SQL when it is not a bare identifier.
255
+ */
256
+ export async function ensureBinding(name) {
257
+ await registerBinding(name);
258
+ return name;
259
+ }
260
+
261
+ /**
262
+ * Forget a loaded binding: drops its table so its memory is freed. The next
263
+ * query or ensureBinding() downloads it again.
264
+ */
265
+ export async function unloadBinding(name) {
266
+ const pending = registered.get(name);
267
+ if (!pending) return;
268
+ registered.delete(name);
269
+ await pending.catch(() => {});
270
+ await withConnection(conn => conn.query("DROP TABLE IF EXISTS " + quoteIdent(name)));
271
+ }
272
+
273
+ /**
274
+ * The binding's parquet bytes, for an app with its own reader (e.g.
275
+ * read_parquet options such as file_row_number). Not registered anywhere.
276
+ * `fresh` skips caches; `onProgress(loaded, total)` reports the download.
277
+ */
278
+ export async function fetchBindingBytes(name, opts) {
279
+ const fresh = !!(opts && opts.fresh) || staleBindings.has(name);
280
+ const onProgress = opts && opts.onProgress;
281
+ let record = null;
282
+ try {
283
+ return await downloadBinding(name, fresh, patch => {
284
+ record = { ...record, ...patch };
285
+ if (onProgress) onProgress(record);
286
+ });
287
+ } finally {
288
+ if (onProgress) onProgress(null);
289
+ }
290
+ }
291
+
163
292
  /** Replace a binding's loaded bytes with what the server has now. */
164
293
  async function reloadBinding(name) {
294
+ const previous = registered.get(name);
295
+ if (!previous) {
296
+ // Nothing on this page reads it yet: load it fresh when a query does.
297
+ staleBindings.add(name);
298
+ return;
299
+ }
165
300
  // Let a load in flight settle first, or its (older) bytes could land after
166
301
  // the fresh ones and win.
167
- const previous = registered.get(name);
168
- if (previous) await previous.catch(() => {});
302
+ await previous.catch(() => {});
169
303
  const load = loadBinding(name, true);
170
304
  registered.set(name, load);
171
305
  try {
172
306
  await load;
173
307
  } catch (error) {
174
- registered.delete(name);
308
+ if (registered.get(name) === load) registered.delete(name);
175
309
  throw error;
176
310
  }
177
311
  }
@@ -342,49 +476,18 @@ function bindingIndex() {
342
476
  return indexPromise;
343
477
  }
344
478
 
345
- // Arrow → plain JS, with the types apps actually want. DuckDB hands DATE and
346
- // TIMESTAMP columns to JS as epoch milliseconds; v1 apps (JSON over the wire)
347
- // always saw strings, and a migrated formatDate() that did
348
- // new Date(`${d}T00:00:00`) on a number threw and unmounted the whole app
349
- // (apps.md §15.4). So: DATE → "YYYY-MM-DD", TIMESTAMP → ISO 8601, BigInt →
350
- // Number when it fits. Everything else passes through untouched.
351
- function columnDecoder(field) {
352
- const type = String(field.type);
353
- if (type.startsWith("Date")) {
354
- return v => (v == null ? v : new Date(Number(v)).toISOString().slice(0, 10));
355
- }
356
- if (type.startsWith("Timestamp")) {
357
- return v => (v == null ? v : new Date(Number(v)).toISOString());
358
- }
359
- return v =>
360
- typeof v === "bigint" && v >= Number.MIN_SAFE_INTEGER && v <= Number.MAX_SAFE_INTEGER
361
- ? Number(v)
362
- : v;
363
- }
364
-
365
- function toPlainRows(table, cap) {
366
- const rows = [];
367
- const fields = table.schema.fields.map(f => f.name);
368
- const decoders = table.schema.fields.map(columnDecoder);
369
- const limit = cap == null ? Infinity : cap;
370
- for (const row of table) {
371
- if (rows.length >= limit) break;
372
- const out = {};
373
- for (let i = 0; i < fields.length; i++) {
374
- out[fields[i]] = decoders[i](row[fields[i]]);
375
- }
376
- rows.push(out);
377
- }
378
- return { rows, fields };
379
- }
380
-
381
- async function runSql(sql, rowLimit) {
479
+ async function runSql(sql, rowLimit, format) {
382
480
  const cap = rowLimit === null ? Infinity : (rowLimit ?? DEFAULT_ROW_LIMIT);
383
481
  const db = await getDb();
384
482
  const conn = await db.connect();
385
483
  try {
386
484
  const table = await conn.query(sql);
387
485
  const rowCount = table.numRows;
486
+ if (format === "arrow") {
487
+ // The Arrow Table as DuckDB returned it: no row objects, no
488
+ // normalization, no row limit.
489
+ return { rows: table, fields: table.schema.fields.map(f => f.name), rowCount, truncated: false };
490
+ }
388
491
  const { rows, fields } = toPlainRows(table, cap);
389
492
  return { rows, fields, rowCount, truncated: rowCount > rows.length };
390
493
  } finally {
@@ -453,14 +556,12 @@ function hookProgress(record, state) {
453
556
  }
454
557
 
455
558
  export function useQuery(name, opts) {
559
+ bootEngine();
456
560
  const rowLimit = opts ? opts.rowLimit : undefined;
457
561
  const state = useAsyncQuery(
458
562
  async () => {
459
563
  await registerBinding(name);
460
- const r = await runSql(
461
- 'SELECT * FROM "' + name.replace(/"/g, '""') + '"',
462
- rowLimit,
463
- );
564
+ const r = await runSql("SELECT * FROM " + quoteIdent(name), rowLimit);
464
565
  return { data: r.rows, fields: r.fields, truncated: r.truncated, rowCount: r.rowCount };
465
566
  },
466
567
  [name, rowLimit],
@@ -502,24 +603,39 @@ export function useViewer() {
502
603
  return state;
503
604
  }
504
605
 
505
- export function useDuckDB(sql, opts) {
506
- const rowLimit = opts ? opts.rowLimit : undefined;
507
- const state = useAsyncQuery(async () => {
508
- // Register everything the server staged, so SQL can join across
509
- // bindings by name without declaring them first. A binding that fails to
510
- // load must not sink the whole query set silently: remember why, and when
511
- // the SQL then fails (typically DuckDB's "Table … does not exist"),
512
- // surface the load failure — it is the actual cause.
513
- const names = await bindingIndex();
514
- const failures = new Map();
515
- await Promise.all(
516
- names.map(n => registerBinding(n).catch(e => failures.set(n, e))),
517
- );
606
+ /**
607
+ * Run `sql` with the bindings it reads loaded — and only those. The ones
608
+ * its text names start downloading together, alongside the engine boot; a
609
+ * table the hint missed shows up as DuckDB's "Table with name X does not
610
+ * exist", which loads X and runs the query again. Joins across bindings
611
+ * work by name without declaring anything, as before, and an app with 34
612
+ * bindings no longer downloads all 34 to draw one chart.
613
+ *
614
+ * A binding that fails to load must not sink the query silently: when the
615
+ * SQL then fails, the load failure is what surfaces — it is the cause.
616
+ */
617
+ async function runOverBindings(sql, rowLimit, format) {
618
+ const names = await bindingIndex();
619
+ const failures = new Map();
620
+ const attempted = new Set();
621
+ const load = n => {
622
+ attempted.add(n);
623
+ return registerBinding(n).catch(e => {
624
+ failures.set(n, e);
625
+ });
626
+ };
627
+ await Promise.all((readsCatalog(sql) ? names : mentionedBindings(sql, names)).map(load));
628
+ for (;;) {
518
629
  try {
519
- const r = await runSql(sql, rowLimit);
520
- return { data: r.rows, fields: r.fields, truncated: r.truncated, rowCount: r.rowCount };
630
+ return await runSql(sql, rowLimit, format);
521
631
  } catch (error) {
522
632
  const message = error instanceof Error ? error.message : String(error);
633
+ const missing = missingBinding(message, names);
634
+ if (missing && !attempted.has(missing)) {
635
+ await load(missing);
636
+ if (failures.has(missing)) throw failures.get(missing);
637
+ continue;
638
+ }
523
639
  for (const [n, cause] of failures) {
524
640
  if (message.includes(n)) throw cause;
525
641
  }
@@ -531,12 +647,23 @@ export function useDuckDB(sql, opts) {
531
647
  }
532
648
  throw error;
533
649
  }
534
- }, [sql, rowLimit], refreshing => refreshing.size > 0);
650
+ }
651
+ }
652
+
653
+ export function useDuckDB(sql, opts) {
654
+ bootEngine();
655
+ const rowLimit = opts ? opts.rowLimit : undefined;
656
+ const format = opts && opts.format === "arrow" ? "arrow" : "rows";
657
+ const state = useAsyncQuery(async () => {
658
+ const r = await runOverBindings(sql, rowLimit, format);
659
+ return { data: r.rows, fields: r.fields, truncated: r.truncated, rowCount: r.rowCount };
660
+ }, [sql, rowLimit, format], refreshing => refreshing.size > 0);
535
661
  // Every binding the query can read: the combined record says what the
536
662
  // slowest of them is waiting on; `bindings` has each one.
537
663
  const bindings = useBindingProgress();
538
664
  const combined = combineBindingProgress(bindings);
539
- // A query over every binding refreshes every binding.
665
+ // A query over every binding refreshes every binding (the ones this page
666
+ // has not loaded yet are rebuilt and fetched fresh when first read).
540
667
  const refresh = React.useCallback(() => refreshBindings(), []);
541
668
  return {
542
669
  ...state,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@makoai/app-sdk",
3
- "version": "2.7.4",
3
+ "version": "2.8.0",
4
4
  "description": "Mako app SDK: data bindings (useQuery/useDuckDB), URL state, theme, and the house dashboard kit (/ui).",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -42,6 +42,7 @@
42
42
  "index.js",
43
43
  "index.d.ts",
44
44
  "progress.js",
45
+ "data-core.js",
45
46
  "vite.js",
46
47
  "vite.d.ts",
47
48
  "credentials.js",
@@ -61,5 +62,8 @@
61
62
  },
62
63
  "scripts": {
63
64
  "test": "node --test"
65
+ },
66
+ "devDependencies": {
67
+ "apache-arrow": "17.0.0"
64
68
  }
65
69
  }
package/vite.d.ts CHANGED
@@ -12,9 +12,11 @@ export interface MakoDataOptions {
12
12
  /** Repo root. Default: the nearest ancestor holding .mako/ or .git. */
13
13
  repoRoot?: string;
14
14
  /**
15
- * Re-fetch a cached parquet after this long (ms). Default: 5 minutes. A
16
- * cache built from different binding text (or dbt environment) is never
17
- * served, whatever this says — `Infinity` is safe while you edit SQL.
15
+ * A cached parquet of the current binding text is always served at once;
16
+ * after this long (ms) it is also rebuilt in the background for the next
17
+ * load. Default: 5 minutes. A cache built from different binding text (or
18
+ * dbt environment) is never served — `Infinity` is safe while you edit
19
+ * SQL. `refresh()` / `?refresh` waits for the rebuild.
18
20
  */
19
21
  revalidateMs?: number;
20
22
  /**
package/vite.js CHANGED
@@ -388,9 +388,18 @@ export function makoData(options = {}) {
388
388
  }
389
389
  }
390
390
 
391
+ // Atomic: a background revalidation can land while a page is still
392
+ // streaming the previous copy, which keeps reading the old file.
391
393
  function writeCache(name, buf, meta) {
392
394
  fs.mkdirSync(ctx.cacheDir, { recursive: true });
393
- fs.writeFileSync(cachePath(name), buf);
395
+ const tmp = `${cachePath(name)}.${process.pid}.${Date.now()}.tmp`;
396
+ fs.writeFileSync(tmp, buf);
397
+ try {
398
+ fs.renameSync(tmp, cachePath(name));
399
+ } catch {
400
+ fs.rmSync(tmp, { force: true });
401
+ fs.writeFileSync(cachePath(name), buf);
402
+ }
394
403
  if (meta) fs.writeFileSync(metaPath(name), JSON.stringify(meta));
395
404
  else fs.rmSync(metaPath(name), { force: true });
396
405
  }
@@ -406,6 +415,24 @@ export function makoData(options = {}) {
406
415
  let devBuildUnsupported = false;
407
416
  const inflight = new Map();
408
417
 
418
+ // One background rebuild per binding at a time; a failure is logged and
419
+ // the next load tries again.
420
+ const revalidating = new Set();
421
+ function revalidate(name, source) {
422
+ if (revalidating.has(name)) return;
423
+ revalidating.add(name);
424
+ (async () => {
425
+ const built = source !== null ? (await devBuild(name, source, false))?.buf : null;
426
+ if (!built) writeCache(name, await fetchArtifact(name), null);
427
+ })()
428
+ .catch(error => {
429
+ server.config.logger.error(
430
+ ` mako-data: background revalidate of ${name}: ${error instanceof Error ? error.message : String(error)}`,
431
+ );
432
+ })
433
+ .finally(() => revalidating.delete(name));
434
+ }
435
+
409
436
  // Build the LOCAL binding text through the API. Concurrent asks for the
410
437
  // same text share one request (a page mounting several tables at once).
411
438
  function devBuild(name, source, refresh, onPhase) {
@@ -631,8 +658,15 @@ export function makoData(options = {}) {
631
658
  let stream = null;
632
659
  try {
633
660
  const stat = fs.statSync(cached, { throwIfNoEntry: false });
634
- const fresh = stat && Date.now() - stat.mtimeMs < revalidateMs;
635
- if (!refresh && fresh && cacheMatches()) return serveCached("cache");
661
+ // A copy built from this text is served at once, however old: past
662
+ // `revalidateMs` it is rebuilt in the background for the next load
663
+ // (stale-while-revalidate), so a page reload never waits on the API
664
+ // for data it already has. `?refresh` (the SDK's refresh()) waits.
665
+ if (!refresh && stat && cacheMatches()) {
666
+ const fresh = Date.now() - stat.mtimeMs < revalidateMs;
667
+ if (!fresh) revalidate(name, source);
668
+ return serveCached(fresh ? "cache" : "revalidating");
669
+ }
636
670
  if (wantsProgress(req)) {
637
671
  stream = progressStream(res);
638
672
  stream.emit({ phase: "checking" });