@makoai/app-sdk 2.7.4 → 2.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -29,8 +29,8 @@ import { useQuery, useDuckDB, useSearchParams, useTheme } from "@makoai/app-sdk"
29
29
  // Rows of bindings/<name>.sql, materialized to parquet by Mako.
30
30
  const { data, loading, error } = useQuery("latest_sales");
31
31
 
32
- // Analytical SQL over every binding (DuckDB-WASM in the browser; table
33
- // names are binding names).
32
+ // Analytical SQL over the bindings (DuckDB-WASM in the browser; table
33
+ // names are binding names). Only the bindings the SQL names download.
34
34
  const totals = useDuckDB("select country, sum(revenue) r from latest_sales group by 1");
35
35
 
36
36
  // Rematerialize on demand: re-runs the binding's query on the warehouse,
@@ -42,6 +42,34 @@ const { refresh, refreshing } = useQuery("latest_sales");
42
42
  </button>
43
43
  ```
44
44
 
45
+ ### Which SDK is running: `SDK_VERSION` (2.9+)
46
+
47
+ `import { SDK_VERSION } from "@makoai/app-sdk"` is this package's version.
48
+ Importing the SDK also sets `window.__MAKO_APP_SDK__` to it. In DevTools,
49
+ switch the console context to the app's frame and read
50
+ `__MAKO_APP_SDK__` to see which release a deployed app's build resolved.
51
+
52
+ ### Preview rows while the parquet loads (2.9+, opt-in)
53
+
54
+ ```tsx
55
+ const { data, loading, preview } = useQuery("latest_sales", { snapshot: true });
56
+ // loading && preview → paint preview.data as a skeleton table
57
+ // !loading → data is the full binding; preview is null
58
+ ```
59
+
60
+ Every build writes its first 500 rows (200 KiB cap) next to the parquet as
61
+ `__data/<name>.snapshot.json`. With `snapshot: true` the hook fetches it once
62
+ per page (skipped when the parquet is already loaded) and exposes it as
63
+ `preview` = `{ data, fields, rowCount, truncated, builtAt }`. `data` and
64
+ `loading` behave exactly as without the option, and a failed load clears
65
+ `preview`. Preview values have the same types as `data` (DATE →
66
+ `"YYYY-MM-DD"`, TIMESTAMP → ISO 8601, a 64-bit integer past 2^53 → BigInt).
67
+ The snapshot is the first N rows of the build stream in the query's own
68
+ order, with no `ORDER BY` guarantee. Use it for a skeleton or first paint.
69
+ Never aggregate it. `preview` is null when a binding has no snapshot (built
70
+ before 2.9's server release, live, a laptop draft still building, or a
71
+ DECIMAL/LIST/STRUCT column). Without the option no snapshot is fetched.
72
+
45
73
  `useDuckDB(...).refresh()` refreshes every binding; `refreshBinding(name)` /
46
74
  `refreshBindings()` do the same outside a component. A refresh POSTs to
47
75
  `__data/<name>/refresh`, the data URL's sibling, so whoever serves the app's
@@ -55,6 +83,38 @@ or 502 with the query's error.
55
83
  `useTheme` follows the OS preference. Theme tokens (`--background`,
56
84
  `--chart-1`, …) match the ones the scaffold's `styles.css` declares.
57
85
 
86
+ ### Custom data work: one engine (2.8)
87
+
88
+ Never import `@duckdb/duckdb-wasm`, `hyparquet` or `apache-arrow` parquet
89
+ readers in an app. Use the page's engine through the SDK:
90
+
91
+ ```ts
92
+ import { getDuckDB, withConnection, ensureBinding, unloadBinding, fetchBindingBytes, useDuckDB } from "@makoai/app-sdk";
93
+
94
+ const big = useDuckDB("select lat, lng from listings", { format: "arrow" }); // data: Arrow Table
95
+ const table = await ensureBinding("valuations"); // loaded once, returns "valuations"
96
+ const n = await withConnection(c => c.query(`select count(*) from ${table}`));
97
+ await unloadBinding("valuations"); // free its memory
98
+ const bytes = await fetchBindingBytes("valuations", { onProgress: p => p && console.log(p.phase, p.bytes, p.bytesTotal) });
99
+ const db = await getDuckDB(); // shared: never close() or terminate()
100
+ await db.registerFileBuffer("valuations.parquet", bytes); // read_parquet with your own options
101
+ ```
102
+
103
+ ### Loading cost (2.8)
104
+
105
+ - `useDuckDB` downloads only the bindings its SQL names. A missed name (for
106
+ example `query_table('x')`) loads when DuckDB reports the table missing,
107
+ and the query re-runs. `SHOW TABLES` / `information_schema` load all.
108
+ - Each binding is decoded once into a DuckDB table and its file buffer is
109
+ freed. A refresh swaps the table in one commit, so a running query never
110
+ sees it missing. A table costs about 3x the parquet in memory (10 MB of
111
+ parquet, 149k rows: 33 MB).
112
+ - The DuckDB engine and its parquet extension start loading on the first
113
+ data hook render, in parallel with the first binding download.
114
+ - Rows decode column by column: 149k rows x 16 columns in 190 ms, down
115
+ from 740 ms. Values are unchanged (DATE as `YYYY-MM-DD`, TIMESTAMP as
116
+ ISO 8601, BigInt as Number when safe).
117
+
58
118
  Data arrives from `__data/<name>.parquet`, relative to the page — the same
59
119
  path in Mako's sandbox, in a published app, and on a laptop.
60
120
 
@@ -214,8 +274,9 @@ import { makoData } from "@makoai/app-sdk/vite";
214
274
  export default defineConfig({ plugins: [react(), makoData()] });
215
275
  ```
216
276
 
217
- `makoData()` answers `__data/index.json` (the app's `bindings/*.sql`) and
218
- `__data/<name>.parquet` during `vite dev` from the Mako API, and what it asks
277
+ `makoData()` answers `__data/index.json` (the app's `bindings/*.sql`),
278
+ `__data/<name>.parquet` and `__data/<name>.snapshot.json` during `vite dev`
279
+ from the Mako API, and what it asks
219
280
  for is **your local binding file**: it sends the text of
220
281
  `bindings/<name>.sql` and gets back the parquet of exactly that query, run
221
282
  read-only through the workspace connection its front matter names. When the
@@ -225,15 +286,21 @@ from your text and hands it back without storing it — nobody else, and no
225
286
  published viewer, ever sees uncommitted SQL. Building needs edit access to the
226
287
  app; read-only members get committed artifacts. `POST __data/<name>/refresh`
227
288
  (the SDK's `refresh()`) rebuilds from the local text on demand.
289
+ `__data/<name>.snapshot.json` is asked for by the same text: the API answers
290
+ with the snapshot of exactly the stored artifact that text maps to, and 404
291
+ while a draft still needs building. An edited query never paints the
292
+ committed query's preview. Snapshots are not cached locally.
228
293
 
229
294
  Builds run as jobs: the plugin asks for one (`async`), gets a job id back at
230
295
  once, and polls it until the parquet is ready — so a query that runs for
231
296
  minutes is waited for instead of failing when a proxy cuts the request at
232
297
  100 s. `pollIntervalMs` / `buildTimeoutMs` tune the wait (1 s, 30 min).
233
298
 
234
- Results are cached under `node_modules/.mako-data/` for five minutes
235
- (`revalidateMs`; `?refresh` bypasses), next to a fingerprint of the text they
236
- were built from: after an edit the cache is never served, however long
299
+ Results are cached under `node_modules/.mako-data/` next to a fingerprint of
300
+ the text they were built from. A cached copy of the current text is served
301
+ at once. Once it is older than `revalidateMs` (five minutes), it is also
302
+ rebuilt in the background for the next load. `?refresh` waits for a
303
+ build. A copy from other text is never served: after an edit the cache is never served, however long
237
304
  `revalidateMs` is, and while the API is unreachable a stale copy is served
238
305
  only if it was built from the same text. It is `apply: "serve"` only —
239
306
  production builds never load it.
package/data-core.js ADDED
@@ -0,0 +1,300 @@
1
+ // Pure helpers behind @makoai/app-sdk's data hooks — no React and no
2
+ // DuckDB, so `node --test` can cover them without installing anything.
3
+
4
+ /** A SQL identifier, double-quoted (DuckDB folds case either way). */
5
+ export function quoteIdent(name) {
6
+ return '"' + String(name).replace(/"/g, '""') + '"';
7
+ }
8
+
9
+ /** A SQL string literal. */
10
+ export function quoteString(value) {
11
+ return "'" + String(value).replace(/'/g, "''") + "'";
12
+ }
13
+
14
+ // String literals and comments first (skipped), then quoted and bare
15
+ // identifiers (kept).
16
+ const SQL_TOKENS = /'(?:[^']|'')*'|--[^\n]*|\/\*[\s\S]*?\*\/|"((?:[^"]|"")+)"|([A-Za-z_][A-Za-z0-9_$]*)/g;
17
+
18
+ /**
19
+ * The staged bindings whose names appear as identifiers in `sql`. A hint,
20
+ * not a parse: a column that shares a binding's name costs one extra
21
+ * download; a reference it misses is caught by the retry on DuckDB's
22
+ * "Table with name … does not exist" error.
23
+ */
24
+ export function mentionedBindings(sql, names) {
25
+ if (!names.length) return [];
26
+ const byLower = new Map();
27
+ for (const n of names) {
28
+ const key = n.toLowerCase();
29
+ if (!byLower.has(key)) byLower.set(key, n);
30
+ }
31
+ const found = new Set();
32
+ for (const m of String(sql).matchAll(SQL_TOKENS)) {
33
+ const ident = m[1] != null ? m[1].replace(/""/g, '"') : m[2];
34
+ if (ident == null) continue;
35
+ const hit = byLower.get(ident.toLowerCase());
36
+ if (hit) found.add(hit);
37
+ }
38
+ return [...found];
39
+ }
40
+
41
+ // Queries that list tables instead of naming them (SHOW TABLES,
42
+ // information_schema, duckdb_tables() …) see every binding, as they did when
43
+ // all of them were registered up front.
44
+ const CATALOG_QUERY = /\b(?:information_schema|duckdb_(?:tables|columns|views|constraints|indexes)|sqlite_master|pg_catalog|show_tables)\b|\bshow\s+(?:all\s+)?tables\b/i;
45
+ export function readsCatalog(sql) {
46
+ return CATALOG_QUERY.test(String(sql));
47
+ }
48
+
49
+ /**
50
+ * The binding a DuckDB "Table with name X does not exist" error is about —
51
+ * X itself when the index lists it, else its case-insensitive match (DuckDB
52
+ * folds identifier case). null for any other error, or a name no binding has.
53
+ */
54
+ export function missingBinding(message, names) {
55
+ const m = /Table with name (.+?) does not exist/.exec(String(message));
56
+ if (!m) return null;
57
+ const table = m[1];
58
+ if (names.includes(table)) return table;
59
+ const lower = table.toLowerCase();
60
+ return names.find(n => n.toLowerCase() === lower) ?? null;
61
+ }
62
+
63
+ // Arrow → plain JS, with the types apps actually want. DuckDB hands DATE and
64
+ // TIMESTAMP columns to JS as epoch milliseconds; v1 apps (JSON over the wire)
65
+ // always saw strings, and a migrated formatDate() that did
66
+ // new Date(`${d}T00:00:00`) on a number threw and unmounted the whole app
67
+ // (apps.md §15.4). So: DATE → "YYYY-MM-DD", TIMESTAMP → ISO 8601, BigInt →
68
+ // Number when it fits. Everything else passes through untouched.
69
+ const DATE = 1;
70
+ const TIMESTAMP = 2;
71
+ const OTHER = 0;
72
+ function columnKind(field) {
73
+ const type = String(field.type);
74
+ if (type.startsWith("Date")) return DATE;
75
+ if (type.startsWith("Timestamp")) return TIMESTAMP;
76
+ return OTHER;
77
+ }
78
+
79
+ // Fast paths for the two column types that dominate decode time. Each
80
+ // reads Arrow's buffers directly and yields exactly what Arrow's getter
81
+ // would; a chunk it does not fully understand returns null and the column
82
+ // takes the general path (Arrow's own iterator).
83
+ const UTF8 = 5; // arrow Type.Utf8
84
+ const TIMESTAMP_TYPE = 10; // arrow Type.Timestamp
85
+ const MAX_SAFE = Number.MAX_SAFE_INTEGER;
86
+ let textDecoder = null;
87
+ function plainChunks(source, typeId) {
88
+ const chunks = source.data;
89
+ if (!Array.isArray(chunks) || !chunks.length) return null;
90
+ for (const d of chunks) {
91
+ if (!d || !d.type || d.type.typeId !== typeId || d.offset !== 0) return null;
92
+ if (d.nullCount > 0 && typeof d.getValid !== "function") return null;
93
+ }
94
+ return chunks;
95
+ }
96
+
97
+ // Utf8: Arrow runs a TextDecoder per cell. A chunk whose bytes are all
98
+ // ASCII decodes in one call and slices by offset — the same strings.
99
+ function asciiColumn(source, n) {
100
+ const chunks = plainChunks(source, UTF8);
101
+ if (!chunks) return null;
102
+ textDecoder ??= new TextDecoder();
103
+ const out = new Array(n);
104
+ let r = 0;
105
+ for (const d of chunks) {
106
+ const { values, valueOffsets, length } = d;
107
+ if (!(values instanceof Uint8Array) || !(valueOffsets instanceof Int32Array)) return null;
108
+ const start = valueOffsets[0];
109
+ const end = valueOffsets[length];
110
+ for (let i = start; i < end; i++) if (values[i] > 0x7f) return null;
111
+ const text = textDecoder.decode(values.subarray(start, end));
112
+ const nulls = d.nullCount > 0;
113
+ for (let i = 0; i < length && r < n; i++) {
114
+ out[r++] = nulls && !d.getValid(i)
115
+ ? null
116
+ : text.slice(valueOffsets[i] - start, valueOffsets[i + 1] - start);
117
+ }
118
+ }
119
+ return out;
120
+ }
121
+
122
+ // Timestamp: Arrow divides BigInts per cell (getTimestampMicrosecond is
123
+ // `Number(v / 1000n) + Number(v % 1000n) / 1000`). Within ±2^53 the same
124
+ // quotient and remainder are exact in doubles, so the same two IEEE
125
+ // operations give the same millisecond value. A value outside that range
126
+ // sends the column to the general path.
127
+ function timestampColumn(source, n) {
128
+ const chunks = plainChunks(source, TIMESTAMP_TYPE);
129
+ if (!chunks) return null;
130
+ const out = new Array(n);
131
+ let r = 0;
132
+ for (const d of chunks) {
133
+ const { values, length } = d;
134
+ if (!(values instanceof BigInt64Array)) return null;
135
+ const unit = d.type.unit; // 0 s, 1 ms, 2 µs, 3 ns
136
+ if (unit !== 0 && unit !== 1 && unit !== 2 && unit !== 3) return null;
137
+ const nulls = d.nullCount > 0;
138
+ for (let i = 0; i < length && r < n; i++) {
139
+ if (nulls && !d.getValid(i)) {
140
+ out[r++] = null;
141
+ continue;
142
+ }
143
+ const v = Number(values[i]);
144
+ if (v > MAX_SAFE || v < -MAX_SAFE) return null;
145
+ let ms;
146
+ if (unit === 0) ms = 1000 * v;
147
+ else if (unit === 1) ms = v;
148
+ else {
149
+ const div = unit === 2 ? 1000 : 1000000;
150
+ const rem = v % div;
151
+ ms = (v - rem) / div + rem / div;
152
+ }
153
+ out[r++] = new Date(ms).toISOString();
154
+ }
155
+ }
156
+ return out;
157
+ }
158
+
159
+ /**
160
+ * An Arrow table as `{ rows, fields }`: one plain object per row, keyed by
161
+ * column name, values normalized as above, at most `cap` rows.
162
+ *
163
+ * Column by column: each column's values come from Arrow's own vector
164
+ * iterator (the same getter a row proxy's `row[name]` calls), so the output
165
+ * is identical to walking `for (const row of table)` — without a Proxy trap
166
+ * and a field-name lookup per cell, and with each distinct DATE formatted
167
+ * once. 149k rows × 16 columns: 740 ms before, 190 ms after (Chromium).
168
+ */
169
+ export function toPlainRows(table, cap) {
170
+ const schemaFields = table.schema.fields;
171
+ const fields = schemaFields.map(f => f.name);
172
+ const limit = cap == null ? Infinity : cap;
173
+ const n = Math.max(0, Math.min(table.numRows, limit));
174
+ const rows = new Array(n);
175
+ for (let r = 0; r < n; r++) rows[r] = {};
176
+ for (let c = 0; c < fields.length; c++) {
177
+ const name = fields[c];
178
+ // A row proxy resolves a name to its FIRST column; a duplicated name
179
+ // reads that column's values (decoded with this column's type).
180
+ const source = table.getChildAt(fields.indexOf(name));
181
+ const kind = columnKind(schemaFields[c]);
182
+ let r = 0;
183
+ if (n === 0 || !source) {
184
+ for (; r < n; r++) rows[r][name] = undefined;
185
+ continue;
186
+ }
187
+ const fast =
188
+ kind === OTHER ? asciiColumn(source, n) : kind === TIMESTAMP ? timestampColumn(source, n) : null;
189
+ if (fast) {
190
+ for (; r < n; r++) rows[r][name] = fast[r];
191
+ continue;
192
+ }
193
+ const dates = kind === DATE ? new Map() : null;
194
+ for (const v of source) {
195
+ if (r >= n) break;
196
+ let out = v;
197
+ if (v != null) {
198
+ if (kind === DATE) {
199
+ out = dates.get(v);
200
+ if (out === undefined) {
201
+ out = new Date(Number(v)).toISOString().slice(0, 10);
202
+ dates.set(v, out);
203
+ }
204
+ } else if (kind === TIMESTAMP) out = new Date(Number(v)).toISOString();
205
+ else if (typeof v === "bigint" && v >= Number.MIN_SAFE_INTEGER && v <= Number.MAX_SAFE_INTEGER)
206
+ out = Number(v);
207
+ }
208
+ rows[r++][name] = out;
209
+ }
210
+ }
211
+ return { rows, fields };
212
+ }
213
+
214
+ // ---------------------------------------------------------------------------
215
+ // Snapshot: `__data/<name>.snapshot.json`, the first N rows of the build
216
+ // stream, written next to the parquet (api/src/services/binding-snapshot.ts).
217
+ // The server already applied toPlainRows's rules (DATE → "YYYY-MM-DD",
218
+ // TIMESTAMP → ISO 8601). JSON cannot carry two things the parquet path can
219
+ // produce, so the server writes them as strings and this decoder restores
220
+ // them: a 64-bit integer past 2^53 (BigInt) and a non-finite float (NaN,
221
+ // ±Infinity). The server writes no snapshot at all for a column type JSON
222
+ // cannot reproduce (DECIMAL, LIST, STRUCT, …), and any type this decoder does
223
+ // not know rejects the whole snapshot — the app then waits for the parquet.
224
+ // ---------------------------------------------------------------------------
225
+ export const SNAPSHOT_VERSION = 1;
226
+
227
+ const SNAPSHOT_INT64 = new Set(["BIGINT", "UBIGINT"]);
228
+ const SNAPSHOT_FLOAT = new Set(["DOUBLE", "FLOAT"]);
229
+ const SNAPSHOT_PLAIN = new Set([
230
+ "BOOLEAN",
231
+ "TINYINT",
232
+ "SMALLINT",
233
+ "INTEGER",
234
+ "UTINYINT",
235
+ "USMALLINT",
236
+ "UINTEGER",
237
+ "VARCHAR",
238
+ "DATE",
239
+ "TIMESTAMP",
240
+ "TIMESTAMP_S",
241
+ "TIMESTAMP_MS",
242
+ "TIMESTAMP_NS",
243
+ "TIMESTAMP WITH TIME ZONE",
244
+ ]);
245
+
246
+ function snapshotDecoder(type) {
247
+ if (SNAPSHOT_INT64.has(type)) {
248
+ return v => (typeof v === "string" ? BigInt(v) : v);
249
+ }
250
+ if (SNAPSHOT_FLOAT.has(type)) {
251
+ return v => (typeof v === "string" ? Number(v) : v);
252
+ }
253
+ if (SNAPSHOT_PLAIN.has(type)) return v => v;
254
+ return null;
255
+ }
256
+
257
+ export function snapshotUrl(name) {
258
+ return "__data/" + encodeURIComponent(name) + ".snapshot.json";
259
+ }
260
+
261
+ /**
262
+ * A snapshot body as `{ data, fields, rowCount, truncated, builtAt }`, rows
263
+ * decoded to what the parquet path yields for the same values. Null when the
264
+ * body is malformed, from another version, or names a type this SDK does not
265
+ * decode — the caller then waits for the parquet as if there were none.
266
+ */
267
+ export function parseBindingSnapshot(body, rowLimit) {
268
+ if (!body || typeof body !== "object" || body.version !== SNAPSHOT_VERSION) return null;
269
+ if (!Array.isArray(body.rows) || !Array.isArray(body.fields)) return null;
270
+ const rowCount = Number(body.rowCount);
271
+ if (!Number.isInteger(rowCount) || rowCount < 0) return null;
272
+ const names = [];
273
+ const decoders = [];
274
+ for (const field of body.fields) {
275
+ if (!field || typeof field.name !== "string" || !field.name) return null;
276
+ const decode = snapshotDecoder(String(field.type));
277
+ if (!decode) return null;
278
+ names.push(field.name);
279
+ decoders.push(decode);
280
+ }
281
+ const cap = rowLimit === null ? Infinity : rowLimit == null ? Infinity : rowLimit;
282
+ const data = [];
283
+ for (const row of body.rows) {
284
+ if (!row || typeof row !== "object" || Array.isArray(row)) return null;
285
+ if (data.length >= cap) break;
286
+ const out = {};
287
+ for (let i = 0; i < names.length; i++) {
288
+ const v = row[names[i]];
289
+ out[names[i]] = v == null ? null : decoders[i](v);
290
+ }
291
+ data.push(out);
292
+ }
293
+ return {
294
+ data,
295
+ fields: names,
296
+ rowCount,
297
+ truncated: data.length < rowCount,
298
+ builtAt: typeof body.builtAt === "string" ? body.builtAt : null,
299
+ };
300
+ }
package/index.d.ts CHANGED
@@ -135,6 +135,56 @@ export interface DuckDBState<Row = Record<string, unknown>>
135
135
  export interface QueryOptions {
136
136
  rowLimit?: number | null;
137
137
  }
138
+ export interface UseQueryOptions extends QueryOptions {
139
+ /**
140
+ * Opt in to the preview: fetch `__data/<name>.snapshot.json` (the first
141
+ * rows of the last build, no DuckDB-WASM needed) while the parquet loads,
142
+ * and expose it as `preview`. `data`, `loading`, `phase` and every other
143
+ * field behave exactly as without it. Default false.
144
+ */
145
+ snapshot?: boolean;
146
+ }
147
+ /**
148
+ * The first rows of the binding's last build, for a skeleton or a first
149
+ * paint. The first N rows of the build stream in the query's own order: not
150
+ * a sample, not sorted, not filtered. Never aggregate it.
151
+ */
152
+ export interface QueryPreview<Row = Record<string, unknown>> {
153
+ /** At most a few hundred rows, typed exactly as `data` will be. */
154
+ data: Row[];
155
+ fields: string[];
156
+ /** Rows in the whole build, not in `data`. */
157
+ rowCount: number;
158
+ /** True when `data` holds fewer rows than `rowCount`. */
159
+ truncated: boolean;
160
+ /** ISO 8601, when the snapshot's build ran; null when unknown. */
161
+ builtAt: string | null;
162
+ }
163
+ export interface DuckDBOptions extends QueryOptions {
164
+ /**
165
+ * "arrow": `data` is DuckDB's Arrow Table as returned (no row objects, no
166
+ * date/BigInt normalization, `rowLimit` ignored). Default "rows".
167
+ */
168
+ format?: "rows" | "arrow";
169
+ }
170
+
171
+ /**
172
+ * The page's shared DuckDB-WASM engine (`AsyncDuckDB` from
173
+ * @duckdb/duckdb-wasm 1.29), typed loosely so the SDK needs no dependency.
174
+ * Never close() or terminate() it: every hook on the page reads from it.
175
+ */
176
+ export interface SharedDuckDB {
177
+ connect(): Promise<DuckDBConnection>;
178
+ registerFileBuffer(name: string, buffer: Uint8Array): Promise<void>;
179
+ dropFile(name: string): Promise<void>;
180
+ [key: string]: unknown;
181
+ }
182
+ export interface DuckDBConnection {
183
+ /** Resolves with an Arrow Table. */
184
+ query(sql: string): Promise<any>;
185
+ close(): Promise<void>;
186
+ [key: string]: unknown;
187
+ }
138
188
  export interface MakoLocation {
139
189
  pathname: string;
140
190
  search: string;
@@ -143,20 +193,45 @@ export interface MakoLocation {
143
193
  searchParams: URLSearchParams;
144
194
  }
145
195
 
146
- /** Rows of a named data binding (bindings/<name>.sql, materialized). */
196
+ /**
197
+ * Rows of a named data binding (bindings/<name>.sql, materialized).
198
+ *
199
+ * With `{ snapshot: true }` the result also carries `preview`: the snapshot
200
+ * rows while `loading` is true and no error has arrived, null otherwise.
201
+ * `data` stays null until the full parquet is in, so code that sums or
202
+ * filters `data` never sees a prefix.
203
+ */
147
204
  export function useQuery<Row = Record<string, unknown>>(
148
205
  name: string,
149
- opts?: QueryOptions,
206
+ opts: UseQueryOptions & { snapshot: true },
150
207
  ): QueryState<Row> & {
151
208
  refresh: () => Promise<RefreshResult>;
152
209
  /** The binding's own progress record (null when it is not loading). */
153
210
  progress: BindingProgress | null;
211
+ preview: QueryPreview<Row> | null;
154
212
  };
213
+ export function useQuery<Row = Record<string, unknown>>(
214
+ name: string,
215
+ opts?: UseQueryOptions,
216
+ ): QueryState<Row> & {
217
+ refresh: () => Promise<RefreshResult>;
218
+ /** The binding's own progress record (null when it is not loading). */
219
+ progress: BindingProgress | null;
220
+ };
221
+
222
+ /** This package's version at runtime; pages also get `window.__MAKO_APP_SDK__`. */
223
+ export const SDK_VERSION: string;
155
224
 
156
- /** Analytical SQL over the app's bindings; table names are binding names. */
225
+ /**
226
+ * Analytical SQL over the app's bindings; table names are binding names.
227
+ * Since 2.8 only the bindings the SQL names are downloaded (each decoded
228
+ * once into a DuckDB table); one it misses loads on DuckDB's "table does
229
+ * not exist" and the query re-runs. Catalog queries (SHOW TABLES,
230
+ * information_schema) load every binding.
231
+ */
157
232
  export function useDuckDB<Row = Record<string, unknown>>(
158
233
  sql: string,
159
- opts?: QueryOptions,
234
+ opts?: QueryOptions & { format?: "rows" },
160
235
  ): DuckDBState<Row> & {
161
236
  refresh: () => Promise<RefreshResult[]>;
162
237
  /** The slowest pending binding's wait, with sums; null when none. */
@@ -164,6 +239,18 @@ export function useDuckDB<Row = Record<string, unknown>>(
164
239
  /** Every pending binding's progress, by name. */
165
240
  bindings: Record<string, BindingProgress>;
166
241
  };
242
+ export function useDuckDB(
243
+ sql: string,
244
+ opts: DuckDBOptions & { format: "arrow" },
245
+ ): Omit<DuckDBState, "data"> & {
246
+ /** The Arrow Table (apache-arrow 17) DuckDB returned. */
247
+ data: any | null;
248
+ refresh: () => Promise<RefreshResult[]>;
249
+ /** The slowest pending binding's wait, with sums; null when none. */
250
+ progress: CombinedBindingProgress | null;
251
+ /** Every pending binding's progress, by name. */
252
+ bindings: Record<string, BindingProgress>;
253
+ };
167
254
 
168
255
  export interface RefreshOptions {
169
256
  /**
@@ -173,6 +260,42 @@ export interface RefreshOptions {
173
260
  onProgress?: (progress: BindingProgress | null) => void;
174
261
  }
175
262
 
263
+ /**
264
+ * The page's shared DuckDB, booted on first call. Use it for anything the
265
+ * hooks do not cover instead of importing @duckdb/duckdb-wasm, hyparquet or
266
+ * apache-arrow readers. Never close or terminate it.
267
+ */
268
+ export function getDuckDB(): Promise<SharedDuckDB>;
269
+
270
+ /** Run `fn` on a fresh connection to the shared DuckDB; closed afterwards. */
271
+ export function withConnection<T>(fn: (conn: DuckDBConnection) => Promise<T>): Promise<T>;
272
+
273
+ /**
274
+ * Load a binding into the shared DuckDB as a table (once per page;
275
+ * concurrent callers share the download). Resolves with the table name.
276
+ */
277
+ export function ensureBinding(name: string): Promise<string>;
278
+
279
+ /** Drop a loaded binding's table to free memory; the next read reloads it. */
280
+ export function unloadBinding(name: string): Promise<void>;
281
+
282
+ /**
283
+ * The binding's raw parquet bytes, not registered anywhere — for an app with
284
+ * its own reader (register them via getDuckDB().registerFileBuffer to use
285
+ * read_parquet options). Rejects like a failed load (`status`, message).
286
+ */
287
+ export function fetchBindingBytes(
288
+ name: string,
289
+ opts?: {
290
+ fresh?: boolean;
291
+ /**
292
+ * The same phases a hook reports (requesting → resolving/signing or a
293
+ * build → downloading with `bytes` / `bytesTotal`), then `null`.
294
+ */
295
+ onProgress?: (progress: BindingProgress | null) => void;
296
+ },
297
+ ): Promise<Uint8Array>;
298
+
176
299
  /**
177
300
  * Rematerialize one binding on demand and reload it everywhere it is used.
178
301
  * Concurrent calls for the same binding share one request.
package/index.js CHANGED
@@ -16,6 +16,31 @@ export {
16
16
  bindingPhaseStage,
17
17
  } from "./progress.js";
18
18
 
19
+ import {
20
+ missingBinding,
21
+ mentionedBindings,
22
+ quoteIdent,
23
+ readsCatalog,
24
+ quoteString,
25
+ toPlainRows,
26
+ parseBindingSnapshot,
27
+ snapshotUrl,
28
+ } from "./data-core.js";
29
+
30
+ /**
31
+ * This package's version, visible at runtime: `window.__MAKO_APP_SDK__`
32
+ * in a page (or this export) says which SDK a deployed app actually runs.
33
+ * Kept in step with package.json by snapshot.test.mjs.
34
+ */
35
+ export const SDK_VERSION = "2.9.0";
36
+ if (typeof window !== "undefined") {
37
+ try {
38
+ window.__MAKO_APP_SDK__ = SDK_VERSION;
39
+ } catch {
40
+ /* a frozen window: nothing to stamp */
41
+ }
42
+ }
43
+
19
44
  const DEFAULT_ROW_LIMIT = 500000;
20
45
  const DUCKDB_ESM =
21
46
  "https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.29.0/+esm";
@@ -23,21 +48,48 @@ const DUCKDB_ESM =
23
48
  // ---------------------------------------------------------------------------
24
49
  // DuckDB: one lazy instance per page, bindings registered on first use. The
25
50
  // engine arrives from the CDN at runtime, so this package needs no install
26
- // step and no dependencies of its own.
51
+ // step and no dependencies of its own. The first data hook to render starts
52
+ // the boot (WASM download + instantiate, ~1 s) so it overlaps the first
53
+ // parquet download instead of waiting for it.
27
54
  // ---------------------------------------------------------------------------
28
55
  let dbPromise = null;
29
56
  function getDb() {
30
- dbPromise ??= (async () => {
31
- const duckdb = await import(/* @vite-ignore */ DUCKDB_ESM);
32
- const bundle = await duckdb.selectBundle(duckdb.getJsDelivrBundles());
33
- const worker = await duckdb.createWorker(bundle.mainWorker);
34
- const db = new duckdb.AsyncDuckDB(new duckdb.VoidLogger(), worker);
35
- await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
36
- return db;
37
- })();
57
+ if (!dbPromise) {
58
+ dbPromise = (async () => {
59
+ const duckdb = await import(/* @vite-ignore */ DUCKDB_ESM);
60
+ const bundle = await duckdb.selectBundle(duckdb.getJsDelivrBundles());
61
+ const worker = await duckdb.createWorker(bundle.mainWorker);
62
+ const db = new duckdb.AsyncDuckDB(new duckdb.VoidLogger(), worker);
63
+ await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
64
+ // read_parquet lives in an extension DuckDB fetches on first use
65
+ // (~660 kB from extensions.duckdb.org). Load it now, while the
66
+ // binding bytes are still downloading, not after they arrive.
67
+ const conn = await db.connect();
68
+ try {
69
+ await conn.query("LOAD parquet");
70
+ } catch {
71
+ /* autoload on first read_parquet still applies */
72
+ } finally {
73
+ await conn.close();
74
+ }
75
+ return db;
76
+ })();
77
+ // A failed boot (CDN unreachable) is retried by the next query.
78
+ dbPromise.catch(() => {
79
+ dbPromise = null;
80
+ });
81
+ }
38
82
  return dbPromise;
39
83
  }
40
84
 
85
+ /** Start the engine early; a no-op outside a browser and once started. */
86
+ function bootEngine() {
87
+ if (typeof window === "undefined" || typeof Worker === "undefined") return;
88
+ getDb().catch(() => {
89
+ /* the query that needs it reports the failure */
90
+ });
91
+ }
92
+
41
93
  // The server answers failures with JSON { error, hint, retryAfterMs } —
42
94
  // show that, not a bare status code (and never let it reach DuckDB, where it
43
95
  // would surface as a baffling Catalog Error).
@@ -107,43 +159,66 @@ export function useBindingProgress(name) {
107
159
  return name === undefined ? all : (all[name] ?? null);
108
160
  }
109
161
 
110
- async function loadBinding(name, bust) {
162
+ // Each binding becomes a TABLE, decoded from its parquet once: a view over
163
+ // read_parquet re-decoded the file on every query (SELECT * of a 149k-row
164
+ // binding: 366 ms through a view, 174 ms from a table). The file buffer is
165
+ // dropped as soon as the table exists. CREATE OR REPLACE swaps the new table
166
+ // in with one commit, so a query running during a reload reads the old rows
167
+ // or the new ones, never a missing table. Every load registers its bytes
168
+ // under its own file name, so a reload never drops a file another load is
169
+ // still reading.
170
+
171
+ /**
172
+ * A binding's parquet bytes, phase-aware: asks for NDJSON phase events (a
173
+ * build reports each step, a stored file resolving → signing and a signed
174
+ * URL) and reports them through `report(patch)`. A server that predates the
175
+ * events answers with the bytes, as it always did.
176
+ */
177
+ async function downloadBinding(name, bust, report) {
111
178
  const path = "__data/" + encodeURIComponent(name) + ".parquet";
112
179
  const what = 'Data for binding "' + name + '" could not be loaded';
113
- setProgress(name, { phase: "requesting", rowsWritten: null, bytes: null, bytesTotal: null });
180
+ report({ phase: "requesting", rowsWritten: null, bytes: null, bytesTotal: null });
181
+ let res = await fetch(path + (bust ? "?refresh=" + Date.now() : ""), {
182
+ headers: { accept: PROGRESS_ACCEPT },
183
+ });
184
+ if (isProgressResponse(res)) {
185
+ const final = await readProgressStream(res, e => report(progressPatch(e)));
186
+ if (final.phase === "failed") throw progressError(final, what);
187
+ report({ phase: "downloading", rowsWritten: null, bytes: 0 });
188
+ // No url: the server streams the bytes itself — the same path, asked
189
+ // plainly, serves them (the build already staged or cached them).
190
+ res = await fetch(final.url || path);
191
+ }
192
+ if (!res.ok) throw await responseError(res, what);
193
+ return readBytes(res, (bytes, bytesTotal) =>
194
+ report({ phase: "downloading", rowsWritten: null, bytes, bytesTotal }),
195
+ );
196
+ }
197
+
198
+ let fileSeq = 0;
199
+ async function loadBinding(name, bust) {
200
+ const db = getDb(); // boot in parallel with the download
201
+ db.catch(() => {});
114
202
  try {
115
- // Ask for phase events: a build (laptop, sandbox live binding) reports
116
- // each step, a stored file reports resolving → signing and hands back
117
- // the signed URL. A server that predates them answers as it always did.
118
- let res = await fetch(path + (bust ? "?refresh=" + Date.now() : ""), {
119
- headers: { accept: PROGRESS_ACCEPT },
120
- });
121
- if (isProgressResponse(res)) {
122
- const final = await readProgressStream(res, e => setProgress(name, progressPatch(e)));
123
- if (final.phase === "failed") throw progressError(final, what);
124
- setProgress(name, { phase: "downloading", rowsWritten: null, bytes: 0 });
125
- // No url: the server streams the bytes itself — the same path, asked
126
- // plainly, serves them (the build already staged or cached them).
127
- res = await fetch(final.url || path);
128
- }
129
- if (!res.ok) throw await responseError(res, what);
130
- const buf = await readBytes(res, (bytes, bytesTotal) =>
131
- setProgress(name, { phase: "downloading", rowsWritten: null, bytes, bytesTotal }),
132
- );
203
+ const buf = await downloadBinding(name, bust, patch => setProgress(name, patch));
133
204
  setProgress(name, { phase: "loading_duckdb" });
134
- const db = await getDb();
135
- if (bust) await db.dropFile(name + ".parquet").catch(() => {});
136
- await db.registerFileBuffer(name + ".parquet", buf);
137
- const conn = await db.connect();
205
+ const engine = await db;
206
+ const file = "mako-" + ++fileSeq + ".parquet";
207
+ await engine.registerFileBuffer(file, buf);
138
208
  try {
139
- // A view per binding so SQL can name tables by binding name.
140
- await conn.query(
141
- 'CREATE OR REPLACE VIEW "' + name.replace(/"/g, '""') +
142
- "\" AS SELECT * FROM read_parquet('" + name + ".parquet')",
143
- );
209
+ const conn = await engine.connect();
210
+ try {
211
+ await conn.query(
212
+ "CREATE OR REPLACE TABLE " + quoteIdent(name) +
213
+ " AS SELECT * FROM read_parquet(" + quoteString(file) + ")",
214
+ );
215
+ } finally {
216
+ await conn.close();
217
+ }
144
218
  } finally {
145
- await conn.close();
219
+ await engine.dropFile(file).catch(() => {});
146
220
  }
221
+ loadedBindings.add(name);
147
222
  } finally {
148
223
  // A refresh owns the record until its reload is done (refreshBinding).
149
224
  if (!refreshes.has(name)) setProgress(name, null);
@@ -151,27 +226,105 @@ async function loadBinding(name, bust) {
151
226
  }
152
227
 
153
228
  const registered = new Map(); // name -> Promise<void>
229
+ const loadedBindings = new Set(); // names whose table is in DuckDB
230
+ // Refreshed on the server while never loaded on this page: the next load
231
+ // must skip any cache between here and the server.
232
+ const staleBindings = new Set();
154
233
  function registerBinding(name) {
155
234
  if (!registered.has(name)) {
156
- const load = loadBinding(name, false);
235
+ const load = loadBinding(name, staleBindings.delete(name));
157
236
  registered.set(name, load);
158
- load.catch(() => registered.delete(name));
237
+ load.catch(() => {
238
+ if (registered.get(name) === load) registered.delete(name);
239
+ });
159
240
  }
160
241
  return registered.get(name);
161
242
  }
162
243
 
244
+ // ---------------------------------------------------------------------------
245
+ // Escape hatches — the same engine and the same loader the hooks use, for
246
+ // apps that need more than useQuery/useDuckDB. One DuckDB per page: an app
247
+ // that boots its own pays a second ~1 s boot and a second copy of the data.
248
+ // ---------------------------------------------------------------------------
249
+
250
+ /**
251
+ * The page's shared DuckDB (an AsyncDuckDB), booted on first call. Never
252
+ * close or terminate it: every hook on the page reads from it.
253
+ */
254
+ export function getDuckDB() {
255
+ return getDb();
256
+ }
257
+
258
+ /** Run `fn` with a fresh connection to the shared DuckDB; always closed after. */
259
+ export async function withConnection(fn) {
260
+ const db = await getDb();
261
+ const conn = await db.connect();
262
+ try {
263
+ return await fn(conn);
264
+ } finally {
265
+ await conn.close();
266
+ }
267
+ }
268
+
269
+ /**
270
+ * Make sure a binding's table exists in the shared DuckDB, loading it once
271
+ * per page (concurrent callers share the download). Resolves with the table
272
+ * name — the binding name; quote it in SQL when it is not a bare identifier.
273
+ */
274
+ export async function ensureBinding(name) {
275
+ await registerBinding(name);
276
+ return name;
277
+ }
278
+
279
+ /**
280
+ * Forget a loaded binding: drops its table so its memory is freed. The next
281
+ * query or ensureBinding() downloads it again.
282
+ */
283
+ export async function unloadBinding(name) {
284
+ const pending = registered.get(name);
285
+ if (!pending) return;
286
+ registered.delete(name);
287
+ loadedBindings.delete(name);
288
+ await pending.catch(() => {});
289
+ await withConnection(conn => conn.query("DROP TABLE IF EXISTS " + quoteIdent(name)));
290
+ }
291
+
292
+ /**
293
+ * The binding's parquet bytes, for an app with its own reader (e.g.
294
+ * read_parquet options such as file_row_number). Not registered anywhere.
295
+ * `fresh` skips caches; `onProgress(loaded, total)` reports the download.
296
+ */
297
+ export async function fetchBindingBytes(name, opts) {
298
+ const fresh = !!(opts && opts.fresh) || staleBindings.has(name);
299
+ const onProgress = opts && opts.onProgress;
300
+ let record = null;
301
+ try {
302
+ return await downloadBinding(name, fresh, patch => {
303
+ record = { ...record, ...patch };
304
+ if (onProgress) onProgress(record);
305
+ });
306
+ } finally {
307
+ if (onProgress) onProgress(null);
308
+ }
309
+ }
310
+
163
311
  /** Replace a binding's loaded bytes with what the server has now. */
164
312
  async function reloadBinding(name) {
313
+ const previous = registered.get(name);
314
+ if (!previous) {
315
+ // Nothing on this page reads it yet: load it fresh when a query does.
316
+ staleBindings.add(name);
317
+ return;
318
+ }
165
319
  // Let a load in flight settle first, or its (older) bytes could land after
166
320
  // the fresh ones and win.
167
- const previous = registered.get(name);
168
- if (previous) await previous.catch(() => {});
321
+ await previous.catch(() => {});
169
322
  const load = loadBinding(name, true);
170
323
  registered.set(name, load);
171
324
  try {
172
325
  await load;
173
326
  } catch (error) {
174
- registered.delete(name);
327
+ if (registered.get(name) === load) registered.delete(name);
175
328
  throw error;
176
329
  }
177
330
  }
@@ -342,49 +495,18 @@ function bindingIndex() {
342
495
  return indexPromise;
343
496
  }
344
497
 
345
- // Arrow → plain JS, with the types apps actually want. DuckDB hands DATE and
346
- // TIMESTAMP columns to JS as epoch milliseconds; v1 apps (JSON over the wire)
347
- // always saw strings, and a migrated formatDate() that did
348
- // new Date(`${d}T00:00:00`) on a number threw and unmounted the whole app
349
- // (apps.md §15.4). So: DATE → "YYYY-MM-DD", TIMESTAMP → ISO 8601, BigInt →
350
- // Number when it fits. Everything else passes through untouched.
351
- function columnDecoder(field) {
352
- const type = String(field.type);
353
- if (type.startsWith("Date")) {
354
- return v => (v == null ? v : new Date(Number(v)).toISOString().slice(0, 10));
355
- }
356
- if (type.startsWith("Timestamp")) {
357
- return v => (v == null ? v : new Date(Number(v)).toISOString());
358
- }
359
- return v =>
360
- typeof v === "bigint" && v >= Number.MIN_SAFE_INTEGER && v <= Number.MAX_SAFE_INTEGER
361
- ? Number(v)
362
- : v;
363
- }
364
-
365
- function toPlainRows(table, cap) {
366
- const rows = [];
367
- const fields = table.schema.fields.map(f => f.name);
368
- const decoders = table.schema.fields.map(columnDecoder);
369
- const limit = cap == null ? Infinity : cap;
370
- for (const row of table) {
371
- if (rows.length >= limit) break;
372
- const out = {};
373
- for (let i = 0; i < fields.length; i++) {
374
- out[fields[i]] = decoders[i](row[fields[i]]);
375
- }
376
- rows.push(out);
377
- }
378
- return { rows, fields };
379
- }
380
-
381
- async function runSql(sql, rowLimit) {
498
+ async function runSql(sql, rowLimit, format) {
382
499
  const cap = rowLimit === null ? Infinity : (rowLimit ?? DEFAULT_ROW_LIMIT);
383
500
  const db = await getDb();
384
501
  const conn = await db.connect();
385
502
  try {
386
503
  const table = await conn.query(sql);
387
504
  const rowCount = table.numRows;
505
+ if (format === "arrow") {
506
+ // The Arrow Table as DuckDB returned it: no row objects, no
507
+ // normalization, no row limit.
508
+ return { rows: table, fields: table.schema.fields.map(f => f.name), rowCount, truncated: false };
509
+ }
388
510
  const { rows, fields } = toPlainRows(table, cap);
389
511
  return { rows, fields, rowCount, truncated: rowCount > rows.length };
390
512
  } finally {
@@ -452,15 +574,57 @@ function hookProgress(record, state) {
452
574
  };
453
575
  }
454
576
 
577
+ // ---------------------------------------------------------------------------
578
+ // Snapshot preview — opt-in with useQuery(name, { snapshot: true }). The
579
+ // server writes the first N rows of each build next to its parquet as
580
+ // `__data/<name>.snapshot.json`. It is the first N rows of the build stream
581
+ // in whatever order the query produced them, so it is a skeleton for the
582
+ // first paint, never data: `data` stays null and `loading` stays true until
583
+ // the table is in DuckDB (through every load phase), and `preview`
584
+ // disappears the moment either the rows or an error arrive. One request per
585
+ // binding per page, none once the table is already loaded; a binding
586
+ // without a snapshot is one cheap 404.
587
+ // ---------------------------------------------------------------------------
588
+ const snapshots = new Map(); // name -> Promise<body | null>
589
+ function loadSnapshot(name) {
590
+ if (!snapshots.has(name)) {
591
+ snapshots.set(
592
+ name,
593
+ fetch(snapshotUrl(name))
594
+ .then(r => (r.ok ? r.json() : null))
595
+ .catch(() => null),
596
+ );
597
+ }
598
+ return snapshots.get(name);
599
+ }
600
+
601
+ function useSnapshotPreview(name, enabled, rowLimit) {
602
+ const [preview, setPreview] = React.useState(null);
603
+ React.useEffect(() => {
604
+ setPreview(null);
605
+ if (!enabled || loadedBindings.has(name)) return;
606
+ let active = true;
607
+ loadSnapshot(name).then(body => {
608
+ if (!active || loadedBindings.has(name)) return;
609
+ const parsed = parseBindingSnapshot(body, rowLimit);
610
+ if (parsed) setPreview(parsed);
611
+ });
612
+ return () => {
613
+ active = false;
614
+ };
615
+ }, [name, enabled, rowLimit]);
616
+ return preview;
617
+ }
618
+
455
619
  export function useQuery(name, opts) {
620
+ bootEngine();
456
621
  const rowLimit = opts ? opts.rowLimit : undefined;
622
+ const withSnapshot = Boolean(opts && opts.snapshot);
623
+ const preview = useSnapshotPreview(name, withSnapshot, rowLimit);
457
624
  const state = useAsyncQuery(
458
625
  async () => {
459
626
  await registerBinding(name);
460
- const r = await runSql(
461
- 'SELECT * FROM "' + name.replace(/"/g, '""') + '"',
462
- rowLimit,
463
- );
627
+ const r = await runSql("SELECT * FROM " + quoteIdent(name), rowLimit);
464
628
  return { data: r.rows, fields: r.fields, truncated: r.truncated, rowCount: r.rowCount };
465
629
  },
466
630
  [name, rowLimit],
@@ -468,7 +632,12 @@ export function useQuery(name, opts) {
468
632
  );
469
633
  const progress = useBindingProgress(name);
470
634
  const refresh = React.useCallback(() => refreshBinding(name), [name]);
471
- return { ...state, ...hookProgress(progress, state), progress, refresh };
635
+ const result = { ...state, ...hookProgress(progress, state), progress, refresh };
636
+ if (!withSnapshot) return result;
637
+ // Only while nothing final is on screen: not next to rows, not next to an
638
+ // error (a failed load must not leave preview rows looking final).
639
+ result.preview = state.loading && state.data === null && state.error === null ? preview : null;
640
+ return result;
472
641
  }
473
642
 
474
643
  /**
@@ -502,24 +671,39 @@ export function useViewer() {
502
671
  return state;
503
672
  }
504
673
 
505
- export function useDuckDB(sql, opts) {
506
- const rowLimit = opts ? opts.rowLimit : undefined;
507
- const state = useAsyncQuery(async () => {
508
- // Register everything the server staged, so SQL can join across
509
- // bindings by name without declaring them first. A binding that fails to
510
- // load must not sink the whole query set silently: remember why, and when
511
- // the SQL then fails (typically DuckDB's "Table … does not exist"),
512
- // surface the load failure — it is the actual cause.
513
- const names = await bindingIndex();
514
- const failures = new Map();
515
- await Promise.all(
516
- names.map(n => registerBinding(n).catch(e => failures.set(n, e))),
517
- );
674
+ /**
675
+ * Run `sql` with the bindings it reads loaded — and only those. The ones
676
+ * its text names start downloading together, alongside the engine boot; a
677
+ * table the hint missed shows up as DuckDB's "Table with name X does not
678
+ * exist", which loads X and runs the query again. Joins across bindings
679
+ * work by name without declaring anything, as before, and an app with 34
680
+ * bindings no longer downloads all 34 to draw one chart.
681
+ *
682
+ * A binding that fails to load must not sink the query silently: when the
683
+ * SQL then fails, the load failure is what surfaces — it is the cause.
684
+ */
685
+ async function runOverBindings(sql, rowLimit, format) {
686
+ const names = await bindingIndex();
687
+ const failures = new Map();
688
+ const attempted = new Set();
689
+ const load = n => {
690
+ attempted.add(n);
691
+ return registerBinding(n).catch(e => {
692
+ failures.set(n, e);
693
+ });
694
+ };
695
+ await Promise.all((readsCatalog(sql) ? names : mentionedBindings(sql, names)).map(load));
696
+ for (;;) {
518
697
  try {
519
- const r = await runSql(sql, rowLimit);
520
- return { data: r.rows, fields: r.fields, truncated: r.truncated, rowCount: r.rowCount };
698
+ return await runSql(sql, rowLimit, format);
521
699
  } catch (error) {
522
700
  const message = error instanceof Error ? error.message : String(error);
701
+ const missing = missingBinding(message, names);
702
+ if (missing && !attempted.has(missing)) {
703
+ await load(missing);
704
+ if (failures.has(missing)) throw failures.get(missing);
705
+ continue;
706
+ }
523
707
  for (const [n, cause] of failures) {
524
708
  if (message.includes(n)) throw cause;
525
709
  }
@@ -531,12 +715,23 @@ export function useDuckDB(sql, opts) {
531
715
  }
532
716
  throw error;
533
717
  }
534
- }, [sql, rowLimit], refreshing => refreshing.size > 0);
718
+ }
719
+ }
720
+
721
+ export function useDuckDB(sql, opts) {
722
+ bootEngine();
723
+ const rowLimit = opts ? opts.rowLimit : undefined;
724
+ const format = opts && opts.format === "arrow" ? "arrow" : "rows";
725
+ const state = useAsyncQuery(async () => {
726
+ const r = await runOverBindings(sql, rowLimit, format);
727
+ return { data: r.rows, fields: r.fields, truncated: r.truncated, rowCount: r.rowCount };
728
+ }, [sql, rowLimit, format], refreshing => refreshing.size > 0);
535
729
  // Every binding the query can read: the combined record says what the
536
730
  // slowest of them is waiting on; `bindings` has each one.
537
731
  const bindings = useBindingProgress();
538
732
  const combined = combineBindingProgress(bindings);
539
- // A query over every binding refreshes every binding.
733
+ // A query over every binding refreshes every binding (the ones this page
734
+ // has not loaded yet are rebuilt and fetched fresh when first read).
540
735
  const refresh = React.useCallback(() => refreshBindings(), []);
541
736
  return {
542
737
  ...state,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@makoai/app-sdk",
3
- "version": "2.7.4",
3
+ "version": "2.9.0",
4
4
  "description": "Mako app SDK: data bindings (useQuery/useDuckDB), URL state, theme, and the house dashboard kit (/ui).",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -42,6 +42,7 @@
42
42
  "index.js",
43
43
  "index.d.ts",
44
44
  "progress.js",
45
+ "data-core.js",
45
46
  "vite.js",
46
47
  "vite.d.ts",
47
48
  "credentials.js",
@@ -61,5 +62,8 @@
61
62
  },
62
63
  "scripts": {
63
64
  "test": "node --test"
65
+ },
66
+ "devDependencies": {
67
+ "apache-arrow": "17.0.0"
64
68
  }
65
69
  }
package/vite.d.ts CHANGED
@@ -12,9 +12,11 @@ export interface MakoDataOptions {
12
12
  /** Repo root. Default: the nearest ancestor holding .mako/ or .git. */
13
13
  repoRoot?: string;
14
14
  /**
15
- * Re-fetch a cached parquet after this long (ms). Default: 5 minutes. A
16
- * cache built from different binding text (or dbt environment) is never
17
- * served, whatever this says — `Infinity` is safe while you edit SQL.
15
+ * A cached parquet of the current binding text is always served at once;
16
+ * after this long (ms) it is also rebuilt in the background for the next
17
+ * load. Default: 5 minutes. A cache built from different binding text (or
18
+ * dbt environment) is never served — `Infinity` is safe while you edit
19
+ * SQL. `refresh()` / `?refresh` waits for the rebuild.
18
20
  */
19
21
  revalidateMs?: number;
20
22
  /**
package/vite.js CHANGED
@@ -388,9 +388,18 @@ export function makoData(options = {}) {
388
388
  }
389
389
  }
390
390
 
391
+ // Atomic: a background revalidation can land while a page is still
392
+ // streaming the previous copy, which keeps reading the old file.
391
393
  function writeCache(name, buf, meta) {
392
394
  fs.mkdirSync(ctx.cacheDir, { recursive: true });
393
- fs.writeFileSync(cachePath(name), buf);
395
+ const tmp = `${cachePath(name)}.${process.pid}.${Date.now()}.tmp`;
396
+ fs.writeFileSync(tmp, buf);
397
+ try {
398
+ fs.renameSync(tmp, cachePath(name));
399
+ } catch {
400
+ fs.rmSync(tmp, { force: true });
401
+ fs.writeFileSync(cachePath(name), buf);
402
+ }
394
403
  if (meta) fs.writeFileSync(metaPath(name), JSON.stringify(meta));
395
404
  else fs.rmSync(metaPath(name), { force: true });
396
405
  }
@@ -406,6 +415,24 @@ export function makoData(options = {}) {
406
415
  let devBuildUnsupported = false;
407
416
  const inflight = new Map();
408
417
 
418
+ // One background rebuild per binding at a time; a failure is logged and
419
+ // the next load tries again.
420
+ const revalidating = new Set();
421
+ function revalidate(name, source) {
422
+ if (revalidating.has(name)) return;
423
+ revalidating.add(name);
424
+ (async () => {
425
+ const built = source !== null ? (await devBuild(name, source, false))?.buf : null;
426
+ if (!built) writeCache(name, await fetchArtifact(name), null);
427
+ })()
428
+ .catch(error => {
429
+ server.config.logger.error(
430
+ ` mako-data: background revalidate of ${name}: ${error instanceof Error ? error.message : String(error)}`,
431
+ );
432
+ })
433
+ .finally(() => revalidating.delete(name));
434
+ }
435
+
409
436
  // Build the LOCAL binding text through the API. Concurrent asks for the
410
437
  // same text share one request (a page mounting several tables at once).
411
438
  function devBuild(name, source, refresh, onPhase) {
@@ -582,12 +609,55 @@ export function makoData(options = {}) {
582
609
  }
583
610
  }
584
611
 
612
+ // `__data/<name>.snapshot.json` — the preview for
613
+ // useQuery(name, { snapshot: true }). It must describe the parquet
614
+ // this server is serving, never another definition's: with a local
615
+ // binding file the API is asked by its TEXT (dev-snapshot answers only
616
+ // for the stored artifact dev-build would answer that text with, and
617
+ // 404 while a draft still needs building); without one, the committed
618
+ // artifact by name, as fetchArtifact serves it. Never cached here, and
619
+ // every failure is a 404: the SDK then just waits for the parquet.
620
+ async function serveSnapshot(name, res) {
621
+ const none = () => json(res, 404, { error: "Snapshot not available" });
622
+ if (problems.length) return none();
623
+ const source = options.materialize === false || devBuildUnsupported ? null : readSource(name);
624
+ const base = `${appBase()}/bindings/${encodeURIComponent(name)}`;
625
+ try {
626
+ const r =
627
+ source === null
628
+ ? await apiFetch(`${base}/snapshot`, { headers: await headers() })
629
+ : await apiFetch(`${base}/dev-snapshot`, {
630
+ method: "POST",
631
+ headers: { ...(await headers()), "content-type": "application/json" },
632
+ body: JSON.stringify({
633
+ source,
634
+ ...(ctx.dbtEnvironment ? { dbtEnvironment: ctx.dbtEnvironment } : {}),
635
+ }),
636
+ });
637
+ if (!r.ok) return none();
638
+ const buf = Buffer.from(await r.arrayBuffer());
639
+ res.setHeader("content-type", "application/json; charset=utf-8");
640
+ res.setHeader("content-length", String(buf.length));
641
+ res.setHeader("cache-control", "no-store");
642
+ res.setHeader("x-mako-data", "api");
643
+ return res.end(buf);
644
+ } catch {
645
+ return none();
646
+ }
647
+ }
648
+
585
649
  server.middlewares.use(async (req, res, next) => {
586
650
  const [pathname, query = ""] = (req.url || "").split("?");
587
651
  if (pathname === "/__data/index.json") {
588
652
  return json(res, 200, listBindings(ctx.bindingsDir));
589
653
  }
590
654
  if (pathname === "/__data/viewer.json") return serveViewer(res);
655
+ const snap = /^\/__data\/([^/]+)\.snapshot\.json$/.exec(pathname);
656
+ if (snap) {
657
+ const name = decodeURIComponent(snap[1]);
658
+ if (!BINDING_NAME.test(name)) return json(res, 400, { error: "invalid binding name" });
659
+ return serveSnapshot(name, res);
660
+ }
591
661
  const match = /^\/__data\/([^/]+)(\.parquet|\/refresh)$/.exec(pathname);
592
662
  if (!match) return next();
593
663
  const name = decodeURIComponent(match[1]);
@@ -631,8 +701,15 @@ export function makoData(options = {}) {
631
701
  let stream = null;
632
702
  try {
633
703
  const stat = fs.statSync(cached, { throwIfNoEntry: false });
634
- const fresh = stat && Date.now() - stat.mtimeMs < revalidateMs;
635
- if (!refresh && fresh && cacheMatches()) return serveCached("cache");
704
+ // A copy built from this text is served at once, however old: past
705
+ // `revalidateMs` it is rebuilt in the background for the next load
706
+ // (stale-while-revalidate), so a page reload never waits on the API
707
+ // for data it already has. `?refresh` (the SDK's refresh()) waits.
708
+ if (!refresh && stat && cacheMatches()) {
709
+ const fresh = Date.now() - stat.mtimeMs < revalidateMs;
710
+ if (!fresh) revalidate(name, source);
711
+ return serveCached(fresh ? "cache" : "revalidating");
712
+ }
636
713
  if (wantsProgress(req)) {
637
714
  stream = progressStream(res);
638
715
  stream.emit({ phase: "checking" });