@makoai/app-sdk 2.7.4 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -7
- package/data-core.js +300 -0
- package/index.d.ts +127 -4
- package/index.js +298 -103
- package/package.json +5 -1
- package/vite.d.ts +5 -3
- package/vite.js +80 -3
package/README.md
CHANGED
|
@@ -29,8 +29,8 @@ import { useQuery, useDuckDB, useSearchParams, useTheme } from "@makoai/app-sdk"
|
|
|
29
29
|
// Rows of bindings/<name>.sql, materialized to parquet by Mako.
|
|
30
30
|
const { data, loading, error } = useQuery("latest_sales");
|
|
31
31
|
|
|
32
|
-
// Analytical SQL over
|
|
33
|
-
// names are binding names).
|
|
32
|
+
// Analytical SQL over the bindings (DuckDB-WASM in the browser; table
|
|
33
|
+
// names are binding names). Only the bindings the SQL names download.
|
|
34
34
|
const totals = useDuckDB("select country, sum(revenue) r from latest_sales group by 1");
|
|
35
35
|
|
|
36
36
|
// Rematerialize on demand: re-runs the binding's query on the warehouse,
|
|
@@ -42,6 +42,34 @@ const { refresh, refreshing } = useQuery("latest_sales");
|
|
|
42
42
|
</button>
|
|
43
43
|
```
|
|
44
44
|
|
|
45
|
+
### Which SDK is running: `SDK_VERSION` (2.9+)
|
|
46
|
+
|
|
47
|
+
`import { SDK_VERSION } from "@makoai/app-sdk"` is this package's version.
|
|
48
|
+
Importing the SDK also sets `window.__MAKO_APP_SDK__` to it. In DevTools,
|
|
49
|
+
switch the console context to the app's frame and read
|
|
50
|
+
`__MAKO_APP_SDK__` to see which release a deployed app's build resolved.
|
|
51
|
+
|
|
52
|
+
### Preview rows while the parquet loads (2.9+, opt-in)
|
|
53
|
+
|
|
54
|
+
```tsx
|
|
55
|
+
const { data, loading, preview } = useQuery("latest_sales", { snapshot: true });
|
|
56
|
+
// loading && preview → paint preview.data as a skeleton table
|
|
57
|
+
// !loading → data is the full binding; preview is null
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Every build writes its first 500 rows (200 KiB cap) next to the parquet as
|
|
61
|
+
`__data/<name>.snapshot.json`. With `snapshot: true` the hook fetches it once
|
|
62
|
+
per page (skipped when the parquet is already loaded) and exposes it as
|
|
63
|
+
`preview` = `{ data, fields, rowCount, truncated, builtAt }`. `data` and
|
|
64
|
+
`loading` behave exactly as without the option, and a failed load clears
|
|
65
|
+
`preview`. Preview values have the same types as `data` (DATE →
|
|
66
|
+
`"YYYY-MM-DD"`, TIMESTAMP → ISO 8601, a 64-bit integer past 2^53 → BigInt).
|
|
67
|
+
The snapshot is the first N rows of the build stream in the query's own
|
|
68
|
+
order, with no `ORDER BY` guarantee. Use it for a skeleton or first paint.
|
|
69
|
+
Never aggregate it. `preview` is null when a binding has no snapshot (built
|
|
70
|
+
before 2.9's server release, live, a laptop draft still building, or a
|
|
71
|
+
DECIMAL/LIST/STRUCT column). Without the option no snapshot is fetched.
|
|
72
|
+
|
|
45
73
|
`useDuckDB(...).refresh()` refreshes every binding; `refreshBinding(name)` /
|
|
46
74
|
`refreshBindings()` do the same outside a component. A refresh POSTs to
|
|
47
75
|
`__data/<name>/refresh`, the data URL's sibling, so whoever serves the app's
|
|
@@ -55,6 +83,38 @@ or 502 with the query's error.
|
|
|
55
83
|
`useTheme` follows the OS preference. Theme tokens (`--background`,
|
|
56
84
|
`--chart-1`, …) match the ones the scaffold's `styles.css` declares.
|
|
57
85
|
|
|
86
|
+
### Custom data work: one engine (2.8)
|
|
87
|
+
|
|
88
|
+
Never import `@duckdb/duckdb-wasm`, `hyparquet` or `apache-arrow` parquet
|
|
89
|
+
readers in an app. Use the page's engine through the SDK:
|
|
90
|
+
|
|
91
|
+
```ts
|
|
92
|
+
import { getDuckDB, withConnection, ensureBinding, unloadBinding, fetchBindingBytes, useDuckDB } from "@makoai/app-sdk";
|
|
93
|
+
|
|
94
|
+
const big = useDuckDB("select lat, lng from listings", { format: "arrow" }); // data: Arrow Table
|
|
95
|
+
const table = await ensureBinding("valuations"); // loaded once, returns "valuations"
|
|
96
|
+
const n = await withConnection(c => c.query(`select count(*) from ${table}`));
|
|
97
|
+
await unloadBinding("valuations"); // free its memory
|
|
98
|
+
const bytes = await fetchBindingBytes("valuations", { onProgress: p => p && console.log(p.phase, p.bytes, p.bytesTotal) });
|
|
99
|
+
const db = await getDuckDB(); // shared: never close() or terminate()
|
|
100
|
+
await db.registerFileBuffer("valuations.parquet", bytes); // read_parquet with your own options
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
### Loading cost (2.8)
|
|
104
|
+
|
|
105
|
+
- `useDuckDB` downloads only the bindings its SQL names. A missed name (for
|
|
106
|
+
example `query_table('x')`) loads when DuckDB reports the table missing,
|
|
107
|
+
and the query re-runs. `SHOW TABLES` / `information_schema` load all.
|
|
108
|
+
- Each binding is decoded once into a DuckDB table and its file buffer is
|
|
109
|
+
freed. A refresh swaps the table in one commit, so a running query never
|
|
110
|
+
sees it missing. A table costs about 3x the parquet in memory (10 MB of
|
|
111
|
+
parquet, 149k rows: 33 MB).
|
|
112
|
+
- The DuckDB engine and its parquet extension start loading on the first
|
|
113
|
+
data hook render, in parallel with the first binding download.
|
|
114
|
+
- Rows decode column by column: 149k rows x 16 columns in 190 ms, down
|
|
115
|
+
from 740 ms. Values are unchanged (DATE as `YYYY-MM-DD`, TIMESTAMP as
|
|
116
|
+
ISO 8601, BigInt as Number when safe).
|
|
117
|
+
|
|
58
118
|
Data arrives from `__data/<name>.parquet`, relative to the page — the same
|
|
59
119
|
path in Mako's sandbox, in a published app, and on a laptop.
|
|
60
120
|
|
|
@@ -214,8 +274,9 @@ import { makoData } from "@makoai/app-sdk/vite";
|
|
|
214
274
|
export default defineConfig({ plugins: [react(), makoData()] });
|
|
215
275
|
```
|
|
216
276
|
|
|
217
|
-
`makoData()` answers `__data/index.json` (the app's `bindings/*.sql`)
|
|
218
|
-
`__data/<name>.parquet` during `vite dev`
|
|
277
|
+
`makoData()` answers `__data/index.json` (the app's `bindings/*.sql`),
|
|
278
|
+
`__data/<name>.parquet` and `__data/<name>.snapshot.json` during `vite dev`
|
|
279
|
+
from the Mako API, and what it asks
|
|
219
280
|
for is **your local binding file**: it sends the text of
|
|
220
281
|
`bindings/<name>.sql` and gets back the parquet of exactly that query, run
|
|
221
282
|
read-only through the workspace connection its front matter names. When the
|
|
@@ -225,15 +286,21 @@ from your text and hands it back without storing it — nobody else, and no
|
|
|
225
286
|
published viewer, ever sees uncommitted SQL. Building needs edit access to the
|
|
226
287
|
app; read-only members get committed artifacts. `POST __data/<name>/refresh`
|
|
227
288
|
(the SDK's `refresh()`) rebuilds from the local text on demand.
|
|
289
|
+
`__data/<name>.snapshot.json` is asked for by the same text: the API answers
|
|
290
|
+
with the snapshot of exactly the stored artifact that text maps to, and 404
|
|
291
|
+
while a draft still needs building. An edited query never paints the
|
|
292
|
+
committed query's preview. Snapshots are not cached locally.
|
|
228
293
|
|
|
229
294
|
Builds run as jobs: the plugin asks for one (`async`), gets a job id back at
|
|
230
295
|
once, and polls it until the parquet is ready — so a query that runs for
|
|
231
296
|
minutes is waited for instead of failing when a proxy cuts the request at
|
|
232
297
|
100 s. `pollIntervalMs` / `buildTimeoutMs` tune the wait (1 s, 30 min).
|
|
233
298
|
|
|
234
|
-
Results are cached under `node_modules/.mako-data/`
|
|
235
|
-
|
|
236
|
-
|
|
299
|
+
Results are cached under `node_modules/.mako-data/` next to a fingerprint of
|
|
300
|
+
the text they were built from. A cached copy of the current text is served
|
|
301
|
+
at once. Once it is older than `revalidateMs` (five minutes), it is also
|
|
302
|
+
rebuilt in the background for the next load. `?refresh` waits for a
|
|
303
|
+
build. A copy from other text is never served: after an edit the cache is never served, however long
|
|
237
304
|
`revalidateMs` is, and while the API is unreachable a stale copy is served
|
|
238
305
|
only if it was built from the same text. It is `apply: "serve"` only —
|
|
239
306
|
production builds never load it.
|
package/data-core.js
ADDED
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
// Pure helpers behind @makoai/app-sdk's data hooks — no React and no
|
|
2
|
+
// DuckDB, so `node --test` can cover them without installing anything.
|
|
3
|
+
|
|
4
|
+
/** A SQL identifier, double-quoted (DuckDB folds case either way). */
|
|
5
|
+
export function quoteIdent(name) {
|
|
6
|
+
return '"' + String(name).replace(/"/g, '""') + '"';
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
/** A SQL string literal. */
|
|
10
|
+
export function quoteString(value) {
|
|
11
|
+
return "'" + String(value).replace(/'/g, "''") + "'";
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
// String literals and comments first (skipped), then quoted and bare
|
|
15
|
+
// identifiers (kept).
|
|
16
|
+
const SQL_TOKENS = /'(?:[^']|'')*'|--[^\n]*|\/\*[\s\S]*?\*\/|"((?:[^"]|"")+)"|([A-Za-z_][A-Za-z0-9_$]*)/g;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* The staged bindings whose names appear as identifiers in `sql`. A hint,
|
|
20
|
+
* not a parse: a column that shares a binding's name costs one extra
|
|
21
|
+
* download; a reference it misses is caught by the retry on DuckDB's
|
|
22
|
+
* "Table with name … does not exist" error.
|
|
23
|
+
*/
|
|
24
|
+
export function mentionedBindings(sql, names) {
|
|
25
|
+
if (!names.length) return [];
|
|
26
|
+
const byLower = new Map();
|
|
27
|
+
for (const n of names) {
|
|
28
|
+
const key = n.toLowerCase();
|
|
29
|
+
if (!byLower.has(key)) byLower.set(key, n);
|
|
30
|
+
}
|
|
31
|
+
const found = new Set();
|
|
32
|
+
for (const m of String(sql).matchAll(SQL_TOKENS)) {
|
|
33
|
+
const ident = m[1] != null ? m[1].replace(/""/g, '"') : m[2];
|
|
34
|
+
if (ident == null) continue;
|
|
35
|
+
const hit = byLower.get(ident.toLowerCase());
|
|
36
|
+
if (hit) found.add(hit);
|
|
37
|
+
}
|
|
38
|
+
return [...found];
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// Queries that list tables instead of naming them (SHOW TABLES,
|
|
42
|
+
// information_schema, duckdb_tables() …) see every binding, as they did when
|
|
43
|
+
// all of them were registered up front.
|
|
44
|
+
const CATALOG_QUERY = /\b(?:information_schema|duckdb_(?:tables|columns|views|constraints|indexes)|sqlite_master|pg_catalog|show_tables)\b|\bshow\s+(?:all\s+)?tables\b/i;
|
|
45
|
+
export function readsCatalog(sql) {
|
|
46
|
+
return CATALOG_QUERY.test(String(sql));
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* The binding a DuckDB "Table with name X does not exist" error is about —
|
|
51
|
+
* X itself when the index lists it, else its case-insensitive match (DuckDB
|
|
52
|
+
* folds identifier case). null for any other error, or a name no binding has.
|
|
53
|
+
*/
|
|
54
|
+
export function missingBinding(message, names) {
|
|
55
|
+
const m = /Table with name (.+?) does not exist/.exec(String(message));
|
|
56
|
+
if (!m) return null;
|
|
57
|
+
const table = m[1];
|
|
58
|
+
if (names.includes(table)) return table;
|
|
59
|
+
const lower = table.toLowerCase();
|
|
60
|
+
return names.find(n => n.toLowerCase() === lower) ?? null;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// Arrow → plain JS, with the types apps actually want. DuckDB hands DATE and
|
|
64
|
+
// TIMESTAMP columns to JS as epoch milliseconds; v1 apps (JSON over the wire)
|
|
65
|
+
// always saw strings, and a migrated formatDate() that did
|
|
66
|
+
// new Date(`${d}T00:00:00`) on a number threw and unmounted the whole app
|
|
67
|
+
// (apps.md §15.4). So: DATE → "YYYY-MM-DD", TIMESTAMP → ISO 8601, BigInt →
|
|
68
|
+
// Number when it fits. Everything else passes through untouched.
|
|
69
|
+
const DATE = 1;
|
|
70
|
+
const TIMESTAMP = 2;
|
|
71
|
+
const OTHER = 0;
|
|
72
|
+
function columnKind(field) {
|
|
73
|
+
const type = String(field.type);
|
|
74
|
+
if (type.startsWith("Date")) return DATE;
|
|
75
|
+
if (type.startsWith("Timestamp")) return TIMESTAMP;
|
|
76
|
+
return OTHER;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// Fast paths for the two column types that dominate decode time. Each
|
|
80
|
+
// reads Arrow's buffers directly and yields exactly what Arrow's getter
|
|
81
|
+
// would; a chunk it does not fully understand returns null and the column
|
|
82
|
+
// takes the general path (Arrow's own iterator).
|
|
83
|
+
const UTF8 = 5; // arrow Type.Utf8
|
|
84
|
+
const TIMESTAMP_TYPE = 10; // arrow Type.Timestamp
|
|
85
|
+
const MAX_SAFE = Number.MAX_SAFE_INTEGER;
|
|
86
|
+
let textDecoder = null;
|
|
87
|
+
function plainChunks(source, typeId) {
|
|
88
|
+
const chunks = source.data;
|
|
89
|
+
if (!Array.isArray(chunks) || !chunks.length) return null;
|
|
90
|
+
for (const d of chunks) {
|
|
91
|
+
if (!d || !d.type || d.type.typeId !== typeId || d.offset !== 0) return null;
|
|
92
|
+
if (d.nullCount > 0 && typeof d.getValid !== "function") return null;
|
|
93
|
+
}
|
|
94
|
+
return chunks;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// Utf8: Arrow runs a TextDecoder per cell. A chunk whose bytes are all
|
|
98
|
+
// ASCII decodes in one call and slices by offset — the same strings.
|
|
99
|
+
function asciiColumn(source, n) {
|
|
100
|
+
const chunks = plainChunks(source, UTF8);
|
|
101
|
+
if (!chunks) return null;
|
|
102
|
+
textDecoder ??= new TextDecoder();
|
|
103
|
+
const out = new Array(n);
|
|
104
|
+
let r = 0;
|
|
105
|
+
for (const d of chunks) {
|
|
106
|
+
const { values, valueOffsets, length } = d;
|
|
107
|
+
if (!(values instanceof Uint8Array) || !(valueOffsets instanceof Int32Array)) return null;
|
|
108
|
+
const start = valueOffsets[0];
|
|
109
|
+
const end = valueOffsets[length];
|
|
110
|
+
for (let i = start; i < end; i++) if (values[i] > 0x7f) return null;
|
|
111
|
+
const text = textDecoder.decode(values.subarray(start, end));
|
|
112
|
+
const nulls = d.nullCount > 0;
|
|
113
|
+
for (let i = 0; i < length && r < n; i++) {
|
|
114
|
+
out[r++] = nulls && !d.getValid(i)
|
|
115
|
+
? null
|
|
116
|
+
: text.slice(valueOffsets[i] - start, valueOffsets[i + 1] - start);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
return out;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// Timestamp: Arrow divides BigInts per cell (getTimestampMicrosecond is
|
|
123
|
+
// `Number(v / 1000n) + Number(v % 1000n) / 1000`). Within ±2^53 the same
|
|
124
|
+
// quotient and remainder are exact in doubles, so the same two IEEE
|
|
125
|
+
// operations give the same millisecond value. A value outside that range
|
|
126
|
+
// sends the column to the general path.
|
|
127
|
+
function timestampColumn(source, n) {
|
|
128
|
+
const chunks = plainChunks(source, TIMESTAMP_TYPE);
|
|
129
|
+
if (!chunks) return null;
|
|
130
|
+
const out = new Array(n);
|
|
131
|
+
let r = 0;
|
|
132
|
+
for (const d of chunks) {
|
|
133
|
+
const { values, length } = d;
|
|
134
|
+
if (!(values instanceof BigInt64Array)) return null;
|
|
135
|
+
const unit = d.type.unit; // 0 s, 1 ms, 2 µs, 3 ns
|
|
136
|
+
if (unit !== 0 && unit !== 1 && unit !== 2 && unit !== 3) return null;
|
|
137
|
+
const nulls = d.nullCount > 0;
|
|
138
|
+
for (let i = 0; i < length && r < n; i++) {
|
|
139
|
+
if (nulls && !d.getValid(i)) {
|
|
140
|
+
out[r++] = null;
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
143
|
+
const v = Number(values[i]);
|
|
144
|
+
if (v > MAX_SAFE || v < -MAX_SAFE) return null;
|
|
145
|
+
let ms;
|
|
146
|
+
if (unit === 0) ms = 1000 * v;
|
|
147
|
+
else if (unit === 1) ms = v;
|
|
148
|
+
else {
|
|
149
|
+
const div = unit === 2 ? 1000 : 1000000;
|
|
150
|
+
const rem = v % div;
|
|
151
|
+
ms = (v - rem) / div + rem / div;
|
|
152
|
+
}
|
|
153
|
+
out[r++] = new Date(ms).toISOString();
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
return out;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* An Arrow table as `{ rows, fields }`: one plain object per row, keyed by
|
|
161
|
+
* column name, values normalized as above, at most `cap` rows.
|
|
162
|
+
*
|
|
163
|
+
* Column by column: each column's values come from Arrow's own vector
|
|
164
|
+
* iterator (the same getter a row proxy's `row[name]` calls), so the output
|
|
165
|
+
* is identical to walking `for (const row of table)` — without a Proxy trap
|
|
166
|
+
* and a field-name lookup per cell, and with each distinct DATE formatted
|
|
167
|
+
* once. 149k rows × 16 columns: 740 ms before, 190 ms after (Chromium).
|
|
168
|
+
*/
|
|
169
|
+
export function toPlainRows(table, cap) {
|
|
170
|
+
const schemaFields = table.schema.fields;
|
|
171
|
+
const fields = schemaFields.map(f => f.name);
|
|
172
|
+
const limit = cap == null ? Infinity : cap;
|
|
173
|
+
const n = Math.max(0, Math.min(table.numRows, limit));
|
|
174
|
+
const rows = new Array(n);
|
|
175
|
+
for (let r = 0; r < n; r++) rows[r] = {};
|
|
176
|
+
for (let c = 0; c < fields.length; c++) {
|
|
177
|
+
const name = fields[c];
|
|
178
|
+
// A row proxy resolves a name to its FIRST column; a duplicated name
|
|
179
|
+
// reads that column's values (decoded with this column's type).
|
|
180
|
+
const source = table.getChildAt(fields.indexOf(name));
|
|
181
|
+
const kind = columnKind(schemaFields[c]);
|
|
182
|
+
let r = 0;
|
|
183
|
+
if (n === 0 || !source) {
|
|
184
|
+
for (; r < n; r++) rows[r][name] = undefined;
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
const fast =
|
|
188
|
+
kind === OTHER ? asciiColumn(source, n) : kind === TIMESTAMP ? timestampColumn(source, n) : null;
|
|
189
|
+
if (fast) {
|
|
190
|
+
for (; r < n; r++) rows[r][name] = fast[r];
|
|
191
|
+
continue;
|
|
192
|
+
}
|
|
193
|
+
const dates = kind === DATE ? new Map() : null;
|
|
194
|
+
for (const v of source) {
|
|
195
|
+
if (r >= n) break;
|
|
196
|
+
let out = v;
|
|
197
|
+
if (v != null) {
|
|
198
|
+
if (kind === DATE) {
|
|
199
|
+
out = dates.get(v);
|
|
200
|
+
if (out === undefined) {
|
|
201
|
+
out = new Date(Number(v)).toISOString().slice(0, 10);
|
|
202
|
+
dates.set(v, out);
|
|
203
|
+
}
|
|
204
|
+
} else if (kind === TIMESTAMP) out = new Date(Number(v)).toISOString();
|
|
205
|
+
else if (typeof v === "bigint" && v >= Number.MIN_SAFE_INTEGER && v <= Number.MAX_SAFE_INTEGER)
|
|
206
|
+
out = Number(v);
|
|
207
|
+
}
|
|
208
|
+
rows[r++][name] = out;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
return { rows, fields };
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// ---------------------------------------------------------------------------
|
|
215
|
+
// Snapshot: `__data/<name>.snapshot.json`, the first N rows of the build
|
|
216
|
+
// stream, written next to the parquet (api/src/services/binding-snapshot.ts).
|
|
217
|
+
// The server already applied toPlainRows's rules (DATE → "YYYY-MM-DD",
|
|
218
|
+
// TIMESTAMP → ISO 8601). JSON cannot carry two things the parquet path can
|
|
219
|
+
// produce, so the server writes them as strings and this decoder restores
|
|
220
|
+
// them: a 64-bit integer past 2^53 (BigInt) and a non-finite float (NaN,
|
|
221
|
+
// ±Infinity). The server writes no snapshot at all for a column type JSON
|
|
222
|
+
// cannot reproduce (DECIMAL, LIST, STRUCT, …), and any type this decoder does
|
|
223
|
+
// not know rejects the whole snapshot — the app then waits for the parquet.
|
|
224
|
+
// ---------------------------------------------------------------------------
|
|
225
|
+
export const SNAPSHOT_VERSION = 1;
|
|
226
|
+
|
|
227
|
+
const SNAPSHOT_INT64 = new Set(["BIGINT", "UBIGINT"]);
|
|
228
|
+
const SNAPSHOT_FLOAT = new Set(["DOUBLE", "FLOAT"]);
|
|
229
|
+
const SNAPSHOT_PLAIN = new Set([
|
|
230
|
+
"BOOLEAN",
|
|
231
|
+
"TINYINT",
|
|
232
|
+
"SMALLINT",
|
|
233
|
+
"INTEGER",
|
|
234
|
+
"UTINYINT",
|
|
235
|
+
"USMALLINT",
|
|
236
|
+
"UINTEGER",
|
|
237
|
+
"VARCHAR",
|
|
238
|
+
"DATE",
|
|
239
|
+
"TIMESTAMP",
|
|
240
|
+
"TIMESTAMP_S",
|
|
241
|
+
"TIMESTAMP_MS",
|
|
242
|
+
"TIMESTAMP_NS",
|
|
243
|
+
"TIMESTAMP WITH TIME ZONE",
|
|
244
|
+
]);
|
|
245
|
+
|
|
246
|
+
function snapshotDecoder(type) {
|
|
247
|
+
if (SNAPSHOT_INT64.has(type)) {
|
|
248
|
+
return v => (typeof v === "string" ? BigInt(v) : v);
|
|
249
|
+
}
|
|
250
|
+
if (SNAPSHOT_FLOAT.has(type)) {
|
|
251
|
+
return v => (typeof v === "string" ? Number(v) : v);
|
|
252
|
+
}
|
|
253
|
+
if (SNAPSHOT_PLAIN.has(type)) return v => v;
|
|
254
|
+
return null;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
export function snapshotUrl(name) {
|
|
258
|
+
return "__data/" + encodeURIComponent(name) + ".snapshot.json";
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* A snapshot body as `{ data, fields, rowCount, truncated, builtAt }`, rows
|
|
263
|
+
* decoded to what the parquet path yields for the same values. Null when the
|
|
264
|
+
* body is malformed, from another version, or names a type this SDK does not
|
|
265
|
+
* decode — the caller then waits for the parquet as if there were none.
|
|
266
|
+
*/
|
|
267
|
+
export function parseBindingSnapshot(body, rowLimit) {
|
|
268
|
+
if (!body || typeof body !== "object" || body.version !== SNAPSHOT_VERSION) return null;
|
|
269
|
+
if (!Array.isArray(body.rows) || !Array.isArray(body.fields)) return null;
|
|
270
|
+
const rowCount = Number(body.rowCount);
|
|
271
|
+
if (!Number.isInteger(rowCount) || rowCount < 0) return null;
|
|
272
|
+
const names = [];
|
|
273
|
+
const decoders = [];
|
|
274
|
+
for (const field of body.fields) {
|
|
275
|
+
if (!field || typeof field.name !== "string" || !field.name) return null;
|
|
276
|
+
const decode = snapshotDecoder(String(field.type));
|
|
277
|
+
if (!decode) return null;
|
|
278
|
+
names.push(field.name);
|
|
279
|
+
decoders.push(decode);
|
|
280
|
+
}
|
|
281
|
+
const cap = rowLimit === null ? Infinity : rowLimit == null ? Infinity : rowLimit;
|
|
282
|
+
const data = [];
|
|
283
|
+
for (const row of body.rows) {
|
|
284
|
+
if (!row || typeof row !== "object" || Array.isArray(row)) return null;
|
|
285
|
+
if (data.length >= cap) break;
|
|
286
|
+
const out = {};
|
|
287
|
+
for (let i = 0; i < names.length; i++) {
|
|
288
|
+
const v = row[names[i]];
|
|
289
|
+
out[names[i]] = v == null ? null : decoders[i](v);
|
|
290
|
+
}
|
|
291
|
+
data.push(out);
|
|
292
|
+
}
|
|
293
|
+
return {
|
|
294
|
+
data,
|
|
295
|
+
fields: names,
|
|
296
|
+
rowCount,
|
|
297
|
+
truncated: data.length < rowCount,
|
|
298
|
+
builtAt: typeof body.builtAt === "string" ? body.builtAt : null,
|
|
299
|
+
};
|
|
300
|
+
}
|
package/index.d.ts
CHANGED
|
@@ -135,6 +135,56 @@ export interface DuckDBState<Row = Record<string, unknown>>
|
|
|
135
135
|
export interface QueryOptions {
|
|
136
136
|
rowLimit?: number | null;
|
|
137
137
|
}
|
|
138
|
+
export interface UseQueryOptions extends QueryOptions {
|
|
139
|
+
/**
|
|
140
|
+
* Opt in to the preview: fetch `__data/<name>.snapshot.json` (the first
|
|
141
|
+
* rows of the last build, no DuckDB-WASM needed) while the parquet loads,
|
|
142
|
+
* and expose it as `preview`. `data`, `loading`, `phase` and every other
|
|
143
|
+
* field behave exactly as without it. Default false.
|
|
144
|
+
*/
|
|
145
|
+
snapshot?: boolean;
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* The first rows of the binding's last build, for a skeleton or a first
|
|
149
|
+
* paint. The first N rows of the build stream in the query's own order: not
|
|
150
|
+
* a sample, not sorted, not filtered. Never aggregate it.
|
|
151
|
+
*/
|
|
152
|
+
export interface QueryPreview<Row = Record<string, unknown>> {
|
|
153
|
+
/** At most a few hundred rows, typed exactly as `data` will be. */
|
|
154
|
+
data: Row[];
|
|
155
|
+
fields: string[];
|
|
156
|
+
/** Rows in the whole build, not in `data`. */
|
|
157
|
+
rowCount: number;
|
|
158
|
+
/** True when `data` holds fewer rows than `rowCount`. */
|
|
159
|
+
truncated: boolean;
|
|
160
|
+
/** ISO 8601, when the snapshot's build ran; null when unknown. */
|
|
161
|
+
builtAt: string | null;
|
|
162
|
+
}
|
|
163
|
+
export interface DuckDBOptions extends QueryOptions {
|
|
164
|
+
/**
|
|
165
|
+
* "arrow": `data` is DuckDB's Arrow Table as returned (no row objects, no
|
|
166
|
+
* date/BigInt normalization, `rowLimit` ignored). Default "rows".
|
|
167
|
+
*/
|
|
168
|
+
format?: "rows" | "arrow";
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* The page's shared DuckDB-WASM engine (`AsyncDuckDB` from
|
|
173
|
+
* @duckdb/duckdb-wasm 1.29), typed loosely so the SDK needs no dependency.
|
|
174
|
+
* Never close() or terminate() it: every hook on the page reads from it.
|
|
175
|
+
*/
|
|
176
|
+
export interface SharedDuckDB {
|
|
177
|
+
connect(): Promise<DuckDBConnection>;
|
|
178
|
+
registerFileBuffer(name: string, buffer: Uint8Array): Promise<void>;
|
|
179
|
+
dropFile(name: string): Promise<void>;
|
|
180
|
+
[key: string]: unknown;
|
|
181
|
+
}
|
|
182
|
+
export interface DuckDBConnection {
|
|
183
|
+
/** Resolves with an Arrow Table. */
|
|
184
|
+
query(sql: string): Promise<any>;
|
|
185
|
+
close(): Promise<void>;
|
|
186
|
+
[key: string]: unknown;
|
|
187
|
+
}
|
|
138
188
|
export interface MakoLocation {
|
|
139
189
|
pathname: string;
|
|
140
190
|
search: string;
|
|
@@ -143,20 +193,45 @@ export interface MakoLocation {
|
|
|
143
193
|
searchParams: URLSearchParams;
|
|
144
194
|
}
|
|
145
195
|
|
|
146
|
-
/**
|
|
196
|
+
/**
|
|
197
|
+
* Rows of a named data binding (bindings/<name>.sql, materialized).
|
|
198
|
+
*
|
|
199
|
+
* With `{ snapshot: true }` the result also carries `preview`: the snapshot
|
|
200
|
+
* rows while `loading` is true and no error has arrived, null otherwise.
|
|
201
|
+
* `data` stays null until the full parquet is in, so code that sums or
|
|
202
|
+
* filters `data` never sees a prefix.
|
|
203
|
+
*/
|
|
147
204
|
export function useQuery<Row = Record<string, unknown>>(
|
|
148
205
|
name: string,
|
|
149
|
-
opts
|
|
206
|
+
opts: UseQueryOptions & { snapshot: true },
|
|
150
207
|
): QueryState<Row> & {
|
|
151
208
|
refresh: () => Promise<RefreshResult>;
|
|
152
209
|
/** The binding's own progress record (null when it is not loading). */
|
|
153
210
|
progress: BindingProgress | null;
|
|
211
|
+
preview: QueryPreview<Row> | null;
|
|
154
212
|
};
|
|
213
|
+
export function useQuery<Row = Record<string, unknown>>(
|
|
214
|
+
name: string,
|
|
215
|
+
opts?: UseQueryOptions,
|
|
216
|
+
): QueryState<Row> & {
|
|
217
|
+
refresh: () => Promise<RefreshResult>;
|
|
218
|
+
/** The binding's own progress record (null when it is not loading). */
|
|
219
|
+
progress: BindingProgress | null;
|
|
220
|
+
};
|
|
221
|
+
|
|
222
|
+
/** This package's version at runtime; pages also get `window.__MAKO_APP_SDK__`. */
|
|
223
|
+
export const SDK_VERSION: string;
|
|
155
224
|
|
|
156
|
-
/**
|
|
225
|
+
/**
|
|
226
|
+
* Analytical SQL over the app's bindings; table names are binding names.
|
|
227
|
+
* Since 2.8 only the bindings the SQL names are downloaded (each decoded
|
|
228
|
+
* once into a DuckDB table); one it misses loads on DuckDB's "table does
|
|
229
|
+
* not exist" and the query re-runs. Catalog queries (SHOW TABLES,
|
|
230
|
+
* information_schema) load every binding.
|
|
231
|
+
*/
|
|
157
232
|
export function useDuckDB<Row = Record<string, unknown>>(
|
|
158
233
|
sql: string,
|
|
159
|
-
opts?: QueryOptions,
|
|
234
|
+
opts?: QueryOptions & { format?: "rows" },
|
|
160
235
|
): DuckDBState<Row> & {
|
|
161
236
|
refresh: () => Promise<RefreshResult[]>;
|
|
162
237
|
/** The slowest pending binding's wait, with sums; null when none. */
|
|
@@ -164,6 +239,18 @@ export function useDuckDB<Row = Record<string, unknown>>(
|
|
|
164
239
|
/** Every pending binding's progress, by name. */
|
|
165
240
|
bindings: Record<string, BindingProgress>;
|
|
166
241
|
};
|
|
242
|
+
export function useDuckDB(
|
|
243
|
+
sql: string,
|
|
244
|
+
opts: DuckDBOptions & { format: "arrow" },
|
|
245
|
+
): Omit<DuckDBState, "data"> & {
|
|
246
|
+
/** The Arrow Table (apache-arrow 17) DuckDB returned. */
|
|
247
|
+
data: any | null;
|
|
248
|
+
refresh: () => Promise<RefreshResult[]>;
|
|
249
|
+
/** The slowest pending binding's wait, with sums; null when none. */
|
|
250
|
+
progress: CombinedBindingProgress | null;
|
|
251
|
+
/** Every pending binding's progress, by name. */
|
|
252
|
+
bindings: Record<string, BindingProgress>;
|
|
253
|
+
};
|
|
167
254
|
|
|
168
255
|
export interface RefreshOptions {
|
|
169
256
|
/**
|
|
@@ -173,6 +260,42 @@ export interface RefreshOptions {
|
|
|
173
260
|
onProgress?: (progress: BindingProgress | null) => void;
|
|
174
261
|
}
|
|
175
262
|
|
|
263
|
+
/**
|
|
264
|
+
* The page's shared DuckDB, booted on first call. Use it for anything the
|
|
265
|
+
* hooks do not cover instead of importing @duckdb/duckdb-wasm, hyparquet or
|
|
266
|
+
* apache-arrow readers. Never close or terminate it.
|
|
267
|
+
*/
|
|
268
|
+
export function getDuckDB(): Promise<SharedDuckDB>;
|
|
269
|
+
|
|
270
|
+
/** Run `fn` on a fresh connection to the shared DuckDB; closed afterwards. */
|
|
271
|
+
export function withConnection<T>(fn: (conn: DuckDBConnection) => Promise<T>): Promise<T>;
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Load a binding into the shared DuckDB as a table (once per page;
|
|
275
|
+
* concurrent callers share the download). Resolves with the table name.
|
|
276
|
+
*/
|
|
277
|
+
export function ensureBinding(name: string): Promise<string>;
|
|
278
|
+
|
|
279
|
+
/** Drop a loaded binding's table to free memory; the next read reloads it. */
|
|
280
|
+
export function unloadBinding(name: string): Promise<void>;
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* The binding's raw parquet bytes, not registered anywhere — for an app with
|
|
284
|
+
* its own reader (register them via getDuckDB().registerFileBuffer to use
|
|
285
|
+
* read_parquet options). Rejects like a failed load (`status`, message).
|
|
286
|
+
*/
|
|
287
|
+
export function fetchBindingBytes(
|
|
288
|
+
name: string,
|
|
289
|
+
opts?: {
|
|
290
|
+
fresh?: boolean;
|
|
291
|
+
/**
|
|
292
|
+
* The same phases a hook reports (requesting → resolving/signing or a
|
|
293
|
+
* build → downloading with `bytes` / `bytesTotal`), then `null`.
|
|
294
|
+
*/
|
|
295
|
+
onProgress?: (progress: BindingProgress | null) => void;
|
|
296
|
+
},
|
|
297
|
+
): Promise<Uint8Array>;
|
|
298
|
+
|
|
176
299
|
/**
|
|
177
300
|
* Rematerialize one binding on demand and reload it everywhere it is used.
|
|
178
301
|
* Concurrent calls for the same binding share one request.
|
package/index.js
CHANGED
|
@@ -16,6 +16,31 @@ export {
|
|
|
16
16
|
bindingPhaseStage,
|
|
17
17
|
} from "./progress.js";
|
|
18
18
|
|
|
19
|
+
import {
|
|
20
|
+
missingBinding,
|
|
21
|
+
mentionedBindings,
|
|
22
|
+
quoteIdent,
|
|
23
|
+
readsCatalog,
|
|
24
|
+
quoteString,
|
|
25
|
+
toPlainRows,
|
|
26
|
+
parseBindingSnapshot,
|
|
27
|
+
snapshotUrl,
|
|
28
|
+
} from "./data-core.js";
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* This package's version, visible at runtime: `window.__MAKO_APP_SDK__`
|
|
32
|
+
* in a page (or this export) says which SDK a deployed app actually runs.
|
|
33
|
+
* Kept in step with package.json by snapshot.test.mjs.
|
|
34
|
+
*/
|
|
35
|
+
export const SDK_VERSION = "2.9.0";
|
|
36
|
+
if (typeof window !== "undefined") {
|
|
37
|
+
try {
|
|
38
|
+
window.__MAKO_APP_SDK__ = SDK_VERSION;
|
|
39
|
+
} catch {
|
|
40
|
+
/* a frozen window: nothing to stamp */
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
19
44
|
const DEFAULT_ROW_LIMIT = 500000;
|
|
20
45
|
const DUCKDB_ESM =
|
|
21
46
|
"https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.29.0/+esm";
|
|
@@ -23,21 +48,48 @@ const DUCKDB_ESM =
|
|
|
23
48
|
// ---------------------------------------------------------------------------
|
|
24
49
|
// DuckDB: one lazy instance per page, bindings registered on first use. The
|
|
25
50
|
// engine arrives from the CDN at runtime, so this package needs no install
|
|
26
|
-
// step and no dependencies of its own.
|
|
51
|
+
// step and no dependencies of its own. The first data hook to render starts
|
|
52
|
+
// the boot (WASM download + instantiate, ~1 s) so it overlaps the first
|
|
53
|
+
// parquet download instead of waiting for it.
|
|
27
54
|
// ---------------------------------------------------------------------------
|
|
28
55
|
let dbPromise = null;
|
|
29
56
|
function getDb() {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
57
|
+
if (!dbPromise) {
|
|
58
|
+
dbPromise = (async () => {
|
|
59
|
+
const duckdb = await import(/* @vite-ignore */ DUCKDB_ESM);
|
|
60
|
+
const bundle = await duckdb.selectBundle(duckdb.getJsDelivrBundles());
|
|
61
|
+
const worker = await duckdb.createWorker(bundle.mainWorker);
|
|
62
|
+
const db = new duckdb.AsyncDuckDB(new duckdb.VoidLogger(), worker);
|
|
63
|
+
await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
|
|
64
|
+
// read_parquet lives in an extension DuckDB fetches on first use
|
|
65
|
+
// (~660 kB from extensions.duckdb.org). Load it now, while the
|
|
66
|
+
// binding bytes are still downloading, not after they arrive.
|
|
67
|
+
const conn = await db.connect();
|
|
68
|
+
try {
|
|
69
|
+
await conn.query("LOAD parquet");
|
|
70
|
+
} catch {
|
|
71
|
+
/* autoload on first read_parquet still applies */
|
|
72
|
+
} finally {
|
|
73
|
+
await conn.close();
|
|
74
|
+
}
|
|
75
|
+
return db;
|
|
76
|
+
})();
|
|
77
|
+
// A failed boot (CDN unreachable) is retried by the next query.
|
|
78
|
+
dbPromise.catch(() => {
|
|
79
|
+
dbPromise = null;
|
|
80
|
+
});
|
|
81
|
+
}
|
|
38
82
|
return dbPromise;
|
|
39
83
|
}
|
|
40
84
|
|
|
85
|
+
/** Start the engine early; a no-op outside a browser and once started. */
|
|
86
|
+
function bootEngine() {
|
|
87
|
+
if (typeof window === "undefined" || typeof Worker === "undefined") return;
|
|
88
|
+
getDb().catch(() => {
|
|
89
|
+
/* the query that needs it reports the failure */
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
|
|
41
93
|
// The server answers failures with JSON { error, hint, retryAfterMs } —
|
|
42
94
|
// show that, not a bare status code (and never let it reach DuckDB, where it
|
|
43
95
|
// would surface as a baffling Catalog Error).
|
|
@@ -107,43 +159,66 @@ export function useBindingProgress(name) {
|
|
|
107
159
|
return name === undefined ? all : (all[name] ?? null);
|
|
108
160
|
}
|
|
109
161
|
|
|
110
|
-
|
|
162
|
+
// Each binding becomes a TABLE, decoded from its parquet once: a view over
|
|
163
|
+
// read_parquet re-decoded the file on every query (SELECT * of a 149k-row
|
|
164
|
+
// binding: 366 ms through a view, 174 ms from a table). The file buffer is
|
|
165
|
+
// dropped as soon as the table exists. CREATE OR REPLACE swaps the new table
|
|
166
|
+
// in with one commit, so a query running during a reload reads the old rows
|
|
167
|
+
// or the new ones, never a missing table. Every load registers its bytes
|
|
168
|
+
// under its own file name, so a reload never drops a file another load is
|
|
169
|
+
// still reading.
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* A binding's parquet bytes, phase-aware: asks for NDJSON phase events (a
|
|
173
|
+
* build reports each step, a stored file resolving → signing and a signed
|
|
174
|
+
* URL) and reports them through `report(patch)`. A server that predates the
|
|
175
|
+
* events answers with the bytes, as it always did.
|
|
176
|
+
*/
|
|
177
|
+
async function downloadBinding(name, bust, report) {
|
|
111
178
|
const path = "__data/" + encodeURIComponent(name) + ".parquet";
|
|
112
179
|
const what = 'Data for binding "' + name + '" could not be loaded';
|
|
113
|
-
|
|
180
|
+
report({ phase: "requesting", rowsWritten: null, bytes: null, bytesTotal: null });
|
|
181
|
+
let res = await fetch(path + (bust ? "?refresh=" + Date.now() : ""), {
|
|
182
|
+
headers: { accept: PROGRESS_ACCEPT },
|
|
183
|
+
});
|
|
184
|
+
if (isProgressResponse(res)) {
|
|
185
|
+
const final = await readProgressStream(res, e => report(progressPatch(e)));
|
|
186
|
+
if (final.phase === "failed") throw progressError(final, what);
|
|
187
|
+
report({ phase: "downloading", rowsWritten: null, bytes: 0 });
|
|
188
|
+
// No url: the server streams the bytes itself — the same path, asked
|
|
189
|
+
// plainly, serves them (the build already staged or cached them).
|
|
190
|
+
res = await fetch(final.url || path);
|
|
191
|
+
}
|
|
192
|
+
if (!res.ok) throw await responseError(res, what);
|
|
193
|
+
return readBytes(res, (bytes, bytesTotal) =>
|
|
194
|
+
report({ phase: "downloading", rowsWritten: null, bytes, bytesTotal }),
|
|
195
|
+
);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
let fileSeq = 0;
|
|
199
|
+
async function loadBinding(name, bust) {
|
|
200
|
+
const db = getDb(); // boot in parallel with the download
|
|
201
|
+
db.catch(() => {});
|
|
114
202
|
try {
|
|
115
|
-
|
|
116
|
-
// each step, a stored file reports resolving → signing and hands back
|
|
117
|
-
// the signed URL. A server that predates them answers as it always did.
|
|
118
|
-
let res = await fetch(path + (bust ? "?refresh=" + Date.now() : ""), {
|
|
119
|
-
headers: { accept: PROGRESS_ACCEPT },
|
|
120
|
-
});
|
|
121
|
-
if (isProgressResponse(res)) {
|
|
122
|
-
const final = await readProgressStream(res, e => setProgress(name, progressPatch(e)));
|
|
123
|
-
if (final.phase === "failed") throw progressError(final, what);
|
|
124
|
-
setProgress(name, { phase: "downloading", rowsWritten: null, bytes: 0 });
|
|
125
|
-
// No url: the server streams the bytes itself — the same path, asked
|
|
126
|
-
// plainly, serves them (the build already staged or cached them).
|
|
127
|
-
res = await fetch(final.url || path);
|
|
128
|
-
}
|
|
129
|
-
if (!res.ok) throw await responseError(res, what);
|
|
130
|
-
const buf = await readBytes(res, (bytes, bytesTotal) =>
|
|
131
|
-
setProgress(name, { phase: "downloading", rowsWritten: null, bytes, bytesTotal }),
|
|
132
|
-
);
|
|
203
|
+
const buf = await downloadBinding(name, bust, patch => setProgress(name, patch));
|
|
133
204
|
setProgress(name, { phase: "loading_duckdb" });
|
|
134
|
-
const
|
|
135
|
-
|
|
136
|
-
await
|
|
137
|
-
const conn = await db.connect();
|
|
205
|
+
const engine = await db;
|
|
206
|
+
const file = "mako-" + ++fileSeq + ".parquet";
|
|
207
|
+
await engine.registerFileBuffer(file, buf);
|
|
138
208
|
try {
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
"
|
|
143
|
-
|
|
209
|
+
const conn = await engine.connect();
|
|
210
|
+
try {
|
|
211
|
+
await conn.query(
|
|
212
|
+
"CREATE OR REPLACE TABLE " + quoteIdent(name) +
|
|
213
|
+
" AS SELECT * FROM read_parquet(" + quoteString(file) + ")",
|
|
214
|
+
);
|
|
215
|
+
} finally {
|
|
216
|
+
await conn.close();
|
|
217
|
+
}
|
|
144
218
|
} finally {
|
|
145
|
-
await
|
|
219
|
+
await engine.dropFile(file).catch(() => {});
|
|
146
220
|
}
|
|
221
|
+
loadedBindings.add(name);
|
|
147
222
|
} finally {
|
|
148
223
|
// A refresh owns the record until its reload is done (refreshBinding).
|
|
149
224
|
if (!refreshes.has(name)) setProgress(name, null);
|
|
@@ -151,27 +226,105 @@ async function loadBinding(name, bust) {
|
|
|
151
226
|
}
|
|
152
227
|
|
|
153
228
|
const registered = new Map(); // name -> Promise<void>
|
|
229
|
+
const loadedBindings = new Set(); // names whose table is in DuckDB
|
|
230
|
+
// Refreshed on the server while never loaded on this page: the next load
|
|
231
|
+
// must skip any cache between here and the server.
|
|
232
|
+
const staleBindings = new Set();
|
|
154
233
|
function registerBinding(name) {
|
|
155
234
|
if (!registered.has(name)) {
|
|
156
|
-
const load = loadBinding(name,
|
|
235
|
+
const load = loadBinding(name, staleBindings.delete(name));
|
|
157
236
|
registered.set(name, load);
|
|
158
|
-
load.catch(() =>
|
|
237
|
+
load.catch(() => {
|
|
238
|
+
if (registered.get(name) === load) registered.delete(name);
|
|
239
|
+
});
|
|
159
240
|
}
|
|
160
241
|
return registered.get(name);
|
|
161
242
|
}
|
|
162
243
|
|
|
244
|
+
// ---------------------------------------------------------------------------
|
|
245
|
+
// Escape hatches — the same engine and the same loader the hooks use, for
|
|
246
|
+
// apps that need more than useQuery/useDuckDB. One DuckDB per page: an app
|
|
247
|
+
// that boots its own pays a second ~1 s boot and a second copy of the data.
|
|
248
|
+
// ---------------------------------------------------------------------------
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* The page's shared DuckDB (an AsyncDuckDB), booted on first call. Never
|
|
252
|
+
* close or terminate it: every hook on the page reads from it.
|
|
253
|
+
*/
|
|
254
|
+
export function getDuckDB() {
|
|
255
|
+
return getDb();
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** Run `fn` with a fresh connection to the shared DuckDB; always closed after. */
|
|
259
|
+
export async function withConnection(fn) {
|
|
260
|
+
const db = await getDb();
|
|
261
|
+
const conn = await db.connect();
|
|
262
|
+
try {
|
|
263
|
+
return await fn(conn);
|
|
264
|
+
} finally {
|
|
265
|
+
await conn.close();
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* Make sure a binding's table exists in the shared DuckDB, loading it once
|
|
271
|
+
* per page (concurrent callers share the download). Resolves with the table
|
|
272
|
+
* name — the binding name; quote it in SQL when it is not a bare identifier.
|
|
273
|
+
*/
|
|
274
|
+
export async function ensureBinding(name) {
|
|
275
|
+
await registerBinding(name);
|
|
276
|
+
return name;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* Forget a loaded binding: drops its table so its memory is freed. The next
|
|
281
|
+
* query or ensureBinding() downloads it again.
|
|
282
|
+
*/
|
|
283
|
+
export async function unloadBinding(name) {
|
|
284
|
+
const pending = registered.get(name);
|
|
285
|
+
if (!pending) return;
|
|
286
|
+
registered.delete(name);
|
|
287
|
+
loadedBindings.delete(name);
|
|
288
|
+
await pending.catch(() => {});
|
|
289
|
+
await withConnection(conn => conn.query("DROP TABLE IF EXISTS " + quoteIdent(name)));
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* The binding's parquet bytes, for an app with its own reader (e.g.
|
|
294
|
+
* read_parquet options such as file_row_number). Not registered anywhere.
|
|
295
|
+
* `fresh` skips caches; `onProgress(loaded, total)` reports the download.
|
|
296
|
+
*/
|
|
297
|
+
export async function fetchBindingBytes(name, opts) {
|
|
298
|
+
const fresh = !!(opts && opts.fresh) || staleBindings.has(name);
|
|
299
|
+
const onProgress = opts && opts.onProgress;
|
|
300
|
+
let record = null;
|
|
301
|
+
try {
|
|
302
|
+
return await downloadBinding(name, fresh, patch => {
|
|
303
|
+
record = { ...record, ...patch };
|
|
304
|
+
if (onProgress) onProgress(record);
|
|
305
|
+
});
|
|
306
|
+
} finally {
|
|
307
|
+
if (onProgress) onProgress(null);
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
163
311
|
/** Replace a binding's loaded bytes with what the server has now. */
|
|
164
312
|
async function reloadBinding(name) {
|
|
313
|
+
const previous = registered.get(name);
|
|
314
|
+
if (!previous) {
|
|
315
|
+
// Nothing on this page reads it yet: load it fresh when a query does.
|
|
316
|
+
staleBindings.add(name);
|
|
317
|
+
return;
|
|
318
|
+
}
|
|
165
319
|
// Let a load in flight settle first, or its (older) bytes could land after
|
|
166
320
|
// the fresh ones and win.
|
|
167
|
-
|
|
168
|
-
if (previous) await previous.catch(() => {});
|
|
321
|
+
await previous.catch(() => {});
|
|
169
322
|
const load = loadBinding(name, true);
|
|
170
323
|
registered.set(name, load);
|
|
171
324
|
try {
|
|
172
325
|
await load;
|
|
173
326
|
} catch (error) {
|
|
174
|
-
registered.delete(name);
|
|
327
|
+
if (registered.get(name) === load) registered.delete(name);
|
|
175
328
|
throw error;
|
|
176
329
|
}
|
|
177
330
|
}
|
|
@@ -342,49 +495,18 @@ function bindingIndex() {
|
|
|
342
495
|
return indexPromise;
|
|
343
496
|
}
|
|
344
497
|
|
|
345
|
-
|
|
346
|
-
// TIMESTAMP columns to JS as epoch milliseconds; v1 apps (JSON over the wire)
|
|
347
|
-
// always saw strings, and a migrated formatDate() that did
|
|
348
|
-
// new Date(`${d}T00:00:00`) on a number threw and unmounted the whole app
|
|
349
|
-
// (apps.md §15.4). So: DATE → "YYYY-MM-DD", TIMESTAMP → ISO 8601, BigInt →
|
|
350
|
-
// Number when it fits. Everything else passes through untouched.
|
|
351
|
-
function columnDecoder(field) {
|
|
352
|
-
const type = String(field.type);
|
|
353
|
-
if (type.startsWith("Date")) {
|
|
354
|
-
return v => (v == null ? v : new Date(Number(v)).toISOString().slice(0, 10));
|
|
355
|
-
}
|
|
356
|
-
if (type.startsWith("Timestamp")) {
|
|
357
|
-
return v => (v == null ? v : new Date(Number(v)).toISOString());
|
|
358
|
-
}
|
|
359
|
-
return v =>
|
|
360
|
-
typeof v === "bigint" && v >= Number.MIN_SAFE_INTEGER && v <= Number.MAX_SAFE_INTEGER
|
|
361
|
-
? Number(v)
|
|
362
|
-
: v;
|
|
363
|
-
}
|
|
364
|
-
|
|
365
|
-
function toPlainRows(table, cap) {
|
|
366
|
-
const rows = [];
|
|
367
|
-
const fields = table.schema.fields.map(f => f.name);
|
|
368
|
-
const decoders = table.schema.fields.map(columnDecoder);
|
|
369
|
-
const limit = cap == null ? Infinity : cap;
|
|
370
|
-
for (const row of table) {
|
|
371
|
-
if (rows.length >= limit) break;
|
|
372
|
-
const out = {};
|
|
373
|
-
for (let i = 0; i < fields.length; i++) {
|
|
374
|
-
out[fields[i]] = decoders[i](row[fields[i]]);
|
|
375
|
-
}
|
|
376
|
-
rows.push(out);
|
|
377
|
-
}
|
|
378
|
-
return { rows, fields };
|
|
379
|
-
}
|
|
380
|
-
|
|
381
|
-
async function runSql(sql, rowLimit) {
|
|
498
|
+
async function runSql(sql, rowLimit, format) {
|
|
382
499
|
const cap = rowLimit === null ? Infinity : (rowLimit ?? DEFAULT_ROW_LIMIT);
|
|
383
500
|
const db = await getDb();
|
|
384
501
|
const conn = await db.connect();
|
|
385
502
|
try {
|
|
386
503
|
const table = await conn.query(sql);
|
|
387
504
|
const rowCount = table.numRows;
|
|
505
|
+
if (format === "arrow") {
|
|
506
|
+
// The Arrow Table as DuckDB returned it: no row objects, no
|
|
507
|
+
// normalization, no row limit.
|
|
508
|
+
return { rows: table, fields: table.schema.fields.map(f => f.name), rowCount, truncated: false };
|
|
509
|
+
}
|
|
388
510
|
const { rows, fields } = toPlainRows(table, cap);
|
|
389
511
|
return { rows, fields, rowCount, truncated: rowCount > rows.length };
|
|
390
512
|
} finally {
|
|
@@ -452,15 +574,57 @@ function hookProgress(record, state) {
|
|
|
452
574
|
};
|
|
453
575
|
}
|
|
454
576
|
|
|
577
|
+
// ---------------------------------------------------------------------------
|
|
578
|
+
// Snapshot preview — opt-in with useQuery(name, { snapshot: true }). The
|
|
579
|
+
// server writes the first N rows of each build next to its parquet as
|
|
580
|
+
// `__data/<name>.snapshot.json`. It is the first N rows of the build stream
|
|
581
|
+
// in whatever order the query produced them, so it is a skeleton for the
|
|
582
|
+
// first paint, never data: `data` stays null and `loading` stays true until
|
|
583
|
+
// the table is in DuckDB (through every load phase), and `preview`
|
|
584
|
+
// disappears the moment either the rows or an error arrive. One request per
|
|
585
|
+
// binding per page, none once the table is already loaded; a binding
|
|
586
|
+
// without a snapshot is one cheap 404.
|
|
587
|
+
// ---------------------------------------------------------------------------
|
|
588
|
+
const snapshots = new Map(); // name -> Promise<body | null>
|
|
589
|
+
function loadSnapshot(name) {
|
|
590
|
+
if (!snapshots.has(name)) {
|
|
591
|
+
snapshots.set(
|
|
592
|
+
name,
|
|
593
|
+
fetch(snapshotUrl(name))
|
|
594
|
+
.then(r => (r.ok ? r.json() : null))
|
|
595
|
+
.catch(() => null),
|
|
596
|
+
);
|
|
597
|
+
}
|
|
598
|
+
return snapshots.get(name);
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
function useSnapshotPreview(name, enabled, rowLimit) {
|
|
602
|
+
const [preview, setPreview] = React.useState(null);
|
|
603
|
+
React.useEffect(() => {
|
|
604
|
+
setPreview(null);
|
|
605
|
+
if (!enabled || loadedBindings.has(name)) return;
|
|
606
|
+
let active = true;
|
|
607
|
+
loadSnapshot(name).then(body => {
|
|
608
|
+
if (!active || loadedBindings.has(name)) return;
|
|
609
|
+
const parsed = parseBindingSnapshot(body, rowLimit);
|
|
610
|
+
if (parsed) setPreview(parsed);
|
|
611
|
+
});
|
|
612
|
+
return () => {
|
|
613
|
+
active = false;
|
|
614
|
+
};
|
|
615
|
+
}, [name, enabled, rowLimit]);
|
|
616
|
+
return preview;
|
|
617
|
+
}
|
|
618
|
+
|
|
455
619
|
export function useQuery(name, opts) {
|
|
620
|
+
bootEngine();
|
|
456
621
|
const rowLimit = opts ? opts.rowLimit : undefined;
|
|
622
|
+
const withSnapshot = Boolean(opts && opts.snapshot);
|
|
623
|
+
const preview = useSnapshotPreview(name, withSnapshot, rowLimit);
|
|
457
624
|
const state = useAsyncQuery(
|
|
458
625
|
async () => {
|
|
459
626
|
await registerBinding(name);
|
|
460
|
-
const r = await runSql(
|
|
461
|
-
'SELECT * FROM "' + name.replace(/"/g, '""') + '"',
|
|
462
|
-
rowLimit,
|
|
463
|
-
);
|
|
627
|
+
const r = await runSql("SELECT * FROM " + quoteIdent(name), rowLimit);
|
|
464
628
|
return { data: r.rows, fields: r.fields, truncated: r.truncated, rowCount: r.rowCount };
|
|
465
629
|
},
|
|
466
630
|
[name, rowLimit],
|
|
@@ -468,7 +632,12 @@ export function useQuery(name, opts) {
|
|
|
468
632
|
);
|
|
469
633
|
const progress = useBindingProgress(name);
|
|
470
634
|
const refresh = React.useCallback(() => refreshBinding(name), [name]);
|
|
471
|
-
|
|
635
|
+
const result = { ...state, ...hookProgress(progress, state), progress, refresh };
|
|
636
|
+
if (!withSnapshot) return result;
|
|
637
|
+
// Only while nothing final is on screen: not next to rows, not next to an
|
|
638
|
+
// error (a failed load must not leave preview rows looking final).
|
|
639
|
+
result.preview = state.loading && state.data === null && state.error === null ? preview : null;
|
|
640
|
+
return result;
|
|
472
641
|
}
|
|
473
642
|
|
|
474
643
|
/**
|
|
@@ -502,24 +671,39 @@ export function useViewer() {
|
|
|
502
671
|
return state;
|
|
503
672
|
}
|
|
504
673
|
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
674
|
+
/**
|
|
675
|
+
* Run `sql` with the bindings it reads loaded — and only those. The ones
|
|
676
|
+
* its text names start downloading together, alongside the engine boot; a
|
|
677
|
+
* table the hint missed shows up as DuckDB's "Table with name X does not
|
|
678
|
+
* exist", which loads X and runs the query again. Joins across bindings
|
|
679
|
+
* work by name without declaring anything, as before, and an app with 34
|
|
680
|
+
* bindings no longer downloads all 34 to draw one chart.
|
|
681
|
+
*
|
|
682
|
+
* A binding that fails to load must not sink the query silently: when the
|
|
683
|
+
* SQL then fails, the load failure is what surfaces — it is the cause.
|
|
684
|
+
*/
|
|
685
|
+
async function runOverBindings(sql, rowLimit, format) {
|
|
686
|
+
const names = await bindingIndex();
|
|
687
|
+
const failures = new Map();
|
|
688
|
+
const attempted = new Set();
|
|
689
|
+
const load = n => {
|
|
690
|
+
attempted.add(n);
|
|
691
|
+
return registerBinding(n).catch(e => {
|
|
692
|
+
failures.set(n, e);
|
|
693
|
+
});
|
|
694
|
+
};
|
|
695
|
+
await Promise.all((readsCatalog(sql) ? names : mentionedBindings(sql, names)).map(load));
|
|
696
|
+
for (;;) {
|
|
518
697
|
try {
|
|
519
|
-
|
|
520
|
-
return { data: r.rows, fields: r.fields, truncated: r.truncated, rowCount: r.rowCount };
|
|
698
|
+
return await runSql(sql, rowLimit, format);
|
|
521
699
|
} catch (error) {
|
|
522
700
|
const message = error instanceof Error ? error.message : String(error);
|
|
701
|
+
const missing = missingBinding(message, names);
|
|
702
|
+
if (missing && !attempted.has(missing)) {
|
|
703
|
+
await load(missing);
|
|
704
|
+
if (failures.has(missing)) throw failures.get(missing);
|
|
705
|
+
continue;
|
|
706
|
+
}
|
|
523
707
|
for (const [n, cause] of failures) {
|
|
524
708
|
if (message.includes(n)) throw cause;
|
|
525
709
|
}
|
|
@@ -531,12 +715,23 @@ export function useDuckDB(sql, opts) {
|
|
|
531
715
|
}
|
|
532
716
|
throw error;
|
|
533
717
|
}
|
|
534
|
-
}
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
export function useDuckDB(sql, opts) {
|
|
722
|
+
bootEngine();
|
|
723
|
+
const rowLimit = opts ? opts.rowLimit : undefined;
|
|
724
|
+
const format = opts && opts.format === "arrow" ? "arrow" : "rows";
|
|
725
|
+
const state = useAsyncQuery(async () => {
|
|
726
|
+
const r = await runOverBindings(sql, rowLimit, format);
|
|
727
|
+
return { data: r.rows, fields: r.fields, truncated: r.truncated, rowCount: r.rowCount };
|
|
728
|
+
}, [sql, rowLimit, format], refreshing => refreshing.size > 0);
|
|
535
729
|
// Every binding the query can read: the combined record says what the
|
|
536
730
|
// slowest of them is waiting on; `bindings` has each one.
|
|
537
731
|
const bindings = useBindingProgress();
|
|
538
732
|
const combined = combineBindingProgress(bindings);
|
|
539
|
-
// A query over every binding refreshes every binding
|
|
733
|
+
// A query over every binding refreshes every binding (the ones this page
|
|
734
|
+
// has not loaded yet are rebuilt and fetched fresh when first read).
|
|
540
735
|
const refresh = React.useCallback(() => refreshBindings(), []);
|
|
541
736
|
return {
|
|
542
737
|
...state,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@makoai/app-sdk",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.9.0",
|
|
4
4
|
"description": "Mako app SDK: data bindings (useQuery/useDuckDB), URL state, theme, and the house dashboard kit (/ui).",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -42,6 +42,7 @@
|
|
|
42
42
|
"index.js",
|
|
43
43
|
"index.d.ts",
|
|
44
44
|
"progress.js",
|
|
45
|
+
"data-core.js",
|
|
45
46
|
"vite.js",
|
|
46
47
|
"vite.d.ts",
|
|
47
48
|
"credentials.js",
|
|
@@ -61,5 +62,8 @@
|
|
|
61
62
|
},
|
|
62
63
|
"scripts": {
|
|
63
64
|
"test": "node --test"
|
|
65
|
+
},
|
|
66
|
+
"devDependencies": {
|
|
67
|
+
"apache-arrow": "17.0.0"
|
|
64
68
|
}
|
|
65
69
|
}
|
package/vite.d.ts
CHANGED
|
@@ -12,9 +12,11 @@ export interface MakoDataOptions {
|
|
|
12
12
|
/** Repo root. Default: the nearest ancestor holding .mako/ or .git. */
|
|
13
13
|
repoRoot?: string;
|
|
14
14
|
/**
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
15
|
+
* A cached parquet of the current binding text is always served at once;
|
|
16
|
+
* after this long (ms) it is also rebuilt in the background for the next
|
|
17
|
+
* load. Default: 5 minutes. A cache built from different binding text (or
|
|
18
|
+
* dbt environment) is never served — `Infinity` is safe while you edit
|
|
19
|
+
* SQL. `refresh()` / `?refresh` waits for the rebuild.
|
|
18
20
|
*/
|
|
19
21
|
revalidateMs?: number;
|
|
20
22
|
/**
|
package/vite.js
CHANGED
|
@@ -388,9 +388,18 @@ export function makoData(options = {}) {
|
|
|
388
388
|
}
|
|
389
389
|
}
|
|
390
390
|
|
|
391
|
+
// Atomic: a background revalidation can land while a page is still
|
|
392
|
+
// streaming the previous copy, which keeps reading the old file.
|
|
391
393
|
function writeCache(name, buf, meta) {
|
|
392
394
|
fs.mkdirSync(ctx.cacheDir, { recursive: true });
|
|
393
|
-
|
|
395
|
+
const tmp = `${cachePath(name)}.${process.pid}.${Date.now()}.tmp`;
|
|
396
|
+
fs.writeFileSync(tmp, buf);
|
|
397
|
+
try {
|
|
398
|
+
fs.renameSync(tmp, cachePath(name));
|
|
399
|
+
} catch {
|
|
400
|
+
fs.rmSync(tmp, { force: true });
|
|
401
|
+
fs.writeFileSync(cachePath(name), buf);
|
|
402
|
+
}
|
|
394
403
|
if (meta) fs.writeFileSync(metaPath(name), JSON.stringify(meta));
|
|
395
404
|
else fs.rmSync(metaPath(name), { force: true });
|
|
396
405
|
}
|
|
@@ -406,6 +415,24 @@ export function makoData(options = {}) {
|
|
|
406
415
|
let devBuildUnsupported = false;
|
|
407
416
|
const inflight = new Map();
|
|
408
417
|
|
|
418
|
+
// One background rebuild per binding at a time; a failure is logged and
|
|
419
|
+
// the next load tries again.
|
|
420
|
+
const revalidating = new Set();
|
|
421
|
+
function revalidate(name, source) {
|
|
422
|
+
if (revalidating.has(name)) return;
|
|
423
|
+
revalidating.add(name);
|
|
424
|
+
(async () => {
|
|
425
|
+
const built = source !== null ? (await devBuild(name, source, false))?.buf : null;
|
|
426
|
+
if (!built) writeCache(name, await fetchArtifact(name), null);
|
|
427
|
+
})()
|
|
428
|
+
.catch(error => {
|
|
429
|
+
server.config.logger.error(
|
|
430
|
+
` mako-data: background revalidate of ${name}: ${error instanceof Error ? error.message : String(error)}`,
|
|
431
|
+
);
|
|
432
|
+
})
|
|
433
|
+
.finally(() => revalidating.delete(name));
|
|
434
|
+
}
|
|
435
|
+
|
|
409
436
|
// Build the LOCAL binding text through the API. Concurrent asks for the
|
|
410
437
|
// same text share one request (a page mounting several tables at once).
|
|
411
438
|
function devBuild(name, source, refresh, onPhase) {
|
|
@@ -582,12 +609,55 @@ export function makoData(options = {}) {
|
|
|
582
609
|
}
|
|
583
610
|
}
|
|
584
611
|
|
|
612
|
+
// `__data/<name>.snapshot.json` — the preview for
|
|
613
|
+
// useQuery(name, { snapshot: true }). It must describe the parquet
|
|
614
|
+
// this server is serving, never another definition's: with a local
|
|
615
|
+
// binding file the API is asked by its TEXT (dev-snapshot answers only
|
|
616
|
+
// for the stored artifact dev-build would answer that text with, and
|
|
617
|
+
// 404 while a draft still needs building); without one, the committed
|
|
618
|
+
// artifact by name, as fetchArtifact serves it. Never cached here, and
|
|
619
|
+
// every failure is a 404: the SDK then just waits for the parquet.
|
|
620
|
+
async function serveSnapshot(name, res) {
|
|
621
|
+
const none = () => json(res, 404, { error: "Snapshot not available" });
|
|
622
|
+
if (problems.length) return none();
|
|
623
|
+
const source = options.materialize === false || devBuildUnsupported ? null : readSource(name);
|
|
624
|
+
const base = `${appBase()}/bindings/${encodeURIComponent(name)}`;
|
|
625
|
+
try {
|
|
626
|
+
const r =
|
|
627
|
+
source === null
|
|
628
|
+
? await apiFetch(`${base}/snapshot`, { headers: await headers() })
|
|
629
|
+
: await apiFetch(`${base}/dev-snapshot`, {
|
|
630
|
+
method: "POST",
|
|
631
|
+
headers: { ...(await headers()), "content-type": "application/json" },
|
|
632
|
+
body: JSON.stringify({
|
|
633
|
+
source,
|
|
634
|
+
...(ctx.dbtEnvironment ? { dbtEnvironment: ctx.dbtEnvironment } : {}),
|
|
635
|
+
}),
|
|
636
|
+
});
|
|
637
|
+
if (!r.ok) return none();
|
|
638
|
+
const buf = Buffer.from(await r.arrayBuffer());
|
|
639
|
+
res.setHeader("content-type", "application/json; charset=utf-8");
|
|
640
|
+
res.setHeader("content-length", String(buf.length));
|
|
641
|
+
res.setHeader("cache-control", "no-store");
|
|
642
|
+
res.setHeader("x-mako-data", "api");
|
|
643
|
+
return res.end(buf);
|
|
644
|
+
} catch {
|
|
645
|
+
return none();
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
|
|
585
649
|
server.middlewares.use(async (req, res, next) => {
|
|
586
650
|
const [pathname, query = ""] = (req.url || "").split("?");
|
|
587
651
|
if (pathname === "/__data/index.json") {
|
|
588
652
|
return json(res, 200, listBindings(ctx.bindingsDir));
|
|
589
653
|
}
|
|
590
654
|
if (pathname === "/__data/viewer.json") return serveViewer(res);
|
|
655
|
+
const snap = /^\/__data\/([^/]+)\.snapshot\.json$/.exec(pathname);
|
|
656
|
+
if (snap) {
|
|
657
|
+
const name = decodeURIComponent(snap[1]);
|
|
658
|
+
if (!BINDING_NAME.test(name)) return json(res, 400, { error: "invalid binding name" });
|
|
659
|
+
return serveSnapshot(name, res);
|
|
660
|
+
}
|
|
591
661
|
const match = /^\/__data\/([^/]+)(\.parquet|\/refresh)$/.exec(pathname);
|
|
592
662
|
if (!match) return next();
|
|
593
663
|
const name = decodeURIComponent(match[1]);
|
|
@@ -631,8 +701,15 @@ export function makoData(options = {}) {
|
|
|
631
701
|
let stream = null;
|
|
632
702
|
try {
|
|
633
703
|
const stat = fs.statSync(cached, { throwIfNoEntry: false });
|
|
634
|
-
|
|
635
|
-
|
|
704
|
+
// A copy built from this text is served at once, however old: past
|
|
705
|
+
// `revalidateMs` it is rebuilt in the background for the next load
|
|
706
|
+
// (stale-while-revalidate), so a page reload never waits on the API
|
|
707
|
+
// for data it already has. `?refresh` (the SDK's refresh()) waits.
|
|
708
|
+
if (!refresh && stat && cacheMatches()) {
|
|
709
|
+
const fresh = Date.now() - stat.mtimeMs < revalidateMs;
|
|
710
|
+
if (!fresh) revalidate(name, source);
|
|
711
|
+
return serveCached(fresh ? "cache" : "revalidating");
|
|
712
|
+
}
|
|
636
713
|
if (wantsProgress(req)) {
|
|
637
714
|
stream = progressStream(res);
|
|
638
715
|
stream.emit({ phase: "checking" });
|