@makoai/app-sdk 2.7.4 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -5
- package/data-core.js +212 -0
- package/index.d.ts +81 -2
- package/index.js +229 -102
- package/package.json +5 -1
- package/vite.d.ts +5 -3
- package/vite.js +37 -3
package/README.md
CHANGED
|
@@ -29,8 +29,8 @@ import { useQuery, useDuckDB, useSearchParams, useTheme } from "@makoai/app-sdk"
|
|
|
29
29
|
// Rows of bindings/<name>.sql, materialized to parquet by Mako.
|
|
30
30
|
const { data, loading, error } = useQuery("latest_sales");
|
|
31
31
|
|
|
32
|
-
// Analytical SQL over
|
|
33
|
-
// names are binding names).
|
|
32
|
+
// Analytical SQL over the bindings (DuckDB-WASM in the browser; table
|
|
33
|
+
// names are binding names). Only the bindings the SQL names download.
|
|
34
34
|
const totals = useDuckDB("select country, sum(revenue) r from latest_sales group by 1");
|
|
35
35
|
|
|
36
36
|
// Rematerialize on demand: re-runs the binding's query on the warehouse,
|
|
@@ -55,6 +55,38 @@ or 502 with the query's error.
|
|
|
55
55
|
`useTheme` follows the OS preference. Theme tokens (`--background`,
|
|
56
56
|
`--chart-1`, …) match the ones the scaffold's `styles.css` declares.
|
|
57
57
|
|
|
58
|
+
### Custom data work: one engine (2.8)
|
|
59
|
+
|
|
60
|
+
Never import `@duckdb/duckdb-wasm`, `hyparquet` or `apache-arrow` parquet
|
|
61
|
+
readers in an app. Use the page's engine through the SDK:
|
|
62
|
+
|
|
63
|
+
```ts
|
|
64
|
+
import { getDuckDB, withConnection, ensureBinding, unloadBinding, fetchBindingBytes, useDuckDB } from "@makoai/app-sdk";
|
|
65
|
+
|
|
66
|
+
const big = useDuckDB("select lat, lng from listings", { format: "arrow" }); // data: Arrow Table
|
|
67
|
+
const table = await ensureBinding("valuations"); // loaded once, returns "valuations"
|
|
68
|
+
const n = await withConnection(c => c.query(`select count(*) from ${table}`));
|
|
69
|
+
await unloadBinding("valuations"); // free its memory
|
|
70
|
+
const bytes = await fetchBindingBytes("valuations", { onProgress: p => p && console.log(p.phase, p.bytes, p.bytesTotal) });
|
|
71
|
+
const db = await getDuckDB(); // shared: never close() or terminate()
|
|
72
|
+
await db.registerFileBuffer("valuations.parquet", bytes); // read_parquet with your own options
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
### Loading cost (2.8)
|
|
76
|
+
|
|
77
|
+
- `useDuckDB` downloads only the bindings its SQL names. A missed name (for
|
|
78
|
+
example `query_table('x')`) loads when DuckDB reports the table missing,
|
|
79
|
+
and the query re-runs. `SHOW TABLES` / `information_schema` load all.
|
|
80
|
+
- Each binding is decoded once into a DuckDB table and its file buffer is
|
|
81
|
+
freed. A refresh swaps the table in one commit, so a running query never
|
|
82
|
+
sees it missing. A table costs about 3x the parquet in memory (10 MB of
|
|
83
|
+
parquet, 149k rows: 33 MB).
|
|
84
|
+
- The DuckDB engine and its parquet extension start loading on the first
|
|
85
|
+
data hook render, in parallel with the first binding download.
|
|
86
|
+
- Rows decode column by column: 149k rows x 16 columns in 190 ms, down
|
|
87
|
+
from 740 ms. Values are unchanged (DATE as `YYYY-MM-DD`, TIMESTAMP as
|
|
88
|
+
ISO 8601, BigInt as Number when safe).
|
|
89
|
+
|
|
58
90
|
Data arrives from `__data/<name>.parquet`, relative to the page — the same
|
|
59
91
|
path in Mako's sandbox, in a published app, and on a laptop.
|
|
60
92
|
|
|
@@ -231,9 +263,11 @@ once, and polls it until the parquet is ready — so a query that runs for
|
|
|
231
263
|
minutes is waited for instead of failing when a proxy cuts the request at
|
|
232
264
|
100 s. `pollIntervalMs` / `buildTimeoutMs` tune the wait (1 s, 30 min).
|
|
233
265
|
|
|
234
|
-
Results are cached under `node_modules/.mako-data/`
|
|
235
|
-
|
|
236
|
-
|
|
266
|
+
Results are cached under `node_modules/.mako-data/` next to a fingerprint of
|
|
267
|
+
the text they were built from. A cached copy of the current text is served
|
|
268
|
+
at once. Once it is older than `revalidateMs` (five minutes), it is also
|
|
269
|
+
rebuilt in the background for the next load. `?refresh` waits for a
|
|
270
|
+
build. A copy from other text is never served: after an edit the cache is never served, however long
|
|
237
271
|
`revalidateMs` is, and while the API is unreachable a stale copy is served
|
|
238
272
|
only if it was built from the same text. It is `apply: "serve"` only —
|
|
239
273
|
production builds never load it.
|
package/data-core.js
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
// Pure helpers behind @makoai/app-sdk's data hooks — no React and no
|
|
2
|
+
// DuckDB, so `node --test` can cover them without installing anything.
|
|
3
|
+
|
|
4
|
+
/** A SQL identifier, double-quoted (DuckDB folds case either way). */
|
|
5
|
+
export function quoteIdent(name) {
|
|
6
|
+
return '"' + String(name).replace(/"/g, '""') + '"';
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
/** A SQL string literal. */
|
|
10
|
+
export function quoteString(value) {
|
|
11
|
+
return "'" + String(value).replace(/'/g, "''") + "'";
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
// String literals and comments first (skipped), then quoted and bare
|
|
15
|
+
// identifiers (kept).
|
|
16
|
+
const SQL_TOKENS = /'(?:[^']|'')*'|--[^\n]*|\/\*[\s\S]*?\*\/|"((?:[^"]|"")+)"|([A-Za-z_][A-Za-z0-9_$]*)/g;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* The staged bindings whose names appear as identifiers in `sql`. A hint,
|
|
20
|
+
* not a parse: a column that shares a binding's name costs one extra
|
|
21
|
+
* download; a reference it misses is caught by the retry on DuckDB's
|
|
22
|
+
* "Table with name … does not exist" error.
|
|
23
|
+
*/
|
|
24
|
+
export function mentionedBindings(sql, names) {
|
|
25
|
+
if (!names.length) return [];
|
|
26
|
+
const byLower = new Map();
|
|
27
|
+
for (const n of names) {
|
|
28
|
+
const key = n.toLowerCase();
|
|
29
|
+
if (!byLower.has(key)) byLower.set(key, n);
|
|
30
|
+
}
|
|
31
|
+
const found = new Set();
|
|
32
|
+
for (const m of String(sql).matchAll(SQL_TOKENS)) {
|
|
33
|
+
const ident = m[1] != null ? m[1].replace(/""/g, '"') : m[2];
|
|
34
|
+
if (ident == null) continue;
|
|
35
|
+
const hit = byLower.get(ident.toLowerCase());
|
|
36
|
+
if (hit) found.add(hit);
|
|
37
|
+
}
|
|
38
|
+
return [...found];
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// Queries that list tables instead of naming them (SHOW TABLES,
|
|
42
|
+
// information_schema, duckdb_tables() …) see every binding, as they did when
|
|
43
|
+
// all of them were registered up front.
|
|
44
|
+
const CATALOG_QUERY = /\b(?:information_schema|duckdb_(?:tables|columns|views|constraints|indexes)|sqlite_master|pg_catalog|show_tables)\b|\bshow\s+(?:all\s+)?tables\b/i;
|
|
45
|
+
export function readsCatalog(sql) {
|
|
46
|
+
return CATALOG_QUERY.test(String(sql));
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* The binding a DuckDB "Table with name X does not exist" error is about —
|
|
51
|
+
* X itself when the index lists it, else its case-insensitive match (DuckDB
|
|
52
|
+
* folds identifier case). null for any other error, or a name no binding has.
|
|
53
|
+
*/
|
|
54
|
+
export function missingBinding(message, names) {
|
|
55
|
+
const m = /Table with name (.+?) does not exist/.exec(String(message));
|
|
56
|
+
if (!m) return null;
|
|
57
|
+
const table = m[1];
|
|
58
|
+
if (names.includes(table)) return table;
|
|
59
|
+
const lower = table.toLowerCase();
|
|
60
|
+
return names.find(n => n.toLowerCase() === lower) ?? null;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// Arrow → plain JS, with the types apps actually want. DuckDB hands DATE and
|
|
64
|
+
// TIMESTAMP columns to JS as epoch milliseconds; v1 apps (JSON over the wire)
|
|
65
|
+
// always saw strings, and a migrated formatDate() that did
|
|
66
|
+
// new Date(`${d}T00:00:00`) on a number threw and unmounted the whole app
|
|
67
|
+
// (apps.md §15.4). So: DATE → "YYYY-MM-DD", TIMESTAMP → ISO 8601, BigInt →
|
|
68
|
+
// Number when it fits. Everything else passes through untouched.
|
|
69
|
+
const DATE = 1;
|
|
70
|
+
const TIMESTAMP = 2;
|
|
71
|
+
const OTHER = 0;
|
|
72
|
+
function columnKind(field) {
|
|
73
|
+
const type = String(field.type);
|
|
74
|
+
if (type.startsWith("Date")) return DATE;
|
|
75
|
+
if (type.startsWith("Timestamp")) return TIMESTAMP;
|
|
76
|
+
return OTHER;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// Fast paths for the two column types that dominate decode time. Each
|
|
80
|
+
// reads Arrow's buffers directly and yields exactly what Arrow's getter
|
|
81
|
+
// would; a chunk it does not fully understand returns null and the column
|
|
82
|
+
// takes the general path (Arrow's own iterator).
|
|
83
|
+
const UTF8 = 5; // arrow Type.Utf8
|
|
84
|
+
const TIMESTAMP_TYPE = 10; // arrow Type.Timestamp
|
|
85
|
+
const MAX_SAFE = Number.MAX_SAFE_INTEGER;
|
|
86
|
+
let textDecoder = null;
|
|
87
|
+
function plainChunks(source, typeId) {
|
|
88
|
+
const chunks = source.data;
|
|
89
|
+
if (!Array.isArray(chunks) || !chunks.length) return null;
|
|
90
|
+
for (const d of chunks) {
|
|
91
|
+
if (!d || !d.type || d.type.typeId !== typeId || d.offset !== 0) return null;
|
|
92
|
+
if (d.nullCount > 0 && typeof d.getValid !== "function") return null;
|
|
93
|
+
}
|
|
94
|
+
return chunks;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// Utf8: Arrow runs a TextDecoder per cell. A chunk whose bytes are all
|
|
98
|
+
// ASCII decodes in one call and slices by offset — the same strings.
|
|
99
|
+
function asciiColumn(source, n) {
|
|
100
|
+
const chunks = plainChunks(source, UTF8);
|
|
101
|
+
if (!chunks) return null;
|
|
102
|
+
textDecoder ??= new TextDecoder();
|
|
103
|
+
const out = new Array(n);
|
|
104
|
+
let r = 0;
|
|
105
|
+
for (const d of chunks) {
|
|
106
|
+
const { values, valueOffsets, length } = d;
|
|
107
|
+
if (!(values instanceof Uint8Array) || !(valueOffsets instanceof Int32Array)) return null;
|
|
108
|
+
const start = valueOffsets[0];
|
|
109
|
+
const end = valueOffsets[length];
|
|
110
|
+
for (let i = start; i < end; i++) if (values[i] > 0x7f) return null;
|
|
111
|
+
const text = textDecoder.decode(values.subarray(start, end));
|
|
112
|
+
const nulls = d.nullCount > 0;
|
|
113
|
+
for (let i = 0; i < length && r < n; i++) {
|
|
114
|
+
out[r++] = nulls && !d.getValid(i)
|
|
115
|
+
? null
|
|
116
|
+
: text.slice(valueOffsets[i] - start, valueOffsets[i + 1] - start);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
return out;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// Timestamp: Arrow divides BigInts per cell (getTimestampMicrosecond is
|
|
123
|
+
// `Number(v / 1000n) + Number(v % 1000n) / 1000`). Within ±2^53 the same
|
|
124
|
+
// quotient and remainder are exact in doubles, so the same two IEEE
|
|
125
|
+
// operations give the same millisecond value. A value outside that range
|
|
126
|
+
// sends the column to the general path.
|
|
127
|
+
function timestampColumn(source, n) {
|
|
128
|
+
const chunks = plainChunks(source, TIMESTAMP_TYPE);
|
|
129
|
+
if (!chunks) return null;
|
|
130
|
+
const out = new Array(n);
|
|
131
|
+
let r = 0;
|
|
132
|
+
for (const d of chunks) {
|
|
133
|
+
const { values, length } = d;
|
|
134
|
+
if (!(values instanceof BigInt64Array)) return null;
|
|
135
|
+
const unit = d.type.unit; // 0 s, 1 ms, 2 µs, 3 ns
|
|
136
|
+
if (unit !== 0 && unit !== 1 && unit !== 2 && unit !== 3) return null;
|
|
137
|
+
const nulls = d.nullCount > 0;
|
|
138
|
+
for (let i = 0; i < length && r < n; i++) {
|
|
139
|
+
if (nulls && !d.getValid(i)) {
|
|
140
|
+
out[r++] = null;
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
143
|
+
const v = Number(values[i]);
|
|
144
|
+
if (v > MAX_SAFE || v < -MAX_SAFE) return null;
|
|
145
|
+
let ms;
|
|
146
|
+
if (unit === 0) ms = 1000 * v;
|
|
147
|
+
else if (unit === 1) ms = v;
|
|
148
|
+
else {
|
|
149
|
+
const div = unit === 2 ? 1000 : 1000000;
|
|
150
|
+
const rem = v % div;
|
|
151
|
+
ms = (v - rem) / div + rem / div;
|
|
152
|
+
}
|
|
153
|
+
out[r++] = new Date(ms).toISOString();
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
return out;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* An Arrow table as `{ rows, fields }`: one plain object per row, keyed by
|
|
161
|
+
* column name, values normalized as above, at most `cap` rows.
|
|
162
|
+
*
|
|
163
|
+
* Column by column: each column's values come from Arrow's own vector
|
|
164
|
+
* iterator (the same getter a row proxy's `row[name]` calls), so the output
|
|
165
|
+
* is identical to walking `for (const row of table)` — without a Proxy trap
|
|
166
|
+
* and a field-name lookup per cell, and with each distinct DATE formatted
|
|
167
|
+
* once. 149k rows × 16 columns: 740 ms before, 190 ms after (Chromium).
|
|
168
|
+
*/
|
|
169
|
+
export function toPlainRows(table, cap) {
|
|
170
|
+
const schemaFields = table.schema.fields;
|
|
171
|
+
const fields = schemaFields.map(f => f.name);
|
|
172
|
+
const limit = cap == null ? Infinity : cap;
|
|
173
|
+
const n = Math.max(0, Math.min(table.numRows, limit));
|
|
174
|
+
const rows = new Array(n);
|
|
175
|
+
for (let r = 0; r < n; r++) rows[r] = {};
|
|
176
|
+
for (let c = 0; c < fields.length; c++) {
|
|
177
|
+
const name = fields[c];
|
|
178
|
+
// A row proxy resolves a name to its FIRST column; a duplicated name
|
|
179
|
+
// reads that column's values (decoded with this column's type).
|
|
180
|
+
const source = table.getChildAt(fields.indexOf(name));
|
|
181
|
+
const kind = columnKind(schemaFields[c]);
|
|
182
|
+
let r = 0;
|
|
183
|
+
if (n === 0 || !source) {
|
|
184
|
+
for (; r < n; r++) rows[r][name] = undefined;
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
const fast =
|
|
188
|
+
kind === OTHER ? asciiColumn(source, n) : kind === TIMESTAMP ? timestampColumn(source, n) : null;
|
|
189
|
+
if (fast) {
|
|
190
|
+
for (; r < n; r++) rows[r][name] = fast[r];
|
|
191
|
+
continue;
|
|
192
|
+
}
|
|
193
|
+
const dates = kind === DATE ? new Map() : null;
|
|
194
|
+
for (const v of source) {
|
|
195
|
+
if (r >= n) break;
|
|
196
|
+
let out = v;
|
|
197
|
+
if (v != null) {
|
|
198
|
+
if (kind === DATE) {
|
|
199
|
+
out = dates.get(v);
|
|
200
|
+
if (out === undefined) {
|
|
201
|
+
out = new Date(Number(v)).toISOString().slice(0, 10);
|
|
202
|
+
dates.set(v, out);
|
|
203
|
+
}
|
|
204
|
+
} else if (kind === TIMESTAMP) out = new Date(Number(v)).toISOString();
|
|
205
|
+
else if (typeof v === "bigint" && v >= Number.MIN_SAFE_INTEGER && v <= Number.MAX_SAFE_INTEGER)
|
|
206
|
+
out = Number(v);
|
|
207
|
+
}
|
|
208
|
+
rows[r++][name] = out;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
return { rows, fields };
|
|
212
|
+
}
|
package/index.d.ts
CHANGED
|
@@ -135,6 +135,31 @@ export interface DuckDBState<Row = Record<string, unknown>>
|
|
|
135
135
|
export interface QueryOptions {
|
|
136
136
|
rowLimit?: number | null;
|
|
137
137
|
}
|
|
138
|
+
export interface DuckDBOptions extends QueryOptions {
|
|
139
|
+
/**
|
|
140
|
+
* "arrow": `data` is DuckDB's Arrow Table as returned (no row objects, no
|
|
141
|
+
* date/BigInt normalization, `rowLimit` ignored). Default "rows".
|
|
142
|
+
*/
|
|
143
|
+
format?: "rows" | "arrow";
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* The page's shared DuckDB-WASM engine (`AsyncDuckDB` from
|
|
148
|
+
* @duckdb/duckdb-wasm 1.29), typed loosely so the SDK needs no dependency.
|
|
149
|
+
* Never close() or terminate() it: every hook on the page reads from it.
|
|
150
|
+
*/
|
|
151
|
+
export interface SharedDuckDB {
|
|
152
|
+
connect(): Promise<DuckDBConnection>;
|
|
153
|
+
registerFileBuffer(name: string, buffer: Uint8Array): Promise<void>;
|
|
154
|
+
dropFile(name: string): Promise<void>;
|
|
155
|
+
[key: string]: unknown;
|
|
156
|
+
}
|
|
157
|
+
export interface DuckDBConnection {
|
|
158
|
+
/** Resolves with an Arrow Table. */
|
|
159
|
+
query(sql: string): Promise<any>;
|
|
160
|
+
close(): Promise<void>;
|
|
161
|
+
[key: string]: unknown;
|
|
162
|
+
}
|
|
138
163
|
export interface MakoLocation {
|
|
139
164
|
pathname: string;
|
|
140
165
|
search: string;
|
|
@@ -153,10 +178,16 @@ export function useQuery<Row = Record<string, unknown>>(
|
|
|
153
178
|
progress: BindingProgress | null;
|
|
154
179
|
};
|
|
155
180
|
|
|
156
|
-
/**
|
|
181
|
+
/**
|
|
182
|
+
* Analytical SQL over the app's bindings; table names are binding names.
|
|
183
|
+
* Since 2.8 only the bindings the SQL names are downloaded (each decoded
|
|
184
|
+
* once into a DuckDB table); one it misses loads on DuckDB's "table does
|
|
185
|
+
* not exist" and the query re-runs. Catalog queries (SHOW TABLES,
|
|
186
|
+
* information_schema) load every binding.
|
|
187
|
+
*/
|
|
157
188
|
export function useDuckDB<Row = Record<string, unknown>>(
|
|
158
189
|
sql: string,
|
|
159
|
-
opts?: QueryOptions,
|
|
190
|
+
opts?: QueryOptions & { format?: "rows" },
|
|
160
191
|
): DuckDBState<Row> & {
|
|
161
192
|
refresh: () => Promise<RefreshResult[]>;
|
|
162
193
|
/** The slowest pending binding's wait, with sums; null when none. */
|
|
@@ -164,6 +195,18 @@ export function useDuckDB<Row = Record<string, unknown>>(
|
|
|
164
195
|
/** Every pending binding's progress, by name. */
|
|
165
196
|
bindings: Record<string, BindingProgress>;
|
|
166
197
|
};
|
|
198
|
+
export function useDuckDB(
|
|
199
|
+
sql: string,
|
|
200
|
+
opts: DuckDBOptions & { format: "arrow" },
|
|
201
|
+
): Omit<DuckDBState, "data"> & {
|
|
202
|
+
/** The Arrow Table (apache-arrow 17) DuckDB returned. */
|
|
203
|
+
data: any | null;
|
|
204
|
+
refresh: () => Promise<RefreshResult[]>;
|
|
205
|
+
/** The slowest pending binding's wait, with sums; null when none. */
|
|
206
|
+
progress: CombinedBindingProgress | null;
|
|
207
|
+
/** Every pending binding's progress, by name. */
|
|
208
|
+
bindings: Record<string, BindingProgress>;
|
|
209
|
+
};
|
|
167
210
|
|
|
168
211
|
export interface RefreshOptions {
|
|
169
212
|
/**
|
|
@@ -173,6 +216,42 @@ export interface RefreshOptions {
|
|
|
173
216
|
onProgress?: (progress: BindingProgress | null) => void;
|
|
174
217
|
}
|
|
175
218
|
|
|
219
|
+
/**
|
|
220
|
+
* The page's shared DuckDB, booted on first call. Use it for anything the
|
|
221
|
+
* hooks do not cover instead of importing @duckdb/duckdb-wasm, hyparquet or
|
|
222
|
+
* apache-arrow readers. Never close or terminate it.
|
|
223
|
+
*/
|
|
224
|
+
export function getDuckDB(): Promise<SharedDuckDB>;
|
|
225
|
+
|
|
226
|
+
/** Run `fn` on a fresh connection to the shared DuckDB; closed afterwards. */
|
|
227
|
+
export function withConnection<T>(fn: (conn: DuckDBConnection) => Promise<T>): Promise<T>;
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Load a binding into the shared DuckDB as a table (once per page;
|
|
231
|
+
* concurrent callers share the download). Resolves with the table name.
|
|
232
|
+
*/
|
|
233
|
+
export function ensureBinding(name: string): Promise<string>;
|
|
234
|
+
|
|
235
|
+
/** Drop a loaded binding's table to free memory; the next read reloads it. */
|
|
236
|
+
export function unloadBinding(name: string): Promise<void>;
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* The binding's raw parquet bytes, not registered anywhere — for an app with
|
|
240
|
+
* its own reader (register them via getDuckDB().registerFileBuffer to use
|
|
241
|
+
* read_parquet options). Rejects like a failed load (`status`, message).
|
|
242
|
+
*/
|
|
243
|
+
export function fetchBindingBytes(
|
|
244
|
+
name: string,
|
|
245
|
+
opts?: {
|
|
246
|
+
fresh?: boolean;
|
|
247
|
+
/**
|
|
248
|
+
* The same phases a hook reports (requesting → resolving/signing or a
|
|
249
|
+
* build → downloading with `bytes` / `bytesTotal`), then `null`.
|
|
250
|
+
*/
|
|
251
|
+
onProgress?: (progress: BindingProgress | null) => void;
|
|
252
|
+
},
|
|
253
|
+
): Promise<Uint8Array>;
|
|
254
|
+
|
|
176
255
|
/**
|
|
177
256
|
* Rematerialize one binding on demand and reload it everywhere it is used.
|
|
178
257
|
* Concurrent calls for the same binding share one request.
|
package/index.js
CHANGED
|
@@ -16,6 +16,15 @@ export {
|
|
|
16
16
|
bindingPhaseStage,
|
|
17
17
|
} from "./progress.js";
|
|
18
18
|
|
|
19
|
+
import {
|
|
20
|
+
missingBinding,
|
|
21
|
+
mentionedBindings,
|
|
22
|
+
quoteIdent,
|
|
23
|
+
readsCatalog,
|
|
24
|
+
quoteString,
|
|
25
|
+
toPlainRows,
|
|
26
|
+
} from "./data-core.js";
|
|
27
|
+
|
|
19
28
|
const DEFAULT_ROW_LIMIT = 500000;
|
|
20
29
|
const DUCKDB_ESM =
|
|
21
30
|
"https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.29.0/+esm";
|
|
@@ -23,21 +32,48 @@ const DUCKDB_ESM =
|
|
|
23
32
|
// ---------------------------------------------------------------------------
|
|
24
33
|
// DuckDB: one lazy instance per page, bindings registered on first use. The
|
|
25
34
|
// engine arrives from the CDN at runtime, so this package needs no install
|
|
26
|
-
// step and no dependencies of its own.
|
|
35
|
+
// step and no dependencies of its own. The first data hook to render starts
|
|
36
|
+
// the boot (WASM download + instantiate, ~1 s) so it overlaps the first
|
|
37
|
+
// parquet download instead of waiting for it.
|
|
27
38
|
// ---------------------------------------------------------------------------
|
|
28
39
|
let dbPromise = null;
|
|
29
40
|
function getDb() {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
41
|
+
if (!dbPromise) {
|
|
42
|
+
dbPromise = (async () => {
|
|
43
|
+
const duckdb = await import(/* @vite-ignore */ DUCKDB_ESM);
|
|
44
|
+
const bundle = await duckdb.selectBundle(duckdb.getJsDelivrBundles());
|
|
45
|
+
const worker = await duckdb.createWorker(bundle.mainWorker);
|
|
46
|
+
const db = new duckdb.AsyncDuckDB(new duckdb.VoidLogger(), worker);
|
|
47
|
+
await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
|
|
48
|
+
// read_parquet lives in an extension DuckDB fetches on first use
|
|
49
|
+
// (~660 kB from extensions.duckdb.org). Load it now, while the
|
|
50
|
+
// binding bytes are still downloading, not after they arrive.
|
|
51
|
+
const conn = await db.connect();
|
|
52
|
+
try {
|
|
53
|
+
await conn.query("LOAD parquet");
|
|
54
|
+
} catch {
|
|
55
|
+
/* autoload on first read_parquet still applies */
|
|
56
|
+
} finally {
|
|
57
|
+
await conn.close();
|
|
58
|
+
}
|
|
59
|
+
return db;
|
|
60
|
+
})();
|
|
61
|
+
// A failed boot (CDN unreachable) is retried by the next query.
|
|
62
|
+
dbPromise.catch(() => {
|
|
63
|
+
dbPromise = null;
|
|
64
|
+
});
|
|
65
|
+
}
|
|
38
66
|
return dbPromise;
|
|
39
67
|
}
|
|
40
68
|
|
|
69
|
+
/** Start the engine early; a no-op outside a browser and once started. */
|
|
70
|
+
function bootEngine() {
|
|
71
|
+
if (typeof window === "undefined" || typeof Worker === "undefined") return;
|
|
72
|
+
getDb().catch(() => {
|
|
73
|
+
/* the query that needs it reports the failure */
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
|
|
41
77
|
// The server answers failures with JSON { error, hint, retryAfterMs } —
|
|
42
78
|
// show that, not a bare status code (and never let it reach DuckDB, where it
|
|
43
79
|
// would surface as a baffling Catalog Error).
|
|
@@ -107,42 +143,64 @@ export function useBindingProgress(name) {
|
|
|
107
143
|
return name === undefined ? all : (all[name] ?? null);
|
|
108
144
|
}
|
|
109
145
|
|
|
110
|
-
|
|
146
|
+
// Each binding becomes a TABLE, decoded from its parquet once: a view over
|
|
147
|
+
// read_parquet re-decoded the file on every query (SELECT * of a 149k-row
|
|
148
|
+
// binding: 366 ms through a view, 174 ms from a table). The file buffer is
|
|
149
|
+
// dropped as soon as the table exists. CREATE OR REPLACE swaps the new table
|
|
150
|
+
// in with one commit, so a query running during a reload reads the old rows
|
|
151
|
+
// or the new ones, never a missing table. Every load registers its bytes
|
|
152
|
+
// under its own file name, so a reload never drops a file another load is
|
|
153
|
+
// still reading.
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* A binding's parquet bytes, phase-aware: asks for NDJSON phase events (a
|
|
157
|
+
* build reports each step, a stored file resolving → signing and a signed
|
|
158
|
+
* URL) and reports them through `report(patch)`. A server that predates the
|
|
159
|
+
* events answers with the bytes, as it always did.
|
|
160
|
+
*/
|
|
161
|
+
async function downloadBinding(name, bust, report) {
|
|
111
162
|
const path = "__data/" + encodeURIComponent(name) + ".parquet";
|
|
112
163
|
const what = 'Data for binding "' + name + '" could not be loaded';
|
|
113
|
-
|
|
164
|
+
report({ phase: "requesting", rowsWritten: null, bytes: null, bytesTotal: null });
|
|
165
|
+
let res = await fetch(path + (bust ? "?refresh=" + Date.now() : ""), {
|
|
166
|
+
headers: { accept: PROGRESS_ACCEPT },
|
|
167
|
+
});
|
|
168
|
+
if (isProgressResponse(res)) {
|
|
169
|
+
const final = await readProgressStream(res, e => report(progressPatch(e)));
|
|
170
|
+
if (final.phase === "failed") throw progressError(final, what);
|
|
171
|
+
report({ phase: "downloading", rowsWritten: null, bytes: 0 });
|
|
172
|
+
// No url: the server streams the bytes itself — the same path, asked
|
|
173
|
+
// plainly, serves them (the build already staged or cached them).
|
|
174
|
+
res = await fetch(final.url || path);
|
|
175
|
+
}
|
|
176
|
+
if (!res.ok) throw await responseError(res, what);
|
|
177
|
+
return readBytes(res, (bytes, bytesTotal) =>
|
|
178
|
+
report({ phase: "downloading", rowsWritten: null, bytes, bytesTotal }),
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
let fileSeq = 0;
|
|
183
|
+
async function loadBinding(name, bust) {
|
|
184
|
+
const db = getDb(); // boot in parallel with the download
|
|
185
|
+
db.catch(() => {});
|
|
114
186
|
try {
|
|
115
|
-
|
|
116
|
-
// each step, a stored file reports resolving → signing and hands back
|
|
117
|
-
// the signed URL. A server that predates them answers as it always did.
|
|
118
|
-
let res = await fetch(path + (bust ? "?refresh=" + Date.now() : ""), {
|
|
119
|
-
headers: { accept: PROGRESS_ACCEPT },
|
|
120
|
-
});
|
|
121
|
-
if (isProgressResponse(res)) {
|
|
122
|
-
const final = await readProgressStream(res, e => setProgress(name, progressPatch(e)));
|
|
123
|
-
if (final.phase === "failed") throw progressError(final, what);
|
|
124
|
-
setProgress(name, { phase: "downloading", rowsWritten: null, bytes: 0 });
|
|
125
|
-
// No url: the server streams the bytes itself — the same path, asked
|
|
126
|
-
// plainly, serves them (the build already staged or cached them).
|
|
127
|
-
res = await fetch(final.url || path);
|
|
128
|
-
}
|
|
129
|
-
if (!res.ok) throw await responseError(res, what);
|
|
130
|
-
const buf = await readBytes(res, (bytes, bytesTotal) =>
|
|
131
|
-
setProgress(name, { phase: "downloading", rowsWritten: null, bytes, bytesTotal }),
|
|
132
|
-
);
|
|
187
|
+
const buf = await downloadBinding(name, bust, patch => setProgress(name, patch));
|
|
133
188
|
setProgress(name, { phase: "loading_duckdb" });
|
|
134
|
-
const
|
|
135
|
-
|
|
136
|
-
await
|
|
137
|
-
const conn = await db.connect();
|
|
189
|
+
const engine = await db;
|
|
190
|
+
const file = "mako-" + ++fileSeq + ".parquet";
|
|
191
|
+
await engine.registerFileBuffer(file, buf);
|
|
138
192
|
try {
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
"
|
|
143
|
-
|
|
193
|
+
const conn = await engine.connect();
|
|
194
|
+
try {
|
|
195
|
+
await conn.query(
|
|
196
|
+
"CREATE OR REPLACE TABLE " + quoteIdent(name) +
|
|
197
|
+
" AS SELECT * FROM read_parquet(" + quoteString(file) + ")",
|
|
198
|
+
);
|
|
199
|
+
} finally {
|
|
200
|
+
await conn.close();
|
|
201
|
+
}
|
|
144
202
|
} finally {
|
|
145
|
-
await
|
|
203
|
+
await engine.dropFile(file).catch(() => {});
|
|
146
204
|
}
|
|
147
205
|
} finally {
|
|
148
206
|
// A refresh owns the record until its reload is done (refreshBinding).
|
|
@@ -151,27 +209,103 @@ async function loadBinding(name, bust) {
|
|
|
151
209
|
}
|
|
152
210
|
|
|
153
211
|
const registered = new Map(); // name -> Promise<void>
|
|
212
|
+
// Refreshed on the server while never loaded on this page: the next load
|
|
213
|
+
// must skip any cache between here and the server.
|
|
214
|
+
const staleBindings = new Set();
|
|
154
215
|
function registerBinding(name) {
|
|
155
216
|
if (!registered.has(name)) {
|
|
156
|
-
const load = loadBinding(name,
|
|
217
|
+
const load = loadBinding(name, staleBindings.delete(name));
|
|
157
218
|
registered.set(name, load);
|
|
158
|
-
load.catch(() =>
|
|
219
|
+
load.catch(() => {
|
|
220
|
+
if (registered.get(name) === load) registered.delete(name);
|
|
221
|
+
});
|
|
159
222
|
}
|
|
160
223
|
return registered.get(name);
|
|
161
224
|
}
|
|
162
225
|
|
|
226
|
+
// ---------------------------------------------------------------------------
|
|
227
|
+
// Escape hatches — the same engine and the same loader the hooks use, for
|
|
228
|
+
// apps that need more than useQuery/useDuckDB. One DuckDB per page: an app
|
|
229
|
+
// that boots its own pays a second ~1 s boot and a second copy of the data.
|
|
230
|
+
// ---------------------------------------------------------------------------
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* The page's shared DuckDB (an AsyncDuckDB), booted on first call. Never
|
|
234
|
+
* close or terminate it: every hook on the page reads from it.
|
|
235
|
+
*/
|
|
236
|
+
export function getDuckDB() {
|
|
237
|
+
return getDb();
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/** Run `fn` with a fresh connection to the shared DuckDB; always closed after. */
|
|
241
|
+
export async function withConnection(fn) {
|
|
242
|
+
const db = await getDb();
|
|
243
|
+
const conn = await db.connect();
|
|
244
|
+
try {
|
|
245
|
+
return await fn(conn);
|
|
246
|
+
} finally {
|
|
247
|
+
await conn.close();
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Make sure a binding's table exists in the shared DuckDB, loading it once
|
|
253
|
+
* per page (concurrent callers share the download). Resolves with the table
|
|
254
|
+
* name — the binding name; quote it in SQL when it is not a bare identifier.
|
|
255
|
+
*/
|
|
256
|
+
export async function ensureBinding(name) {
|
|
257
|
+
await registerBinding(name);
|
|
258
|
+
return name;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* Forget a loaded binding: drops its table so its memory is freed. The next
|
|
263
|
+
* query or ensureBinding() downloads it again.
|
|
264
|
+
*/
|
|
265
|
+
export async function unloadBinding(name) {
|
|
266
|
+
const pending = registered.get(name);
|
|
267
|
+
if (!pending) return;
|
|
268
|
+
registered.delete(name);
|
|
269
|
+
await pending.catch(() => {});
|
|
270
|
+
await withConnection(conn => conn.query("DROP TABLE IF EXISTS " + quoteIdent(name)));
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* The binding's parquet bytes, for an app with its own reader (e.g.
|
|
275
|
+
* read_parquet options such as file_row_number). Not registered anywhere.
|
|
276
|
+
* `fresh` skips caches; `onProgress(loaded, total)` reports the download.
|
|
277
|
+
*/
|
|
278
|
+
export async function fetchBindingBytes(name, opts) {
|
|
279
|
+
const fresh = !!(opts && opts.fresh) || staleBindings.has(name);
|
|
280
|
+
const onProgress = opts && opts.onProgress;
|
|
281
|
+
let record = null;
|
|
282
|
+
try {
|
|
283
|
+
return await downloadBinding(name, fresh, patch => {
|
|
284
|
+
record = { ...record, ...patch };
|
|
285
|
+
if (onProgress) onProgress(record);
|
|
286
|
+
});
|
|
287
|
+
} finally {
|
|
288
|
+
if (onProgress) onProgress(null);
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
|
|
163
292
|
/** Replace a binding's loaded bytes with what the server has now. */
|
|
164
293
|
async function reloadBinding(name) {
|
|
294
|
+
const previous = registered.get(name);
|
|
295
|
+
if (!previous) {
|
|
296
|
+
// Nothing on this page reads it yet: load it fresh when a query does.
|
|
297
|
+
staleBindings.add(name);
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
165
300
|
// Let a load in flight settle first, or its (older) bytes could land after
|
|
166
301
|
// the fresh ones and win.
|
|
167
|
-
|
|
168
|
-
if (previous) await previous.catch(() => {});
|
|
302
|
+
await previous.catch(() => {});
|
|
169
303
|
const load = loadBinding(name, true);
|
|
170
304
|
registered.set(name, load);
|
|
171
305
|
try {
|
|
172
306
|
await load;
|
|
173
307
|
} catch (error) {
|
|
174
|
-
registered.delete(name);
|
|
308
|
+
if (registered.get(name) === load) registered.delete(name);
|
|
175
309
|
throw error;
|
|
176
310
|
}
|
|
177
311
|
}
|
|
@@ -342,49 +476,18 @@ function bindingIndex() {
|
|
|
342
476
|
return indexPromise;
|
|
343
477
|
}
|
|
344
478
|
|
|
345
|
-
|
|
346
|
-
// TIMESTAMP columns to JS as epoch milliseconds; v1 apps (JSON over the wire)
|
|
347
|
-
// always saw strings, and a migrated formatDate() that did
|
|
348
|
-
// new Date(`${d}T00:00:00`) on a number threw and unmounted the whole app
|
|
349
|
-
// (apps.md §15.4). So: DATE → "YYYY-MM-DD", TIMESTAMP → ISO 8601, BigInt →
|
|
350
|
-
// Number when it fits. Everything else passes through untouched.
|
|
351
|
-
function columnDecoder(field) {
|
|
352
|
-
const type = String(field.type);
|
|
353
|
-
if (type.startsWith("Date")) {
|
|
354
|
-
return v => (v == null ? v : new Date(Number(v)).toISOString().slice(0, 10));
|
|
355
|
-
}
|
|
356
|
-
if (type.startsWith("Timestamp")) {
|
|
357
|
-
return v => (v == null ? v : new Date(Number(v)).toISOString());
|
|
358
|
-
}
|
|
359
|
-
return v =>
|
|
360
|
-
typeof v === "bigint" && v >= Number.MIN_SAFE_INTEGER && v <= Number.MAX_SAFE_INTEGER
|
|
361
|
-
? Number(v)
|
|
362
|
-
: v;
|
|
363
|
-
}
|
|
364
|
-
|
|
365
|
-
function toPlainRows(table, cap) {
|
|
366
|
-
const rows = [];
|
|
367
|
-
const fields = table.schema.fields.map(f => f.name);
|
|
368
|
-
const decoders = table.schema.fields.map(columnDecoder);
|
|
369
|
-
const limit = cap == null ? Infinity : cap;
|
|
370
|
-
for (const row of table) {
|
|
371
|
-
if (rows.length >= limit) break;
|
|
372
|
-
const out = {};
|
|
373
|
-
for (let i = 0; i < fields.length; i++) {
|
|
374
|
-
out[fields[i]] = decoders[i](row[fields[i]]);
|
|
375
|
-
}
|
|
376
|
-
rows.push(out);
|
|
377
|
-
}
|
|
378
|
-
return { rows, fields };
|
|
379
|
-
}
|
|
380
|
-
|
|
381
|
-
async function runSql(sql, rowLimit) {
|
|
479
|
+
async function runSql(sql, rowLimit, format) {
|
|
382
480
|
const cap = rowLimit === null ? Infinity : (rowLimit ?? DEFAULT_ROW_LIMIT);
|
|
383
481
|
const db = await getDb();
|
|
384
482
|
const conn = await db.connect();
|
|
385
483
|
try {
|
|
386
484
|
const table = await conn.query(sql);
|
|
387
485
|
const rowCount = table.numRows;
|
|
486
|
+
if (format === "arrow") {
|
|
487
|
+
// The Arrow Table as DuckDB returned it: no row objects, no
|
|
488
|
+
// normalization, no row limit.
|
|
489
|
+
return { rows: table, fields: table.schema.fields.map(f => f.name), rowCount, truncated: false };
|
|
490
|
+
}
|
|
388
491
|
const { rows, fields } = toPlainRows(table, cap);
|
|
389
492
|
return { rows, fields, rowCount, truncated: rowCount > rows.length };
|
|
390
493
|
} finally {
|
|
@@ -453,14 +556,12 @@ function hookProgress(record, state) {
|
|
|
453
556
|
}
|
|
454
557
|
|
|
455
558
|
export function useQuery(name, opts) {
|
|
559
|
+
bootEngine();
|
|
456
560
|
const rowLimit = opts ? opts.rowLimit : undefined;
|
|
457
561
|
const state = useAsyncQuery(
|
|
458
562
|
async () => {
|
|
459
563
|
await registerBinding(name);
|
|
460
|
-
const r = await runSql(
|
|
461
|
-
'SELECT * FROM "' + name.replace(/"/g, '""') + '"',
|
|
462
|
-
rowLimit,
|
|
463
|
-
);
|
|
564
|
+
const r = await runSql("SELECT * FROM " + quoteIdent(name), rowLimit);
|
|
464
565
|
return { data: r.rows, fields: r.fields, truncated: r.truncated, rowCount: r.rowCount };
|
|
465
566
|
},
|
|
466
567
|
[name, rowLimit],
|
|
@@ -502,24 +603,39 @@ export function useViewer() {
|
|
|
502
603
|
return state;
|
|
503
604
|
}
|
|
504
605
|
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
606
|
+
/**
|
|
607
|
+
* Run `sql` with the bindings it reads loaded — and only those. The ones
|
|
608
|
+
* its text names start downloading together, alongside the engine boot; a
|
|
609
|
+
* table the hint missed shows up as DuckDB's "Table with name X does not
|
|
610
|
+
* exist", which loads X and runs the query again. Joins across bindings
|
|
611
|
+
* work by name without declaring anything, as before, and an app with 34
|
|
612
|
+
* bindings no longer downloads all 34 to draw one chart.
|
|
613
|
+
*
|
|
614
|
+
* A binding that fails to load must not sink the query silently: when the
|
|
615
|
+
* SQL then fails, the load failure is what surfaces — it is the cause.
|
|
616
|
+
*/
|
|
617
|
+
async function runOverBindings(sql, rowLimit, format) {
|
|
618
|
+
const names = await bindingIndex();
|
|
619
|
+
const failures = new Map();
|
|
620
|
+
const attempted = new Set();
|
|
621
|
+
const load = n => {
|
|
622
|
+
attempted.add(n);
|
|
623
|
+
return registerBinding(n).catch(e => {
|
|
624
|
+
failures.set(n, e);
|
|
625
|
+
});
|
|
626
|
+
};
|
|
627
|
+
await Promise.all((readsCatalog(sql) ? names : mentionedBindings(sql, names)).map(load));
|
|
628
|
+
for (;;) {
|
|
518
629
|
try {
|
|
519
|
-
|
|
520
|
-
return { data: r.rows, fields: r.fields, truncated: r.truncated, rowCount: r.rowCount };
|
|
630
|
+
return await runSql(sql, rowLimit, format);
|
|
521
631
|
} catch (error) {
|
|
522
632
|
const message = error instanceof Error ? error.message : String(error);
|
|
633
|
+
const missing = missingBinding(message, names);
|
|
634
|
+
if (missing && !attempted.has(missing)) {
|
|
635
|
+
await load(missing);
|
|
636
|
+
if (failures.has(missing)) throw failures.get(missing);
|
|
637
|
+
continue;
|
|
638
|
+
}
|
|
523
639
|
for (const [n, cause] of failures) {
|
|
524
640
|
if (message.includes(n)) throw cause;
|
|
525
641
|
}
|
|
@@ -531,12 +647,23 @@ export function useDuckDB(sql, opts) {
|
|
|
531
647
|
}
|
|
532
648
|
throw error;
|
|
533
649
|
}
|
|
534
|
-
}
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
export function useDuckDB(sql, opts) {
|
|
654
|
+
bootEngine();
|
|
655
|
+
const rowLimit = opts ? opts.rowLimit : undefined;
|
|
656
|
+
const format = opts && opts.format === "arrow" ? "arrow" : "rows";
|
|
657
|
+
const state = useAsyncQuery(async () => {
|
|
658
|
+
const r = await runOverBindings(sql, rowLimit, format);
|
|
659
|
+
return { data: r.rows, fields: r.fields, truncated: r.truncated, rowCount: r.rowCount };
|
|
660
|
+
}, [sql, rowLimit, format], refreshing => refreshing.size > 0);
|
|
535
661
|
// Every binding the query can read: the combined record says what the
|
|
536
662
|
// slowest of them is waiting on; `bindings` has each one.
|
|
537
663
|
const bindings = useBindingProgress();
|
|
538
664
|
const combined = combineBindingProgress(bindings);
|
|
539
|
-
// A query over every binding refreshes every binding
|
|
665
|
+
// A query over every binding refreshes every binding (the ones this page
|
|
666
|
+
// has not loaded yet are rebuilt and fetched fresh when first read).
|
|
540
667
|
const refresh = React.useCallback(() => refreshBindings(), []);
|
|
541
668
|
return {
|
|
542
669
|
...state,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@makoai/app-sdk",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.8.0",
|
|
4
4
|
"description": "Mako app SDK: data bindings (useQuery/useDuckDB), URL state, theme, and the house dashboard kit (/ui).",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -42,6 +42,7 @@
|
|
|
42
42
|
"index.js",
|
|
43
43
|
"index.d.ts",
|
|
44
44
|
"progress.js",
|
|
45
|
+
"data-core.js",
|
|
45
46
|
"vite.js",
|
|
46
47
|
"vite.d.ts",
|
|
47
48
|
"credentials.js",
|
|
@@ -61,5 +62,8 @@
|
|
|
61
62
|
},
|
|
62
63
|
"scripts": {
|
|
63
64
|
"test": "node --test"
|
|
65
|
+
},
|
|
66
|
+
"devDependencies": {
|
|
67
|
+
"apache-arrow": "17.0.0"
|
|
64
68
|
}
|
|
65
69
|
}
|
package/vite.d.ts
CHANGED
|
@@ -12,9 +12,11 @@ export interface MakoDataOptions {
|
|
|
12
12
|
/** Repo root. Default: the nearest ancestor holding .mako/ or .git. */
|
|
13
13
|
repoRoot?: string;
|
|
14
14
|
/**
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
15
|
+
* A cached parquet of the current binding text is always served at once;
|
|
16
|
+
* after this long (ms) it is also rebuilt in the background for the next
|
|
17
|
+
* load. Default: 5 minutes. A cache built from different binding text (or
|
|
18
|
+
* dbt environment) is never served — `Infinity` is safe while you edit
|
|
19
|
+
* SQL. `refresh()` / `?refresh` waits for the rebuild.
|
|
18
20
|
*/
|
|
19
21
|
revalidateMs?: number;
|
|
20
22
|
/**
|
package/vite.js
CHANGED
|
@@ -388,9 +388,18 @@ export function makoData(options = {}) {
|
|
|
388
388
|
}
|
|
389
389
|
}
|
|
390
390
|
|
|
391
|
+
// Atomic: a background revalidation can land while a page is still
|
|
392
|
+
// streaming the previous copy, which keeps reading the old file.
|
|
391
393
|
function writeCache(name, buf, meta) {
|
|
392
394
|
fs.mkdirSync(ctx.cacheDir, { recursive: true });
|
|
393
|
-
|
|
395
|
+
const tmp = `${cachePath(name)}.${process.pid}.${Date.now()}.tmp`;
|
|
396
|
+
fs.writeFileSync(tmp, buf);
|
|
397
|
+
try {
|
|
398
|
+
fs.renameSync(tmp, cachePath(name));
|
|
399
|
+
} catch {
|
|
400
|
+
fs.rmSync(tmp, { force: true });
|
|
401
|
+
fs.writeFileSync(cachePath(name), buf);
|
|
402
|
+
}
|
|
394
403
|
if (meta) fs.writeFileSync(metaPath(name), JSON.stringify(meta));
|
|
395
404
|
else fs.rmSync(metaPath(name), { force: true });
|
|
396
405
|
}
|
|
@@ -406,6 +415,24 @@ export function makoData(options = {}) {
|
|
|
406
415
|
let devBuildUnsupported = false;
|
|
407
416
|
const inflight = new Map();
|
|
408
417
|
|
|
418
|
+
// One background rebuild per binding at a time; a failure is logged and
|
|
419
|
+
// the next load tries again.
|
|
420
|
+
const revalidating = new Set();
|
|
421
|
+
function revalidate(name, source) {
|
|
422
|
+
if (revalidating.has(name)) return;
|
|
423
|
+
revalidating.add(name);
|
|
424
|
+
(async () => {
|
|
425
|
+
const built = source !== null ? (await devBuild(name, source, false))?.buf : null;
|
|
426
|
+
if (!built) writeCache(name, await fetchArtifact(name), null);
|
|
427
|
+
})()
|
|
428
|
+
.catch(error => {
|
|
429
|
+
server.config.logger.error(
|
|
430
|
+
` mako-data: background revalidate of ${name}: ${error instanceof Error ? error.message : String(error)}`,
|
|
431
|
+
);
|
|
432
|
+
})
|
|
433
|
+
.finally(() => revalidating.delete(name));
|
|
434
|
+
}
|
|
435
|
+
|
|
409
436
|
// Build the LOCAL binding text through the API. Concurrent asks for the
|
|
410
437
|
// same text share one request (a page mounting several tables at once).
|
|
411
438
|
function devBuild(name, source, refresh, onPhase) {
|
|
@@ -631,8 +658,15 @@ export function makoData(options = {}) {
|
|
|
631
658
|
let stream = null;
|
|
632
659
|
try {
|
|
633
660
|
const stat = fs.statSync(cached, { throwIfNoEntry: false });
|
|
634
|
-
|
|
635
|
-
|
|
661
|
+
// A copy built from this text is served at once, however old: past
|
|
662
|
+
// `revalidateMs` it is rebuilt in the background for the next load
|
|
663
|
+
// (stale-while-revalidate), so a page reload never waits on the API
|
|
664
|
+
// for data it already has. `?refresh` (the SDK's refresh()) waits.
|
|
665
|
+
if (!refresh && stat && cacheMatches()) {
|
|
666
|
+
const fresh = Date.now() - stat.mtimeMs < revalidateMs;
|
|
667
|
+
if (!fresh) revalidate(name, source);
|
|
668
|
+
return serveCached(fresh ? "cache" : "revalidating");
|
|
669
|
+
}
|
|
636
670
|
if (wantsProgress(req)) {
|
|
637
671
|
stream = progressStream(res);
|
|
638
672
|
stream.emit({ phase: "checking" });
|