@makoai/app-sdk 2.7.2 → 2.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -3
- package/index.d.ts +118 -3
- package/index.js +152 -25
- package/package.json +2 -1
- package/progress.js +233 -0
- package/ui.js +5 -2
- package/vite.js +142 -15
package/README.md
CHANGED
|
@@ -58,6 +58,40 @@ or 502 with the query's error.
|
|
|
58
58
|
Data arrives from `__data/<name>.parquet`, relative to the page — the same
|
|
59
59
|
path in Mako's sandbox, in a published app, and on a laptop.
|
|
60
60
|
|
|
61
|
+
### What a load is waiting on: `phase` (SDK ≥ 2.7.4)
|
|
62
|
+
|
|
63
|
+
`useQuery` and `useDuckDB` return `phase` while `loading` or `refreshing`,
|
|
64
|
+
plus `rowsWritten`, `bytes` and `bytesTotal` when known. Caption a progress
|
|
65
|
+
bar with `bindingPhaseLabel(...)`:
|
|
66
|
+
|
|
67
|
+
```tsx
|
|
68
|
+
import { useQuery, bindingPhaseLabel } from "@makoai/app-sdk";
|
|
69
|
+
|
|
70
|
+
const q = useQuery("latest_sales");
|
|
71
|
+
if (q.loading) return <Spinner label={bindingPhaseLabel(q)} />;
|
|
72
|
+
// "Building from warehouse · 120,000 rows" → "Getting download link"
|
|
73
|
+
// → "Downloading · 3.1 MB of 10.0 MB" → "Loading into DuckDB" → "Running query"
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
| Stage (`bindingPhaseStage`) | Phases | What is happening |
|
|
77
|
+
| --- | --- | --- |
|
|
78
|
+
| `building` | `queued`, `querying_warehouse`, `writing_parquet`, `uploading` | Mako runs the query on the warehouse. Seconds to minutes. Only a refresh, a laptop dev build or a sandbox live binding builds. |
|
|
79
|
+
| `linking` | `resolving`, `signing` | The file exists. Mako finds it and signs its download link. Under a second. |
|
|
80
|
+
| `downloading` | `downloading` | The browser fetches the parquet. |
|
|
81
|
+
| `loading` | `loading_duckdb` | DuckDB-WASM boots and registers the file. |
|
|
82
|
+
| `querying` | `running_sql` | Your SQL runs. |
|
|
83
|
+
| `waiting` | `requesting`, `checking` | Sent; the server has not said which yet. |
|
|
84
|
+
|
|
85
|
+
The SDK learns the server-side phases on the same request: it sends
|
|
86
|
+
`Accept: application/x-ndjson`, and a server that knows phases answers one
|
|
87
|
+
JSON event per line, ending with `{"phase":"ready", …}` (a stored file's
|
|
88
|
+
event carries the signed `url` and `timings`) or `{"phase":"failed",
|
|
89
|
+
"status", "error"}`. A server that predates this answers as before, and the
|
|
90
|
+
SDK still reports `requesting` → `downloading` → `loading_duckdb`.
|
|
91
|
+
`refreshBinding(name, { onProgress })` hears every phase of one refresh;
|
|
92
|
+
`useBindingProgress()` lists every pending binding. `RefreshAllButton`
|
|
93
|
+
captions each running source with its phase.
|
|
94
|
+
|
|
61
95
|
### Who is looking: `useViewer()`
|
|
62
96
|
|
|
63
97
|
```tsx
|
|
@@ -149,12 +183,18 @@ import {
|
|
|
149
183
|
names with a build-time define in `vite.config.ts`:
|
|
150
184
|
|
|
151
185
|
```ts
|
|
152
|
-
import { readdirSync } from "node:fs";
|
|
186
|
+
import { existsSync, readdirSync } from "node:fs";
|
|
187
|
+
const bindingsDir = new URL("./bindings", import.meta.url);
|
|
153
188
|
// inside defineConfig({ … }):
|
|
154
189
|
define: {
|
|
155
190
|
__APP_BINDING_NAMES__: JSON.stringify(
|
|
156
|
-
|
|
157
|
-
|
|
191
|
+
// An app with no bindings/ folder yet (a new shell app) has none:
|
|
192
|
+
// without the existsSync guard the config throws ENOENT and
|
|
193
|
+
// `vite build` fails before it starts.
|
|
194
|
+
existsSync(bindingsDir)
|
|
195
|
+
? readdirSync(bindingsDir)
|
|
196
|
+
.filter(f => f.endsWith(".sql")).map(f => f.slice(0, -4))
|
|
197
|
+
: [],
|
|
158
198
|
),
|
|
159
199
|
},
|
|
160
200
|
```
|
package/index.d.ts
CHANGED
|
@@ -1,5 +1,88 @@
|
|
|
1
1
|
import type * as React from "react";
|
|
2
2
|
|
|
3
|
+
/**
|
|
4
|
+
* What a pending binding load is waiting on (SDK ≥ 2.7.4).
|
|
5
|
+
*
|
|
6
|
+
* - `requesting`: sent, the server has not said yet (or predates phases).
|
|
7
|
+
* - `queued`, `checking`, `querying_warehouse`, `writing_parquet`,
|
|
8
|
+
* `uploading`: Mako is BUILDING the file from the warehouse (seconds to
|
|
9
|
+
* minutes). Only a refresh, a laptop dev build or a sandbox live binding
|
|
10
|
+
* builds; a published app never builds on a read.
|
|
11
|
+
* - `resolving`, `signing`: the file exists; Mako is finding it and signing
|
|
12
|
+
* its download link (well under a second).
|
|
13
|
+
* - `downloading`, `loading_duckdb`, `running_sql`: the browser's own work.
|
|
14
|
+
*/
|
|
15
|
+
export type BindingPhase =
|
|
16
|
+
| "requesting"
|
|
17
|
+
| "queued"
|
|
18
|
+
| "checking"
|
|
19
|
+
| "querying_warehouse"
|
|
20
|
+
| "writing_parquet"
|
|
21
|
+
| "uploading"
|
|
22
|
+
| "resolving"
|
|
23
|
+
| "signing"
|
|
24
|
+
| "downloading"
|
|
25
|
+
| "loading_duckdb"
|
|
26
|
+
| "running_sql";
|
|
27
|
+
|
|
28
|
+
/** The phases grouped by the wait a person cares about. */
|
|
29
|
+
export type BindingStage =
|
|
30
|
+
| "waiting"
|
|
31
|
+
| "building"
|
|
32
|
+
| "linking"
|
|
33
|
+
| "downloading"
|
|
34
|
+
| "loading"
|
|
35
|
+
| "querying";
|
|
36
|
+
|
|
37
|
+
/** One binding's progress while it loads or refreshes. */
|
|
38
|
+
export interface BindingProgress {
|
|
39
|
+
phase: BindingPhase;
|
|
40
|
+
/** Rows written so far while building; null when the server does not say. */
|
|
41
|
+
rowsWritten?: number | null;
|
|
42
|
+
/** Building: the file's size. Downloading: bytes received so far. */
|
|
43
|
+
bytes?: number | null;
|
|
44
|
+
/** Downloading: the file's size, when the server sent Content-Length. */
|
|
45
|
+
bytesTotal?: number | null;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Several bindings at once (useDuckDB): the slowest wait, sums, a count. */
|
|
49
|
+
export interface CombinedBindingProgress {
|
|
50
|
+
phase: BindingPhase;
|
|
51
|
+
rowsWritten: number | null;
|
|
52
|
+
bytes: number | null;
|
|
53
|
+
bytesTotal: number | null;
|
|
54
|
+
pending: number;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export const BINDING_PHASES: readonly BindingPhase[];
|
|
58
|
+
export const BINDING_PHASE_LABELS: Record<BindingPhase, string>;
|
|
59
|
+
/** "Building from warehouse", "Getting download link", "Downloading", "Loading into DuckDB", … */
|
|
60
|
+
export const BINDING_STAGE_LABELS: Record<BindingStage, string>;
|
|
61
|
+
export function bindingPhaseStage(phase: BindingPhase | string | null | undefined): BindingStage | null;
|
|
62
|
+
/**
|
|
63
|
+
* A progress-bar caption: the stage label plus rows or bytes when known,
|
|
64
|
+
* e.g. "Building from warehouse · 120,000 rows", "Downloading · 3.1 of 10.0 MB".
|
|
65
|
+
* Empty when nothing is pending. Pass a hook's state (`bindingPhaseLabel(q)`)
|
|
66
|
+
* or a progress record.
|
|
67
|
+
*/
|
|
68
|
+
export function bindingPhaseLabel(
|
|
69
|
+
progress:
|
|
70
|
+
| BindingPhase
|
|
71
|
+
| {
|
|
72
|
+
phase: BindingPhase | null;
|
|
73
|
+
rowsWritten?: number | null;
|
|
74
|
+
bytes?: number | null;
|
|
75
|
+
bytesTotal?: number | null;
|
|
76
|
+
}
|
|
77
|
+
| null
|
|
78
|
+
| undefined,
|
|
79
|
+
): string;
|
|
80
|
+
|
|
81
|
+
/** One binding's progress, or null when it is not loading. */
|
|
82
|
+
export function useBindingProgress(name: string): BindingProgress | null;
|
|
83
|
+
/** Every pending binding's progress, by name. */
|
|
84
|
+
export function useBindingProgress(): Record<string, BindingProgress>;
|
|
85
|
+
|
|
3
86
|
/** What a refresh built — the server's answer to `POST __data/<name>/refresh`. */
|
|
4
87
|
export interface RefreshResult {
|
|
5
88
|
binding: string;
|
|
@@ -27,6 +110,17 @@ export interface QueryState<Row = Record<string, unknown>> {
|
|
|
27
110
|
/** A refresh is in flight; `data` still holds the previous rows. */
|
|
28
111
|
refreshing: boolean;
|
|
29
112
|
truncated: boolean;
|
|
113
|
+
/**
|
|
114
|
+
* What `loading` / `refreshing` is waiting on; null when neither is true.
|
|
115
|
+
* Caption it with `bindingPhaseLabel(phase)`.
|
|
116
|
+
*/
|
|
117
|
+
phase: BindingPhase | null;
|
|
118
|
+
/** While building: rows written so far, when the server says. */
|
|
119
|
+
rowsWritten: number | null;
|
|
120
|
+
/** While building: file size; while downloading: bytes received. */
|
|
121
|
+
bytes: number | null;
|
|
122
|
+
/** While downloading: the file's size, when known. */
|
|
123
|
+
bytesTotal: number | null;
|
|
30
124
|
/**
|
|
31
125
|
* Rematerialize the data behind this query and re-run it. Rejects with a
|
|
32
126
|
* `RefreshError` — the rows on screen are untouched when it does.
|
|
@@ -53,19 +147,40 @@ export interface MakoLocation {
|
|
|
53
147
|
export function useQuery<Row = Record<string, unknown>>(
|
|
54
148
|
name: string,
|
|
55
149
|
opts?: QueryOptions,
|
|
56
|
-
): QueryState<Row> & {
|
|
150
|
+
): QueryState<Row> & {
|
|
151
|
+
refresh: () => Promise<RefreshResult>;
|
|
152
|
+
/** The binding's own progress record (null when it is not loading). */
|
|
153
|
+
progress: BindingProgress | null;
|
|
154
|
+
};
|
|
57
155
|
|
|
58
156
|
/** Analytical SQL over the app's bindings; table names are binding names. */
|
|
59
157
|
export function useDuckDB<Row = Record<string, unknown>>(
|
|
60
158
|
sql: string,
|
|
61
159
|
opts?: QueryOptions,
|
|
62
|
-
): DuckDBState<Row> & {
|
|
160
|
+
): DuckDBState<Row> & {
|
|
161
|
+
refresh: () => Promise<RefreshResult[]>;
|
|
162
|
+
/** The slowest pending binding's wait, with sums; null when none. */
|
|
163
|
+
progress: CombinedBindingProgress | null;
|
|
164
|
+
/** Every pending binding's progress, by name. */
|
|
165
|
+
bindings: Record<string, BindingProgress>;
|
|
166
|
+
};
|
|
167
|
+
|
|
168
|
+
export interface RefreshOptions {
|
|
169
|
+
/**
|
|
170
|
+
* Every phase of this refresh: the build, then the reload (link,
|
|
171
|
+
* download, DuckDB), then `null` once it settles.
|
|
172
|
+
*/
|
|
173
|
+
onProgress?: (progress: BindingProgress | null) => void;
|
|
174
|
+
}
|
|
63
175
|
|
|
64
176
|
/**
|
|
65
177
|
* Rematerialize one binding on demand and reload it everywhere it is used.
|
|
66
178
|
* Concurrent calls for the same binding share one request.
|
|
67
179
|
*/
|
|
68
|
-
export function refreshBinding(
|
|
180
|
+
export function refreshBinding(
|
|
181
|
+
name: string,
|
|
182
|
+
opts?: RefreshOptions,
|
|
183
|
+
): Promise<RefreshResult>;
|
|
69
184
|
|
|
70
185
|
/**
|
|
71
186
|
* Refresh several bindings — all of them when `names` is omitted. Every one
|
package/index.js
CHANGED
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
// @makoai/app-sdk — see package.json. Plain ESM on purpose: no build step.
|
|
2
2
|
import * as React from "react";
|
|
3
|
+
import {
|
|
4
|
+
PROGRESS_ACCEPT,
|
|
5
|
+
combineBindingProgress,
|
|
6
|
+
isProgressResponse,
|
|
7
|
+
progressError,
|
|
8
|
+
readBytes,
|
|
9
|
+
readProgressStream,
|
|
10
|
+
} from "./progress.js";
|
|
11
|
+
export {
|
|
12
|
+
BINDING_PHASES,
|
|
13
|
+
BINDING_PHASE_LABELS,
|
|
14
|
+
BINDING_STAGE_LABELS,
|
|
15
|
+
bindingPhaseLabel,
|
|
16
|
+
bindingPhaseStage,
|
|
17
|
+
} from "./progress.js";
|
|
3
18
|
|
|
4
19
|
const DEFAULT_ROW_LIMIT = 500000;
|
|
5
20
|
const DUCKDB_ESM =
|
|
@@ -40,26 +55,98 @@ async function responseError(res, what) {
|
|
|
40
55
|
return error;
|
|
41
56
|
}
|
|
42
57
|
|
|
58
|
+
// ---------------------------------------------------------------------------
|
|
59
|
+
// Progress — what each pending binding is waiting on (see progress.js). One
|
|
60
|
+
// record per binding while it loads or refreshes, gone when it settles.
|
|
61
|
+
// Notifications are batched: 34 bindings downloading at once must not
|
|
62
|
+
// re-render every hook on every chunk.
|
|
63
|
+
// ---------------------------------------------------------------------------
|
|
64
|
+
let progressState = {};
|
|
65
|
+
const progressListeners = new Set();
|
|
66
|
+
const progressWatchers = new Map(); // name -> Set<(record|null) => void>
|
|
67
|
+
let progressFlush = null;
|
|
68
|
+
function setProgress(name, patch) {
|
|
69
|
+
const next = { ...progressState };
|
|
70
|
+
if (patch === null) delete next[name];
|
|
71
|
+
else next[name] = { ...progressState[name], ...patch };
|
|
72
|
+
progressState = next;
|
|
73
|
+
const watchers = progressWatchers.get(name);
|
|
74
|
+
if (watchers) for (const w of [...watchers]) w(next[name] ?? null);
|
|
75
|
+
progressFlush ??= setTimeout(() => {
|
|
76
|
+
progressFlush = null;
|
|
77
|
+
for (const l of [...progressListeners]) l();
|
|
78
|
+
}, 50);
|
|
79
|
+
}
|
|
80
|
+
function subscribeProgress(l) {
|
|
81
|
+
progressListeners.add(l);
|
|
82
|
+
return () => progressListeners.delete(l);
|
|
83
|
+
}
|
|
84
|
+
function getProgressState() {
|
|
85
|
+
return progressState;
|
|
86
|
+
}
|
|
87
|
+
function watchProgress(name, fn) {
|
|
88
|
+
if (!progressWatchers.has(name)) progressWatchers.set(name, new Set());
|
|
89
|
+
progressWatchers.get(name).add(fn);
|
|
90
|
+
return () => progressWatchers.get(name)?.delete(fn);
|
|
91
|
+
}
|
|
92
|
+
/** A server event as a progress patch: the phase and whatever it counted. */
|
|
93
|
+
function progressPatch(event) {
|
|
94
|
+
return {
|
|
95
|
+
phase: event.phase,
|
|
96
|
+
...(typeof event.rowsWritten === "number" ? { rowsWritten: event.rowsWritten } : {}),
|
|
97
|
+
...(typeof event.bytes === "number" ? { bytes: event.bytes } : {}),
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Live progress: one binding's record (null when it is not loading), or
|
|
103
|
+
* every pending binding's when called without a name.
|
|
104
|
+
*/
|
|
105
|
+
export function useBindingProgress(name) {
|
|
106
|
+
const all = React.useSyncExternalStore(subscribeProgress, getProgressState, getProgressState);
|
|
107
|
+
return name === undefined ? all : (all[name] ?? null);
|
|
108
|
+
}
|
|
109
|
+
|
|
43
110
|
async function loadBinding(name, bust) {
|
|
44
|
-
const
|
|
45
|
-
|
|
46
|
-
);
|
|
47
|
-
if (!res.ok) {
|
|
48
|
-
throw await responseError(res, 'Data for binding "' + name + '" could not be loaded');
|
|
49
|
-
}
|
|
50
|
-
const buf = new Uint8Array(await res.arrayBuffer());
|
|
51
|
-
const db = await getDb();
|
|
52
|
-
if (bust) await db.dropFile(name + ".parquet").catch(() => {});
|
|
53
|
-
await db.registerFileBuffer(name + ".parquet", buf);
|
|
54
|
-
const conn = await db.connect();
|
|
111
|
+
const path = "__data/" + encodeURIComponent(name) + ".parquet";
|
|
112
|
+
const what = 'Data for binding "' + name + '" could not be loaded';
|
|
113
|
+
setProgress(name, { phase: "requesting", rowsWritten: null, bytes: null, bytesTotal: null });
|
|
55
114
|
try {
|
|
56
|
-
//
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
115
|
+
// Ask for phase events: a build (laptop, sandbox live binding) reports
|
|
116
|
+
// each step, a stored file reports resolving → signing and hands back
|
|
117
|
+
// the signed URL. A server that predates them answers as it always did.
|
|
118
|
+
let res = await fetch(path + (bust ? "?refresh=" + Date.now() : ""), {
|
|
119
|
+
headers: { accept: PROGRESS_ACCEPT },
|
|
120
|
+
});
|
|
121
|
+
if (isProgressResponse(res)) {
|
|
122
|
+
const final = await readProgressStream(res, e => setProgress(name, progressPatch(e)));
|
|
123
|
+
if (final.phase === "failed") throw progressError(final, what);
|
|
124
|
+
setProgress(name, { phase: "downloading", rowsWritten: null, bytes: 0 });
|
|
125
|
+
// No url: the server streams the bytes itself — the same path, asked
|
|
126
|
+
// plainly, serves them (the build already staged or cached them).
|
|
127
|
+
res = await fetch(final.url || path);
|
|
128
|
+
}
|
|
129
|
+
if (!res.ok) throw await responseError(res, what);
|
|
130
|
+
const buf = await readBytes(res, (bytes, bytesTotal) =>
|
|
131
|
+
setProgress(name, { phase: "downloading", rowsWritten: null, bytes, bytesTotal }),
|
|
60
132
|
);
|
|
133
|
+
setProgress(name, { phase: "loading_duckdb" });
|
|
134
|
+
const db = await getDb();
|
|
135
|
+
if (bust) await db.dropFile(name + ".parquet").catch(() => {});
|
|
136
|
+
await db.registerFileBuffer(name + ".parquet", buf);
|
|
137
|
+
const conn = await db.connect();
|
|
138
|
+
try {
|
|
139
|
+
// A view per binding so SQL can name tables by binding name.
|
|
140
|
+
await conn.query(
|
|
141
|
+
'CREATE OR REPLACE VIEW "' + name.replace(/"/g, '""') +
|
|
142
|
+
"\" AS SELECT * FROM read_parquet('" + name + ".parquet')",
|
|
143
|
+
);
|
|
144
|
+
} finally {
|
|
145
|
+
await conn.close();
|
|
146
|
+
}
|
|
61
147
|
} finally {
|
|
62
|
-
|
|
148
|
+
// A refresh owns the record until its reload is done (refreshBinding).
|
|
149
|
+
if (!refreshes.has(name)) setProgress(name, null);
|
|
63
150
|
}
|
|
64
151
|
}
|
|
65
152
|
|
|
@@ -122,17 +209,32 @@ const refreshes = new Map(); // name -> Promise<RefreshResult>
|
|
|
122
209
|
* built; rejects with an Error carrying `status` (403: not allowed here,
|
|
123
210
|
* 429: refreshed too recently — see `retryAfterMs`, 502: the query failed —
|
|
124
211
|
* the message says why). Concurrent calls for the same binding share one
|
|
125
|
-
* request.
|
|
212
|
+
* request. `opts.onProgress` hears every phase of it: the build, then the
|
|
213
|
+
* reload (link, download, DuckDB), then `null` when it is done.
|
|
126
214
|
*/
|
|
127
|
-
export function refreshBinding(name) {
|
|
215
|
+
export function refreshBinding(name, opts) {
|
|
216
|
+
const onProgress = opts && opts.onProgress;
|
|
217
|
+
const unwatch = onProgress ? watchProgress(name, onProgress) : null;
|
|
128
218
|
const pending = refreshes.get(name);
|
|
129
|
-
if (pending)
|
|
219
|
+
if (pending) {
|
|
220
|
+
if (unwatch) pending.finally(unwatch).catch(() => {});
|
|
221
|
+
return pending;
|
|
222
|
+
}
|
|
223
|
+
const what = 'Binding "' + name + '" could not be refreshed';
|
|
130
224
|
const run = (async () => {
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
225
|
+
setProgress(name, { phase: "requesting", rowsWritten: null, bytes: null, bytesTotal: null });
|
|
226
|
+
const res = await fetch("__data/" + encodeURIComponent(name) + "/refresh", {
|
|
227
|
+
method: "POST",
|
|
228
|
+
headers: { accept: PROGRESS_ACCEPT },
|
|
229
|
+
});
|
|
230
|
+
let body;
|
|
231
|
+
if (isProgressResponse(res)) {
|
|
232
|
+
body = await readProgressStream(res, e => setProgress(name, progressPatch(e)));
|
|
233
|
+
if (body.phase === "failed") throw progressError(body, what);
|
|
234
|
+
} else {
|
|
235
|
+
if (!res.ok) throw await responseError(res, what);
|
|
236
|
+
body = await res.json();
|
|
134
237
|
}
|
|
135
|
-
const body = await res.json();
|
|
136
238
|
await reloadBinding(name);
|
|
137
239
|
setBindingState({ ...bindingState, version: bindingState.version + 1 });
|
|
138
240
|
return {
|
|
@@ -148,6 +250,8 @@ export function refreshBinding(name) {
|
|
|
148
250
|
run
|
|
149
251
|
.finally(() => {
|
|
150
252
|
refreshes.delete(name);
|
|
253
|
+
setProgress(name, null);
|
|
254
|
+
if (unwatch) unwatch();
|
|
151
255
|
const refreshing = new Set(bindingState.refreshing);
|
|
152
256
|
refreshing.delete(name);
|
|
153
257
|
setBindingState({ ...bindingState, refreshing });
|
|
@@ -336,6 +440,18 @@ function useAsyncQuery(run, deps, isRefreshing) {
|
|
|
336
440
|
};
|
|
337
441
|
}
|
|
338
442
|
|
|
443
|
+
// A hook's view of progress: the binding's own record while it loads, else
|
|
444
|
+
// `running_sql` while the hook's SQL runs, else nothing.
|
|
445
|
+
function hookProgress(record, state) {
|
|
446
|
+
const p = record ?? (state.loading || state.refreshing ? { phase: "running_sql" } : null);
|
|
447
|
+
return {
|
|
448
|
+
phase: p ? p.phase : null,
|
|
449
|
+
rowsWritten: p && typeof p.rowsWritten === "number" ? p.rowsWritten : null,
|
|
450
|
+
bytes: p && typeof p.bytes === "number" ? p.bytes : null,
|
|
451
|
+
bytesTotal: p && typeof p.bytesTotal === "number" ? p.bytesTotal : null,
|
|
452
|
+
};
|
|
453
|
+
}
|
|
454
|
+
|
|
339
455
|
export function useQuery(name, opts) {
|
|
340
456
|
const rowLimit = opts ? opts.rowLimit : undefined;
|
|
341
457
|
const state = useAsyncQuery(
|
|
@@ -350,8 +466,9 @@ export function useQuery(name, opts) {
|
|
|
350
466
|
[name, rowLimit],
|
|
351
467
|
refreshing => refreshing.has(name),
|
|
352
468
|
);
|
|
469
|
+
const progress = useBindingProgress(name);
|
|
353
470
|
const refresh = React.useCallback(() => refreshBinding(name), [name]);
|
|
354
|
-
return { ...state, refresh };
|
|
471
|
+
return { ...state, ...hookProgress(progress, state), progress, refresh };
|
|
355
472
|
}
|
|
356
473
|
|
|
357
474
|
/**
|
|
@@ -415,9 +532,19 @@ export function useDuckDB(sql, opts) {
|
|
|
415
532
|
throw error;
|
|
416
533
|
}
|
|
417
534
|
}, [sql, rowLimit], refreshing => refreshing.size > 0);
|
|
535
|
+
// Every binding the query can read: the combined record says what the
|
|
536
|
+
// slowest of them is waiting on; `bindings` has each one.
|
|
537
|
+
const bindings = useBindingProgress();
|
|
538
|
+
const combined = combineBindingProgress(bindings);
|
|
418
539
|
// A query over every binding refreshes every binding.
|
|
419
540
|
const refresh = React.useCallback(() => refreshBindings(), []);
|
|
420
|
-
return {
|
|
541
|
+
return {
|
|
542
|
+
...state,
|
|
543
|
+
...hookProgress(combined, state),
|
|
544
|
+
progress: combined,
|
|
545
|
+
bindings,
|
|
546
|
+
refresh,
|
|
547
|
+
};
|
|
421
548
|
}
|
|
422
549
|
|
|
423
550
|
// ---------------------------------------------------------------------------
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@makoai/app-sdk",
|
|
3
|
-
"version": "2.7.
|
|
3
|
+
"version": "2.7.4",
|
|
4
4
|
"description": "Mako app SDK: data bindings (useQuery/useDuckDB), URL state, theme, and the house dashboard kit (/ui).",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -41,6 +41,7 @@
|
|
|
41
41
|
"files": [
|
|
42
42
|
"index.js",
|
|
43
43
|
"index.d.ts",
|
|
44
|
+
"progress.js",
|
|
44
45
|
"vite.js",
|
|
45
46
|
"vite.d.ts",
|
|
46
47
|
"credentials.js",
|
package/progress.js
ADDED
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
// What a binding load is waiting on — the pure half (no React, no DuckDB),
|
|
2
|
+
// so `node --test` covers it without installing anything.
|
|
3
|
+
//
|
|
4
|
+
// From 2.7.4 the SDK asks every `__data` request for phase events
|
|
5
|
+
// (`Accept: application/x-ndjson`). A server that knows them answers one
|
|
6
|
+
// JSON object per line on the same connection and ends with exactly one
|
|
7
|
+
// `ready` or `failed`; a server that predates them answers as before
|
|
8
|
+
// (parquet bytes, a 302 to them, or JSON), and the SDK still knows when the
|
|
9
|
+
// download and the DuckDB load start. See api/src/apps/binding-progress.ts.
|
|
10
|
+
|
|
11
|
+
export const PROGRESS_ACCEPT = "application/x-ndjson, */*;q=0.5";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Every phase, in the order a full rebuild passes through them. Server
|
|
15
|
+
* phases come from the stream; `requesting`, `downloading`,
|
|
16
|
+
* `loading_duckdb` and `running_sql` are the SDK's own.
|
|
17
|
+
*/
|
|
18
|
+
export const BINDING_PHASES = [
|
|
19
|
+
"requesting",
|
|
20
|
+
"queued",
|
|
21
|
+
"checking",
|
|
22
|
+
"querying_warehouse",
|
|
23
|
+
"writing_parquet",
|
|
24
|
+
"uploading",
|
|
25
|
+
"resolving",
|
|
26
|
+
"signing",
|
|
27
|
+
"downloading",
|
|
28
|
+
"loading_duckdb",
|
|
29
|
+
"running_sql",
|
|
30
|
+
];
|
|
31
|
+
|
|
32
|
+
const STAGE_OF = {
|
|
33
|
+
requesting: "waiting",
|
|
34
|
+
queued: "building",
|
|
35
|
+
checking: "waiting",
|
|
36
|
+
querying_warehouse: "building",
|
|
37
|
+
writing_parquet: "building",
|
|
38
|
+
uploading: "building",
|
|
39
|
+
resolving: "linking",
|
|
40
|
+
signing: "linking",
|
|
41
|
+
downloading: "downloading",
|
|
42
|
+
loading_duckdb: "loading",
|
|
43
|
+
running_sql: "querying",
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* The wait a person cares about: `building` (the warehouse is running the
|
|
48
|
+
* query — seconds to minutes), `linking` (Mako is finding and signing the
|
|
49
|
+
* stored file — well under a second), `downloading`, `loading` (into
|
|
50
|
+
* DuckDB), `querying` (your SQL), or `waiting` before the server says.
|
|
51
|
+
*/
|
|
52
|
+
export function bindingPhaseStage(phase) {
|
|
53
|
+
return STAGE_OF[phase] ?? null;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export const BINDING_STAGE_LABELS = {
|
|
57
|
+
waiting: "Contacting Mako",
|
|
58
|
+
building: "Building from warehouse",
|
|
59
|
+
linking: "Getting download link",
|
|
60
|
+
downloading: "Downloading",
|
|
61
|
+
loading: "Loading into DuckDB",
|
|
62
|
+
querying: "Running query",
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
/** Finer wording per phase, for a tooltip or a second line. */
|
|
66
|
+
export const BINDING_PHASE_LABELS = {
|
|
67
|
+
requesting: "Contacting Mako",
|
|
68
|
+
queued: "Waiting for a build slot",
|
|
69
|
+
checking: "Checking the binding",
|
|
70
|
+
querying_warehouse: "Querying the warehouse",
|
|
71
|
+
writing_parquet: "Writing parquet",
|
|
72
|
+
uploading: "Storing the file",
|
|
73
|
+
resolving: "Finding the file",
|
|
74
|
+
signing: "Signing the download link",
|
|
75
|
+
downloading: "Downloading",
|
|
76
|
+
loading_duckdb: "Loading into DuckDB",
|
|
77
|
+
running_sql: "Running query",
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
function formatBytes(n) {
|
|
81
|
+
if (n < 1024) return n + " B";
|
|
82
|
+
if (n < 1024 * 1024) return Math.round(n / 1024) + " KB";
|
|
83
|
+
return (n / (1024 * 1024)).toFixed(1) + " MB";
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* A caption for a progress bar: the stage label, plus rows or bytes when
|
|
88
|
+
* known. Takes a phase name or a progress record (`useQuery().progress`).
|
|
89
|
+
* Empty string when nothing is pending.
|
|
90
|
+
*/
|
|
91
|
+
export function bindingPhaseLabel(progress) {
|
|
92
|
+
const p = typeof progress === "string" ? { phase: progress } : progress;
|
|
93
|
+
const stage = p ? bindingPhaseStage(p.phase) : null;
|
|
94
|
+
if (!stage) return "";
|
|
95
|
+
const label = BINDING_STAGE_LABELS[stage];
|
|
96
|
+
if (stage === "building" && typeof p.rowsWritten === "number") {
|
|
97
|
+
return label + " · " + p.rowsWritten.toLocaleString("en-US") + " rows";
|
|
98
|
+
}
|
|
99
|
+
if (stage === "downloading" && typeof p.bytes === "number") {
|
|
100
|
+
return (
|
|
101
|
+
label +
|
|
102
|
+
" · " +
|
|
103
|
+
formatBytes(p.bytes) +
|
|
104
|
+
(typeof p.bytesTotal === "number" ? " of " + formatBytes(p.bytesTotal) : "")
|
|
105
|
+
);
|
|
106
|
+
}
|
|
107
|
+
return label;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// Most worth showing first when several bindings are pending at once: a
|
|
111
|
+
// warehouse build dwarfs everything, a download is the next longest.
|
|
112
|
+
const STAGE_WEIGHT = { building: 5, waiting: 4, downloading: 3, linking: 2, loading: 1, querying: 0 };
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* One record for many pending bindings (useDuckDB): the phase of the one
|
|
116
|
+
* furthest from done, rows summed over the builds, bytes over the
|
|
117
|
+
* downloads, and how many are pending.
|
|
118
|
+
*/
|
|
119
|
+
export function combineBindingProgress(records) {
|
|
120
|
+
const list = Object.values(records ?? {}).filter(Boolean);
|
|
121
|
+
if (!list.length) return null;
|
|
122
|
+
let top = list[0];
|
|
123
|
+
for (const r of list) {
|
|
124
|
+
const w = STAGE_WEIGHT[bindingPhaseStage(r.phase)] ?? -1;
|
|
125
|
+
if (w > (STAGE_WEIGHT[bindingPhaseStage(top.phase)] ?? -1)) top = r;
|
|
126
|
+
}
|
|
127
|
+
const stage = bindingPhaseStage(top.phase);
|
|
128
|
+
const same = list.filter(r => bindingPhaseStage(r.phase) === stage);
|
|
129
|
+
const sum = key => {
|
|
130
|
+
const known = same.filter(r => typeof r[key] === "number");
|
|
131
|
+
return known.length ? known.reduce((n, r) => n + r[key], 0) : null;
|
|
132
|
+
};
|
|
133
|
+
return {
|
|
134
|
+
phase: top.phase,
|
|
135
|
+
rowsWritten: sum("rowsWritten"),
|
|
136
|
+
bytes: sum("bytes"),
|
|
137
|
+
bytesTotal: same.every(r => typeof r.bytesTotal === "number") ? sum("bytesTotal") : null,
|
|
138
|
+
pending: list.length,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** Did the server answer with phase events? */
|
|
143
|
+
export function isProgressResponse(res) {
|
|
144
|
+
const type = (res.headers && res.headers.get("content-type")) || "";
|
|
145
|
+
return type.split(";")[0].trim().toLowerCase() === "application/x-ndjson";
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Read a phase stream: `onEvent` for every line, resolves with the terminal
|
|
150
|
+
* event (`ready` or `failed`). A stream that ends without one resolves as
|
|
151
|
+
* `failed` with status 502 — the connection broke before the server said.
|
|
152
|
+
*/
|
|
153
|
+
export async function readProgressStream(res, onEvent) {
|
|
154
|
+
const reader = res.body.getReader();
|
|
155
|
+
const decoder = new TextDecoder();
|
|
156
|
+
let buffered = "";
|
|
157
|
+
const handle = line => {
|
|
158
|
+
const text = line.trim();
|
|
159
|
+
if (!text) return null;
|
|
160
|
+
let event;
|
|
161
|
+
try {
|
|
162
|
+
event = JSON.parse(text);
|
|
163
|
+
} catch {
|
|
164
|
+
return null;
|
|
165
|
+
}
|
|
166
|
+
if (event && (event.phase === "ready" || event.phase === "failed")) return event;
|
|
167
|
+
if (event && event.phase) onEvent(event);
|
|
168
|
+
return null;
|
|
169
|
+
};
|
|
170
|
+
for (;;) {
|
|
171
|
+
const { done, value } = await reader.read();
|
|
172
|
+
buffered += decoder.decode(value ?? new Uint8Array(), { stream: !done });
|
|
173
|
+
let nl;
|
|
174
|
+
while ((nl = buffered.indexOf("\n")) !== -1) {
|
|
175
|
+
const final = handle(buffered.slice(0, nl));
|
|
176
|
+
buffered = buffered.slice(nl + 1);
|
|
177
|
+
if (final) {
|
|
178
|
+
reader.cancel().catch(() => {});
|
|
179
|
+
return final;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
if (done) break;
|
|
183
|
+
}
|
|
184
|
+
return (
|
|
185
|
+
handle(buffered) ?? {
|
|
186
|
+
phase: "failed",
|
|
187
|
+
status: 502,
|
|
188
|
+
error: "The connection closed before the server finished",
|
|
189
|
+
}
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* The response body as bytes, reporting `(received, total)` as it arrives.
|
|
195
|
+
* `total` is null when the server sent no Content-Length.
|
|
196
|
+
*/
|
|
197
|
+
export async function readBytes(res, onProgress) {
|
|
198
|
+
const total = Number(res.headers.get("content-length")) || null;
|
|
199
|
+
if (!res.body || !res.body.getReader) {
|
|
200
|
+
const buf = new Uint8Array(await res.arrayBuffer());
|
|
201
|
+
onProgress(buf.length, total);
|
|
202
|
+
return buf;
|
|
203
|
+
}
|
|
204
|
+
const reader = res.body.getReader();
|
|
205
|
+
const chunks = [];
|
|
206
|
+
let received = 0;
|
|
207
|
+
onProgress(0, total);
|
|
208
|
+
for (;;) {
|
|
209
|
+
const { done, value } = await reader.read();
|
|
210
|
+
if (done) break;
|
|
211
|
+
chunks.push(value);
|
|
212
|
+
received += value.length;
|
|
213
|
+
onProgress(received, total);
|
|
214
|
+
}
|
|
215
|
+
if (chunks.length === 1) return chunks[0];
|
|
216
|
+
const out = new Uint8Array(received);
|
|
217
|
+
let at = 0;
|
|
218
|
+
for (const c of chunks) {
|
|
219
|
+
out.set(c, at);
|
|
220
|
+
at += c.length;
|
|
221
|
+
}
|
|
222
|
+
return out;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/** The Error a `failed` event becomes — the same shape as an HTTP failure. */
|
|
226
|
+
export function progressError(event, what) {
|
|
227
|
+
const detail = [event.error, event.hint].filter(Boolean).join(" — ");
|
|
228
|
+
const status = typeof event.status === "number" ? event.status : 502;
|
|
229
|
+
const error = new Error(what + " (HTTP " + status + ")" + (detail ? ": " + detail : ""));
|
|
230
|
+
error.status = status;
|
|
231
|
+
if (typeof event.retryAfterMs === "number") error.retryAfterMs = event.retryAfterMs;
|
|
232
|
+
return error;
|
|
233
|
+
}
|
package/ui.js
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
// in through props.
|
|
11
11
|
import * as React from "react";
|
|
12
12
|
import { createPortal } from "react-dom";
|
|
13
|
-
import { refreshBinding } from "./index.js";
|
|
13
|
+
import { bindingPhaseLabel, refreshBinding, useBindingProgress } from "./index.js";
|
|
14
14
|
import { formatSeconds as fmtSecs, normalizeTone, refreshOrder } from "./ui-core.js";
|
|
15
15
|
|
|
16
16
|
export { normalizeTone, refreshOrder };
|
|
@@ -446,6 +446,9 @@ export function RefreshAllButton({
|
|
|
446
446
|
const [open, setOpen] = React.useState(false);
|
|
447
447
|
const [now, setNow] = React.useState(() => Date.now());
|
|
448
448
|
const closeRef = React.useRef(null);
|
|
449
|
+
// What each running source is waiting on: the warehouse, the link, the
|
|
450
|
+
// download or DuckDB (progress.js).
|
|
451
|
+
const progress = useBindingProgress();
|
|
449
452
|
|
|
450
453
|
const running = state.status === "running";
|
|
451
454
|
const total = sources.length;
|
|
@@ -562,7 +565,7 @@ export function RefreshAllButton({
|
|
|
562
565
|
? ""
|
|
563
566
|
: "Queued"
|
|
564
567
|
: st === "running"
|
|
565
|
-
? "Running"
|
|
568
|
+
? bindingPhaseLabel(progress[name]) || "Running"
|
|
566
569
|
: st === "done"
|
|
567
570
|
? took[name] != null
|
|
568
571
|
? fmtSecs(took[name])
|
package/vite.js
CHANGED
|
@@ -156,6 +156,54 @@ function json(res, status, body) {
|
|
|
156
156
|
res.end(JSON.stringify(body));
|
|
157
157
|
}
|
|
158
158
|
|
|
159
|
+
// The SDK (>= 2.7.4) asks for phase events with Accept: application/x-ndjson
|
|
160
|
+
// and then hears each step of a build on the same request (see progress.js).
|
|
161
|
+
function wantsProgress(req) {
|
|
162
|
+
return String(req.headers?.accept ?? "")
|
|
163
|
+
.split(",")
|
|
164
|
+
.some((part) => part.split(";")[0].trim().toLowerCase() === "application/x-ndjson");
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// Start a phase stream; returns { emit, end(final) }.
|
|
168
|
+
function progressStream(res) {
|
|
169
|
+
const started = Date.now();
|
|
170
|
+
res.statusCode = 200;
|
|
171
|
+
res.setHeader("content-type", "application/x-ndjson");
|
|
172
|
+
res.setHeader("cache-control", "no-store");
|
|
173
|
+
const write = (event) => {
|
|
174
|
+
try {
|
|
175
|
+
res.write(JSON.stringify({ ...event, elapsedMs: Date.now() - started }) + "\n");
|
|
176
|
+
} catch {
|
|
177
|
+
// The page went away; the build carries on and fills the cache.
|
|
178
|
+
}
|
|
179
|
+
};
|
|
180
|
+
return {
|
|
181
|
+
emit: write,
|
|
182
|
+
end(final) {
|
|
183
|
+
write(final);
|
|
184
|
+
res.end();
|
|
185
|
+
},
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// One build's phases, for everyone waiting on it (a page mounting several
|
|
190
|
+
// tables at once shares one request): the latest phase first, then the rest.
|
|
191
|
+
function phaseHub(first) {
|
|
192
|
+
const listeners = new Set(first ? [first] : []);
|
|
193
|
+
let last = null;
|
|
194
|
+
return {
|
|
195
|
+
emit(event) {
|
|
196
|
+
last = event;
|
|
197
|
+
for (const l of [...listeners]) l(event);
|
|
198
|
+
},
|
|
199
|
+
watch(fn) {
|
|
200
|
+
if (!fn) return;
|
|
201
|
+
listeners.add(fn);
|
|
202
|
+
if (last) fn(last);
|
|
203
|
+
},
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
|
|
159
207
|
export function makoData(options = {}) {
|
|
160
208
|
return {
|
|
161
209
|
name: "mako-data",
|
|
@@ -216,7 +264,7 @@ export function makoData(options = {}) {
|
|
|
216
264
|
// edge's 524 — is retried with backoff until the overall deadline; only
|
|
217
265
|
// a definitive 4xx (the job is gone, access was revoked) ends the wait.
|
|
218
266
|
const isTransient = (status) => status === 429 || status >= 500;
|
|
219
|
-
async function awaitJob(name, jobId) {
|
|
267
|
+
async function awaitJob(name, jobId, onPhase) {
|
|
220
268
|
const url = `${appBase()}/binding-jobs/${encodeURIComponent(jobId)}`;
|
|
221
269
|
const deadline = Date.now() + buildTimeoutMs;
|
|
222
270
|
let wait = pollMs;
|
|
@@ -232,6 +280,14 @@ export function makoData(options = {}) {
|
|
|
232
280
|
const body = await res.json().catch(() => ({}));
|
|
233
281
|
if (res.ok) {
|
|
234
282
|
if (body.status === "ready") return body;
|
|
283
|
+
// The job row carries the worker's phase (binding-jobs.ts).
|
|
284
|
+
if (onPhase && body.status !== "error") {
|
|
285
|
+
onPhase({
|
|
286
|
+
phase: body.phase ?? (body.status === "queued" ? "queued" : "checking"),
|
|
287
|
+
...(typeof body.rowsWritten === "number" ? { rowsWritten: body.rowsWritten } : {}),
|
|
288
|
+
...(typeof body.bytes === "number" ? { bytes: body.bytes } : {}),
|
|
289
|
+
});
|
|
290
|
+
}
|
|
235
291
|
if (body.status === "error") {
|
|
236
292
|
const error = new Error(body.error || `build ${name} failed`);
|
|
237
293
|
error.status = body.errorStatus ?? 502;
|
|
@@ -272,13 +328,13 @@ export function makoData(options = {}) {
|
|
|
272
328
|
}
|
|
273
329
|
// POST …/materialize, asynchronously when the API supports it.
|
|
274
330
|
// Resolves to the materialize answer ({ rowCount, byteSize, materializedAt }).
|
|
275
|
-
async function materialize(name) {
|
|
331
|
+
async function materialize(name, onPhase) {
|
|
276
332
|
const res = await apiFetch(
|
|
277
333
|
`${appBase()}/bindings/${encodeURIComponent(name)}/materialize?async=1`,
|
|
278
334
|
{ method: "POST", headers: await headers() },
|
|
279
335
|
);
|
|
280
336
|
const body = await res.json().catch(() => ({}));
|
|
281
|
-
if (res.status === 202 && body.jobId) return awaitJob(name, body.jobId);
|
|
337
|
+
if (res.status === 202 && body.jobId) return awaitJob(name, body.jobId, onPhase);
|
|
282
338
|
if (!res.ok) {
|
|
283
339
|
const error = new Error(body.error || `materialize ${name}: HTTP ${res.status}`);
|
|
284
340
|
error.status = res.status;
|
|
@@ -287,12 +343,12 @@ export function makoData(options = {}) {
|
|
|
287
343
|
return body;
|
|
288
344
|
}
|
|
289
345
|
|
|
290
|
-
async function fetchArtifact(name) {
|
|
346
|
+
async function fetchArtifact(name, onPhase) {
|
|
291
347
|
const url = `${appBase()}/bindings/${encodeURIComponent(name)}/artifact`;
|
|
292
348
|
let res = await apiFetch(url, { headers: await headers() });
|
|
293
349
|
if (res.status === 404 && options.materialize !== false) {
|
|
294
350
|
// Never materialized (or a live binding): build it now, then read.
|
|
295
|
-
await materialize(name);
|
|
351
|
+
await materialize(name, onPhase);
|
|
296
352
|
res = await apiFetch(url, { headers: await headers() });
|
|
297
353
|
}
|
|
298
354
|
if (!res.ok) {
|
|
@@ -352,11 +408,15 @@ export function makoData(options = {}) {
|
|
|
352
408
|
|
|
353
409
|
// Build the LOCAL binding text through the API. Concurrent asks for the
|
|
354
410
|
// same text share one request (a page mounting several tables at once).
|
|
355
|
-
function devBuild(name, source, refresh) {
|
|
411
|
+
function devBuild(name, source, refresh, onPhase) {
|
|
356
412
|
const fingerprint = bindingFingerprint(source, ctx.dbtEnvironment);
|
|
357
413
|
const key = `${name}:${fingerprint}:${refresh ? 1 : 0}`;
|
|
358
414
|
let pending = inflight.get(key);
|
|
359
|
-
if (pending)
|
|
415
|
+
if (pending) {
|
|
416
|
+
pending.hub.watch(onPhase);
|
|
417
|
+
return pending;
|
|
418
|
+
}
|
|
419
|
+
const hub = phaseHub(onPhase);
|
|
360
420
|
pending = (async () => {
|
|
361
421
|
const res = await apiFetch(`${appBase()}/bindings/${encodeURIComponent(name)}/dev-build`, {
|
|
362
422
|
method: "POST",
|
|
@@ -372,7 +432,7 @@ export function makoData(options = {}) {
|
|
|
372
432
|
if (res.status === 202) {
|
|
373
433
|
const queued = await res.json().catch(() => ({}));
|
|
374
434
|
if (!queued.jobId) throw new Error(`build ${name}: HTTP 202 without a job id`);
|
|
375
|
-
jobDone = await awaitJob(name, queued.jobId);
|
|
435
|
+
jobDone = await awaitJob(name, queued.jobId, hub.emit);
|
|
376
436
|
}
|
|
377
437
|
if (!jobDone && !res.ok) {
|
|
378
438
|
const text = await res.text().catch(() => "");
|
|
@@ -427,17 +487,47 @@ export function makoData(options = {}) {
|
|
|
427
487
|
writeCache(name, buf, meta);
|
|
428
488
|
return { buf, meta };
|
|
429
489
|
})().finally(() => inflight.delete(key));
|
|
490
|
+
pending.hub = hub;
|
|
430
491
|
inflight.set(key, pending);
|
|
431
492
|
return pending;
|
|
432
493
|
}
|
|
433
494
|
|
|
495
|
+
// Bytes a phase stream just built, handed to the page's follow-up GET
|
|
496
|
+
// (`?take=<id>`) once, so that GET never builds again — whatever
|
|
497
|
+
// revalidateMs says, and also when the bytes are a stale fallback.
|
|
498
|
+
const takes = new Map();
|
|
499
|
+
function keepForTake(buf) {
|
|
500
|
+
const id = createHash("sha256").update(String(Math.random()) + Date.now()).digest("hex").slice(0, 16);
|
|
501
|
+
takes.set(id, buf);
|
|
502
|
+
setTimeout(() => takes.delete(id), 60_000).unref?.();
|
|
503
|
+
return id;
|
|
504
|
+
}
|
|
505
|
+
|
|
434
506
|
// POST __data/<name>/refresh — the SDK's refresh(): rebuild the LOCAL
|
|
435
507
|
// binding through the API (the committed one is re-materialized and
|
|
436
508
|
// stored, an edited one is built as a draft), and cache what came back.
|
|
437
|
-
async function refreshBinding(name, res) {
|
|
509
|
+
async function refreshBinding(name, req, res) {
|
|
438
510
|
const source = options.materialize === false || devBuildUnsupported ? null : readSource(name);
|
|
511
|
+
const stream = wantsProgress(req) ? progressStream(res) : null;
|
|
512
|
+
if (stream) {
|
|
513
|
+
stream.emit({ phase: "checking" });
|
|
514
|
+
// The same answers as below, as the stream's terminal event.
|
|
515
|
+
res = {
|
|
516
|
+
code: 200,
|
|
517
|
+
set statusCode(code) {
|
|
518
|
+
this.code = code;
|
|
519
|
+
},
|
|
520
|
+
setHeader() {},
|
|
521
|
+
end(text) {
|
|
522
|
+
const body = JSON.parse(text);
|
|
523
|
+
const ok = this.code >= 200 && this.code < 300;
|
|
524
|
+
stream.end({ ...body, phase: ok ? "ready" : "failed", ...(ok ? {} : { status: this.code }) });
|
|
525
|
+
},
|
|
526
|
+
};
|
|
527
|
+
}
|
|
528
|
+
const onPhase = stream ? stream.emit : undefined;
|
|
439
529
|
try {
|
|
440
|
-
const built = source === null ? null : await devBuild(name, source, true);
|
|
530
|
+
const built = source === null ? null : await devBuild(name, source, true, onPhase);
|
|
441
531
|
if (built) {
|
|
442
532
|
return json(res, 200, {
|
|
443
533
|
success: true,
|
|
@@ -450,7 +540,7 @@ export function makoData(options = {}) {
|
|
|
450
540
|
}
|
|
451
541
|
let body;
|
|
452
542
|
try {
|
|
453
|
-
body = await materialize(name);
|
|
543
|
+
body = await materialize(name, onPhase);
|
|
454
544
|
} catch (error) {
|
|
455
545
|
return json(res, error?.status ?? 502, {
|
|
456
546
|
success: false,
|
|
@@ -512,7 +602,16 @@ export function makoData(options = {}) {
|
|
|
512
602
|
hint: "Run `mako login` in this repo, or put MAKO_API_URL and MAKO_API_KEY in the repo's .env (see AGENTS.md).",
|
|
513
603
|
});
|
|
514
604
|
}
|
|
515
|
-
if (isRefresh) return refreshBinding(name, res);
|
|
605
|
+
if (isRefresh) return refreshBinding(name, req, res);
|
|
606
|
+
const take = /(^|&)take=([0-9a-f]{16})(&|$)/.exec(query);
|
|
607
|
+
if (take && takes.has(take[2])) {
|
|
608
|
+
const buf = takes.get(take[2]);
|
|
609
|
+
takes.delete(take[2]);
|
|
610
|
+
res.setHeader("content-type", "application/vnd.apache.parquet");
|
|
611
|
+
res.setHeader("content-length", String(buf.length));
|
|
612
|
+
res.setHeader("x-mako-data", "api");
|
|
613
|
+
return res.end(buf);
|
|
614
|
+
}
|
|
516
615
|
const cached = cachePath(name);
|
|
517
616
|
const refresh = /(^|&)refresh(=|&|$)/.test(query);
|
|
518
617
|
const source = options.materialize === false || devBuildUnsupported ? null : readSource(name);
|
|
@@ -527,16 +626,31 @@ export function makoData(options = {}) {
|
|
|
527
626
|
res.setHeader("x-mako-data", how);
|
|
528
627
|
return fs.createReadStream(cached).pipe(res);
|
|
529
628
|
};
|
|
629
|
+
// A build is about to happen: a page that asked hears its phases,
|
|
630
|
+
// then a "ready" whose url takes the bytes (keepForTake).
|
|
631
|
+
let stream = null;
|
|
530
632
|
try {
|
|
531
633
|
const stat = fs.statSync(cached, { throwIfNoEntry: false });
|
|
532
634
|
const fresh = stat && Date.now() - stat.mtimeMs < revalidateMs;
|
|
533
635
|
if (!refresh && fresh && cacheMatches()) return serveCached("cache");
|
|
636
|
+
if (wantsProgress(req)) {
|
|
637
|
+
stream = progressStream(res);
|
|
638
|
+
stream.emit({ phase: "checking" });
|
|
639
|
+
}
|
|
640
|
+
const onPhase = stream ? stream.emit : undefined;
|
|
534
641
|
let buf = null;
|
|
535
|
-
if (source !== null) buf = (await devBuild(name, source, false))?.buf ?? null;
|
|
642
|
+
if (source !== null) buf = (await devBuild(name, source, false, onPhase))?.buf ?? null;
|
|
536
643
|
if (!buf) {
|
|
537
|
-
buf = await fetchArtifact(name);
|
|
644
|
+
buf = await fetchArtifact(name, onPhase);
|
|
538
645
|
writeCache(name, buf, null);
|
|
539
646
|
}
|
|
647
|
+
if (stream) {
|
|
648
|
+
return stream.end({
|
|
649
|
+
phase: "ready",
|
|
650
|
+
binding: name,
|
|
651
|
+
url: `__data/${encodeURIComponent(name)}.parquet?take=${keepForTake(buf)}`,
|
|
652
|
+
});
|
|
653
|
+
}
|
|
540
654
|
res.setHeader("content-type", "application/vnd.apache.parquet");
|
|
541
655
|
res.setHeader("content-length", String(buf.length));
|
|
542
656
|
res.setHeader("x-mako-data", "api");
|
|
@@ -546,7 +660,20 @@ export function makoData(options = {}) {
|
|
|
546
660
|
server.config.logger.error(` mako-data: ${message}`);
|
|
547
661
|
// Stale beats nothing while offline — but only data built from THIS
|
|
548
662
|
// text; an old query's parquet is not a stale copy of a new one.
|
|
549
|
-
|
|
663
|
+
const stale = fs.existsSync(cached) && cacheMatches();
|
|
664
|
+
if (stream) {
|
|
665
|
+
return stream.end(
|
|
666
|
+
stale
|
|
667
|
+
? {
|
|
668
|
+
phase: "ready",
|
|
669
|
+
binding: name,
|
|
670
|
+
stale: true,
|
|
671
|
+
url: `__data/${encodeURIComponent(name)}.parquet?take=${keepForTake(fs.readFileSync(cached))}`,
|
|
672
|
+
}
|
|
673
|
+
: { phase: "failed", status: error?.status ?? 502, error: message },
|
|
674
|
+
);
|
|
675
|
+
}
|
|
676
|
+
if (stale) return serveCached("stale");
|
|
550
677
|
json(res, error?.status ?? 502, { error: message });
|
|
551
678
|
}
|
|
552
679
|
});
|