@makoai/app-sdk 2.7.2 → 2.7.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -58,6 +58,40 @@ or 502 with the query's error.
58
58
  Data arrives from `__data/<name>.parquet`, relative to the page — the same
59
59
  path in Mako's sandbox, in a published app, and on a laptop.
60
60
 
61
+ ### What a load is waiting on: `phase` (SDK ≥ 2.7.4)
62
+
63
+ `useQuery` and `useDuckDB` return `phase` while `loading` or `refreshing`,
64
+ plus `rowsWritten`, `bytes` and `bytesTotal` when known. Caption a progress
65
+ bar with `bindingPhaseLabel(...)`:
66
+
67
+ ```tsx
68
+ import { useQuery, bindingPhaseLabel } from "@makoai/app-sdk";
69
+
70
+ const q = useQuery("latest_sales");
71
+ if (q.loading) return <Spinner label={bindingPhaseLabel(q)} />;
72
+ // "Building from warehouse · 120,000 rows" → "Getting download link"
73
+ // → "Downloading · 3.1 MB of 10.0 MB" → "Loading into DuckDB" → "Running query"
74
+ ```
75
+
76
+ | Stage (`bindingPhaseStage`) | Phases | What is happening |
77
+ | --- | --- | --- |
78
+ | `building` | `queued`, `querying_warehouse`, `writing_parquet`, `uploading` | Mako runs the query on the warehouse. Seconds to minutes. Only a refresh, a laptop dev build or a sandbox live binding builds. |
79
+ | `linking` | `resolving`, `signing` | The file exists. Mako finds it and signs its download link. Under a second. |
80
+ | `downloading` | `downloading` | The browser fetches the parquet. |
81
+ | `loading` | `loading_duckdb` | DuckDB-WASM boots and registers the file. |
82
+ | `querying` | `running_sql` | Your SQL runs. |
83
+ | `waiting` | `requesting`, `checking` | Sent; the server has not said which yet. |
84
+
85
+ The SDK learns the server-side phases on the same request: it sends
86
+ `Accept: application/x-ndjson`, and a server that knows phases answers one
87
+ JSON event per line, ending with `{"phase":"ready", …}` (a stored file's
88
+ event carries the signed `url` and `timings`) or `{"phase":"failed",
89
+ "status", "error"}`. A server that predates this answers as before, and the
90
+ SDK still reports `requesting` → `downloading` → `loading_duckdb`.
91
+ `refreshBinding(name, { onProgress })` hears every phase of one refresh;
92
+ `useBindingProgress()` lists every pending binding. `RefreshAllButton`
93
+ captions each running source with its phase.
94
+
61
95
  ### Who is looking: `useViewer()`
62
96
 
63
97
  ```tsx
@@ -149,12 +183,18 @@ import {
149
183
  names with a build-time define in `vite.config.ts`:
150
184
 
151
185
  ```ts
152
- import { readdirSync } from "node:fs";
186
+ import { existsSync, readdirSync } from "node:fs";
187
+ const bindingsDir = new URL("./bindings", import.meta.url);
153
188
  // inside defineConfig({ … }):
154
189
  define: {
155
190
  __APP_BINDING_NAMES__: JSON.stringify(
156
- readdirSync(new URL("./bindings", import.meta.url))
157
- .filter(f => f.endsWith(".sql")).map(f => f.slice(0, -4)),
191
+ // An app with no bindings/ folder yet (a new shell app) has none:
192
+ // without the existsSync guard the config throws ENOENT and
193
+ // `vite build` fails before it starts.
194
+ existsSync(bindingsDir)
195
+ ? readdirSync(bindingsDir)
196
+ .filter(f => f.endsWith(".sql")).map(f => f.slice(0, -4))
197
+ : [],
158
198
  ),
159
199
  },
160
200
  ```
package/index.d.ts CHANGED
@@ -1,5 +1,88 @@
1
1
  import type * as React from "react";
2
2
 
3
+ /**
4
+ * What a pending binding load is waiting on (SDK ≥ 2.7.4).
5
+ *
6
+ * - `requesting`: sent, the server has not said yet (or predates phases).
7
+ * - `queued`, `checking`, `querying_warehouse`, `writing_parquet`,
8
+ * `uploading`: Mako is BUILDING the file from the warehouse (seconds to
9
+ * minutes). Only a refresh, a laptop dev build or a sandbox live binding
10
+ * builds; a published app never builds on a read.
11
+ * - `resolving`, `signing`: the file exists; Mako is finding it and signing
12
+ * its download link (well under a second).
13
+ * - `downloading`, `loading_duckdb`, `running_sql`: the browser's own work.
14
+ */
15
+ export type BindingPhase =
16
+ | "requesting"
17
+ | "queued"
18
+ | "checking"
19
+ | "querying_warehouse"
20
+ | "writing_parquet"
21
+ | "uploading"
22
+ | "resolving"
23
+ | "signing"
24
+ | "downloading"
25
+ | "loading_duckdb"
26
+ | "running_sql";
27
+
28
+ /** The phases grouped by the wait a person cares about. */
29
+ export type BindingStage =
30
+ | "waiting"
31
+ | "building"
32
+ | "linking"
33
+ | "downloading"
34
+ | "loading"
35
+ | "querying";
36
+
37
+ /** One binding's progress while it loads or refreshes. */
38
+ export interface BindingProgress {
39
+ phase: BindingPhase;
40
+ /** Rows written so far while building; null when the server does not say. */
41
+ rowsWritten?: number | null;
42
+ /** Building: the file's size. Downloading: bytes received so far. */
43
+ bytes?: number | null;
44
+ /** Downloading: the file's size, when the server sent Content-Length. */
45
+ bytesTotal?: number | null;
46
+ }
47
+
48
+ /** Several bindings at once (useDuckDB): the slowest wait, sums, a count. */
49
+ export interface CombinedBindingProgress {
50
+ phase: BindingPhase;
51
+ rowsWritten: number | null;
52
+ bytes: number | null;
53
+ bytesTotal: number | null;
54
+ pending: number;
55
+ }
56
+
57
+ export const BINDING_PHASES: readonly BindingPhase[];
58
+ export const BINDING_PHASE_LABELS: Record<BindingPhase, string>;
59
+ /** "Building from warehouse", "Getting download link", "Downloading", "Loading into DuckDB", … */
60
+ export const BINDING_STAGE_LABELS: Record<BindingStage, string>;
61
+ export function bindingPhaseStage(phase: BindingPhase | string | null | undefined): BindingStage | null;
62
+ /**
63
+ * A progress-bar caption: the stage label plus rows or bytes when known,
64
+ * e.g. "Building from warehouse · 120,000 rows", "Downloading · 3.1 of 10.0 MB".
65
+ * Empty when nothing is pending. Pass a hook's state (`bindingPhaseLabel(q)`)
66
+ * or a progress record.
67
+ */
68
+ export function bindingPhaseLabel(
69
+ progress:
70
+ | BindingPhase
71
+ | {
72
+ phase: BindingPhase | null;
73
+ rowsWritten?: number | null;
74
+ bytes?: number | null;
75
+ bytesTotal?: number | null;
76
+ }
77
+ | null
78
+ | undefined,
79
+ ): string;
80
+
81
+ /** One binding's progress, or null when it is not loading. */
82
+ export function useBindingProgress(name: string): BindingProgress | null;
83
+ /** Every pending binding's progress, by name. */
84
+ export function useBindingProgress(): Record<string, BindingProgress>;
85
+
3
86
  /** What a refresh built — the server's answer to `POST __data/<name>/refresh`. */
4
87
  export interface RefreshResult {
5
88
  binding: string;
@@ -27,6 +110,17 @@ export interface QueryState<Row = Record<string, unknown>> {
27
110
  /** A refresh is in flight; `data` still holds the previous rows. */
28
111
  refreshing: boolean;
29
112
  truncated: boolean;
113
+ /**
114
+ * What `loading` / `refreshing` is waiting on; null when neither is true.
115
+ * Caption it with `bindingPhaseLabel(phase)`.
116
+ */
117
+ phase: BindingPhase | null;
118
+ /** While building: rows written so far, when the server says. */
119
+ rowsWritten: number | null;
120
+ /** While building: file size; while downloading: bytes received. */
121
+ bytes: number | null;
122
+ /** While downloading: the file's size, when known. */
123
+ bytesTotal: number | null;
30
124
  /**
31
125
  * Rematerialize the data behind this query and re-run it. Rejects with a
32
126
  * `RefreshError` — the rows on screen are untouched when it does.
@@ -53,19 +147,40 @@ export interface MakoLocation {
53
147
  export function useQuery<Row = Record<string, unknown>>(
54
148
  name: string,
55
149
  opts?: QueryOptions,
56
- ): QueryState<Row> & { refresh: () => Promise<RefreshResult> };
150
+ ): QueryState<Row> & {
151
+ refresh: () => Promise<RefreshResult>;
152
+ /** The binding's own progress record (null when it is not loading). */
153
+ progress: BindingProgress | null;
154
+ };
57
155
 
58
156
  /** Analytical SQL over the app's bindings; table names are binding names. */
59
157
  export function useDuckDB<Row = Record<string, unknown>>(
60
158
  sql: string,
61
159
  opts?: QueryOptions,
62
- ): DuckDBState<Row> & { refresh: () => Promise<RefreshResult[]> };
160
+ ): DuckDBState<Row> & {
161
+ refresh: () => Promise<RefreshResult[]>;
162
+ /** The slowest pending binding's wait, with sums; null when none. */
163
+ progress: CombinedBindingProgress | null;
164
+ /** Every pending binding's progress, by name. */
165
+ bindings: Record<string, BindingProgress>;
166
+ };
167
+
168
+ export interface RefreshOptions {
169
+ /**
170
+ * Every phase of this refresh: the build, then the reload (link,
171
+ * download, DuckDB), then `null` once it settles.
172
+ */
173
+ onProgress?: (progress: BindingProgress | null) => void;
174
+ }
63
175
 
64
176
  /**
65
177
  * Rematerialize one binding on demand and reload it everywhere it is used.
66
178
  * Concurrent calls for the same binding share one request.
67
179
  */
68
- export function refreshBinding(name: string): Promise<RefreshResult>;
180
+ export function refreshBinding(
181
+ name: string,
182
+ opts?: RefreshOptions,
183
+ ): Promise<RefreshResult>;
69
184
 
70
185
  /**
71
186
  * Refresh several bindings — all of them when `names` is omitted. Every one
package/index.js CHANGED
@@ -1,5 +1,20 @@
1
1
  // @makoai/app-sdk — see package.json. Plain ESM on purpose: no build step.
2
2
  import * as React from "react";
3
+ import {
4
+ PROGRESS_ACCEPT,
5
+ combineBindingProgress,
6
+ isProgressResponse,
7
+ progressError,
8
+ readBytes,
9
+ readProgressStream,
10
+ } from "./progress.js";
11
+ export {
12
+ BINDING_PHASES,
13
+ BINDING_PHASE_LABELS,
14
+ BINDING_STAGE_LABELS,
15
+ bindingPhaseLabel,
16
+ bindingPhaseStage,
17
+ } from "./progress.js";
3
18
 
4
19
  const DEFAULT_ROW_LIMIT = 500000;
5
20
  const DUCKDB_ESM =
@@ -40,26 +55,98 @@ async function responseError(res, what) {
40
55
  return error;
41
56
  }
42
57
 
58
+ // ---------------------------------------------------------------------------
59
+ // Progress — what each pending binding is waiting on (see progress.js). One
60
+ // record per binding while it loads or refreshes, gone when it settles.
61
+ // Notifications are batched: 34 bindings downloading at once must not
62
+ // re-render every hook on every chunk.
63
+ // ---------------------------------------------------------------------------
64
+ let progressState = {};
65
+ const progressListeners = new Set();
66
+ const progressWatchers = new Map(); // name -> Set<(record|null) => void>
67
+ let progressFlush = null;
68
+ function setProgress(name, patch) {
69
+ const next = { ...progressState };
70
+ if (patch === null) delete next[name];
71
+ else next[name] = { ...progressState[name], ...patch };
72
+ progressState = next;
73
+ const watchers = progressWatchers.get(name);
74
+ if (watchers) for (const w of [...watchers]) w(next[name] ?? null);
75
+ progressFlush ??= setTimeout(() => {
76
+ progressFlush = null;
77
+ for (const l of [...progressListeners]) l();
78
+ }, 50);
79
+ }
80
+ function subscribeProgress(l) {
81
+ progressListeners.add(l);
82
+ return () => progressListeners.delete(l);
83
+ }
84
+ function getProgressState() {
85
+ return progressState;
86
+ }
87
+ function watchProgress(name, fn) {
88
+ if (!progressWatchers.has(name)) progressWatchers.set(name, new Set());
89
+ progressWatchers.get(name).add(fn);
90
+ return () => progressWatchers.get(name)?.delete(fn);
91
+ }
92
+ /** A server event as a progress patch: the phase and whatever it counted. */
93
+ function progressPatch(event) {
94
+ return {
95
+ phase: event.phase,
96
+ ...(typeof event.rowsWritten === "number" ? { rowsWritten: event.rowsWritten } : {}),
97
+ ...(typeof event.bytes === "number" ? { bytes: event.bytes } : {}),
98
+ };
99
+ }
100
+
101
+ /**
102
+ * Live progress: one binding's record (null when it is not loading), or
103
+ * every pending binding's when called without a name.
104
+ */
105
+ export function useBindingProgress(name) {
106
+ const all = React.useSyncExternalStore(subscribeProgress, getProgressState, getProgressState);
107
+ return name === undefined ? all : (all[name] ?? null);
108
+ }
109
+
43
110
  async function loadBinding(name, bust) {
44
- const res = await fetch(
45
- "__data/" + encodeURIComponent(name) + ".parquet" + (bust ? "?refresh=" + Date.now() : ""),
46
- );
47
- if (!res.ok) {
48
- throw await responseError(res, 'Data for binding "' + name + '" could not be loaded');
49
- }
50
- const buf = new Uint8Array(await res.arrayBuffer());
51
- const db = await getDb();
52
- if (bust) await db.dropFile(name + ".parquet").catch(() => {});
53
- await db.registerFileBuffer(name + ".parquet", buf);
54
- const conn = await db.connect();
111
+ const path = "__data/" + encodeURIComponent(name) + ".parquet";
112
+ const what = 'Data for binding "' + name + '" could not be loaded';
113
+ setProgress(name, { phase: "requesting", rowsWritten: null, bytes: null, bytesTotal: null });
55
114
  try {
56
- // A view per binding so SQL can name tables by binding name.
57
- await conn.query(
58
- 'CREATE OR REPLACE VIEW "' + name.replace(/"/g, '""') +
59
- "\" AS SELECT * FROM read_parquet('" + name + ".parquet')",
115
+ // Ask for phase events: a build (laptop, sandbox live binding) reports
116
+ // each step, a stored file reports resolving → signing and hands back
117
+ // the signed URL. A server that predates them answers as it always did.
118
+ let res = await fetch(path + (bust ? "?refresh=" + Date.now() : ""), {
119
+ headers: { accept: PROGRESS_ACCEPT },
120
+ });
121
+ if (isProgressResponse(res)) {
122
+ const final = await readProgressStream(res, e => setProgress(name, progressPatch(e)));
123
+ if (final.phase === "failed") throw progressError(final, what);
124
+ setProgress(name, { phase: "downloading", rowsWritten: null, bytes: 0 });
125
+ // No url: the server streams the bytes itself — the same path, asked
126
+ // plainly, serves them (the build already staged or cached them).
127
+ res = await fetch(final.url || path);
128
+ }
129
+ if (!res.ok) throw await responseError(res, what);
130
+ const buf = await readBytes(res, (bytes, bytesTotal) =>
131
+ setProgress(name, { phase: "downloading", rowsWritten: null, bytes, bytesTotal }),
60
132
  );
133
+ setProgress(name, { phase: "loading_duckdb" });
134
+ const db = await getDb();
135
+ if (bust) await db.dropFile(name + ".parquet").catch(() => {});
136
+ await db.registerFileBuffer(name + ".parquet", buf);
137
+ const conn = await db.connect();
138
+ try {
139
+ // A view per binding so SQL can name tables by binding name.
140
+ await conn.query(
141
+ 'CREATE OR REPLACE VIEW "' + name.replace(/"/g, '""') +
142
+ "\" AS SELECT * FROM read_parquet('" + name + ".parquet')",
143
+ );
144
+ } finally {
145
+ await conn.close();
146
+ }
61
147
  } finally {
62
- await conn.close();
148
+ // A refresh owns the record until its reload is done (refreshBinding).
149
+ if (!refreshes.has(name)) setProgress(name, null);
63
150
  }
64
151
  }
65
152
 
@@ -122,17 +209,32 @@ const refreshes = new Map(); // name -> Promise<RefreshResult>
122
209
  * built; rejects with an Error carrying `status` (403: not allowed here,
123
210
  * 429: refreshed too recently — see `retryAfterMs`, 502: the query failed —
124
211
  * the message says why). Concurrent calls for the same binding share one
125
- * request.
212
+ * request. `opts.onProgress` hears every phase of it: the build, then the
213
+ * reload (link, download, DuckDB), then `null` when it is done.
126
214
  */
127
- export function refreshBinding(name) {
215
+ export function refreshBinding(name, opts) {
216
+ const onProgress = opts && opts.onProgress;
217
+ const unwatch = onProgress ? watchProgress(name, onProgress) : null;
128
218
  const pending = refreshes.get(name);
129
- if (pending) return pending;
219
+ if (pending) {
220
+ if (unwatch) pending.finally(unwatch).catch(() => {});
221
+ return pending;
222
+ }
223
+ const what = 'Binding "' + name + '" could not be refreshed';
130
224
  const run = (async () => {
131
- const res = await fetch("__data/" + encodeURIComponent(name) + "/refresh", { method: "POST" });
132
- if (!res.ok) {
133
- throw await responseError(res, 'Binding "' + name + '" could not be refreshed');
225
+ setProgress(name, { phase: "requesting", rowsWritten: null, bytes: null, bytesTotal: null });
226
+ const res = await fetch("__data/" + encodeURIComponent(name) + "/refresh", {
227
+ method: "POST",
228
+ headers: { accept: PROGRESS_ACCEPT },
229
+ });
230
+ let body;
231
+ if (isProgressResponse(res)) {
232
+ body = await readProgressStream(res, e => setProgress(name, progressPatch(e)));
233
+ if (body.phase === "failed") throw progressError(body, what);
234
+ } else {
235
+ if (!res.ok) throw await responseError(res, what);
236
+ body = await res.json();
134
237
  }
135
- const body = await res.json();
136
238
  await reloadBinding(name);
137
239
  setBindingState({ ...bindingState, version: bindingState.version + 1 });
138
240
  return {
@@ -148,6 +250,8 @@ export function refreshBinding(name) {
148
250
  run
149
251
  .finally(() => {
150
252
  refreshes.delete(name);
253
+ setProgress(name, null);
254
+ if (unwatch) unwatch();
151
255
  const refreshing = new Set(bindingState.refreshing);
152
256
  refreshing.delete(name);
153
257
  setBindingState({ ...bindingState, refreshing });
@@ -336,6 +440,18 @@ function useAsyncQuery(run, deps, isRefreshing) {
336
440
  };
337
441
  }
338
442
 
443
+ // A hook's view of progress: the binding's own record while it loads, else
444
+ // `running_sql` while the hook's SQL runs, else nothing.
445
+ function hookProgress(record, state) {
446
+ const p = record ?? (state.loading || state.refreshing ? { phase: "running_sql" } : null);
447
+ return {
448
+ phase: p ? p.phase : null,
449
+ rowsWritten: p && typeof p.rowsWritten === "number" ? p.rowsWritten : null,
450
+ bytes: p && typeof p.bytes === "number" ? p.bytes : null,
451
+ bytesTotal: p && typeof p.bytesTotal === "number" ? p.bytesTotal : null,
452
+ };
453
+ }
454
+
339
455
  export function useQuery(name, opts) {
340
456
  const rowLimit = opts ? opts.rowLimit : undefined;
341
457
  const state = useAsyncQuery(
@@ -350,8 +466,9 @@ export function useQuery(name, opts) {
350
466
  [name, rowLimit],
351
467
  refreshing => refreshing.has(name),
352
468
  );
469
+ const progress = useBindingProgress(name);
353
470
  const refresh = React.useCallback(() => refreshBinding(name), [name]);
354
- return { ...state, refresh };
471
+ return { ...state, ...hookProgress(progress, state), progress, refresh };
355
472
  }
356
473
 
357
474
  /**
@@ -415,9 +532,19 @@ export function useDuckDB(sql, opts) {
415
532
  throw error;
416
533
  }
417
534
  }, [sql, rowLimit], refreshing => refreshing.size > 0);
535
+ // Every binding the query can read: the combined record says what the
536
+ // slowest of them is waiting on; `bindings` has each one.
537
+ const bindings = useBindingProgress();
538
+ const combined = combineBindingProgress(bindings);
418
539
  // A query over every binding refreshes every binding.
419
540
  const refresh = React.useCallback(() => refreshBindings(), []);
420
- return { ...state, refresh };
541
+ return {
542
+ ...state,
543
+ ...hookProgress(combined, state),
544
+ progress: combined,
545
+ bindings,
546
+ refresh,
547
+ };
421
548
  }
422
549
 
423
550
  // ---------------------------------------------------------------------------
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@makoai/app-sdk",
3
- "version": "2.7.2",
3
+ "version": "2.7.4",
4
4
  "description": "Mako app SDK: data bindings (useQuery/useDuckDB), URL state, theme, and the house dashboard kit (/ui).",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -41,6 +41,7 @@
41
41
  "files": [
42
42
  "index.js",
43
43
  "index.d.ts",
44
+ "progress.js",
44
45
  "vite.js",
45
46
  "vite.d.ts",
46
47
  "credentials.js",
package/progress.js ADDED
@@ -0,0 +1,233 @@
1
+ // What a binding load is waiting on — the pure half (no React, no DuckDB),
2
+ // so `node --test` covers it without installing anything.
3
+ //
4
+ // From 2.7.4 the SDK asks every `__data` request for phase events
5
+ // (`Accept: application/x-ndjson`). A server that knows them answers one
6
+ // JSON object per line on the same connection and ends with exactly one
7
+ // `ready` or `failed`; a server that predates them answers as before
8
+ // (parquet bytes, a 302 to them, or JSON), and the SDK still knows when the
9
+ // download and the DuckDB load start. See api/src/apps/binding-progress.ts.
10
+
11
+ export const PROGRESS_ACCEPT = "application/x-ndjson, */*;q=0.5";
12
+
13
+ /**
14
+ * Every phase, in the order a full rebuild passes through them. Server
15
+ * phases come from the stream; `requesting`, `downloading`,
16
+ * `loading_duckdb` and `running_sql` are the SDK's own.
17
+ */
18
+ export const BINDING_PHASES = [
19
+ "requesting",
20
+ "queued",
21
+ "checking",
22
+ "querying_warehouse",
23
+ "writing_parquet",
24
+ "uploading",
25
+ "resolving",
26
+ "signing",
27
+ "downloading",
28
+ "loading_duckdb",
29
+ "running_sql",
30
+ ];
31
+
32
+ const STAGE_OF = {
33
+ requesting: "waiting",
34
+ queued: "building",
35
+ checking: "waiting",
36
+ querying_warehouse: "building",
37
+ writing_parquet: "building",
38
+ uploading: "building",
39
+ resolving: "linking",
40
+ signing: "linking",
41
+ downloading: "downloading",
42
+ loading_duckdb: "loading",
43
+ running_sql: "querying",
44
+ };
45
+
46
+ /**
47
+ * The wait a person cares about: `building` (the warehouse is running the
48
+ * query — seconds to minutes), `linking` (Mako is finding and signing the
49
+ * stored file — well under a second), `downloading`, `loading` (into
50
+ * DuckDB), `querying` (your SQL), or `waiting` before the server says.
51
+ */
52
+ export function bindingPhaseStage(phase) {
53
+ return STAGE_OF[phase] ?? null;
54
+ }
55
+
56
+ export const BINDING_STAGE_LABELS = {
57
+ waiting: "Contacting Mako",
58
+ building: "Building from warehouse",
59
+ linking: "Getting download link",
60
+ downloading: "Downloading",
61
+ loading: "Loading into DuckDB",
62
+ querying: "Running query",
63
+ };
64
+
65
+ /** Finer wording per phase, for a tooltip or a second line. */
66
+ export const BINDING_PHASE_LABELS = {
67
+ requesting: "Contacting Mako",
68
+ queued: "Waiting for a build slot",
69
+ checking: "Checking the binding",
70
+ querying_warehouse: "Querying the warehouse",
71
+ writing_parquet: "Writing parquet",
72
+ uploading: "Storing the file",
73
+ resolving: "Finding the file",
74
+ signing: "Signing the download link",
75
+ downloading: "Downloading",
76
+ loading_duckdb: "Loading into DuckDB",
77
+ running_sql: "Running query",
78
+ };
79
+
80
+ function formatBytes(n) {
81
+ if (n < 1024) return n + " B";
82
+ if (n < 1024 * 1024) return Math.round(n / 1024) + " KB";
83
+ return (n / (1024 * 1024)).toFixed(1) + " MB";
84
+ }
85
+
86
+ /**
87
+ * A caption for a progress bar: the stage label, plus rows or bytes when
88
+ * known. Takes a phase name or a progress record (`useQuery().progress`).
89
+ * Empty string when nothing is pending.
90
+ */
91
+ export function bindingPhaseLabel(progress) {
92
+ const p = typeof progress === "string" ? { phase: progress } : progress;
93
+ const stage = p ? bindingPhaseStage(p.phase) : null;
94
+ if (!stage) return "";
95
+ const label = BINDING_STAGE_LABELS[stage];
96
+ if (stage === "building" && typeof p.rowsWritten === "number") {
97
+ return label + " · " + p.rowsWritten.toLocaleString("en-US") + " rows";
98
+ }
99
+ if (stage === "downloading" && typeof p.bytes === "number") {
100
+ return (
101
+ label +
102
+ " · " +
103
+ formatBytes(p.bytes) +
104
+ (typeof p.bytesTotal === "number" ? " of " + formatBytes(p.bytesTotal) : "")
105
+ );
106
+ }
107
+ return label;
108
+ }
109
+
110
+ // Most worth showing first when several bindings are pending at once: a
111
+ // warehouse build dwarfs everything, a download is the next longest.
112
+ const STAGE_WEIGHT = { building: 5, waiting: 4, downloading: 3, linking: 2, loading: 1, querying: 0 };
113
+
114
+ /**
115
+ * One record for many pending bindings (useDuckDB): the phase of the one
116
+ * furthest from done, rows summed over the builds, bytes over the
117
+ * downloads, and how many are pending.
118
+ */
119
+ export function combineBindingProgress(records) {
120
+ const list = Object.values(records ?? {}).filter(Boolean);
121
+ if (!list.length) return null;
122
+ let top = list[0];
123
+ for (const r of list) {
124
+ const w = STAGE_WEIGHT[bindingPhaseStage(r.phase)] ?? -1;
125
+ if (w > (STAGE_WEIGHT[bindingPhaseStage(top.phase)] ?? -1)) top = r;
126
+ }
127
+ const stage = bindingPhaseStage(top.phase);
128
+ const same = list.filter(r => bindingPhaseStage(r.phase) === stage);
129
+ const sum = key => {
130
+ const known = same.filter(r => typeof r[key] === "number");
131
+ return known.length ? known.reduce((n, r) => n + r[key], 0) : null;
132
+ };
133
+ return {
134
+ phase: top.phase,
135
+ rowsWritten: sum("rowsWritten"),
136
+ bytes: sum("bytes"),
137
+ bytesTotal: same.every(r => typeof r.bytesTotal === "number") ? sum("bytesTotal") : null,
138
+ pending: list.length,
139
+ };
140
+ }
141
+
142
+ /** Did the server answer with phase events? */
143
+ export function isProgressResponse(res) {
144
+ const type = (res.headers && res.headers.get("content-type")) || "";
145
+ return type.split(";")[0].trim().toLowerCase() === "application/x-ndjson";
146
+ }
147
+
148
+ /**
149
+ * Read a phase stream: `onEvent` for every line, resolves with the terminal
150
+ * event (`ready` or `failed`). A stream that ends without one resolves as
151
+ * `failed` with status 502 — the connection broke before the server said.
152
+ */
153
+ export async function readProgressStream(res, onEvent) {
154
+ const reader = res.body.getReader();
155
+ const decoder = new TextDecoder();
156
+ let buffered = "";
157
+ const handle = line => {
158
+ const text = line.trim();
159
+ if (!text) return null;
160
+ let event;
161
+ try {
162
+ event = JSON.parse(text);
163
+ } catch {
164
+ return null;
165
+ }
166
+ if (event && (event.phase === "ready" || event.phase === "failed")) return event;
167
+ if (event && event.phase) onEvent(event);
168
+ return null;
169
+ };
170
+ for (;;) {
171
+ const { done, value } = await reader.read();
172
+ buffered += decoder.decode(value ?? new Uint8Array(), { stream: !done });
173
+ let nl;
174
+ while ((nl = buffered.indexOf("\n")) !== -1) {
175
+ const final = handle(buffered.slice(0, nl));
176
+ buffered = buffered.slice(nl + 1);
177
+ if (final) {
178
+ reader.cancel().catch(() => {});
179
+ return final;
180
+ }
181
+ }
182
+ if (done) break;
183
+ }
184
+ return (
185
+ handle(buffered) ?? {
186
+ phase: "failed",
187
+ status: 502,
188
+ error: "The connection closed before the server finished",
189
+ }
190
+ );
191
+ }
192
+
193
+ /**
194
+ * The response body as bytes, reporting `(received, total)` as it arrives.
195
+ * `total` is null when the server sent no Content-Length.
196
+ */
197
+ export async function readBytes(res, onProgress) {
198
+ const total = Number(res.headers.get("content-length")) || null;
199
+ if (!res.body || !res.body.getReader) {
200
+ const buf = new Uint8Array(await res.arrayBuffer());
201
+ onProgress(buf.length, total);
202
+ return buf;
203
+ }
204
+ const reader = res.body.getReader();
205
+ const chunks = [];
206
+ let received = 0;
207
+ onProgress(0, total);
208
+ for (;;) {
209
+ const { done, value } = await reader.read();
210
+ if (done) break;
211
+ chunks.push(value);
212
+ received += value.length;
213
+ onProgress(received, total);
214
+ }
215
+ if (chunks.length === 1) return chunks[0];
216
+ const out = new Uint8Array(received);
217
+ let at = 0;
218
+ for (const c of chunks) {
219
+ out.set(c, at);
220
+ at += c.length;
221
+ }
222
+ return out;
223
+ }
224
+
225
+ /** The Error a `failed` event becomes — the same shape as an HTTP failure. */
226
+ export function progressError(event, what) {
227
+ const detail = [event.error, event.hint].filter(Boolean).join(" — ");
228
+ const status = typeof event.status === "number" ? event.status : 502;
229
+ const error = new Error(what + " (HTTP " + status + ")" + (detail ? ": " + detail : ""));
230
+ error.status = status;
231
+ if (typeof event.retryAfterMs === "number") error.retryAfterMs = event.retryAfterMs;
232
+ return error;
233
+ }
package/ui.js CHANGED
@@ -10,7 +10,7 @@
10
10
  // in through props.
11
11
  import * as React from "react";
12
12
  import { createPortal } from "react-dom";
13
- import { refreshBinding } from "./index.js";
13
+ import { bindingPhaseLabel, refreshBinding, useBindingProgress } from "./index.js";
14
14
  import { formatSeconds as fmtSecs, normalizeTone, refreshOrder } from "./ui-core.js";
15
15
 
16
16
  export { normalizeTone, refreshOrder };
@@ -446,6 +446,9 @@ export function RefreshAllButton({
446
446
  const [open, setOpen] = React.useState(false);
447
447
  const [now, setNow] = React.useState(() => Date.now());
448
448
  const closeRef = React.useRef(null);
449
+ // What each running source is waiting on: the warehouse, the link, the
450
+ // download or DuckDB (progress.js).
451
+ const progress = useBindingProgress();
449
452
 
450
453
  const running = state.status === "running";
451
454
  const total = sources.length;
@@ -562,7 +565,7 @@ export function RefreshAllButton({
562
565
  ? ""
563
566
  : "Queued"
564
567
  : st === "running"
565
- ? "Running"
568
+ ? bindingPhaseLabel(progress[name]) || "Running"
566
569
  : st === "done"
567
570
  ? took[name] != null
568
571
  ? fmtSecs(took[name])
package/vite.js CHANGED
@@ -156,6 +156,54 @@ function json(res, status, body) {
156
156
  res.end(JSON.stringify(body));
157
157
  }
158
158
 
159
+ // The SDK (>= 2.7.4) asks for phase events with Accept: application/x-ndjson
160
+ // and then hears each step of a build on the same request (see progress.js).
161
+ function wantsProgress(req) {
162
+ return String(req.headers?.accept ?? "")
163
+ .split(",")
164
+ .some((part) => part.split(";")[0].trim().toLowerCase() === "application/x-ndjson");
165
+ }
166
+
167
+ // Start a phase stream; returns { emit, end(final) }.
168
+ function progressStream(res) {
169
+ const started = Date.now();
170
+ res.statusCode = 200;
171
+ res.setHeader("content-type", "application/x-ndjson");
172
+ res.setHeader("cache-control", "no-store");
173
+ const write = (event) => {
174
+ try {
175
+ res.write(JSON.stringify({ ...event, elapsedMs: Date.now() - started }) + "\n");
176
+ } catch {
177
+ // The page went away; the build carries on and fills the cache.
178
+ }
179
+ };
180
+ return {
181
+ emit: write,
182
+ end(final) {
183
+ write(final);
184
+ res.end();
185
+ },
186
+ };
187
+ }
188
+
189
+ // One build's phases, for everyone waiting on it (a page mounting several
190
+ // tables at once shares one request): the latest phase first, then the rest.
191
+ function phaseHub(first) {
192
+ const listeners = new Set(first ? [first] : []);
193
+ let last = null;
194
+ return {
195
+ emit(event) {
196
+ last = event;
197
+ for (const l of [...listeners]) l(event);
198
+ },
199
+ watch(fn) {
200
+ if (!fn) return;
201
+ listeners.add(fn);
202
+ if (last) fn(last);
203
+ },
204
+ };
205
+ }
206
+
159
207
  export function makoData(options = {}) {
160
208
  return {
161
209
  name: "mako-data",
@@ -216,7 +264,7 @@ export function makoData(options = {}) {
216
264
  // edge's 524 — is retried with backoff until the overall deadline; only
217
265
  // a definitive 4xx (the job is gone, access was revoked) ends the wait.
218
266
  const isTransient = (status) => status === 429 || status >= 500;
219
- async function awaitJob(name, jobId) {
267
+ async function awaitJob(name, jobId, onPhase) {
220
268
  const url = `${appBase()}/binding-jobs/${encodeURIComponent(jobId)}`;
221
269
  const deadline = Date.now() + buildTimeoutMs;
222
270
  let wait = pollMs;
@@ -232,6 +280,14 @@ export function makoData(options = {}) {
232
280
  const body = await res.json().catch(() => ({}));
233
281
  if (res.ok) {
234
282
  if (body.status === "ready") return body;
283
+ // The job row carries the worker's phase (binding-jobs.ts).
284
+ if (onPhase && body.status !== "error") {
285
+ onPhase({
286
+ phase: body.phase ?? (body.status === "queued" ? "queued" : "checking"),
287
+ ...(typeof body.rowsWritten === "number" ? { rowsWritten: body.rowsWritten } : {}),
288
+ ...(typeof body.bytes === "number" ? { bytes: body.bytes } : {}),
289
+ });
290
+ }
235
291
  if (body.status === "error") {
236
292
  const error = new Error(body.error || `build ${name} failed`);
237
293
  error.status = body.errorStatus ?? 502;
@@ -272,13 +328,13 @@ export function makoData(options = {}) {
272
328
  }
273
329
  // POST …/materialize, asynchronously when the API supports it.
274
330
  // Resolves to the materialize answer ({ rowCount, byteSize, materializedAt }).
275
- async function materialize(name) {
331
+ async function materialize(name, onPhase) {
276
332
  const res = await apiFetch(
277
333
  `${appBase()}/bindings/${encodeURIComponent(name)}/materialize?async=1`,
278
334
  { method: "POST", headers: await headers() },
279
335
  );
280
336
  const body = await res.json().catch(() => ({}));
281
- if (res.status === 202 && body.jobId) return awaitJob(name, body.jobId);
337
+ if (res.status === 202 && body.jobId) return awaitJob(name, body.jobId, onPhase);
282
338
  if (!res.ok) {
283
339
  const error = new Error(body.error || `materialize ${name}: HTTP ${res.status}`);
284
340
  error.status = res.status;
@@ -287,12 +343,12 @@ export function makoData(options = {}) {
287
343
  return body;
288
344
  }
289
345
 
290
- async function fetchArtifact(name) {
346
+ async function fetchArtifact(name, onPhase) {
291
347
  const url = `${appBase()}/bindings/${encodeURIComponent(name)}/artifact`;
292
348
  let res = await apiFetch(url, { headers: await headers() });
293
349
  if (res.status === 404 && options.materialize !== false) {
294
350
  // Never materialized (or a live binding): build it now, then read.
295
- await materialize(name);
351
+ await materialize(name, onPhase);
296
352
  res = await apiFetch(url, { headers: await headers() });
297
353
  }
298
354
  if (!res.ok) {
@@ -352,11 +408,15 @@ export function makoData(options = {}) {
352
408
 
353
409
  // Build the LOCAL binding text through the API. Concurrent asks for the
354
410
  // same text share one request (a page mounting several tables at once).
355
- function devBuild(name, source, refresh) {
411
+ function devBuild(name, source, refresh, onPhase) {
356
412
  const fingerprint = bindingFingerprint(source, ctx.dbtEnvironment);
357
413
  const key = `${name}:${fingerprint}:${refresh ? 1 : 0}`;
358
414
  let pending = inflight.get(key);
359
- if (pending) return pending;
415
+ if (pending) {
416
+ pending.hub.watch(onPhase);
417
+ return pending;
418
+ }
419
+ const hub = phaseHub(onPhase);
360
420
  pending = (async () => {
361
421
  const res = await apiFetch(`${appBase()}/bindings/${encodeURIComponent(name)}/dev-build`, {
362
422
  method: "POST",
@@ -372,7 +432,7 @@ export function makoData(options = {}) {
372
432
  if (res.status === 202) {
373
433
  const queued = await res.json().catch(() => ({}));
374
434
  if (!queued.jobId) throw new Error(`build ${name}: HTTP 202 without a job id`);
375
- jobDone = await awaitJob(name, queued.jobId);
435
+ jobDone = await awaitJob(name, queued.jobId, hub.emit);
376
436
  }
377
437
  if (!jobDone && !res.ok) {
378
438
  const text = await res.text().catch(() => "");
@@ -427,17 +487,47 @@ export function makoData(options = {}) {
427
487
  writeCache(name, buf, meta);
428
488
  return { buf, meta };
429
489
  })().finally(() => inflight.delete(key));
490
+ pending.hub = hub;
430
491
  inflight.set(key, pending);
431
492
  return pending;
432
493
  }
433
494
 
495
+ // Bytes a phase stream just built, handed to the page's follow-up GET
496
+ // (`?take=<id>`) once, so that GET never builds again — whatever
497
+ // revalidateMs says, and also when the bytes are a stale fallback.
498
+ const takes = new Map();
499
+ function keepForTake(buf) {
500
+ const id = createHash("sha256").update(String(Math.random()) + Date.now()).digest("hex").slice(0, 16);
501
+ takes.set(id, buf);
502
+ setTimeout(() => takes.delete(id), 60_000).unref?.();
503
+ return id;
504
+ }
505
+
434
506
  // POST __data/<name>/refresh — the SDK's refresh(): rebuild the LOCAL
435
507
  // binding through the API (the committed one is re-materialized and
436
508
  // stored, an edited one is built as a draft), and cache what came back.
437
- async function refreshBinding(name, res) {
509
+ async function refreshBinding(name, req, res) {
438
510
  const source = options.materialize === false || devBuildUnsupported ? null : readSource(name);
511
+ const stream = wantsProgress(req) ? progressStream(res) : null;
512
+ if (stream) {
513
+ stream.emit({ phase: "checking" });
514
+ // The same answers as below, as the stream's terminal event.
515
+ res = {
516
+ code: 200,
517
+ set statusCode(code) {
518
+ this.code = code;
519
+ },
520
+ setHeader() {},
521
+ end(text) {
522
+ const body = JSON.parse(text);
523
+ const ok = this.code >= 200 && this.code < 300;
524
+ stream.end({ ...body, phase: ok ? "ready" : "failed", ...(ok ? {} : { status: this.code }) });
525
+ },
526
+ };
527
+ }
528
+ const onPhase = stream ? stream.emit : undefined;
439
529
  try {
440
- const built = source === null ? null : await devBuild(name, source, true);
530
+ const built = source === null ? null : await devBuild(name, source, true, onPhase);
441
531
  if (built) {
442
532
  return json(res, 200, {
443
533
  success: true,
@@ -450,7 +540,7 @@ export function makoData(options = {}) {
450
540
  }
451
541
  let body;
452
542
  try {
453
- body = await materialize(name);
543
+ body = await materialize(name, onPhase);
454
544
  } catch (error) {
455
545
  return json(res, error?.status ?? 502, {
456
546
  success: false,
@@ -512,7 +602,16 @@ export function makoData(options = {}) {
512
602
  hint: "Run `mako login` in this repo, or put MAKO_API_URL and MAKO_API_KEY in the repo's .env (see AGENTS.md).",
513
603
  });
514
604
  }
515
- if (isRefresh) return refreshBinding(name, res);
605
+ if (isRefresh) return refreshBinding(name, req, res);
606
+ const take = /(^|&)take=([0-9a-f]{16})(&|$)/.exec(query);
607
+ if (take && takes.has(take[2])) {
608
+ const buf = takes.get(take[2]);
609
+ takes.delete(take[2]);
610
+ res.setHeader("content-type", "application/vnd.apache.parquet");
611
+ res.setHeader("content-length", String(buf.length));
612
+ res.setHeader("x-mako-data", "api");
613
+ return res.end(buf);
614
+ }
516
615
  const cached = cachePath(name);
517
616
  const refresh = /(^|&)refresh(=|&|$)/.test(query);
518
617
  const source = options.materialize === false || devBuildUnsupported ? null : readSource(name);
@@ -527,16 +626,31 @@ export function makoData(options = {}) {
527
626
  res.setHeader("x-mako-data", how);
528
627
  return fs.createReadStream(cached).pipe(res);
529
628
  };
629
+ // A build is about to happen: a page that asked hears its phases,
630
+ // then a "ready" whose url takes the bytes (keepForTake).
631
+ let stream = null;
530
632
  try {
531
633
  const stat = fs.statSync(cached, { throwIfNoEntry: false });
532
634
  const fresh = stat && Date.now() - stat.mtimeMs < revalidateMs;
533
635
  if (!refresh && fresh && cacheMatches()) return serveCached("cache");
636
+ if (wantsProgress(req)) {
637
+ stream = progressStream(res);
638
+ stream.emit({ phase: "checking" });
639
+ }
640
+ const onPhase = stream ? stream.emit : undefined;
534
641
  let buf = null;
535
- if (source !== null) buf = (await devBuild(name, source, false))?.buf ?? null;
642
+ if (source !== null) buf = (await devBuild(name, source, false, onPhase))?.buf ?? null;
536
643
  if (!buf) {
537
- buf = await fetchArtifact(name);
644
+ buf = await fetchArtifact(name, onPhase);
538
645
  writeCache(name, buf, null);
539
646
  }
647
+ if (stream) {
648
+ return stream.end({
649
+ phase: "ready",
650
+ binding: name,
651
+ url: `__data/${encodeURIComponent(name)}.parquet?take=${keepForTake(buf)}`,
652
+ });
653
+ }
540
654
  res.setHeader("content-type", "application/vnd.apache.parquet");
541
655
  res.setHeader("content-length", String(buf.length));
542
656
  res.setHeader("x-mako-data", "api");
@@ -546,7 +660,20 @@ export function makoData(options = {}) {
546
660
  server.config.logger.error(` mako-data: ${message}`);
547
661
  // Stale beats nothing while offline — but only data built from THIS
548
662
  // text; an old query's parquet is not a stale copy of a new one.
549
- if (fs.existsSync(cached) && cacheMatches()) return serveCached("stale");
663
+ const stale = fs.existsSync(cached) && cacheMatches();
664
+ if (stream) {
665
+ return stream.end(
666
+ stale
667
+ ? {
668
+ phase: "ready",
669
+ binding: name,
670
+ stale: true,
671
+ url: `__data/${encodeURIComponent(name)}.parquet?take=${keepForTake(fs.readFileSync(cached))}`,
672
+ }
673
+ : { phase: "failed", status: error?.status ?? 502, error: message },
674
+ );
675
+ }
676
+ if (stale) return serveCached("stale");
550
677
  json(res, error?.status ?? 502, { error: message });
551
678
  }
552
679
  });