@makoai/app-sdk 2.7.3 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +73 -5
- package/data-core.js +212 -0
- package/index.d.ts +199 -5
- package/index.js +352 -98
- package/package.json +6 -1
- package/progress.js +233 -0
- package/ui.js +5 -2
- package/vite.d.ts +5 -3
- package/vite.js +179 -18
package/progress.js
ADDED
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
// What a binding load is waiting on — the pure half (no React, no DuckDB),
|
|
2
|
+
// so `node --test` covers it without installing anything.
|
|
3
|
+
//
|
|
4
|
+
// From 2.7.4 the SDK asks every `__data` request for phase events
|
|
5
|
+
// (`Accept: application/x-ndjson`). A server that knows them answers one
|
|
6
|
+
// JSON object per line on the same connection and ends with exactly one
|
|
7
|
+
// `ready` or `failed`; a server that predates them answers as before
|
|
8
|
+
// (parquet bytes, a 302 to them, or JSON), and the SDK still knows when the
|
|
9
|
+
// download and the DuckDB load start. See api/src/apps/binding-progress.ts.
|
|
10
|
+
|
|
11
|
+
export const PROGRESS_ACCEPT = "application/x-ndjson, */*;q=0.5";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Every phase, in the order a full rebuild passes through them. Server
|
|
15
|
+
* phases come from the stream; `requesting`, `downloading`,
|
|
16
|
+
* `loading_duckdb` and `running_sql` are the SDK's own.
|
|
17
|
+
*/
|
|
18
|
+
export const BINDING_PHASES = [
|
|
19
|
+
"requesting",
|
|
20
|
+
"queued",
|
|
21
|
+
"checking",
|
|
22
|
+
"querying_warehouse",
|
|
23
|
+
"writing_parquet",
|
|
24
|
+
"uploading",
|
|
25
|
+
"resolving",
|
|
26
|
+
"signing",
|
|
27
|
+
"downloading",
|
|
28
|
+
"loading_duckdb",
|
|
29
|
+
"running_sql",
|
|
30
|
+
];
|
|
31
|
+
|
|
32
|
+
const STAGE_OF = {
|
|
33
|
+
requesting: "waiting",
|
|
34
|
+
queued: "building",
|
|
35
|
+
checking: "waiting",
|
|
36
|
+
querying_warehouse: "building",
|
|
37
|
+
writing_parquet: "building",
|
|
38
|
+
uploading: "building",
|
|
39
|
+
resolving: "linking",
|
|
40
|
+
signing: "linking",
|
|
41
|
+
downloading: "downloading",
|
|
42
|
+
loading_duckdb: "loading",
|
|
43
|
+
running_sql: "querying",
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* The wait a person cares about: `building` (the warehouse is running the
|
|
48
|
+
* query — seconds to minutes), `linking` (Mako is finding and signing the
|
|
49
|
+
* stored file — well under a second), `downloading`, `loading` (into
|
|
50
|
+
* DuckDB), `querying` (your SQL), or `waiting` before the server says.
|
|
51
|
+
*/
|
|
52
|
+
export function bindingPhaseStage(phase) {
|
|
53
|
+
return STAGE_OF[phase] ?? null;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export const BINDING_STAGE_LABELS = {
|
|
57
|
+
waiting: "Contacting Mako",
|
|
58
|
+
building: "Building from warehouse",
|
|
59
|
+
linking: "Getting download link",
|
|
60
|
+
downloading: "Downloading",
|
|
61
|
+
loading: "Loading into DuckDB",
|
|
62
|
+
querying: "Running query",
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
/** Finer wording per phase, for a tooltip or a second line. */
|
|
66
|
+
export const BINDING_PHASE_LABELS = {
|
|
67
|
+
requesting: "Contacting Mako",
|
|
68
|
+
queued: "Waiting for a build slot",
|
|
69
|
+
checking: "Checking the binding",
|
|
70
|
+
querying_warehouse: "Querying the warehouse",
|
|
71
|
+
writing_parquet: "Writing parquet",
|
|
72
|
+
uploading: "Storing the file",
|
|
73
|
+
resolving: "Finding the file",
|
|
74
|
+
signing: "Signing the download link",
|
|
75
|
+
downloading: "Downloading",
|
|
76
|
+
loading_duckdb: "Loading into DuckDB",
|
|
77
|
+
running_sql: "Running query",
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
function formatBytes(n) {
|
|
81
|
+
if (n < 1024) return n + " B";
|
|
82
|
+
if (n < 1024 * 1024) return Math.round(n / 1024) + " KB";
|
|
83
|
+
return (n / (1024 * 1024)).toFixed(1) + " MB";
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* A caption for a progress bar: the stage label, plus rows or bytes when
|
|
88
|
+
* known. Takes a phase name or a progress record (`useQuery().progress`).
|
|
89
|
+
* Empty string when nothing is pending.
|
|
90
|
+
*/
|
|
91
|
+
export function bindingPhaseLabel(progress) {
|
|
92
|
+
const p = typeof progress === "string" ? { phase: progress } : progress;
|
|
93
|
+
const stage = p ? bindingPhaseStage(p.phase) : null;
|
|
94
|
+
if (!stage) return "";
|
|
95
|
+
const label = BINDING_STAGE_LABELS[stage];
|
|
96
|
+
if (stage === "building" && typeof p.rowsWritten === "number") {
|
|
97
|
+
return label + " · " + p.rowsWritten.toLocaleString("en-US") + " rows";
|
|
98
|
+
}
|
|
99
|
+
if (stage === "downloading" && typeof p.bytes === "number") {
|
|
100
|
+
return (
|
|
101
|
+
label +
|
|
102
|
+
" · " +
|
|
103
|
+
formatBytes(p.bytes) +
|
|
104
|
+
(typeof p.bytesTotal === "number" ? " of " + formatBytes(p.bytesTotal) : "")
|
|
105
|
+
);
|
|
106
|
+
}
|
|
107
|
+
return label;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// Most worth showing first when several bindings are pending at once: a
|
|
111
|
+
// warehouse build dwarfs everything, a download is the next longest.
|
|
112
|
+
const STAGE_WEIGHT = { building: 5, waiting: 4, downloading: 3, linking: 2, loading: 1, querying: 0 };
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* One record for many pending bindings (useDuckDB): the phase of the one
|
|
116
|
+
* furthest from done, rows summed over the builds, bytes over the
|
|
117
|
+
* downloads, and how many are pending.
|
|
118
|
+
*/
|
|
119
|
+
export function combineBindingProgress(records) {
|
|
120
|
+
const list = Object.values(records ?? {}).filter(Boolean);
|
|
121
|
+
if (!list.length) return null;
|
|
122
|
+
let top = list[0];
|
|
123
|
+
for (const r of list) {
|
|
124
|
+
const w = STAGE_WEIGHT[bindingPhaseStage(r.phase)] ?? -1;
|
|
125
|
+
if (w > (STAGE_WEIGHT[bindingPhaseStage(top.phase)] ?? -1)) top = r;
|
|
126
|
+
}
|
|
127
|
+
const stage = bindingPhaseStage(top.phase);
|
|
128
|
+
const same = list.filter(r => bindingPhaseStage(r.phase) === stage);
|
|
129
|
+
const sum = key => {
|
|
130
|
+
const known = same.filter(r => typeof r[key] === "number");
|
|
131
|
+
return known.length ? known.reduce((n, r) => n + r[key], 0) : null;
|
|
132
|
+
};
|
|
133
|
+
return {
|
|
134
|
+
phase: top.phase,
|
|
135
|
+
rowsWritten: sum("rowsWritten"),
|
|
136
|
+
bytes: sum("bytes"),
|
|
137
|
+
bytesTotal: same.every(r => typeof r.bytesTotal === "number") ? sum("bytesTotal") : null,
|
|
138
|
+
pending: list.length,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** Did the server answer with phase events? */
|
|
143
|
+
export function isProgressResponse(res) {
|
|
144
|
+
const type = (res.headers && res.headers.get("content-type")) || "";
|
|
145
|
+
return type.split(";")[0].trim().toLowerCase() === "application/x-ndjson";
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Read a phase stream: `onEvent` for every line, resolves with the terminal
|
|
150
|
+
* event (`ready` or `failed`). A stream that ends without one resolves as
|
|
151
|
+
* `failed` with status 502 — the connection broke before the server said.
|
|
152
|
+
*/
|
|
153
|
+
export async function readProgressStream(res, onEvent) {
|
|
154
|
+
const reader = res.body.getReader();
|
|
155
|
+
const decoder = new TextDecoder();
|
|
156
|
+
let buffered = "";
|
|
157
|
+
const handle = line => {
|
|
158
|
+
const text = line.trim();
|
|
159
|
+
if (!text) return null;
|
|
160
|
+
let event;
|
|
161
|
+
try {
|
|
162
|
+
event = JSON.parse(text);
|
|
163
|
+
} catch {
|
|
164
|
+
return null;
|
|
165
|
+
}
|
|
166
|
+
if (event && (event.phase === "ready" || event.phase === "failed")) return event;
|
|
167
|
+
if (event && event.phase) onEvent(event);
|
|
168
|
+
return null;
|
|
169
|
+
};
|
|
170
|
+
for (;;) {
|
|
171
|
+
const { done, value } = await reader.read();
|
|
172
|
+
buffered += decoder.decode(value ?? new Uint8Array(), { stream: !done });
|
|
173
|
+
let nl;
|
|
174
|
+
while ((nl = buffered.indexOf("\n")) !== -1) {
|
|
175
|
+
const final = handle(buffered.slice(0, nl));
|
|
176
|
+
buffered = buffered.slice(nl + 1);
|
|
177
|
+
if (final) {
|
|
178
|
+
reader.cancel().catch(() => {});
|
|
179
|
+
return final;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
if (done) break;
|
|
183
|
+
}
|
|
184
|
+
return (
|
|
185
|
+
handle(buffered) ?? {
|
|
186
|
+
phase: "failed",
|
|
187
|
+
status: 502,
|
|
188
|
+
error: "The connection closed before the server finished",
|
|
189
|
+
}
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* The response body as bytes, reporting `(received, total)` as it arrives.
|
|
195
|
+
* `total` is null when the server sent no Content-Length.
|
|
196
|
+
*/
|
|
197
|
+
export async function readBytes(res, onProgress) {
|
|
198
|
+
const total = Number(res.headers.get("content-length")) || null;
|
|
199
|
+
if (!res.body || !res.body.getReader) {
|
|
200
|
+
const buf = new Uint8Array(await res.arrayBuffer());
|
|
201
|
+
onProgress(buf.length, total);
|
|
202
|
+
return buf;
|
|
203
|
+
}
|
|
204
|
+
const reader = res.body.getReader();
|
|
205
|
+
const chunks = [];
|
|
206
|
+
let received = 0;
|
|
207
|
+
onProgress(0, total);
|
|
208
|
+
for (;;) {
|
|
209
|
+
const { done, value } = await reader.read();
|
|
210
|
+
if (done) break;
|
|
211
|
+
chunks.push(value);
|
|
212
|
+
received += value.length;
|
|
213
|
+
onProgress(received, total);
|
|
214
|
+
}
|
|
215
|
+
if (chunks.length === 1) return chunks[0];
|
|
216
|
+
const out = new Uint8Array(received);
|
|
217
|
+
let at = 0;
|
|
218
|
+
for (const c of chunks) {
|
|
219
|
+
out.set(c, at);
|
|
220
|
+
at += c.length;
|
|
221
|
+
}
|
|
222
|
+
return out;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/** The Error a `failed` event becomes — the same shape as an HTTP failure. */
|
|
226
|
+
export function progressError(event, what) {
|
|
227
|
+
const detail = [event.error, event.hint].filter(Boolean).join(" — ");
|
|
228
|
+
const status = typeof event.status === "number" ? event.status : 502;
|
|
229
|
+
const error = new Error(what + " (HTTP " + status + ")" + (detail ? ": " + detail : ""));
|
|
230
|
+
error.status = status;
|
|
231
|
+
if (typeof event.retryAfterMs === "number") error.retryAfterMs = event.retryAfterMs;
|
|
232
|
+
return error;
|
|
233
|
+
}
|
package/ui.js
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
// in through props.
|
|
11
11
|
import * as React from "react";
|
|
12
12
|
import { createPortal } from "react-dom";
|
|
13
|
-
import { refreshBinding } from "./index.js";
|
|
13
|
+
import { bindingPhaseLabel, refreshBinding, useBindingProgress } from "./index.js";
|
|
14
14
|
import { formatSeconds as fmtSecs, normalizeTone, refreshOrder } from "./ui-core.js";
|
|
15
15
|
|
|
16
16
|
export { normalizeTone, refreshOrder };
|
|
@@ -446,6 +446,9 @@ export function RefreshAllButton({
|
|
|
446
446
|
const [open, setOpen] = React.useState(false);
|
|
447
447
|
const [now, setNow] = React.useState(() => Date.now());
|
|
448
448
|
const closeRef = React.useRef(null);
|
|
449
|
+
// What each running source is waiting on: the warehouse, the link, the
|
|
450
|
+
// download or DuckDB (progress.js).
|
|
451
|
+
const progress = useBindingProgress();
|
|
449
452
|
|
|
450
453
|
const running = state.status === "running";
|
|
451
454
|
const total = sources.length;
|
|
@@ -562,7 +565,7 @@ export function RefreshAllButton({
|
|
|
562
565
|
? ""
|
|
563
566
|
: "Queued"
|
|
564
567
|
: st === "running"
|
|
565
|
-
? "Running"
|
|
568
|
+
? bindingPhaseLabel(progress[name]) || "Running"
|
|
566
569
|
: st === "done"
|
|
567
570
|
? took[name] != null
|
|
568
571
|
? fmtSecs(took[name])
|
package/vite.d.ts
CHANGED
|
@@ -12,9 +12,11 @@ export interface MakoDataOptions {
|
|
|
12
12
|
/** Repo root. Default: the nearest ancestor holding .mako/ or .git. */
|
|
13
13
|
repoRoot?: string;
|
|
14
14
|
/**
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
15
|
+
* A cached parquet of the current binding text is always served at once;
|
|
16
|
+
* after this long (ms) it is also rebuilt in the background for the next
|
|
17
|
+
* load. Default: 5 minutes. A cache built from different binding text (or
|
|
18
|
+
* dbt environment) is never served — `Infinity` is safe while you edit
|
|
19
|
+
* SQL. `refresh()` / `?refresh` waits for the rebuild.
|
|
18
20
|
*/
|
|
19
21
|
revalidateMs?: number;
|
|
20
22
|
/**
|
package/vite.js
CHANGED
|
@@ -156,6 +156,54 @@ function json(res, status, body) {
|
|
|
156
156
|
res.end(JSON.stringify(body));
|
|
157
157
|
}
|
|
158
158
|
|
|
159
|
+
// The SDK (>= 2.7.4) asks for phase events with Accept: application/x-ndjson
|
|
160
|
+
// and then hears each step of a build on the same request (see progress.js).
|
|
161
|
+
function wantsProgress(req) {
|
|
162
|
+
return String(req.headers?.accept ?? "")
|
|
163
|
+
.split(",")
|
|
164
|
+
.some((part) => part.split(";")[0].trim().toLowerCase() === "application/x-ndjson");
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// Start a phase stream; returns { emit, end(final) }.
|
|
168
|
+
function progressStream(res) {
|
|
169
|
+
const started = Date.now();
|
|
170
|
+
res.statusCode = 200;
|
|
171
|
+
res.setHeader("content-type", "application/x-ndjson");
|
|
172
|
+
res.setHeader("cache-control", "no-store");
|
|
173
|
+
const write = (event) => {
|
|
174
|
+
try {
|
|
175
|
+
res.write(JSON.stringify({ ...event, elapsedMs: Date.now() - started }) + "\n");
|
|
176
|
+
} catch {
|
|
177
|
+
// The page went away; the build carries on and fills the cache.
|
|
178
|
+
}
|
|
179
|
+
};
|
|
180
|
+
return {
|
|
181
|
+
emit: write,
|
|
182
|
+
end(final) {
|
|
183
|
+
write(final);
|
|
184
|
+
res.end();
|
|
185
|
+
},
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// One build's phases, for everyone waiting on it (a page mounting several
|
|
190
|
+
// tables at once shares one request): the latest phase first, then the rest.
|
|
191
|
+
function phaseHub(first) {
|
|
192
|
+
const listeners = new Set(first ? [first] : []);
|
|
193
|
+
let last = null;
|
|
194
|
+
return {
|
|
195
|
+
emit(event) {
|
|
196
|
+
last = event;
|
|
197
|
+
for (const l of [...listeners]) l(event);
|
|
198
|
+
},
|
|
199
|
+
watch(fn) {
|
|
200
|
+
if (!fn) return;
|
|
201
|
+
listeners.add(fn);
|
|
202
|
+
if (last) fn(last);
|
|
203
|
+
},
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
|
|
159
207
|
export function makoData(options = {}) {
|
|
160
208
|
return {
|
|
161
209
|
name: "mako-data",
|
|
@@ -216,7 +264,7 @@ export function makoData(options = {}) {
|
|
|
216
264
|
// edge's 524 — is retried with backoff until the overall deadline; only
|
|
217
265
|
// a definitive 4xx (the job is gone, access was revoked) ends the wait.
|
|
218
266
|
const isTransient = (status) => status === 429 || status >= 500;
|
|
219
|
-
async function awaitJob(name, jobId) {
|
|
267
|
+
async function awaitJob(name, jobId, onPhase) {
|
|
220
268
|
const url = `${appBase()}/binding-jobs/${encodeURIComponent(jobId)}`;
|
|
221
269
|
const deadline = Date.now() + buildTimeoutMs;
|
|
222
270
|
let wait = pollMs;
|
|
@@ -232,6 +280,14 @@ export function makoData(options = {}) {
|
|
|
232
280
|
const body = await res.json().catch(() => ({}));
|
|
233
281
|
if (res.ok) {
|
|
234
282
|
if (body.status === "ready") return body;
|
|
283
|
+
// The job row carries the worker's phase (binding-jobs.ts).
|
|
284
|
+
if (onPhase && body.status !== "error") {
|
|
285
|
+
onPhase({
|
|
286
|
+
phase: body.phase ?? (body.status === "queued" ? "queued" : "checking"),
|
|
287
|
+
...(typeof body.rowsWritten === "number" ? { rowsWritten: body.rowsWritten } : {}),
|
|
288
|
+
...(typeof body.bytes === "number" ? { bytes: body.bytes } : {}),
|
|
289
|
+
});
|
|
290
|
+
}
|
|
235
291
|
if (body.status === "error") {
|
|
236
292
|
const error = new Error(body.error || `build ${name} failed`);
|
|
237
293
|
error.status = body.errorStatus ?? 502;
|
|
@@ -272,13 +328,13 @@ export function makoData(options = {}) {
|
|
|
272
328
|
}
|
|
273
329
|
// POST …/materialize, asynchronously when the API supports it.
|
|
274
330
|
// Resolves to the materialize answer ({ rowCount, byteSize, materializedAt }).
|
|
275
|
-
async function materialize(name) {
|
|
331
|
+
async function materialize(name, onPhase) {
|
|
276
332
|
const res = await apiFetch(
|
|
277
333
|
`${appBase()}/bindings/${encodeURIComponent(name)}/materialize?async=1`,
|
|
278
334
|
{ method: "POST", headers: await headers() },
|
|
279
335
|
);
|
|
280
336
|
const body = await res.json().catch(() => ({}));
|
|
281
|
-
if (res.status === 202 && body.jobId) return awaitJob(name, body.jobId);
|
|
337
|
+
if (res.status === 202 && body.jobId) return awaitJob(name, body.jobId, onPhase);
|
|
282
338
|
if (!res.ok) {
|
|
283
339
|
const error = new Error(body.error || `materialize ${name}: HTTP ${res.status}`);
|
|
284
340
|
error.status = res.status;
|
|
@@ -287,12 +343,12 @@ export function makoData(options = {}) {
|
|
|
287
343
|
return body;
|
|
288
344
|
}
|
|
289
345
|
|
|
290
|
-
async function fetchArtifact(name) {
|
|
346
|
+
async function fetchArtifact(name, onPhase) {
|
|
291
347
|
const url = `${appBase()}/bindings/${encodeURIComponent(name)}/artifact`;
|
|
292
348
|
let res = await apiFetch(url, { headers: await headers() });
|
|
293
349
|
if (res.status === 404 && options.materialize !== false) {
|
|
294
350
|
// Never materialized (or a live binding): build it now, then read.
|
|
295
|
-
await materialize(name);
|
|
351
|
+
await materialize(name, onPhase);
|
|
296
352
|
res = await apiFetch(url, { headers: await headers() });
|
|
297
353
|
}
|
|
298
354
|
if (!res.ok) {
|
|
@@ -332,9 +388,18 @@ export function makoData(options = {}) {
|
|
|
332
388
|
}
|
|
333
389
|
}
|
|
334
390
|
|
|
391
|
+
// Atomic: a background revalidation can land while a page is still
|
|
392
|
+
// streaming the previous copy, which keeps reading the old file.
|
|
335
393
|
function writeCache(name, buf, meta) {
|
|
336
394
|
fs.mkdirSync(ctx.cacheDir, { recursive: true });
|
|
337
|
-
|
|
395
|
+
const tmp = `${cachePath(name)}.${process.pid}.${Date.now()}.tmp`;
|
|
396
|
+
fs.writeFileSync(tmp, buf);
|
|
397
|
+
try {
|
|
398
|
+
fs.renameSync(tmp, cachePath(name));
|
|
399
|
+
} catch {
|
|
400
|
+
fs.rmSync(tmp, { force: true });
|
|
401
|
+
fs.writeFileSync(cachePath(name), buf);
|
|
402
|
+
}
|
|
338
403
|
if (meta) fs.writeFileSync(metaPath(name), JSON.stringify(meta));
|
|
339
404
|
else fs.rmSync(metaPath(name), { force: true });
|
|
340
405
|
}
|
|
@@ -350,13 +415,35 @@ export function makoData(options = {}) {
|
|
|
350
415
|
let devBuildUnsupported = false;
|
|
351
416
|
const inflight = new Map();
|
|
352
417
|
|
|
418
|
+
// One background rebuild per binding at a time; a failure is logged and
|
|
419
|
+
// the next load tries again.
|
|
420
|
+
const revalidating = new Set();
|
|
421
|
+
function revalidate(name, source) {
|
|
422
|
+
if (revalidating.has(name)) return;
|
|
423
|
+
revalidating.add(name);
|
|
424
|
+
(async () => {
|
|
425
|
+
const built = source !== null ? (await devBuild(name, source, false))?.buf : null;
|
|
426
|
+
if (!built) writeCache(name, await fetchArtifact(name), null);
|
|
427
|
+
})()
|
|
428
|
+
.catch(error => {
|
|
429
|
+
server.config.logger.error(
|
|
430
|
+
` mako-data: background revalidate of ${name}: ${error instanceof Error ? error.message : String(error)}`,
|
|
431
|
+
);
|
|
432
|
+
})
|
|
433
|
+
.finally(() => revalidating.delete(name));
|
|
434
|
+
}
|
|
435
|
+
|
|
353
436
|
// Build the LOCAL binding text through the API. Concurrent asks for the
|
|
354
437
|
// same text share one request (a page mounting several tables at once).
|
|
355
|
-
function devBuild(name, source, refresh) {
|
|
438
|
+
function devBuild(name, source, refresh, onPhase) {
|
|
356
439
|
const fingerprint = bindingFingerprint(source, ctx.dbtEnvironment);
|
|
357
440
|
const key = `${name}:${fingerprint}:${refresh ? 1 : 0}`;
|
|
358
441
|
let pending = inflight.get(key);
|
|
359
|
-
if (pending)
|
|
442
|
+
if (pending) {
|
|
443
|
+
pending.hub.watch(onPhase);
|
|
444
|
+
return pending;
|
|
445
|
+
}
|
|
446
|
+
const hub = phaseHub(onPhase);
|
|
360
447
|
pending = (async () => {
|
|
361
448
|
const res = await apiFetch(`${appBase()}/bindings/${encodeURIComponent(name)}/dev-build`, {
|
|
362
449
|
method: "POST",
|
|
@@ -372,7 +459,7 @@ export function makoData(options = {}) {
|
|
|
372
459
|
if (res.status === 202) {
|
|
373
460
|
const queued = await res.json().catch(() => ({}));
|
|
374
461
|
if (!queued.jobId) throw new Error(`build ${name}: HTTP 202 without a job id`);
|
|
375
|
-
jobDone = await awaitJob(name, queued.jobId);
|
|
462
|
+
jobDone = await awaitJob(name, queued.jobId, hub.emit);
|
|
376
463
|
}
|
|
377
464
|
if (!jobDone && !res.ok) {
|
|
378
465
|
const text = await res.text().catch(() => "");
|
|
@@ -427,17 +514,47 @@ export function makoData(options = {}) {
|
|
|
427
514
|
writeCache(name, buf, meta);
|
|
428
515
|
return { buf, meta };
|
|
429
516
|
})().finally(() => inflight.delete(key));
|
|
517
|
+
pending.hub = hub;
|
|
430
518
|
inflight.set(key, pending);
|
|
431
519
|
return pending;
|
|
432
520
|
}
|
|
433
521
|
|
|
522
|
+
// Bytes a phase stream just built, handed to the page's follow-up GET
|
|
523
|
+
// (`?take=<id>`) once, so that GET never builds again — whatever
|
|
524
|
+
// revalidateMs says, and also when the bytes are a stale fallback.
|
|
525
|
+
const takes = new Map();
|
|
526
|
+
function keepForTake(buf) {
|
|
527
|
+
const id = createHash("sha256").update(String(Math.random()) + Date.now()).digest("hex").slice(0, 16);
|
|
528
|
+
takes.set(id, buf);
|
|
529
|
+
setTimeout(() => takes.delete(id), 60_000).unref?.();
|
|
530
|
+
return id;
|
|
531
|
+
}
|
|
532
|
+
|
|
434
533
|
// POST __data/<name>/refresh — the SDK's refresh(): rebuild the LOCAL
|
|
435
534
|
// binding through the API (the committed one is re-materialized and
|
|
436
535
|
// stored, an edited one is built as a draft), and cache what came back.
|
|
437
|
-
async function refreshBinding(name, res) {
|
|
536
|
+
async function refreshBinding(name, req, res) {
|
|
438
537
|
const source = options.materialize === false || devBuildUnsupported ? null : readSource(name);
|
|
538
|
+
const stream = wantsProgress(req) ? progressStream(res) : null;
|
|
539
|
+
if (stream) {
|
|
540
|
+
stream.emit({ phase: "checking" });
|
|
541
|
+
// The same answers as below, as the stream's terminal event.
|
|
542
|
+
res = {
|
|
543
|
+
code: 200,
|
|
544
|
+
set statusCode(code) {
|
|
545
|
+
this.code = code;
|
|
546
|
+
},
|
|
547
|
+
setHeader() {},
|
|
548
|
+
end(text) {
|
|
549
|
+
const body = JSON.parse(text);
|
|
550
|
+
const ok = this.code >= 200 && this.code < 300;
|
|
551
|
+
stream.end({ ...body, phase: ok ? "ready" : "failed", ...(ok ? {} : { status: this.code }) });
|
|
552
|
+
},
|
|
553
|
+
};
|
|
554
|
+
}
|
|
555
|
+
const onPhase = stream ? stream.emit : undefined;
|
|
439
556
|
try {
|
|
440
|
-
const built = source === null ? null : await devBuild(name, source, true);
|
|
557
|
+
const built = source === null ? null : await devBuild(name, source, true, onPhase);
|
|
441
558
|
if (built) {
|
|
442
559
|
return json(res, 200, {
|
|
443
560
|
success: true,
|
|
@@ -450,7 +567,7 @@ export function makoData(options = {}) {
|
|
|
450
567
|
}
|
|
451
568
|
let body;
|
|
452
569
|
try {
|
|
453
|
-
body = await materialize(name);
|
|
570
|
+
body = await materialize(name, onPhase);
|
|
454
571
|
} catch (error) {
|
|
455
572
|
return json(res, error?.status ?? 502, {
|
|
456
573
|
success: false,
|
|
@@ -512,7 +629,16 @@ export function makoData(options = {}) {
|
|
|
512
629
|
hint: "Run `mako login` in this repo, or put MAKO_API_URL and MAKO_API_KEY in the repo's .env (see AGENTS.md).",
|
|
513
630
|
});
|
|
514
631
|
}
|
|
515
|
-
if (isRefresh) return refreshBinding(name, res);
|
|
632
|
+
if (isRefresh) return refreshBinding(name, req, res);
|
|
633
|
+
const take = /(^|&)take=([0-9a-f]{16})(&|$)/.exec(query);
|
|
634
|
+
if (take && takes.has(take[2])) {
|
|
635
|
+
const buf = takes.get(take[2]);
|
|
636
|
+
takes.delete(take[2]);
|
|
637
|
+
res.setHeader("content-type", "application/vnd.apache.parquet");
|
|
638
|
+
res.setHeader("content-length", String(buf.length));
|
|
639
|
+
res.setHeader("x-mako-data", "api");
|
|
640
|
+
return res.end(buf);
|
|
641
|
+
}
|
|
516
642
|
const cached = cachePath(name);
|
|
517
643
|
const refresh = /(^|&)refresh(=|&|$)/.test(query);
|
|
518
644
|
const source = options.materialize === false || devBuildUnsupported ? null : readSource(name);
|
|
@@ -527,16 +653,38 @@ export function makoData(options = {}) {
|
|
|
527
653
|
res.setHeader("x-mako-data", how);
|
|
528
654
|
return fs.createReadStream(cached).pipe(res);
|
|
529
655
|
};
|
|
656
|
+
// A build is about to happen: a page that asked hears its phases,
|
|
657
|
+
// then a "ready" whose url takes the bytes (keepForTake).
|
|
658
|
+
let stream = null;
|
|
530
659
|
try {
|
|
531
660
|
const stat = fs.statSync(cached, { throwIfNoEntry: false });
|
|
532
|
-
|
|
533
|
-
|
|
661
|
+
// A copy built from this text is served at once, however old: past
|
|
662
|
+
// `revalidateMs` it is rebuilt in the background for the next load
|
|
663
|
+
// (stale-while-revalidate), so a page reload never waits on the API
|
|
664
|
+
// for data it already has. `?refresh` (the SDK's refresh()) waits.
|
|
665
|
+
if (!refresh && stat && cacheMatches()) {
|
|
666
|
+
const fresh = Date.now() - stat.mtimeMs < revalidateMs;
|
|
667
|
+
if (!fresh) revalidate(name, source);
|
|
668
|
+
return serveCached(fresh ? "cache" : "revalidating");
|
|
669
|
+
}
|
|
670
|
+
if (wantsProgress(req)) {
|
|
671
|
+
stream = progressStream(res);
|
|
672
|
+
stream.emit({ phase: "checking" });
|
|
673
|
+
}
|
|
674
|
+
const onPhase = stream ? stream.emit : undefined;
|
|
534
675
|
let buf = null;
|
|
535
|
-
if (source !== null) buf = (await devBuild(name, source, false))?.buf ?? null;
|
|
676
|
+
if (source !== null) buf = (await devBuild(name, source, false, onPhase))?.buf ?? null;
|
|
536
677
|
if (!buf) {
|
|
537
|
-
buf = await fetchArtifact(name);
|
|
678
|
+
buf = await fetchArtifact(name, onPhase);
|
|
538
679
|
writeCache(name, buf, null);
|
|
539
680
|
}
|
|
681
|
+
if (stream) {
|
|
682
|
+
return stream.end({
|
|
683
|
+
phase: "ready",
|
|
684
|
+
binding: name,
|
|
685
|
+
url: `__data/${encodeURIComponent(name)}.parquet?take=${keepForTake(buf)}`,
|
|
686
|
+
});
|
|
687
|
+
}
|
|
540
688
|
res.setHeader("content-type", "application/vnd.apache.parquet");
|
|
541
689
|
res.setHeader("content-length", String(buf.length));
|
|
542
690
|
res.setHeader("x-mako-data", "api");
|
|
@@ -546,7 +694,20 @@ export function makoData(options = {}) {
|
|
|
546
694
|
server.config.logger.error(` mako-data: ${message}`);
|
|
547
695
|
// Stale beats nothing while offline — but only data built from THIS
|
|
548
696
|
// text; an old query's parquet is not a stale copy of a new one.
|
|
549
|
-
|
|
697
|
+
const stale = fs.existsSync(cached) && cacheMatches();
|
|
698
|
+
if (stream) {
|
|
699
|
+
return stream.end(
|
|
700
|
+
stale
|
|
701
|
+
? {
|
|
702
|
+
phase: "ready",
|
|
703
|
+
binding: name,
|
|
704
|
+
stale: true,
|
|
705
|
+
url: `__data/${encodeURIComponent(name)}.parquet?take=${keepForTake(fs.readFileSync(cached))}`,
|
|
706
|
+
}
|
|
707
|
+
: { phase: "failed", status: error?.status ?? 502, error: message },
|
|
708
|
+
);
|
|
709
|
+
}
|
|
710
|
+
if (stale) return serveCached("stale");
|
|
550
711
|
json(res, error?.status ?? 502, { error: message });
|
|
551
712
|
}
|
|
552
713
|
});
|