@1agh/maude 0.41.0 → 0.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/studio/api.ts +0 -41
- package/apps/studio/bin/_html-playwright.mjs +26 -4
- package/apps/studio/bin/_pdf-playwright.mjs +13 -2
- package/apps/studio/bin/_png-playwright.mjs +15 -2
- package/apps/studio/bin/_pptx-playwright.mjs +17 -4
- package/apps/studio/bin/_screenshot-playwright.mjs +17 -2
- package/apps/studio/bin/_svg-playwright.mjs +26 -4
- package/apps/studio/bin/screenshot.sh +53 -4
- package/apps/studio/client/app.jsx +25 -54
- package/apps/studio/client/export-center.jsx +426 -0
- package/apps/studio/client/styles/3-shell-maude.css +17 -0
- package/apps/studio/client/styles/4-components.css +150 -0
- package/apps/studio/config.schema.json +2 -2
- package/apps/studio/dist/client.bundle.js +17 -17
- package/apps/studio/dist/styles.css +1 -1
- package/apps/studio/export-dialog.tsx +18 -25
- package/apps/studio/exporters/_runtime.ts +104 -0
- package/apps/studio/exporters/html.ts +12 -20
- package/apps/studio/exporters/index.ts +14 -2
- package/apps/studio/exporters/jobs.ts +334 -0
- package/apps/studio/exporters/pdf.ts +16 -20
- package/apps/studio/exporters/png.ts +12 -20
- package/apps/studio/exporters/pptx.ts +22 -23
- package/apps/studio/exporters/scope.ts +1 -0
- package/apps/studio/exporters/svg.ts +14 -22
- package/apps/studio/exporters/video.ts +15 -17
- package/apps/studio/git/service.ts +3 -1
- package/apps/studio/http.ts +145 -50
- package/apps/studio/server.ts +3 -1
- package/apps/studio/test/canvas-origin-gate.test.ts +6 -0
- package/apps/studio/test/dns-rebinding-guard.test.ts +27 -0
- package/apps/studio/test/export-center.test.tsx +287 -0
- package/apps/studio/test/export-shim-multi-capture.test.ts +83 -0
- package/apps/studio/test/exporters/history.test.ts +32 -3
- package/apps/studio/test/exporters/jobs.test.ts +263 -0
- package/apps/studio/whats-new.json +17 -0
- package/apps/studio/ws.ts +6 -0
- package/cli/lib/gitignore-block.mjs +1 -0
- package/package.json +8 -8
|
@@ -3,8 +3,10 @@
|
|
|
3
3
|
* @scope apps/studio/export-dialog.tsx
|
|
4
4
|
* @purpose Native `<dialog>`-based export modal. Three controls — format,
|
|
5
5
|
* scope, per-format options — plus a Recent tab populated by
|
|
6
|
-
* `/_api/export-history`. Submit
|
|
7
|
-
*
|
|
6
|
+
* `/_api/export-history`. Submit POSTs `/_api/export-jobs` and
|
|
7
|
+
* closes immediately (feature-background-export-notification-
|
|
8
|
+
* center) — the main-shell notification center owns status/
|
|
9
|
+
* progress/completion from there, not this dialog. `⌘E` opens
|
|
8
10
|
* the dialog from anywhere inside the canvas; `⌘⇧E` re-runs the
|
|
9
11
|
* most recent export without opening (T10 fast path).
|
|
10
12
|
*
|
|
@@ -293,7 +295,9 @@ export function ExportDialogProvider({ children }: { children: ReactNode }): Rea
|
|
|
293
295
|
dialogRef.current?.close();
|
|
294
296
|
}, []);
|
|
295
297
|
|
|
296
|
-
//
|
|
298
|
+
// Loaded when the dialog opens. Submit now closes the dialog immediately
|
|
299
|
+
// (the job runs in the background) instead of refreshing this in place —
|
|
300
|
+
// the next open picks up whatever's finished by then.
|
|
297
301
|
const loadHistory = useCallback(async () => {
|
|
298
302
|
try {
|
|
299
303
|
if (isCrossOriginFramed()) {
|
|
@@ -364,10 +368,13 @@ export function ExportDialogProvider({ children }: { children: ReactNode }): Rea
|
|
|
364
368
|
setStatus(null);
|
|
365
369
|
try {
|
|
366
370
|
if (isCrossOriginFramed()) {
|
|
367
|
-
// Bridge through the main shell — the iframe can't reach
|
|
368
|
-
// (canvas origin 403s it). The parent
|
|
369
|
-
//
|
|
370
|
-
|
|
371
|
+
// Bridge through the main shell — the iframe can't reach
|
|
372
|
+
// /_api/export-jobs (canvas origin 403s it, DDR-060). The parent
|
|
373
|
+
// enqueues the job and replies with the id immediately; the
|
|
374
|
+
// trusted main-shell notification center (which already owns the
|
|
375
|
+
// WS connection) is the single place status/progress/completion
|
|
376
|
+
// live from here — this dialog no longer polls or receives bytes.
|
|
377
|
+
const res = await bridgeRequest<{ ok?: boolean; jobId?: string; error?: string }>(
|
|
371
378
|
'export-request',
|
|
372
379
|
'export-result',
|
|
373
380
|
{ payload: { format, scope, options } }
|
|
@@ -376,11 +383,10 @@ export function ExportDialogProvider({ children }: { children: ReactNode }): Rea
|
|
|
376
383
|
setStatus({ text: `Export failed: ${res.error || 'unknown'}`, isError: true });
|
|
377
384
|
return;
|
|
378
385
|
}
|
|
379
|
-
|
|
380
|
-
void loadHistory();
|
|
386
|
+
close();
|
|
381
387
|
return;
|
|
382
388
|
}
|
|
383
|
-
const r = await fetch('/_api/export', {
|
|
389
|
+
const r = await fetch('/_api/export-jobs', {
|
|
384
390
|
method: 'POST',
|
|
385
391
|
headers: { 'content-type': 'application/json' },
|
|
386
392
|
body: JSON.stringify({ format, scope, options }),
|
|
@@ -390,20 +396,7 @@ export function ExportDialogProvider({ children }: { children: ReactNode }): Rea
|
|
|
390
396
|
setStatus({ text: `Export failed: ${text || r.status}`, isError: true });
|
|
391
397
|
return;
|
|
392
398
|
}
|
|
393
|
-
|
|
394
|
-
const filename =
|
|
395
|
-
/filename="([^"]+)"/.exec(disp)?.[1] ?? `export${FORMAT_META[format].defaultExt}`;
|
|
396
|
-
const blob = await r.blob();
|
|
397
|
-
const url = URL.createObjectURL(blob);
|
|
398
|
-
const a = document.createElement('a');
|
|
399
|
-
a.href = url;
|
|
400
|
-
a.download = filename;
|
|
401
|
-
document.body.appendChild(a);
|
|
402
|
-
a.click();
|
|
403
|
-
a.remove();
|
|
404
|
-
URL.revokeObjectURL(url);
|
|
405
|
-
setStatus({ text: `Saved ${filename}`, isError: false });
|
|
406
|
-
void loadHistory();
|
|
399
|
+
close();
|
|
407
400
|
} catch (err) {
|
|
408
401
|
const msg = err instanceof Error ? err.message : String(err);
|
|
409
402
|
setStatus({ text: `Export failed: ${msg}`, isError: true });
|
|
@@ -411,7 +404,7 @@ export function ExportDialogProvider({ children }: { children: ReactNode }): Rea
|
|
|
411
404
|
setSubmitting(false);
|
|
412
405
|
}
|
|
413
406
|
},
|
|
414
|
-
[
|
|
407
|
+
[close]
|
|
415
408
|
);
|
|
416
409
|
|
|
417
410
|
const rerunLast = useCallback(async () => {
|
|
@@ -67,3 +67,107 @@ export function resolveExportRuntime(): string {
|
|
|
67
67
|
}
|
|
68
68
|
return runtime;
|
|
69
69
|
}
|
|
70
|
+
|
|
71
|
+
export interface SpawnShimOptions {
|
|
72
|
+
/** cwd for the spawned shim — callers pass `path.dirname(<SHIM>)`. */
|
|
73
|
+
cwd: string;
|
|
74
|
+
/** Aborts the render — kills the child process. */
|
|
75
|
+
signal?: AbortSignal;
|
|
76
|
+
/** Fired for each `MAUDE_PROGRESS {"current":N,"total":M}` stdout line. */
|
|
77
|
+
onProgress?: (update: { current: number; total: number }) => void;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export interface SpawnShimResult {
|
|
81
|
+
code: number;
|
|
82
|
+
/** stdout lines with `MAUDE_PROGRESS` lines filtered out (written-file paths etc.). */
|
|
83
|
+
stdoutLines: string[];
|
|
84
|
+
stderr: string;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const PROGRESS_LINE = /^MAUDE_PROGRESS (.+)$/;
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Spawn a render shim (`bin/_{png,pdf,svg,html,pptx,video}-playwright.mjs`)
|
|
91
|
+
* via a resolved node/bun runtime, reading stdout INCREMENTALLY (not
|
|
92
|
+
* buffered via `new Response(proc.stdout).text()`) so a `MAUDE_PROGRESS`
|
|
93
|
+
* line written mid-render reaches `onProgress` as soon as it's flushed,
|
|
94
|
+
* instead of only after the whole process exits. Non-progress lines
|
|
95
|
+
* (written-file paths, the existing per-adapter contract) collect into
|
|
96
|
+
* `stdoutLines` in the order they arrived. Exit-code handling stays with the
|
|
97
|
+
* caller — this returns the same shape the old `Promise.all([...text()])`
|
|
98
|
+
* block did, so adapters keep their existing `if (code !== 0) throw …`.
|
|
99
|
+
*/
|
|
100
|
+
export async function spawnShim(args: string[], opts: SpawnShimOptions): Promise<SpawnShimResult> {
|
|
101
|
+
const proc = Bun.spawn([resolveExportRuntime(), ...args], {
|
|
102
|
+
cwd: opts.cwd,
|
|
103
|
+
stdout: 'pipe',
|
|
104
|
+
stderr: 'pipe',
|
|
105
|
+
signal: opts.signal,
|
|
106
|
+
});
|
|
107
|
+
// Bun.spawn's `signal` option kills the child on abort; this listener is a
|
|
108
|
+
// defensive fallback in case a given Bun build ignores it — cheap and
|
|
109
|
+
// idempotent (killing an already-exited process is a no-op).
|
|
110
|
+
const onAbort = () => {
|
|
111
|
+
try {
|
|
112
|
+
proc.kill();
|
|
113
|
+
} catch {
|
|
114
|
+
/* already exited */
|
|
115
|
+
}
|
|
116
|
+
};
|
|
117
|
+
opts.signal?.addEventListener('abort', onAbort);
|
|
118
|
+
|
|
119
|
+
const stderrPromise = new Response(proc.stderr).text();
|
|
120
|
+
const stdoutLines: string[] = [];
|
|
121
|
+
let buffer = '';
|
|
122
|
+
const consumeLine = (line: string) => {
|
|
123
|
+
if (!line) return;
|
|
124
|
+
const match = PROGRESS_LINE.exec(line);
|
|
125
|
+
if (!match) {
|
|
126
|
+
stdoutLines.push(line);
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
try {
|
|
130
|
+
const update = JSON.parse(match[1]) as { current: number; total: number };
|
|
131
|
+
opts.onProgress?.(update);
|
|
132
|
+
} catch {
|
|
133
|
+
/* malformed progress line — drop it rather than treat it as a file path */
|
|
134
|
+
}
|
|
135
|
+
};
|
|
136
|
+
|
|
137
|
+
try {
|
|
138
|
+
const reader = proc.stdout.pipeThrough(new TextDecoderStream()).getReader();
|
|
139
|
+
for (;;) {
|
|
140
|
+
const { done, value } = await reader.read();
|
|
141
|
+
if (done) break;
|
|
142
|
+
buffer += value;
|
|
143
|
+
let newlineIdx = buffer.indexOf('\n');
|
|
144
|
+
while (newlineIdx !== -1) {
|
|
145
|
+
consumeLine(buffer.slice(0, newlineIdx).trim());
|
|
146
|
+
buffer = buffer.slice(newlineIdx + 1);
|
|
147
|
+
newlineIdx = buffer.indexOf('\n');
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
if (buffer.trim()) consumeLine(buffer.trim());
|
|
151
|
+
} finally {
|
|
152
|
+
opts.signal?.removeEventListener('abort', onAbort);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
const [stderr, code] = await Promise.all([stderrPromise, proc.exited]);
|
|
156
|
+
return { code, stdoutLines, stderr };
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* `spawnShim()` + the exit-code check every adapter did identically
|
|
161
|
+
* (`if (code !== 0) throw new Error(...)`; `code review, /flow:done` — six
|
|
162
|
+
* near-copies collapsed into one). `args[0]` is the shim path by convention
|
|
163
|
+
* (every adapter builds it that way), used only for the error message.
|
|
164
|
+
*/
|
|
165
|
+
export async function runShim(args: string[], opts: SpawnShimOptions): Promise<string[]> {
|
|
166
|
+
const { code, stdoutLines, stderr } = await spawnShim(args, opts);
|
|
167
|
+
if (code !== 0) {
|
|
168
|
+
throw new Error(
|
|
169
|
+
`${path.basename(args[0])} exited ${code}: ${stderr.trim() || stdoutLines.join('\n').trim()}`
|
|
170
|
+
);
|
|
171
|
+
}
|
|
172
|
+
return stdoutLines;
|
|
173
|
+
}
|
|
@@ -11,10 +11,11 @@ import { tmpdir } from 'node:os';
|
|
|
11
11
|
import path from 'node:path';
|
|
12
12
|
|
|
13
13
|
import JSZip from 'jszip';
|
|
14
|
-
import { exportShimPath,
|
|
14
|
+
import { exportShimPath, runShim } from './_runtime.ts';
|
|
15
15
|
import {
|
|
16
16
|
canvasShellUrl,
|
|
17
17
|
type ExportContext,
|
|
18
|
+
type ExportHooks,
|
|
18
19
|
type ExportOptions,
|
|
19
20
|
type ExportResult,
|
|
20
21
|
} from './index.ts';
|
|
@@ -27,7 +28,8 @@ async function captureHtml(
|
|
|
27
28
|
target: Extract<Target, { kind: 'element' }>,
|
|
28
29
|
ctx: ExportContext,
|
|
29
30
|
outDir: string,
|
|
30
|
-
timeoutSec: number
|
|
31
|
+
timeoutSec: number,
|
|
32
|
+
hooks?: ExportHooks
|
|
31
33
|
): Promise<string[]> {
|
|
32
34
|
const args = [
|
|
33
35
|
HTML_PLAYWRIGHT,
|
|
@@ -46,29 +48,18 @@ async function captureHtml(
|
|
|
46
48
|
if (target.widen) args.push('--widen-to-artboard', '1');
|
|
47
49
|
args.push('--out', path.join(outDir, `${target.canvasSlug}.html`));
|
|
48
50
|
}
|
|
49
|
-
|
|
51
|
+
return runShim(args, {
|
|
50
52
|
cwd: path.dirname(HTML_PLAYWRIGHT),
|
|
51
|
-
|
|
52
|
-
|
|
53
|
+
signal: hooks?.signal,
|
|
54
|
+
onProgress: hooks?.onProgress,
|
|
53
55
|
});
|
|
54
|
-
const [stdout, stderr] = await Promise.all([
|
|
55
|
-
new Response(proc.stdout).text(),
|
|
56
|
-
new Response(proc.stderr).text(),
|
|
57
|
-
]);
|
|
58
|
-
const code = await proc.exited;
|
|
59
|
-
if (code !== 0) {
|
|
60
|
-
throw new Error(`_html-playwright exited ${code}: ${stderr.trim() || stdout.trim()}`);
|
|
61
|
-
}
|
|
62
|
-
return stdout
|
|
63
|
-
.split('\n')
|
|
64
|
-
.map((s) => s.trim())
|
|
65
|
-
.filter(Boolean);
|
|
66
56
|
}
|
|
67
57
|
|
|
68
58
|
export async function run(
|
|
69
59
|
targets: Target[],
|
|
70
60
|
options: ExportOptions,
|
|
71
|
-
ctx: ExportContext
|
|
61
|
+
ctx: ExportContext,
|
|
62
|
+
hooks?: ExportHooks
|
|
72
63
|
): Promise<ExportResult> {
|
|
73
64
|
if (!targets.length) {
|
|
74
65
|
return { filename: 'export.zip', contentType: 'application/zip', body: new Uint8Array(0) };
|
|
@@ -83,9 +74,10 @@ export async function run(
|
|
|
83
74
|
const tmp = mkdtempSync(path.join(tmpdir(), 'maude-html-'));
|
|
84
75
|
try {
|
|
85
76
|
const written: string[] = [];
|
|
86
|
-
for (
|
|
87
|
-
const paths = await captureHtml(
|
|
77
|
+
for (let i = 0; i < elementTargets.length; i += 1) {
|
|
78
|
+
const paths = await captureHtml(elementTargets[i], ctx, tmp, timeoutSec, hooks);
|
|
88
79
|
written.push(...paths);
|
|
80
|
+
hooks?.onProgress?.({ current: i + 1, total: elementTargets.length });
|
|
89
81
|
}
|
|
90
82
|
if (!written.length) {
|
|
91
83
|
return { filename: 'export.zip', contentType: 'application/zip', body: new Uint8Array(0) };
|
|
@@ -74,8 +74,19 @@ export interface ExportResult {
|
|
|
74
74
|
body: Uint8Array;
|
|
75
75
|
}
|
|
76
76
|
|
|
77
|
+
/** Progress + cancellation hooks threaded through to the shim-spawning layer. */
|
|
78
|
+
export interface ExportHooks {
|
|
79
|
+
onProgress?: (update: { current: number; total: number }) => void;
|
|
80
|
+
signal?: AbortSignal;
|
|
81
|
+
}
|
|
82
|
+
|
|
77
83
|
export interface Adapter {
|
|
78
|
-
run(
|
|
84
|
+
run(
|
|
85
|
+
targets: Target[],
|
|
86
|
+
options: ExportOptions,
|
|
87
|
+
ctx: ExportContext,
|
|
88
|
+
hooks?: ExportHooks
|
|
89
|
+
): Promise<ExportResult>;
|
|
79
90
|
}
|
|
80
91
|
|
|
81
92
|
const REGISTRY: Record<Format, Adapter> = {
|
|
@@ -114,6 +125,7 @@ export async function runExport(args: {
|
|
|
114
125
|
options: ExportOptions;
|
|
115
126
|
resolve: Omit<ResolveScopeArgs, 'scope'>;
|
|
116
127
|
ctx: ExportContext;
|
|
128
|
+
hooks?: ExportHooks;
|
|
117
129
|
}): Promise<ExportResult> {
|
|
118
130
|
// Thread the options bag into the resolver so submit-time selection /
|
|
119
131
|
// active-artboard hints (DDR — selection-passthrough) win over the persisted
|
|
@@ -123,7 +135,7 @@ export async function runExport(args: {
|
|
|
123
135
|
if (!adapter) {
|
|
124
136
|
throw new Error(`unknown format: ${args.format}`);
|
|
125
137
|
}
|
|
126
|
-
return adapter.run(targets, args.options, args.ctx);
|
|
138
|
+
return adapter.run(targets, args.options, args.ctx, args.hooks);
|
|
127
139
|
}
|
|
128
140
|
|
|
129
141
|
export type { Scope, Target };
|
|
@@ -0,0 +1,334 @@
|
|
|
1
|
+
// Background export job queue (feature-background-export-notification-center).
|
|
2
|
+
//
|
|
3
|
+
// Turns the previously-synchronous `POST /_api/export` into a backgroundable
|
|
4
|
+
// operation: `enqueue()` returns immediately with a job id + a Promise that
|
|
5
|
+
// resolves with the same `ExportResult` the old synchronous call produced, so
|
|
6
|
+
// `/_api/export` stays byte-for-byte contract-unchanged (it just awaits the
|
|
7
|
+
// Promise). `POST /_api/export-jobs` returns the id WITHOUT awaiting, and the
|
|
8
|
+
// job progresses queued → running → done|failed in the background, emitting
|
|
9
|
+
// `bus.emit('export:job', job)` on every transition so ws.ts can broadcast a
|
|
10
|
+
// live snapshot to the notification center.
|
|
11
|
+
//
|
|
12
|
+
// Concurrency is capped by a small hand-rolled counting semaphore (same
|
|
13
|
+
// "no new dependency" convention as `writeLocator()` in http.ts) so N
|
|
14
|
+
// Playwright/Chromium-heavy renders don't all launch at once — MAUDE_EXPORT_
|
|
15
|
+
// MAX_CONCURRENT (default 2) lets a quick PNG start while a slow PDF/video is
|
|
16
|
+
// still running, without letting an unbounded number of Chromiums spawn.
|
|
17
|
+
//
|
|
18
|
+
// History persistence: the in-memory `jobs` Map IS the source of truth. The
|
|
19
|
+
// on-disk `_export-history.json` ledger is re-derived from it (done/failed
|
|
20
|
+
// jobs, newest-first, capped) and overwritten in one shot on every
|
|
21
|
+
// completion — no read-modify-write, so concurrent completions can't drop
|
|
22
|
+
// entries (the race the old api.ts loadExportHistory/appendExportHistory
|
|
23
|
+
// pair had). The ledger is seeded from disk once at construction so history
|
|
24
|
+
// survives a server restart even though job state itself doesn't.
|
|
25
|
+
|
|
26
|
+
import { readFileSync } from 'node:fs';
|
|
27
|
+
import { mkdir, rm } from 'node:fs/promises';
|
|
28
|
+
import path from 'node:path';
|
|
29
|
+
|
|
30
|
+
import type { Bus } from '../context.ts';
|
|
31
|
+
import {
|
|
32
|
+
type ExportContext,
|
|
33
|
+
type ExportOptions,
|
|
34
|
+
type ExportResult,
|
|
35
|
+
type Format,
|
|
36
|
+
runExport,
|
|
37
|
+
type Scope,
|
|
38
|
+
} from './index.ts';
|
|
39
|
+
import type { ResolveScopeArgs } from './scope.ts';
|
|
40
|
+
|
|
41
|
+
/** Extended shape of the old Phase 6.5 T10 history entry — additive fields only. */
|
|
42
|
+
export interface ExportHistoryEntry {
|
|
43
|
+
format: string;
|
|
44
|
+
scope: string;
|
|
45
|
+
options?: Record<string, unknown>;
|
|
46
|
+
filename: string;
|
|
47
|
+
at: string;
|
|
48
|
+
id?: string;
|
|
49
|
+
status?: 'done' | 'failed';
|
|
50
|
+
startedAt?: string;
|
|
51
|
+
finishedAt?: string;
|
|
52
|
+
error?: string;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export type ExportJobStatus = 'queued' | 'running' | 'done' | 'failed';
|
|
56
|
+
|
|
57
|
+
export interface ExportJob {
|
|
58
|
+
id: string;
|
|
59
|
+
format: Format;
|
|
60
|
+
scope: Scope;
|
|
61
|
+
options: ExportOptions;
|
|
62
|
+
status: ExportJobStatus;
|
|
63
|
+
progress?: { current: number; total: number };
|
|
64
|
+
createdAt: string;
|
|
65
|
+
startedAt?: string;
|
|
66
|
+
finishedAt?: string;
|
|
67
|
+
filename?: string;
|
|
68
|
+
contentType?: string;
|
|
69
|
+
error?: string;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export interface EnqueueArgs {
|
|
73
|
+
format: Format;
|
|
74
|
+
scope: Scope;
|
|
75
|
+
options: ExportOptions;
|
|
76
|
+
resolve: Omit<ResolveScopeArgs, 'scope'>;
|
|
77
|
+
ctx: ExportContext;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export type DownloadResult =
|
|
81
|
+
| { ok: true; bytes: Uint8Array; filename: string; contentType: string }
|
|
82
|
+
| { ok: false; reason: 'missing' | 'not-done' };
|
|
83
|
+
|
|
84
|
+
export interface ExportJobQueue {
|
|
85
|
+
enqueue(args: EnqueueArgs): { id: string; result: Promise<ExportResult> };
|
|
86
|
+
get(id: string): ExportJob | undefined;
|
|
87
|
+
list(): ExportJob[];
|
|
88
|
+
loadHistory(): ExportHistoryEntry[];
|
|
89
|
+
getBytes(id: string): Promise<DownloadResult>;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const HISTORY_DEPTH = 20;
|
|
93
|
+
const MAX_JOB_AGE_MS = 24 * 60 * 60 * 1000;
|
|
94
|
+
const DEFAULT_JOB_TIMEOUT_MS = 5 * 60 * 1000;
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Security fan-out finding (defender, /flow:done) — MEDIUM: `enqueue()` had
|
|
98
|
+
* no cap on queued/running jobs, so a flood of `POST /_api/export-jobs`
|
|
99
|
+
* (unauthenticated same-origin-omitted-Origin callers pass `sameOriginWrite`)
|
|
100
|
+
* could grow the in-memory Map and fill `_export-jobs/` on disk unbounded —
|
|
101
|
+
* the old synchronous `/_api/export` had natural backpressure (one render at
|
|
102
|
+
* a time from the client's own perspective) that the background queue
|
|
103
|
+
* removed. `MAX_PENDING` bounds queued+running jobs; `enqueue()` throws
|
|
104
|
+
* `ExportQueueFullError` past it and http.ts maps that to 429.
|
|
105
|
+
*/
|
|
106
|
+
const MAX_PENDING = Math.max(1, Number(process.env.MAUDE_EXPORT_MAX_QUEUED) || 20);
|
|
107
|
+
|
|
108
|
+
/** Thrown by `enqueue()` when `MAX_PENDING` queued/running jobs are already in flight. */
|
|
109
|
+
export class ExportQueueFullError extends Error {
|
|
110
|
+
constructor() {
|
|
111
|
+
super('export queue is full — too many pending exports, try again shortly');
|
|
112
|
+
this.name = 'ExportQueueFullError';
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Small counting semaphore — no new dependency, mirrors writeLocator()'s style. */
|
|
117
|
+
class Semaphore {
|
|
118
|
+
private active = 0;
|
|
119
|
+
private readonly waiters: Array<() => void> = [];
|
|
120
|
+
|
|
121
|
+
constructor(private readonly max: number) {}
|
|
122
|
+
|
|
123
|
+
async acquire(): Promise<() => void> {
|
|
124
|
+
if (this.active >= this.max) {
|
|
125
|
+
await new Promise<void>((resolve) => this.waiters.push(resolve));
|
|
126
|
+
}
|
|
127
|
+
this.active += 1;
|
|
128
|
+
let released = false;
|
|
129
|
+
return () => {
|
|
130
|
+
if (released) return;
|
|
131
|
+
released = true;
|
|
132
|
+
this.active -= 1;
|
|
133
|
+
const next = this.waiters.shift();
|
|
134
|
+
if (next) next();
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function isFinished(job: ExportJob): boolean {
|
|
140
|
+
return job.status === 'done' || job.status === 'failed';
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export function createExportJobQueue(bus: Bus, designRoot: string): ExportJobQueue {
|
|
144
|
+
const jobsDir = path.join(designRoot, '_export-jobs');
|
|
145
|
+
const historyPath = path.join(designRoot, '_export-history.json');
|
|
146
|
+
const maxConcurrent = Math.max(1, Number(process.env.MAUDE_EXPORT_MAX_CONCURRENT) || 2);
|
|
147
|
+
const semaphore = new Semaphore(maxConcurrent);
|
|
148
|
+
const jobs = new Map<string, ExportJob>();
|
|
149
|
+
|
|
150
|
+
// Seed the ledger from disk ONCE — the only read of the history file. Every
|
|
151
|
+
// later persist derives fresh from `jobs` and overwrites; there is no
|
|
152
|
+
// subsequent read-then-write step, which is what eliminates the race.
|
|
153
|
+
try {
|
|
154
|
+
const raw = readFileSync(historyPath, 'utf8');
|
|
155
|
+
const parsed = JSON.parse(raw) as unknown;
|
|
156
|
+
if (Array.isArray(parsed)) {
|
|
157
|
+
for (const entry of parsed) {
|
|
158
|
+
if (!entry || typeof entry !== 'object') continue;
|
|
159
|
+
const e = entry as Record<string, unknown>;
|
|
160
|
+
const id = typeof e.id === 'string' && e.id ? e.id : crypto.randomUUID();
|
|
161
|
+
const finishedAt =
|
|
162
|
+
typeof e.finishedAt === 'string'
|
|
163
|
+
? e.finishedAt
|
|
164
|
+
: typeof e.at === 'string'
|
|
165
|
+
? e.at
|
|
166
|
+
: undefined;
|
|
167
|
+
jobs.set(id, {
|
|
168
|
+
id,
|
|
169
|
+
format: e.format as Format,
|
|
170
|
+
scope: e.scope as Scope,
|
|
171
|
+
options: (e.options as ExportOptions) ?? {},
|
|
172
|
+
status: e.status === 'failed' ? 'failed' : 'done',
|
|
173
|
+
createdAt: (e.startedAt as string) ?? finishedAt ?? new Date().toISOString(),
|
|
174
|
+
startedAt: e.startedAt as string | undefined,
|
|
175
|
+
finishedAt,
|
|
176
|
+
filename: e.filename as string | undefined,
|
|
177
|
+
error: e.error as string | undefined,
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
} catch {
|
|
182
|
+
/* no ledger yet, or unreadable — start empty */
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// Orphaned `_export-jobs/*` dirs from a process that died mid-export — job
|
|
186
|
+
// state is in-memory only and doesn't survive a restart, so anything on
|
|
187
|
+
// disk from a prior run is stale. Best-effort, never blocks boot.
|
|
188
|
+
void rm(jobsDir, { recursive: true, force: true })
|
|
189
|
+
.catch(() => {})
|
|
190
|
+
.then(() => mkdir(jobsDir, { recursive: true }).catch(() => {}));
|
|
191
|
+
|
|
192
|
+
function emit(job: ExportJob): void {
|
|
193
|
+
bus.emit('export:job', { ...job });
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
function deriveHistory(): ExportHistoryEntry[] {
|
|
197
|
+
return Array.from(jobs.values())
|
|
198
|
+
.filter(isFinished)
|
|
199
|
+
.sort((a, b) => (b.finishedAt ?? '').localeCompare(a.finishedAt ?? ''))
|
|
200
|
+
.slice(0, HISTORY_DEPTH)
|
|
201
|
+
.map((j) => ({
|
|
202
|
+
id: j.id,
|
|
203
|
+
format: j.format,
|
|
204
|
+
scope: j.scope,
|
|
205
|
+
options: j.options,
|
|
206
|
+
filename: j.filename ?? '',
|
|
207
|
+
at: j.finishedAt ?? j.createdAt,
|
|
208
|
+
status: j.status as 'done' | 'failed',
|
|
209
|
+
startedAt: j.startedAt,
|
|
210
|
+
finishedAt: j.finishedAt,
|
|
211
|
+
error: j.error,
|
|
212
|
+
}));
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
async function persistAndEvict(): Promise<void> {
|
|
216
|
+
const history = deriveHistory();
|
|
217
|
+
await Bun.write(historyPath, JSON.stringify(history, null, 2));
|
|
218
|
+
|
|
219
|
+
// Evict bytes (+ the in-memory record) for anything that rolled past the
|
|
220
|
+
// cap, or aged out, whichever comes first.
|
|
221
|
+
const finished = Array.from(jobs.values())
|
|
222
|
+
.filter(isFinished)
|
|
223
|
+
.sort((a, b) => (b.finishedAt ?? '').localeCompare(a.finishedAt ?? ''));
|
|
224
|
+
const now = Date.now();
|
|
225
|
+
const stale = finished.filter((j, i) => {
|
|
226
|
+
if (i >= HISTORY_DEPTH) return true;
|
|
227
|
+
const finishedAt = j.finishedAt ? Date.parse(j.finishedAt) : Number.NaN;
|
|
228
|
+
return Number.isFinite(finishedAt) && now - finishedAt > MAX_JOB_AGE_MS;
|
|
229
|
+
});
|
|
230
|
+
for (const job of stale) {
|
|
231
|
+
jobs.delete(job.id);
|
|
232
|
+
await rm(path.join(jobsDir, job.id), { recursive: true, force: true }).catch(() => {});
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
function enqueue(args: EnqueueArgs): { id: string; result: Promise<ExportResult> } {
|
|
237
|
+
let pending = 0;
|
|
238
|
+
for (const job of jobs.values()) {
|
|
239
|
+
if (job.status === 'queued' || job.status === 'running') pending += 1;
|
|
240
|
+
}
|
|
241
|
+
if (pending >= MAX_PENDING) throw new ExportQueueFullError();
|
|
242
|
+
|
|
243
|
+
const id = crypto.randomUUID();
|
|
244
|
+
const job: ExportJob = {
|
|
245
|
+
id,
|
|
246
|
+
format: args.format,
|
|
247
|
+
scope: args.scope,
|
|
248
|
+
options: args.options,
|
|
249
|
+
status: 'queued',
|
|
250
|
+
createdAt: new Date().toISOString(),
|
|
251
|
+
};
|
|
252
|
+
jobs.set(id, job);
|
|
253
|
+
emit(job);
|
|
254
|
+
|
|
255
|
+
const controller = new AbortController();
|
|
256
|
+
|
|
257
|
+
const result = (async (): Promise<ExportResult> => {
|
|
258
|
+
const release = await semaphore.acquire();
|
|
259
|
+
// Started here, not at enqueue — the timeout bounds RENDER time (the
|
|
260
|
+
// intent), not queue-wait time. A job can legitimately sit `queued`
|
|
261
|
+
// behind MAUDE_EXPORT_MAX_CONCURRENT other jobs for longer than
|
|
262
|
+
// DEFAULT_JOB_TIMEOUT_MS; arming the timer before the semaphore
|
|
263
|
+
// resolves would abort it before it ever ran (code-review finding,
|
|
264
|
+
// /flow:done).
|
|
265
|
+
const timer = setTimeout(() => controller.abort(), DEFAULT_JOB_TIMEOUT_MS);
|
|
266
|
+
try {
|
|
267
|
+
job.status = 'running';
|
|
268
|
+
job.startedAt = new Date().toISOString();
|
|
269
|
+
emit(job);
|
|
270
|
+
|
|
271
|
+
const res = await runExport({
|
|
272
|
+
format: args.format,
|
|
273
|
+
scope: args.scope,
|
|
274
|
+
options: args.options,
|
|
275
|
+
resolve: args.resolve,
|
|
276
|
+
ctx: args.ctx,
|
|
277
|
+
hooks: {
|
|
278
|
+
signal: controller.signal,
|
|
279
|
+
onProgress: (update) => {
|
|
280
|
+
job.progress = update;
|
|
281
|
+
emit(job);
|
|
282
|
+
},
|
|
283
|
+
},
|
|
284
|
+
});
|
|
285
|
+
|
|
286
|
+
job.status = 'done';
|
|
287
|
+
job.finishedAt = new Date().toISOString();
|
|
288
|
+
job.filename = res.filename;
|
|
289
|
+
job.contentType = res.contentType;
|
|
290
|
+
if (res.body.byteLength) {
|
|
291
|
+
const dir = path.join(jobsDir, id);
|
|
292
|
+
await mkdir(dir, { recursive: true });
|
|
293
|
+
await Bun.write(path.join(dir, res.filename), res.body);
|
|
294
|
+
}
|
|
295
|
+
emit(job);
|
|
296
|
+
await persistAndEvict();
|
|
297
|
+
return res;
|
|
298
|
+
} catch (err) {
|
|
299
|
+
job.status = 'failed';
|
|
300
|
+
job.finishedAt = new Date().toISOString();
|
|
301
|
+
job.error = err instanceof Error ? err.message : String(err);
|
|
302
|
+
emit(job);
|
|
303
|
+
await persistAndEvict();
|
|
304
|
+
throw err;
|
|
305
|
+
} finally {
|
|
306
|
+
clearTimeout(timer);
|
|
307
|
+
release();
|
|
308
|
+
}
|
|
309
|
+
})();
|
|
310
|
+
|
|
311
|
+
return { id, result };
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
return {
|
|
315
|
+
enqueue,
|
|
316
|
+
get: (id) => jobs.get(id),
|
|
317
|
+
list: () => Array.from(jobs.values()).sort((a, b) => b.createdAt.localeCompare(a.createdAt)),
|
|
318
|
+
loadHistory: deriveHistory,
|
|
319
|
+
async getBytes(id) {
|
|
320
|
+
const job = jobs.get(id);
|
|
321
|
+
if (!job) return { ok: false, reason: 'missing' };
|
|
322
|
+
if (job.status !== 'done') return { ok: false, reason: 'not-done' };
|
|
323
|
+
if (!job.filename) return { ok: false, reason: 'missing' };
|
|
324
|
+
const file = Bun.file(path.join(jobsDir, id, job.filename));
|
|
325
|
+
if (!(await file.exists())) return { ok: false, reason: 'missing' };
|
|
326
|
+
return {
|
|
327
|
+
ok: true,
|
|
328
|
+
bytes: new Uint8Array(await file.arrayBuffer()),
|
|
329
|
+
filename: job.filename,
|
|
330
|
+
contentType: job.contentType ?? 'application/octet-stream',
|
|
331
|
+
};
|
|
332
|
+
},
|
|
333
|
+
};
|
|
334
|
+
}
|