@1agh/maude 0.45.2 → 0.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/studio/acp/bridge.ts +43 -1
- package/apps/studio/annotations-snap.ts +50 -10
- package/apps/studio/api.ts +344 -14
- package/apps/studio/artboard-guides-overlay.tsx +270 -0
- package/apps/studio/bin/_pdf-playwright.mjs +35 -4
- package/apps/studio/bin/_png-playwright.mjs +38 -4
- package/apps/studio/bin/_pw-launch.mjs +52 -0
- package/apps/studio/bin/_pw-launch.test.mjs +90 -0
- package/apps/studio/bin/_smart-frames.mjs +419 -0
- package/apps/studio/bin/_smart-frames.test.mjs +140 -0
- package/apps/studio/bin/smart-frames.sh +30 -0
- package/apps/studio/canvas-edit.ts +302 -3
- package/apps/studio/canvas-lib.tsx +203 -11
- package/apps/studio/canvas-shell.tsx +167 -28
- package/apps/studio/client/app.jsx +678 -176
- package/apps/studio/client/panels/ChatPanel.jsx +45 -1
- package/apps/studio/client/panels/SettingsPanel.jsx +211 -0
- package/apps/studio/client/panels/acp-capabilities.js +11 -0
- package/apps/studio/client/styles/6-acp-chat.css +41 -0
- package/apps/studio/dist/client.bundle.js +1398 -1398
- package/apps/studio/dist/comment-mount.js +2 -2
- package/apps/studio/dist/styles.css +1 -1
- package/apps/studio/dom-selection.ts +20 -0
- package/apps/studio/export-dialog.tsx +138 -18
- package/apps/studio/exporters/pdf.ts +332 -18
- package/apps/studio/exporters/png.ts +45 -3
- package/apps/studio/footage/schema.ts +17 -1
- package/apps/studio/generation/gemma-models.ts +224 -0
- package/apps/studio/generation/prefs.ts +43 -0
- package/apps/studio/http.ts +213 -7
- package/apps/studio/print/marks.ts +113 -0
- package/apps/studio/print/units.ts +269 -0
- package/apps/studio/print-overlay-content.tsx +132 -0
- package/apps/studio/test/acp-mode-banner.test.ts +45 -0
- package/apps/studio/test/acp-session-allowed-tools.test.ts +91 -0
- package/apps/studio/test/annotations-snap.test.ts +56 -0
- package/apps/studio/test/artboard-guides-overlay.test.tsx +152 -0
- package/apps/studio/test/artboard-kinds.test.tsx +83 -0
- package/apps/studio/test/artboard-selection-attrs.test.ts +69 -0
- package/apps/studio/test/canvas-meta-api.test.ts +167 -0
- package/apps/studio/test/element-structural-edit.test.ts +250 -0
- package/apps/studio/test/exporters/png.test.ts +49 -1
- package/apps/studio/test/pdf-print-boxes.test.ts +272 -0
- package/apps/studio/test/print-marks.test.ts +113 -0
- package/apps/studio/test/print-units.test.ts +173 -0
- package/apps/studio/test/use-snap-guides.test.ts +81 -0
- package/apps/studio/use-chrome-visibility.tsx +19 -0
- package/apps/studio/use-element-resize.tsx +21 -2
- package/apps/studio/use-snap-guides.tsx +73 -5
- package/apps/studio/use-spacing-handles.tsx +9 -5
- package/apps/studio/whats-new.json +28 -0
- package/cli/commands/design.mjs +9 -1
- package/package.json +8 -8
- package/plugins/design/dependencies.json +35 -0
|
@@ -12,6 +12,7 @@ import { tmpdir } from 'node:os';
|
|
|
12
12
|
import path from 'node:path';
|
|
13
13
|
|
|
14
14
|
import JSZip from 'jszip';
|
|
15
|
+
import { CSS_DPI } from '../print/units.ts';
|
|
15
16
|
import { exportShimPath, runShim } from './_runtime.ts';
|
|
16
17
|
import {
|
|
17
18
|
canvasShellUrl,
|
|
@@ -27,7 +28,8 @@ import type { Target } from './scope.ts';
|
|
|
27
28
|
const PNG_PLAYWRIGHT = exportShimPath('_png-playwright.mjs');
|
|
28
29
|
|
|
29
30
|
interface CaptureOptions {
|
|
30
|
-
|
|
31
|
+
/** Fully-resolved Playwright deviceScaleFactor — see resolveDeviceScale. */
|
|
32
|
+
scale?: number;
|
|
31
33
|
timeoutSec?: number;
|
|
32
34
|
}
|
|
33
35
|
|
|
@@ -42,6 +44,44 @@ export function clampScale(raw: unknown): 1 | 2 | 3 {
|
|
|
42
44
|
return 2;
|
|
43
45
|
}
|
|
44
46
|
|
|
47
|
+
/** feature-2-print-artboards T4 — supported DPI presets (Adobe-conventional
|
|
48
|
+
* print ladder). Exported for unit coverage. */
|
|
49
|
+
export const DPI_PRESETS = [96, 150, 300, 600] as const;
|
|
50
|
+
export type DpiPreset = (typeof DPI_PRESETS)[number];
|
|
51
|
+
|
|
52
|
+
/** Coerce an arbitrary `options.dpi` to the nearest supported preset, or
|
|
53
|
+
* `undefined` when absent/invalid (caller then falls back to `scale`). */
|
|
54
|
+
export function clampDpi(raw: unknown): DpiPreset | undefined {
|
|
55
|
+
if (raw === undefined || raw === null) return undefined;
|
|
56
|
+
const n = Number(raw);
|
|
57
|
+
if (!Number.isFinite(n)) return undefined;
|
|
58
|
+
let best: DpiPreset = DPI_PRESETS[0];
|
|
59
|
+
let bestDelta = Math.abs(n - best);
|
|
60
|
+
for (const preset of DPI_PRESETS) {
|
|
61
|
+
const delta = Math.abs(n - preset);
|
|
62
|
+
if (delta < bestDelta) {
|
|
63
|
+
best = preset;
|
|
64
|
+
bestDelta = delta;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return best;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* The Playwright `deviceScaleFactor` for a capture — `dpi` (physical print
|
|
72
|
+
* resolution) wins when present, else the legacy 1×/2×/3× UI preset.
|
|
73
|
+
* `dpi/96` because authoring is at CSS px @96dpi (print/units.ts CSS_DPI) —
|
|
74
|
+
* this is the ONE place outside that module allowed to divide by 96 (a
|
|
75
|
+
* device-scale-factor derivation, not an mm→px conversion, so it's exempt
|
|
76
|
+
* from the T1 single-source `25.4` lint guard, but stays in lockstep with
|
|
77
|
+
* CSS_DPI by importing it rather than hardcoding 96 again).
|
|
78
|
+
*/
|
|
79
|
+
export function resolveDeviceScale(options: ExportOptions): number {
|
|
80
|
+
const dpi = clampDpi(options.dpi);
|
|
81
|
+
if (dpi !== undefined) return dpi / CSS_DPI;
|
|
82
|
+
return clampScale(options.scale);
|
|
83
|
+
}
|
|
84
|
+
|
|
45
85
|
async function captureElement(
|
|
46
86
|
target: Extract<Target, { kind: 'element' }>,
|
|
47
87
|
ctx: ExportContext,
|
|
@@ -112,8 +152,10 @@ export async function run(
|
|
|
112
152
|
const captureOpts: CaptureOptions = {
|
|
113
153
|
// Default 2× — a single-scale PNG was uselessly small (item 1). The dialog
|
|
114
154
|
// sends an explicit scale; this default covers direct API / curl callers.
|
|
115
|
-
//
|
|
116
|
-
|
|
155
|
+
// feature-2-print-artboards T4 — `options.dpi` (96/150/300/600) wins over
|
|
156
|
+
// the legacy 1–3 preset when present; the shim re-clamps deviceScaleFactor
|
|
157
|
+
// ≤ 8 and guards oversized output (see _png-playwright.mjs).
|
|
158
|
+
scale: resolveDeviceScale(options),
|
|
117
159
|
timeoutSec: (options.timeoutSec as number | undefined) ?? 8,
|
|
118
160
|
};
|
|
119
161
|
|
|
@@ -150,6 +150,10 @@ export interface FootageAnalysis {
|
|
|
150
150
|
summary?: string;
|
|
151
151
|
/** Cross-cutting tags ("rebrand", "exterior", "people"). */
|
|
152
152
|
tags?: string[];
|
|
153
|
+
/** Optional audio/speech note — gist + language of what is said, filled ONLY when
|
|
154
|
+
* an orchestrator (e.g. /design:video-analyze) hands the analyst a transcript.
|
|
155
|
+
* Absent for the default vision-only path (e.g. /design:reel). */
|
|
156
|
+
speech?: string;
|
|
153
157
|
}
|
|
154
158
|
|
|
155
159
|
/** An optional graphic overlay laid over a beat (title card / lower-third / logo). */
|
|
@@ -437,7 +441,18 @@ export function validateFootageAnalysis(input: unknown): ValidationResult {
|
|
|
437
441
|
assertKeys(
|
|
438
442
|
errors,
|
|
439
443
|
input,
|
|
440
|
-
[
|
|
444
|
+
[
|
|
445
|
+
'version',
|
|
446
|
+
'asset',
|
|
447
|
+
'durationSec',
|
|
448
|
+
'width',
|
|
449
|
+
'height',
|
|
450
|
+
'keyframes',
|
|
451
|
+
'shots',
|
|
452
|
+
'summary',
|
|
453
|
+
'tags',
|
|
454
|
+
'speech',
|
|
455
|
+
],
|
|
441
456
|
'root'
|
|
442
457
|
);
|
|
443
458
|
if ('version' in input && input.version != null && typeof input.version !== 'number')
|
|
@@ -449,6 +464,7 @@ export function validateFootageAnalysis(input: unknown): ValidationResult {
|
|
|
449
464
|
intGe(errors, input, 'keyframes', 0, 'root');
|
|
450
465
|
str(errors, input, 'summary', 4000, 'root');
|
|
451
466
|
strArray(errors, input, 'tags', 64, 'root');
|
|
467
|
+
str(errors, input, 'speech', 4000, 'root');
|
|
452
468
|
|
|
453
469
|
const dur = typeof input.durationSec === 'number' ? input.durationSec : Infinity;
|
|
454
470
|
if ('shots' in input && input.shots != null) {
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
// generation/gemma-models.ts — managed local Gemma-4 MLX models for the `gemma`
|
|
2
|
+
// tier of `maude design smart-frames` (feature-scene-aware-keyframes). The
|
|
3
|
+
// "one-click Gemma scout" story, mirroring whisper-models.ts — but with three
|
|
4
|
+
// real differences from whisper:
|
|
5
|
+
//
|
|
6
|
+
// 1. Apple-Silicon only. The scout runs through mlx-vlm (a Python package); the
|
|
7
|
+
// model is useless without it. So the download is GATED on `mlxVlmAvailable`.
|
|
8
|
+
// 2. Multi-file HF snapshot, not a single .bin. mlx-vlm resolves a model from the
|
|
9
|
+
// standard HuggingFace hub cache, so we download THERE (via huggingface_hub,
|
|
10
|
+
// a transitive dep of mlx-vlm) rather than a Maude-private cache — otherwise
|
|
11
|
+
// mlx-vlm wouldn't find it. "downloaded" = the snapshot is in the HF cache.
|
|
12
|
+
// 3. The RUNTIME (mlx-vlm itself) is a manual `pip install` the app can't do for
|
|
13
|
+
// you — the Settings card says so and only the MODEL half is one click.
|
|
14
|
+
//
|
|
15
|
+
// This module owns the registry + availability probes + the HF-cache-aware
|
|
16
|
+
// downloaded check + resolve. The download SPAWN lives here (it shells the mlx
|
|
17
|
+
// Python's huggingface_hub); the http route wraps it with progress state, next to
|
|
18
|
+
// the other provider egress.
|
|
19
|
+
|
|
20
|
+
import { spawn, spawnSync } from 'node:child_process';
|
|
21
|
+
import { existsSync, readdirSync, statSync } from 'node:fs';
|
|
22
|
+
import { homedir } from 'node:os';
|
|
23
|
+
import { join } from 'node:path';
|
|
24
|
+
|
|
25
|
+
export interface GemmaModelDescriptor {
|
|
26
|
+
/** Stable id used by the pref + route. */
|
|
27
|
+
id: string;
|
|
28
|
+
/** The mlx-community HF repo (also the `--model` mlx-vlm resolves). */
|
|
29
|
+
repo: string;
|
|
30
|
+
/** Pinned commit SHA — the download fetches THIS revision, never floating `main`,
|
|
31
|
+
* so a poisoned-main / namespace-reuse push can't land arbitrary model files
|
|
32
|
+
* (DDR-183 supply-chain finding; the whisper baseline pins exact file URLs). */
|
|
33
|
+
revision: string;
|
|
34
|
+
label: string;
|
|
35
|
+
/** Approximate on-disk size (consent copy). */
|
|
36
|
+
sizeMB: number;
|
|
37
|
+
note: string;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// Curated allowlist of Gemma-4 MLX scout models. `repo` + `revision` are the frozen
|
|
41
|
+
// download target (never user input). e4b is the default (better beats); e2b is the
|
|
42
|
+
// small, fast option for lower-RAM Macs. Revisions pinned 2026-07-16.
|
|
43
|
+
export const GEMMA_MODELS: readonly GemmaModelDescriptor[] = [
|
|
44
|
+
{
|
|
45
|
+
id: 'gemma-4-e4b-it-4bit',
|
|
46
|
+
repo: 'mlx-community/gemma-4-e4b-it-4bit',
|
|
47
|
+
revision: '475b9088d29754a3379866cf5aeb6b41acd313c2',
|
|
48
|
+
label: 'Gemma 4 E4B (4-bit)',
|
|
49
|
+
sizeMB: 3300,
|
|
50
|
+
note: 'The recommended scout — best semantic beats. ~3.3 GB. Needs an Apple-Silicon Mac + mlx-vlm.',
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
id: 'gemma-4-e2b-it-4bit',
|
|
54
|
+
repo: 'mlx-community/gemma-4-e2b-it-4bit',
|
|
55
|
+
revision: '238767527555cb75a05732a84dff5d6ba0dd6809',
|
|
56
|
+
label: 'Gemma 4 E2B (4-bit)',
|
|
57
|
+
sizeMB: 1900,
|
|
58
|
+
note: 'Smaller/faster, coarser beats. ~1.9 GB. Good for 16 GB Macs.',
|
|
59
|
+
},
|
|
60
|
+
];
|
|
61
|
+
|
|
62
|
+
export function getGemmaModel(id: unknown): GemmaModelDescriptor | null {
|
|
63
|
+
return GEMMA_MODELS.find((m) => m.id === id) ?? null;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** The HuggingFace hub cache dir mlx-vlm resolves models from (HF_HOME-aware). */
|
|
67
|
+
export function hfHubDir(): string {
|
|
68
|
+
const hfHome = process.env.HF_HOME;
|
|
69
|
+
if (hfHome && hfHome.length > 0) return join(hfHome, 'hub');
|
|
70
|
+
const base = process.env.XDG_CACHE_HOME || join(homedir(), '.cache');
|
|
71
|
+
return join(base, 'huggingface', 'hub');
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** HF snapshot dir name for a repo: `models--<org>--<name>`. */
|
|
75
|
+
function repoCacheDir(repo: string): string {
|
|
76
|
+
return join(hfHubDir(), `models--${repo.replace('/', '--')}`);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** A model counts as downloaded when its HF snapshot dir has a non-empty snapshot. */
|
|
80
|
+
export function gemmaModelDownloaded(m: GemmaModelDescriptor): boolean {
|
|
81
|
+
const snaps = join(repoCacheDir(m.repo), 'snapshots');
|
|
82
|
+
try {
|
|
83
|
+
if (!existsSync(snaps)) return false;
|
|
84
|
+
for (const rev of readdirSync(snaps)) {
|
|
85
|
+
const revDir = join(snaps, rev);
|
|
86
|
+
try {
|
|
87
|
+
if (statSync(revDir).isDirectory() && readdirSync(revDir).length > 0) return true;
|
|
88
|
+
} catch {
|
|
89
|
+
/* skip */
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
} catch {
|
|
93
|
+
/* not downloaded */
|
|
94
|
+
}
|
|
95
|
+
return false;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export interface GemmaModelStatus extends GemmaModelDescriptor {
|
|
99
|
+
downloaded: boolean;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export function listGemmaModels(): GemmaModelStatus[] {
|
|
103
|
+
return GEMMA_MODELS.map((m) => ({ ...m, downloaded: gemmaModelDownloaded(m) }));
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Resolve a Python that can `import mlx_vlm` (honors $MAUDE_MLX_PYTHON), or null. */
|
|
107
|
+
export function resolveMlxPython(): string | null {
|
|
108
|
+
const candidates = [process.env.MAUDE_MLX_PYTHON, 'python3', 'python'].filter(
|
|
109
|
+
Boolean
|
|
110
|
+
) as string[];
|
|
111
|
+
for (const py of candidates) {
|
|
112
|
+
const r = spawnSync(py, ['-c', 'import mlx_vlm'], { stdio: 'ignore' });
|
|
113
|
+
if (r.status === 0) return py;
|
|
114
|
+
}
|
|
115
|
+
return null;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// Availability probes spawn subprocesses (importing mlx_vlm can take hundreds of ms
|
|
119
|
+
// to seconds). The GET /_api/generate/keyframe-model route is un-CSRF-gated (correct
|
|
120
|
+
// for a read), so a cross-origin drive-by could hammer it and stall the single-
|
|
121
|
+
// threaded Bun event loop on synchronous spawns (security-auditor DDR-183 finding).
|
|
122
|
+
// Memoize with a short TTL: DoS-bounded to at most one probe per PROBE_TTL_MS
|
|
123
|
+
// regardless of request rate, while still picking up a mid-session `pip install` /
|
|
124
|
+
// PATH change within the TTL.
|
|
125
|
+
const PROBE_TTL_MS = 30_000;
|
|
126
|
+
const probeCache = new Map<string, { at: number; value: boolean }>();
|
|
127
|
+
|
|
128
|
+
function cachedProbe(key: string, compute: () => boolean): boolean {
|
|
129
|
+
const now = Date.now();
|
|
130
|
+
const hit = probeCache.get(key);
|
|
131
|
+
if (hit && now - hit.at < PROBE_TTL_MS) return hit.value;
|
|
132
|
+
const value = compute();
|
|
133
|
+
probeCache.set(key, { at: now, value });
|
|
134
|
+
return value;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export function mlxVlmAvailable(): boolean {
|
|
138
|
+
return cachedProbe('mlx', () => resolveMlxPython() !== null);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
export function ffmpegAvailable(): boolean {
|
|
142
|
+
return cachedProbe('ffmpeg', () => {
|
|
143
|
+
const finder = process.platform === 'win32' ? 'where' : 'which';
|
|
144
|
+
return (
|
|
145
|
+
spawnSync(finder, ['ffmpeg'], { stdio: 'ignore' }).status === 0 &&
|
|
146
|
+
spawnSync(finder, ['ffprobe'], { stdio: 'ignore' }).status === 0
|
|
147
|
+
);
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Download a Gemma model into the HF hub cache via huggingface_hub (a transitive
|
|
153
|
+
* dep of mlx-vlm, so it's present whenever the scout can actually run). We delegate
|
|
154
|
+
* to Python rather than reimplement multi-file HF snapshot download + LFS/Xet
|
|
155
|
+
* redirect handling in TS — the repo id is frozen (never user input), the runtime
|
|
156
|
+
* is pinned to the mlx Python, and the route is loopback + same-origin only.
|
|
157
|
+
* Progress is coarse (tqdm on stderr → a heartbeat), which the route surfaces as an
|
|
158
|
+
* in-flight state. Rejects if mlx-vlm/Python is absent (you couldn't run the model
|
|
159
|
+
* anyway) or the process exits non-zero.
|
|
160
|
+
*/
|
|
161
|
+
export function downloadGemmaModel(
|
|
162
|
+
id: string,
|
|
163
|
+
onProgress: (received: number, total: number) => void,
|
|
164
|
+
signal?: AbortSignal
|
|
165
|
+
): Promise<void> {
|
|
166
|
+
const m = getGemmaModel(id);
|
|
167
|
+
if (!m) return Promise.reject(new Error(`unknown gemma model: ${id}`));
|
|
168
|
+
const py = resolveMlxPython();
|
|
169
|
+
if (!py)
|
|
170
|
+
return Promise.reject(
|
|
171
|
+
new Error(
|
|
172
|
+
'mlx-vlm not installed — the Gemma scout needs an Apple-Silicon Mac + `pip install mlx-vlm`.'
|
|
173
|
+
)
|
|
174
|
+
);
|
|
175
|
+
|
|
176
|
+
// Refuse a steered HF endpoint (mirror/redirect) — the whisper path pins the host;
|
|
177
|
+
// huggingface_hub otherwise honors HF_ENDPOINT/HF_HUB_ENDPOINT. Keep the pull on
|
|
178
|
+
// the canonical hub (DDR-183 supply-chain finding).
|
|
179
|
+
const endpoint = process.env.HF_ENDPOINT || process.env.HF_HUB_ENDPOINT;
|
|
180
|
+
if (endpoint && !/^https:\/\/huggingface\.co\/?$/.test(endpoint))
|
|
181
|
+
return Promise.reject(new Error(`refusing model download: HF endpoint steered to ${endpoint}`));
|
|
182
|
+
|
|
183
|
+
const total = m.sizeMB * 1024 * 1024;
|
|
184
|
+
return new Promise((resolve, reject) => {
|
|
185
|
+
// snapshot_download(repo, revision=<pinned sha>) — repo AND revision are from the
|
|
186
|
+
// frozen registry (never user input), passed as argv (not interpolated into -c).
|
|
187
|
+
const child = spawn(
|
|
188
|
+
py,
|
|
189
|
+
[
|
|
190
|
+
'-c',
|
|
191
|
+
'import sys; from huggingface_hub import snapshot_download; snapshot_download(sys.argv[1], revision=sys.argv[2])',
|
|
192
|
+
m.repo,
|
|
193
|
+
m.revision,
|
|
194
|
+
],
|
|
195
|
+
{
|
|
196
|
+
stdio: ['ignore', 'ignore', 'pipe'],
|
|
197
|
+
env: { ...process.env, HF_HUB_DISABLE_TELEMETRY: '1' },
|
|
198
|
+
}
|
|
199
|
+
);
|
|
200
|
+
let received = 0;
|
|
201
|
+
const onAbort = () => child.kill('SIGTERM');
|
|
202
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
203
|
+
// tqdm writes percentage lines to stderr; use them as a coarse heartbeat.
|
|
204
|
+
child.stderr?.on('data', (buf: Buffer) => {
|
|
205
|
+
const s = buf.toString();
|
|
206
|
+
const pct = /(\d{1,3})%/.exec(s);
|
|
207
|
+
if (pct) received = Math.min(total, Math.round((Number(pct[1]) / 100) * total));
|
|
208
|
+
onProgress(received, total);
|
|
209
|
+
});
|
|
210
|
+
child.on('error', (err) => {
|
|
211
|
+
signal?.removeEventListener('abort', onAbort);
|
|
212
|
+
reject(err);
|
|
213
|
+
});
|
|
214
|
+
child.on('close', (code) => {
|
|
215
|
+
signal?.removeEventListener('abort', onAbort);
|
|
216
|
+
if (code === 0 && gemmaModelDownloaded(m)) {
|
|
217
|
+
onProgress(total, total);
|
|
218
|
+
resolve();
|
|
219
|
+
} else {
|
|
220
|
+
reject(new Error(`gemma model download failed (exit ${code})`));
|
|
221
|
+
}
|
|
222
|
+
});
|
|
223
|
+
});
|
|
224
|
+
}
|
|
@@ -20,6 +20,16 @@ export function isTranscriptionProvider(v: unknown): v is TranscriptionProvider
|
|
|
20
20
|
return typeof v === 'string' && (TRANSCRIPTION_PROVIDERS as readonly string[]).includes(v);
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
+
/** Scene-aware keyframe engines (feature-scene-aware-keyframes) — the tier the
|
|
24
|
+
* `gemma` scout → `ffmpeg` scene-detect → `blind` Chromium fallback ladder runs
|
|
25
|
+
* in. `auto` self-detects installed deps. */
|
|
26
|
+
export const KEYFRAME_ENGINES = ['auto', 'gemma', 'ffmpeg', 'blind'] as const;
|
|
27
|
+
export type KeyframeEngine = (typeof KEYFRAME_ENGINES)[number];
|
|
28
|
+
|
|
29
|
+
export function isKeyframeEngine(v: unknown): v is KeyframeEngine {
|
|
30
|
+
return typeof v === 'string' && (KEYFRAME_ENGINES as readonly string[]).includes(v);
|
|
31
|
+
}
|
|
32
|
+
|
|
23
33
|
function configPath(repoRoot: string): string {
|
|
24
34
|
return join(repoRoot, '.design', 'config.json');
|
|
25
35
|
}
|
|
@@ -80,3 +90,36 @@ export async function writeTranscriptionProvider(
|
|
|
80
90
|
await Bun.write(configPath(repoRoot), `${JSON.stringify(next, null, 2)}\n`);
|
|
81
91
|
return true;
|
|
82
92
|
}
|
|
93
|
+
|
|
94
|
+
/** The current scene-aware keyframe engine preference, or 'auto' (the default). */
|
|
95
|
+
export function readKeyframeEngine(repoRoot: string): KeyframeEngine {
|
|
96
|
+
const cfg = readConfig(repoRoot);
|
|
97
|
+
const gen = cfg.generation as Record<string, unknown> | undefined;
|
|
98
|
+
const kf = gen?.keyframes as Record<string, unknown> | undefined;
|
|
99
|
+
return isKeyframeEngine(kf?.engine) ? (kf?.engine as KeyframeEngine) : 'auto';
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Persist the keyframe-engine choice into `.design/config.json`, preserving every
|
|
104
|
+
* other field (same additive-merge + fail-closed-on-corrupt discipline as the
|
|
105
|
+
* transcription writer).
|
|
106
|
+
*/
|
|
107
|
+
export async function writeKeyframeEngine(repoRoot: string, engine: string): Promise<boolean> {
|
|
108
|
+
if (!isKeyframeEngine(engine)) throw new Error(`invalid keyframe engine: ${engine}`);
|
|
109
|
+
const p = configPath(repoRoot);
|
|
110
|
+
if (existsSync(p)) {
|
|
111
|
+
try {
|
|
112
|
+
JSON.parse(readFileSync(p, 'utf8'));
|
|
113
|
+
} catch {
|
|
114
|
+
throw new Error(
|
|
115
|
+
'.design/config.json is present but not valid JSON — fix it before changing generation prefs (refusing to overwrite it)'
|
|
116
|
+
);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
const cfg = readConfig(repoRoot);
|
|
120
|
+
const gen = (cfg.generation as Record<string, unknown>) ?? {};
|
|
121
|
+
const keyframes = (gen.keyframes as Record<string, unknown>) ?? {};
|
|
122
|
+
const next = { ...cfg, generation: { ...gen, keyframes: { ...keyframes, engine } } };
|
|
123
|
+
await Bun.write(configPath(repoRoot), `${JSON.stringify(next, null, 2)}\n`);
|
|
124
|
+
return true;
|
|
125
|
+
}
|
package/apps/studio/http.ts
CHANGED
|
@@ -40,6 +40,12 @@ import {
|
|
|
40
40
|
sanitizeReuseText,
|
|
41
41
|
} from './generation/audio-library.ts';
|
|
42
42
|
import { localizeGenAsset } from './generation/download.ts';
|
|
43
|
+
import {
|
|
44
|
+
downloadGemmaModel,
|
|
45
|
+
ffmpegAvailable,
|
|
46
|
+
listGemmaModels,
|
|
47
|
+
mlxVlmAvailable,
|
|
48
|
+
} from './generation/gemma-models.ts';
|
|
43
49
|
import { type GenerationJobQueue, GenerationQueueFullError } from './generation/jobs.ts';
|
|
44
50
|
import {
|
|
45
51
|
configuredProviders,
|
|
@@ -49,8 +55,11 @@ import {
|
|
|
49
55
|
setProviderKey,
|
|
50
56
|
} from './generation/keys.ts';
|
|
51
57
|
import {
|
|
58
|
+
isKeyframeEngine,
|
|
52
59
|
isTranscriptionProvider,
|
|
60
|
+
readKeyframeEngine,
|
|
53
61
|
readTranscriptionProvider,
|
|
62
|
+
writeKeyframeEngine,
|
|
54
63
|
writeTranscriptionProvider,
|
|
55
64
|
} from './generation/prefs.ts';
|
|
56
65
|
import {
|
|
@@ -691,6 +700,12 @@ export function createHttp(
|
|
|
691
700
|
error?: string;
|
|
692
701
|
} | null = null;
|
|
693
702
|
|
|
703
|
+
// feature-scene-aware-keyframes — in-flight Gemma-model download state, polled by
|
|
704
|
+
// the Settings "Scene-aware keyframes" card via GET /_api/generate/keyframe-model.
|
|
705
|
+
// Closure-scoped: one server, one download.
|
|
706
|
+
let keyframeDownload: { id: string; received: number; total: number; error?: string } | null =
|
|
707
|
+
null;
|
|
708
|
+
|
|
694
709
|
// Cache invalidation — when canvas-lib changes, every cached canvas bundle
|
|
695
710
|
// is stale because canvas-lib is inlined into each one via the resolver
|
|
696
711
|
// plugin. Drop the whole cache so the next request rebuilds with the fresh
|
|
@@ -2421,6 +2436,98 @@ export function createHttp(
|
|
|
2421
2436
|
);
|
|
2422
2437
|
},
|
|
2423
2438
|
|
|
2439
|
+
'/_api/set-artboard-kind': async (req: Request) => {
|
|
2440
|
+
// feature-1-artboard-kinds-foundation, T8. POST { canvas, artboardId,
|
|
2441
|
+
// kind: 'digital'|'print'|'web'|'video'|null } → api.setArtboardKindOp.
|
|
2442
|
+
// `null` clears the explicit prop back to the implicit default. Whole-
|
|
2443
|
+
// file undo seq. MAIN-ORIGIN ONLY; same gate pair as set-artboard-style
|
|
2444
|
+
// (the kind-switch surfaces — context menu, Inspector — are shell UI,
|
|
2445
|
+
// never reachable from the untrusted canvas origin).
|
|
2446
|
+
if (req.method !== 'POST') return new Response('Method not allowed', { status: 405 });
|
|
2447
|
+
if (!sameOriginWrite(req))
|
|
2448
|
+
return new Response('cross-origin write rejected', { status: 403 });
|
|
2449
|
+
if (!isLoopbackHost(req.headers.get('host')))
|
|
2450
|
+
return new Response('local request required (DNS-rebinding guard)', { status: 403 });
|
|
2451
|
+
const body = await readJson<{
|
|
2452
|
+
canvas?: unknown;
|
|
2453
|
+
artboardId?: unknown;
|
|
2454
|
+
kind?: unknown;
|
|
2455
|
+
}>(req, 2 * 1024);
|
|
2456
|
+
if (!body) return new Response('body required', { status: 400 });
|
|
2457
|
+
const result = await api.setArtboardKindOp(body);
|
|
2458
|
+
if (!result.ok) {
|
|
2459
|
+
return Response.json(
|
|
2460
|
+
{ ok: false, error: result.error },
|
|
2461
|
+
{ status: result.status, headers: { 'Cache-Control': 'no-store' } }
|
|
2462
|
+
);
|
|
2463
|
+
}
|
|
2464
|
+
return Response.json(
|
|
2465
|
+
{ ok: true, seq: result.seq },
|
|
2466
|
+
{ status: 200, headers: { 'Cache-Control': 'no-store' } }
|
|
2467
|
+
);
|
|
2468
|
+
},
|
|
2469
|
+
|
|
2470
|
+
'/_api/set-artboard-guides': async (req: Request) => {
|
|
2471
|
+
// feature-1-artboard-kinds-foundation, T5. POST { canvas, artboardId,
|
|
2472
|
+
// guides: {...}|null } → api.setArtboardGuidesOp. REPLACE-whole-prop
|
|
2473
|
+
// (see applySetArtboardGuides doc comment) — the caller sends the full
|
|
2474
|
+
// merged object, not a delta. MAIN-ORIGIN ONLY; same gate pair as
|
|
2475
|
+
// set-artboard-style.
|
|
2476
|
+
if (req.method !== 'POST') return new Response('Method not allowed', { status: 405 });
|
|
2477
|
+
if (!sameOriginWrite(req))
|
|
2478
|
+
return new Response('cross-origin write rejected', { status: 403 });
|
|
2479
|
+
if (!isLoopbackHost(req.headers.get('host')))
|
|
2480
|
+
return new Response('local request required (DNS-rebinding guard)', { status: 403 });
|
|
2481
|
+
const body = await readJson<{
|
|
2482
|
+
canvas?: unknown;
|
|
2483
|
+
artboardId?: unknown;
|
|
2484
|
+
guides?: unknown;
|
|
2485
|
+
}>(req, 8 * 1024);
|
|
2486
|
+
if (!body) return new Response('body required', { status: 400 });
|
|
2487
|
+
const result = await api.setArtboardGuidesOp(body);
|
|
2488
|
+
if (!result.ok) {
|
|
2489
|
+
return Response.json(
|
|
2490
|
+
{ ok: false, error: result.error },
|
|
2491
|
+
{ status: result.status, headers: { 'Cache-Control': 'no-store' } }
|
|
2492
|
+
);
|
|
2493
|
+
}
|
|
2494
|
+
return Response.json(
|
|
2495
|
+
{ ok: true, seq: result.seq },
|
|
2496
|
+
{ status: 200, headers: { 'Cache-Control': 'no-store' } }
|
|
2497
|
+
);
|
|
2498
|
+
},
|
|
2499
|
+
|
|
2500
|
+
'/_api/set-artboard-print': async (req: Request) => {
|
|
2501
|
+
// feature-2-print-artboards T2. POST { canvas, artboardId, print:
|
|
2502
|
+
// {paper, orientation?, bleedMm?, marginsMm?}|null } → api.setArtboardPrintOp.
|
|
2503
|
+
// REPLACE-whole-prop (see applySetArtboardPrint's own doc comment) — the
|
|
2504
|
+
// caller sends the full merged object, not a delta. MAIN-ORIGIN ONLY;
|
|
2505
|
+
// same gate pair as set-artboard-guides (Inspector picker is shell UI,
|
|
2506
|
+
// never reachable from the untrusted canvas origin).
|
|
2507
|
+
if (req.method !== 'POST') return new Response('Method not allowed', { status: 405 });
|
|
2508
|
+
if (!sameOriginWrite(req))
|
|
2509
|
+
return new Response('cross-origin write rejected', { status: 403 });
|
|
2510
|
+
if (!isLoopbackHost(req.headers.get('host')))
|
|
2511
|
+
return new Response('local request required (DNS-rebinding guard)', { status: 403 });
|
|
2512
|
+
const body = await readJson<{
|
|
2513
|
+
canvas?: unknown;
|
|
2514
|
+
artboardId?: unknown;
|
|
2515
|
+
print?: unknown;
|
|
2516
|
+
}>(req, 2 * 1024);
|
|
2517
|
+
if (!body) return new Response('body required', { status: 400 });
|
|
2518
|
+
const result = await api.setArtboardPrintOp(body);
|
|
2519
|
+
if (!result.ok) {
|
|
2520
|
+
return Response.json(
|
|
2521
|
+
{ ok: false, error: result.error },
|
|
2522
|
+
{ status: result.status, headers: { 'Cache-Control': 'no-store' } }
|
|
2523
|
+
);
|
|
2524
|
+
}
|
|
2525
|
+
return Response.json(
|
|
2526
|
+
{ ok: true, seq: result.seq },
|
|
2527
|
+
{ status: 200, headers: { 'Cache-Control': 'no-store' } }
|
|
2528
|
+
);
|
|
2529
|
+
},
|
|
2530
|
+
|
|
2424
2531
|
'/_api/delete-artboard': async (req: Request) => {
|
|
2425
2532
|
// Delete an artboard by its `id` prop (Backspace / context-menu on a frame).
|
|
2426
2533
|
// POST { canvas, artboardId } → api.deleteArtboardOp (removes the
|
|
@@ -2942,20 +3049,38 @@ export function createHttp(
|
|
|
2942
3049
|
return new Response('local request required (DNS-rebinding guard)', { status: 403 });
|
|
2943
3050
|
if (req.method === 'GET') {
|
|
2944
3051
|
return Response.json(
|
|
2945
|
-
{
|
|
3052
|
+
{
|
|
3053
|
+
transcriptionProvider: readTranscriptionProvider(ctx.paths.repoRoot),
|
|
3054
|
+
keyframeEngine: readKeyframeEngine(ctx.paths.repoRoot),
|
|
3055
|
+
},
|
|
2946
3056
|
{ headers: { 'Cache-Control': 'no-store' } }
|
|
2947
3057
|
);
|
|
2948
3058
|
}
|
|
2949
3059
|
if (req.method !== 'POST') return new Response('Method not allowed', { status: 405 });
|
|
2950
3060
|
if (!sameOriginWrite(req))
|
|
2951
3061
|
return new Response('cross-origin write rejected', { status: 403 });
|
|
2952
|
-
const body = await readJson<{ transcriptionProvider?: unknown }>(
|
|
2953
|
-
|
|
2954
|
-
|
|
2955
|
-
|
|
2956
|
-
|
|
2957
|
-
});
|
|
3062
|
+
const body = await readJson<{ transcriptionProvider?: unknown; keyframeEngine?: unknown }>(
|
|
3063
|
+
req,
|
|
3064
|
+
4 * 1024
|
|
3065
|
+
);
|
|
3066
|
+
// The panel POSTs ONE pref at a time (transcription engine OR keyframe engine).
|
|
2958
3067
|
try {
|
|
3068
|
+
if ('keyframeEngine' in (body ?? {})) {
|
|
3069
|
+
const engine = body?.keyframeEngine;
|
|
3070
|
+
if (!isKeyframeEngine(engine))
|
|
3071
|
+
return new Response('keyframeEngine must be auto|gemma|ffmpeg|blind', { status: 400 });
|
|
3072
|
+
await writeKeyframeEngine(ctx.paths.repoRoot, engine);
|
|
3073
|
+
reloadConfig(ctx);
|
|
3074
|
+
return Response.json(
|
|
3075
|
+
{ keyframeEngine: engine },
|
|
3076
|
+
{ headers: { 'Cache-Control': 'no-store' } }
|
|
3077
|
+
);
|
|
3078
|
+
}
|
|
3079
|
+
const provider = body?.transcriptionProvider;
|
|
3080
|
+
if (!isTranscriptionProvider(provider))
|
|
3081
|
+
return new Response('transcriptionProvider must be whisper|elevenlabs|groq', {
|
|
3082
|
+
status: 400,
|
|
3083
|
+
});
|
|
2959
3084
|
await writeTranscriptionProvider(ctx.paths.repoRoot, provider);
|
|
2960
3085
|
reloadConfig(ctx);
|
|
2961
3086
|
return Response.json(
|
|
@@ -3177,6 +3302,87 @@ export function createHttp(
|
|
|
3177
3302
|
);
|
|
3178
3303
|
},
|
|
3179
3304
|
|
|
3305
|
+
// feature-scene-aware-keyframes — managed local Gemma-4 MLX scout models: GET
|
|
3306
|
+
// lists the registry + which are downloaded + any in-flight download + the two
|
|
3307
|
+
// availability signals (mlx-vlm runtime, ffmpeg) the Settings card needs to say
|
|
3308
|
+
// which tier will run; POST {id} downloads one via huggingface_hub (gated on
|
|
3309
|
+
// mlx-vlm being installed — the model is useless without it). MAIN-ORIGIN ONLY.
|
|
3310
|
+
'/_api/generate/keyframe-model': async (req: Request) => {
|
|
3311
|
+
if (!isLoopbackHost(req.headers.get('host')))
|
|
3312
|
+
return new Response('local request required (DNS-rebinding guard)', { status: 403 });
|
|
3313
|
+
if (req.method === 'GET') {
|
|
3314
|
+
// The GET builds availability by spawning subprocesses (import mlx_vlm is
|
|
3315
|
+
// heavy). Gate it same-origin so a cross-origin no-cors flood can't drive a
|
|
3316
|
+
// spawn-storm against the loopback server (DDR-183 security finding — the
|
|
3317
|
+
// sibling audio-search GET added this guard for the same reason). The probes
|
|
3318
|
+
// are also TTL-cached in gemma-models.ts as defence-in-depth.
|
|
3319
|
+
if (!sameOriginRead(req))
|
|
3320
|
+
return new Response('cross-origin read rejected', { status: 403 });
|
|
3321
|
+
return Response.json(
|
|
3322
|
+
{
|
|
3323
|
+
models: listGemmaModels(),
|
|
3324
|
+
downloading: keyframeDownload,
|
|
3325
|
+
mlxVlmAvailable: mlxVlmAvailable(),
|
|
3326
|
+
ffmpegAvailable: ffmpegAvailable(),
|
|
3327
|
+
},
|
|
3328
|
+
{ headers: { 'Cache-Control': 'no-store' } }
|
|
3329
|
+
);
|
|
3330
|
+
}
|
|
3331
|
+
if (req.method === 'DELETE') {
|
|
3332
|
+
// Free a wedged download slot (a stalled HF connection) without waiting out
|
|
3333
|
+
// the 60-min abort timeout (DDR-183 F4). Same-origin write-gated.
|
|
3334
|
+
if (!sameOriginWrite(req))
|
|
3335
|
+
return new Response('cross-origin write rejected', { status: 403 });
|
|
3336
|
+
keyframeDownload = null;
|
|
3337
|
+
return Response.json({ cancelled: true }, { headers: { 'Cache-Control': 'no-store' } });
|
|
3338
|
+
}
|
|
3339
|
+
if (req.method !== 'POST') return new Response('Method not allowed', { status: 405 });
|
|
3340
|
+
if (!sameOriginWrite(req))
|
|
3341
|
+
return new Response('cross-origin write rejected', { status: 403 });
|
|
3342
|
+
if (!mlxVlmAvailable())
|
|
3343
|
+
return new Response(
|
|
3344
|
+
'mlx-vlm not installed — the Gemma scout needs an Apple-Silicon Mac + `pip install mlx-vlm`.',
|
|
3345
|
+
{
|
|
3346
|
+
status: 400,
|
|
3347
|
+
}
|
|
3348
|
+
);
|
|
3349
|
+
const body = await readJson<{ id?: unknown }>(req, 4 * 1024);
|
|
3350
|
+
const id = typeof body?.id === 'string' ? body.id : '';
|
|
3351
|
+
if (!listGemmaModels().some((m) => m.id === id))
|
|
3352
|
+
return new Response('unknown model id', { status: 400 });
|
|
3353
|
+
if (keyframeDownload && !keyframeDownload.error)
|
|
3354
|
+
return new Response(`a download is already in progress (${keyframeDownload.id})`, {
|
|
3355
|
+
status: 409,
|
|
3356
|
+
});
|
|
3357
|
+
keyframeDownload = { id, received: 0, total: 0 };
|
|
3358
|
+
void downloadGemmaModel(
|
|
3359
|
+
id,
|
|
3360
|
+
(received, total) => {
|
|
3361
|
+
if (keyframeDownload && keyframeDownload.id === id) {
|
|
3362
|
+
keyframeDownload.received = received;
|
|
3363
|
+
keyframeDownload.total = total;
|
|
3364
|
+
}
|
|
3365
|
+
},
|
|
3366
|
+
AbortSignal.timeout(60 * 60_000) // multi-GB snapshots on a slow link
|
|
3367
|
+
)
|
|
3368
|
+
.then(() => {
|
|
3369
|
+
if (keyframeDownload?.id === id) keyframeDownload = null;
|
|
3370
|
+
})
|
|
3371
|
+
.catch((err) => {
|
|
3372
|
+
if (keyframeDownload?.id === id)
|
|
3373
|
+
keyframeDownload = {
|
|
3374
|
+
id,
|
|
3375
|
+
received: keyframeDownload.received,
|
|
3376
|
+
total: keyframeDownload.total,
|
|
3377
|
+
error: err instanceof Error ? err.message : 'download failed',
|
|
3378
|
+
};
|
|
3379
|
+
});
|
|
3380
|
+
return Response.json(
|
|
3381
|
+
{ started: id },
|
|
3382
|
+
{ status: 202, headers: { 'Cache-Control': 'no-store' } }
|
|
3383
|
+
);
|
|
3384
|
+
},
|
|
3385
|
+
|
|
3180
3386
|
'/_canvas-state': async (req: Request) => {
|
|
3181
3387
|
const url = new URL(req.url);
|
|
3182
3388
|
if (req.method === 'GET') {
|