@1agh/maude 0.45.1 → 0.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/studio/acp/bridge.ts +628 -74
- package/apps/studio/acp/index.ts +172 -23
- package/apps/studio/acp/probe.ts +31 -7
- package/apps/studio/acp/transcript.ts +91 -5
- package/apps/studio/annotations-snap.ts +50 -10
- package/apps/studio/api.ts +344 -14
- package/apps/studio/artboard-guides-overlay.tsx +270 -0
- package/apps/studio/bin/_import-asset.mjs +35 -5
- package/apps/studio/bin/_pdf-playwright.mjs +35 -4
- package/apps/studio/bin/_png-playwright.mjs +38 -4
- package/apps/studio/bin/_pw-launch.mjs +52 -0
- package/apps/studio/bin/_pw-launch.test.mjs +90 -0
- package/apps/studio/bin/_smart-frames.mjs +419 -0
- package/apps/studio/bin/_smart-frames.test.mjs +140 -0
- package/apps/studio/bin/smart-frames.sh +30 -0
- package/apps/studio/canvas-edit.ts +302 -3
- package/apps/studio/canvas-lib.tsx +203 -11
- package/apps/studio/canvas-shell.tsx +167 -28
- package/apps/studio/client/app.jsx +678 -176
- package/apps/studio/client/panels/CapabilityBar.jsx +101 -0
- package/apps/studio/client/panels/ChatPanel.jsx +846 -133
- package/apps/studio/client/panels/ElicitationPrompt.jsx +429 -0
- package/apps/studio/client/panels/PermissionPrompt.jsx +119 -0
- package/apps/studio/client/panels/ReadinessList.jsx +17 -3
- package/apps/studio/client/panels/SettingsPanel.jsx +211 -0
- package/apps/studio/client/panels/ToolGroup.jsx +79 -0
- package/apps/studio/client/panels/acp-capabilities.js +82 -0
- package/apps/studio/client/panels/acp-elicitation.js +194 -0
- package/apps/studio/client/panels/acp-runtime.js +248 -9
- package/apps/studio/client/panels/acp-usage.js +65 -0
- package/apps/studio/client/panels/transcript-view.js +36 -0
- package/apps/studio/client/styles/6-acp-chat.css +689 -11
- package/apps/studio/dist/client.bundle.js +1738 -1736
- package/apps/studio/dist/comment-mount.js +2 -2
- package/apps/studio/dist/styles.css +1 -1
- package/apps/studio/dom-selection.ts +20 -0
- package/apps/studio/export-dialog.tsx +138 -18
- package/apps/studio/exporters/pdf.ts +332 -18
- package/apps/studio/exporters/png.ts +45 -3
- package/apps/studio/footage/schema.ts +17 -1
- package/apps/studio/generation/gemma-models.ts +224 -0
- package/apps/studio/generation/prefs.ts +43 -0
- package/apps/studio/generation/whisper-models.test.ts +28 -1
- package/apps/studio/generation/whisper-models.ts +22 -7
- package/apps/studio/http.ts +246 -9
- package/apps/studio/print/marks.ts +113 -0
- package/apps/studio/print/units.ts +269 -0
- package/apps/studio/print-overlay-content.tsx +132 -0
- package/apps/studio/test/acp-bridge.test.ts +11 -6
- package/apps/studio/test/acp-capabilities.test.ts +123 -0
- package/apps/studio/test/acp-caps-bridge.test.ts +274 -0
- package/apps/studio/test/acp-elicitation-bridge.test.ts +475 -0
- package/apps/studio/test/acp-elicitation.test.ts +251 -0
- package/apps/studio/test/acp-mode-banner.test.ts +45 -0
- package/apps/studio/test/acp-permission-prompt.test.ts +77 -0
- package/apps/studio/test/acp-permission.test.ts +262 -0
- package/apps/studio/test/acp-session-allowed-tools.test.ts +91 -0
- package/apps/studio/test/acp-toolgroup.test.ts +76 -0
- package/apps/studio/test/acp-transcript-view.test.ts +71 -0
- package/apps/studio/test/acp-transcript.test.ts +75 -2
- package/apps/studio/test/acp-usage-bridge.test.ts +136 -0
- package/apps/studio/test/acp-usage.test.ts +143 -0
- package/apps/studio/test/annotations-snap.test.ts +56 -0
- package/apps/studio/test/artboard-guides-overlay.test.tsx +152 -0
- package/apps/studio/test/artboard-kinds.test.tsx +83 -0
- package/apps/studio/test/artboard-selection-attrs.test.ts +69 -0
- package/apps/studio/test/canvas-meta-api.test.ts +167 -0
- package/apps/studio/test/element-structural-edit.test.ts +250 -0
- package/apps/studio/test/exporters/png.test.ts +49 -1
- package/apps/studio/test/fixtures/mock-acp-agent-caps.mjs +158 -0
- package/apps/studio/test/fixtures/mock-acp-agent-elicit-flood.mjs +46 -0
- package/apps/studio/test/fixtures/mock-acp-agent-elicit-url.mjs +42 -0
- package/apps/studio/test/fixtures/mock-acp-agent-elicit.mjs +63 -0
- package/apps/studio/test/fixtures/mock-acp-agent-permission-flood.mjs +51 -0
- package/apps/studio/test/fixtures/mock-acp-agent-permission.mjs +50 -0
- package/apps/studio/test/fixtures/mock-acp-agent-usage.mjs +56 -0
- package/apps/studio/test/import-asset.test.ts +31 -3
- package/apps/studio/test/pdf-print-boxes.test.ts +272 -0
- package/apps/studio/test/print-marks.test.ts +113 -0
- package/apps/studio/test/print-units.test.ts +173 -0
- package/apps/studio/test/use-snap-guides.test.ts +81 -0
- package/apps/studio/use-chrome-visibility.tsx +19 -0
- package/apps/studio/use-element-resize.tsx +21 -2
- package/apps/studio/use-snap-guides.tsx +73 -5
- package/apps/studio/use-spacing-handles.tsx +9 -5
- package/apps/studio/whats-new.json +62 -0
- package/cli/commands/design.mjs +116 -3
- package/cli/lib/pkg-root.mjs +42 -10
- package/cli/lib/pkg-root.test.mjs +33 -1
- package/package.json +11 -9
- package/plugins/design/dependencies.json +35 -0
|
@@ -0,0 +1,419 @@
|
|
|
1
|
+
// _smart-frames.mjs — scene-AWARE keyframe extraction for the footage-analyst's
|
|
2
|
+
// vision pass (feature-scene-aware-keyframes). The engine behind
|
|
3
|
+
// `maude design smart-frames` (DDR-062) and skill `footage-keyframes`.
|
|
4
|
+
//
|
|
5
|
+
// Instead of blind frame-rate sampling (which straddles and misses short but
|
|
6
|
+
// meaningful beats), select frames at scene cuts + semantic action beats + the
|
|
7
|
+
// true endpoints, so the analyzer receives fewer, sharper, context-rich frames.
|
|
8
|
+
//
|
|
9
|
+
// Three tiers, auto-detected, each a graceful fallback of the next:
|
|
10
|
+
// gemma — Gemma-4 MLX scout (semantic beats) + ffmpeg scene cuts. Needs ffmpeg
|
|
11
|
+
// + mlx-vlm + a model (Apple Silicon). Opt-in, richest.
|
|
12
|
+
// ffmpeg — ffmpeg scene detection + endpoints + long-shot midpoints. Needs
|
|
13
|
+
// ffmpeg. The practical default; cross-platform, no model download.
|
|
14
|
+
// blind — delegate to probe-footage (headless Chromium, even-spaced). Zero new
|
|
15
|
+
// deps; the floor so this always works.
|
|
16
|
+
//
|
|
17
|
+
// Emits a SUPERSET of the probe-footage manifest (drop-in for footage-analyst):
|
|
18
|
+
// { asset, durationSec, width, height, method, sceneCuts[], scoutBeats[],
|
|
19
|
+
// outDir, frames:[{index,t,png}] }
|
|
20
|
+
//
|
|
21
|
+
// Pure JS (runs under node OR bun, like probe-footage). Exported pure functions
|
|
22
|
+
// are unit-tested in _smart-frames.test.mjs.
|
|
23
|
+
|
|
24
|
+
import { spawnSync } from 'node:child_process';
|
|
25
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync } from 'node:fs';
|
|
26
|
+
import { dirname, join } from 'node:path';
|
|
27
|
+
import { fileURLToPath } from 'node:url';
|
|
28
|
+
|
|
29
|
+
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
30
|
+
|
|
31
|
+
// ─── dependency probes ───────────────────────────────────────────────────────
|
|
32
|
+
|
|
33
|
+
export function hasCommand(cmd) {
|
|
34
|
+
// `which`/`where` are real binaries → no shell (avoids DEP0190 + injection).
|
|
35
|
+
const finder = process.platform === 'win32' ? 'where' : 'which';
|
|
36
|
+
const r = spawnSync(finder, [cmd], { stdio: 'ignore' });
|
|
37
|
+
return r.status === 0;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Resolve a python that can `import mlx_vlm`, or null. Honors $MAUDE_MLX_PYTHON. */
|
|
41
|
+
export function resolveMlxPython(env = process.env) {
|
|
42
|
+
const candidates = [env.MAUDE_MLX_PYTHON, 'python3', 'python'].filter(Boolean);
|
|
43
|
+
for (const py of candidates) {
|
|
44
|
+
const r = spawnSync(py, ['-c', 'import mlx_vlm'], { stdio: 'ignore' });
|
|
45
|
+
if (r.status === 0) return py;
|
|
46
|
+
}
|
|
47
|
+
return null;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function detectAvailability(env = process.env) {
|
|
51
|
+
return {
|
|
52
|
+
ffmpeg: hasCommand('ffmpeg') && hasCommand('ffprobe'),
|
|
53
|
+
mlxPython: resolveMlxPython(env),
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Pick the tier to actually run. Explicit engine wins (and errors if its deps are
|
|
58
|
+
* missing — no silent downgrade when the user asked for one); `auto` degrades
|
|
59
|
+
* gemma → ffmpeg → blind based on what's installed. */
|
|
60
|
+
export function selectTier(engine, avail) {
|
|
61
|
+
const gemmaOk = Boolean(avail.ffmpeg && avail.mlxPython);
|
|
62
|
+
if (engine && engine !== 'auto') {
|
|
63
|
+
if (engine === 'gemma' && !gemmaOk)
|
|
64
|
+
throw new Error(
|
|
65
|
+
'engine "gemma" needs ffmpeg + mlx-vlm (Apple Silicon). Install both or use --engine ffmpeg|blind|auto.'
|
|
66
|
+
);
|
|
67
|
+
if (engine === 'ffmpeg' && !avail.ffmpeg)
|
|
68
|
+
throw new Error(
|
|
69
|
+
'engine "ffmpeg" needs ffmpeg. Install it (brew install ffmpeg) or use --engine blind|auto.'
|
|
70
|
+
);
|
|
71
|
+
return engine; // blind always allowed
|
|
72
|
+
}
|
|
73
|
+
if (gemmaOk) return 'gemma';
|
|
74
|
+
if (avail.ffmpeg) return 'ffmpeg';
|
|
75
|
+
return 'blind';
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// ─── timestamp math (pure) ───────────────────────────────────────────────────
|
|
79
|
+
|
|
80
|
+
/** Parse ffmpeg `showinfo` stderr → scene-cut timestamps (seconds). */
|
|
81
|
+
export function parseSceneCuts(stderr) {
|
|
82
|
+
const out = [];
|
|
83
|
+
const re = /pts_time:([0-9.]+)/g;
|
|
84
|
+
let m;
|
|
85
|
+
// biome-ignore lint/suspicious/noAssignInExpressions: standard regex-exec loop.
|
|
86
|
+
while ((m = re.exec(stderr))) out.push(Number(m[1]));
|
|
87
|
+
return out.filter((t) => Number.isFinite(t));
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Parse a Gemma scout's text → beat timestamps. Accepts both `TIME=<sec>` and the
|
|
91
|
+
* `M:SS | what` shape the small model tends to emit. */
|
|
92
|
+
export function parseBeats(text, durationSec) {
|
|
93
|
+
const beats = [];
|
|
94
|
+
const reT = /TIME=([0-9.]+)/g;
|
|
95
|
+
let m;
|
|
96
|
+
// biome-ignore lint/suspicious/noAssignInExpressions: standard regex-exec loop.
|
|
97
|
+
while ((m = reT.exec(text))) beats.push({ t: Number(m[1]), what: '' });
|
|
98
|
+
const reMS = /(?:^|\n)\s*(\d+):(\d{2}(?:\.\d+)?)\s*\|\s*([^\n]*)/g;
|
|
99
|
+
// biome-ignore lint/suspicious/noAssignInExpressions: standard regex-exec loop.
|
|
100
|
+
while ((m = reMS.exec(text)))
|
|
101
|
+
beats.push({ t: Number(m[1]) * 60 + Number(m[2]), what: (m[3] || '').trim() });
|
|
102
|
+
return beats.filter((b) => Number.isFinite(b.t) && b.t >= 0 && b.t <= durationSec);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Merge scene cuts + scout beats into a final, deduped, capped timestamp set —
|
|
106
|
+
* always incl. the true first & last frame; long shots sampled more. */
|
|
107
|
+
export function mergeTimestamps({
|
|
108
|
+
durationSec,
|
|
109
|
+
cuts = [],
|
|
110
|
+
beats = [],
|
|
111
|
+
maxFrames = 12,
|
|
112
|
+
dedupSec = 0.4,
|
|
113
|
+
}) {
|
|
114
|
+
const dur = durationSec;
|
|
115
|
+
const pts = new Set([0, Math.max(0, +(dur - 0.05).toFixed(3))]);
|
|
116
|
+
const bounds = Array.from(new Set([0, ...cuts, dur])).sort((a, b) => a - b);
|
|
117
|
+
for (let i = 0; i < bounds.length - 1; i++) {
|
|
118
|
+
const a = bounds[i];
|
|
119
|
+
const b = bounds[i + 1];
|
|
120
|
+
pts.add(+Math.min(a + 0.05, dur).toFixed(3)); // just INSIDE each shot
|
|
121
|
+
const gap = b - a;
|
|
122
|
+
if (gap > 3) {
|
|
123
|
+
const n = Math.floor(gap / 3);
|
|
124
|
+
for (let k = 1; k <= n; k++) pts.add(+(a + (gap * k) / (n + 1)).toFixed(3));
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
for (const beat of beats) pts.add(+beat.t.toFixed(3));
|
|
128
|
+
|
|
129
|
+
const xs = Array.from(pts).sort((a, b) => a - b);
|
|
130
|
+
const deduped = [];
|
|
131
|
+
for (const t of xs)
|
|
132
|
+
if (!deduped.length || t - deduped[deduped.length - 1] >= dedupSec) deduped.push(t);
|
|
133
|
+
|
|
134
|
+
if (deduped.length <= maxFrames) return deduped;
|
|
135
|
+
// too many → keep an even spread across the ordered set
|
|
136
|
+
const capped = [];
|
|
137
|
+
for (let i = 0; i < maxFrames; i++)
|
|
138
|
+
capped.push(deduped[Math.round((i * (deduped.length - 1)) / (maxFrames - 1))]);
|
|
139
|
+
return Array.from(new Set(capped));
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// ─── ffmpeg / ffprobe wrappers ───────────────────────────────────────────────
|
|
143
|
+
|
|
144
|
+
function ffprobeMeta(clip) {
|
|
145
|
+
const r = spawnSync(
|
|
146
|
+
'ffprobe',
|
|
147
|
+
[
|
|
148
|
+
'-v',
|
|
149
|
+
'error',
|
|
150
|
+
'-select_streams',
|
|
151
|
+
'v:0',
|
|
152
|
+
'-show_entries',
|
|
153
|
+
'stream=width,height:format=duration',
|
|
154
|
+
'-of',
|
|
155
|
+
'default=noprint_wrappers=1',
|
|
156
|
+
clip,
|
|
157
|
+
],
|
|
158
|
+
{ encoding: 'utf8' }
|
|
159
|
+
);
|
|
160
|
+
const txt = r.stdout || '';
|
|
161
|
+
const num = (re) => {
|
|
162
|
+
const m = txt.match(re);
|
|
163
|
+
return m ? Number(m[1]) : 0;
|
|
164
|
+
};
|
|
165
|
+
return {
|
|
166
|
+
width: num(/width=(\d+)/),
|
|
167
|
+
height: num(/height=(\d+)/),
|
|
168
|
+
durationSec: num(/duration=([0-9.]+)/),
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function ffmpegSceneCuts(clip, thresh) {
|
|
173
|
+
const r = spawnSync(
|
|
174
|
+
'ffmpeg',
|
|
175
|
+
['-i', clip, '-vf', `select='gt(scene,${thresh})',showinfo`, '-f', 'null', '-'],
|
|
176
|
+
{ encoding: 'utf8' }
|
|
177
|
+
);
|
|
178
|
+
return parseSceneCuts(r.stderr || '');
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function extractFrame(clip, t, outPath) {
|
|
182
|
+
const r = spawnSync(
|
|
183
|
+
'ffmpeg',
|
|
184
|
+
['-y', '-v', 'error', '-ss', String(t), '-i', clip, '-frames:v', '1', outPath],
|
|
185
|
+
{
|
|
186
|
+
encoding: 'utf8',
|
|
187
|
+
}
|
|
188
|
+
);
|
|
189
|
+
return r.status === 0 && existsSync(outPath);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// ─── gemma scout ─────────────────────────────────────────────────────────────
|
|
193
|
+
|
|
194
|
+
const SCOUT_PROMPT = (dur) =>
|
|
195
|
+
`You are a shot-list scout watching a ${dur}-second video clip. List the KEY moments an editor must see: the start of each distinct shot AND any peak action beat (a snap, a catch, a big movement, a reveal) — even inside a continuous shot. Do NOT space them evenly; pick only moments where something meaningful happens or changes. Output ONE line per moment, formatted EXACTLY as: TIME=<seconds> | <a few words>. Seconds must be real numbers between 0 and ${dur}.`;
|
|
196
|
+
|
|
197
|
+
function gemmaScout(clip, durationSec, { python, model, fps }) {
|
|
198
|
+
const r = spawnSync(
|
|
199
|
+
python,
|
|
200
|
+
[
|
|
201
|
+
'-m',
|
|
202
|
+
'mlx_vlm.generate',
|
|
203
|
+
'--model',
|
|
204
|
+
model,
|
|
205
|
+
'--max-tokens',
|
|
206
|
+
'300',
|
|
207
|
+
'--video',
|
|
208
|
+
clip,
|
|
209
|
+
'--fps',
|
|
210
|
+
String(fps),
|
|
211
|
+
'--prompt',
|
|
212
|
+
SCOUT_PROMPT(durationSec),
|
|
213
|
+
],
|
|
214
|
+
{ encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 }
|
|
215
|
+
);
|
|
216
|
+
const text = `${r.stdout || ''}\n${r.stderr || ''}`;
|
|
217
|
+
return { beats: parseBeats(text, durationSec), ok: r.status === 0, raw: text };
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// ─── blind delegate (probe-footage) ──────────────────────────────────────────
|
|
221
|
+
|
|
222
|
+
function blindProbe(argv) {
|
|
223
|
+
// Reuse the existing Chromium extractor verbatim, then re-tag the manifest.
|
|
224
|
+
const script = join(HERE, '_probe-footage-playwright.mjs');
|
|
225
|
+
const runner = hasCommand('node') ? 'node' : 'bun';
|
|
226
|
+
const r = spawnSync(runner, [script, ...argv], { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 });
|
|
227
|
+
if (r.status !== 0) {
|
|
228
|
+
process.stderr.write(r.stderr || 'probe-footage failed\n');
|
|
229
|
+
process.exit(r.status || 4);
|
|
230
|
+
}
|
|
231
|
+
let manifest;
|
|
232
|
+
try {
|
|
233
|
+
const out = r.stdout.trim();
|
|
234
|
+
// probe-footage prints a pretty-printed manifest; parse the last {...} block.
|
|
235
|
+
const start = out.indexOf('{');
|
|
236
|
+
manifest = JSON.parse(start >= 0 ? out.slice(start) : out);
|
|
237
|
+
} catch {
|
|
238
|
+
process.stderr.write('smart-frames: could not parse probe-footage manifest\n');
|
|
239
|
+
process.exit(4);
|
|
240
|
+
}
|
|
241
|
+
return { ...manifest, method: 'blind', sceneCuts: [], scoutBeats: [] };
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// ─── CLI ─────────────────────────────────────────────────────────────────────
|
|
245
|
+
|
|
246
|
+
/** Find the clip on disk, trying the probe-footage-style locations + a raw path. */
|
|
247
|
+
export function clipCandidates(asset, root, designRoot) {
|
|
248
|
+
const c = [asset];
|
|
249
|
+
if (root) {
|
|
250
|
+
c.push(join(root, asset));
|
|
251
|
+
if (!asset.startsWith('assets/')) c.push(join(root, designRoot, 'assets', asset));
|
|
252
|
+
c.push(join(root, designRoot, asset)); // handles `assets/x` → root/.design/assets/x
|
|
253
|
+
}
|
|
254
|
+
return c;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
function resolveClip(asset, root, designRoot) {
|
|
258
|
+
for (const cand of clipCandidates(asset, root, designRoot)) if (existsSync(cand)) return cand;
|
|
259
|
+
return null;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/** Read the app-persisted keyframe-engine pref from <root>/.design/config.json
|
|
263
|
+
* (generation.keyframes.engine). Pure JS — no .ts import. Returns null if absent
|
|
264
|
+
* or unreadable; best-effort, so a standalone terminal run needs no app/config. */
|
|
265
|
+
export function readEnginePref(root) {
|
|
266
|
+
if (!root) return null;
|
|
267
|
+
try {
|
|
268
|
+
const cfg = JSON.parse(readFileSync(join(root, '.design', 'config.json'), 'utf8'));
|
|
269
|
+
const e = cfg?.generation?.keyframes?.engine;
|
|
270
|
+
return ['auto', 'gemma', 'ffmpeg', 'blind'].includes(e) ? e : null;
|
|
271
|
+
} catch {
|
|
272
|
+
return null;
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
function parseArgs(argv) {
|
|
277
|
+
const envEngine = process.env.MAUDE_SMARTFRAMES_ENGINE || '';
|
|
278
|
+
const o = {
|
|
279
|
+
engine: envEngine || 'auto',
|
|
280
|
+
engineExplicit: Boolean(envEngine),
|
|
281
|
+
frames: 12,
|
|
282
|
+
sceneThresh: 0.3,
|
|
283
|
+
scoutFps: 4,
|
|
284
|
+
};
|
|
285
|
+
const rest = [];
|
|
286
|
+
for (let i = 0; i < argv.length; i++) {
|
|
287
|
+
const a = argv[i];
|
|
288
|
+
if (a === '--engine') {
|
|
289
|
+
o.engine = argv[++i];
|
|
290
|
+
o.engineExplicit = true;
|
|
291
|
+
} else if (a === '--frames' || a === '--max-frames') o.frames = Number(argv[++i]);
|
|
292
|
+
else if (a === '--out-dir') o.outDir = argv[++i];
|
|
293
|
+
else if (a === '--root') o.root = argv[++i];
|
|
294
|
+
else if (a === '--design-root') o.designRoot = argv[++i];
|
|
295
|
+
else if (a === '--scene-thresh') o.sceneThresh = Number(argv[++i]);
|
|
296
|
+
else if (a === '--scout-fps') o.scoutFps = Number(argv[++i]);
|
|
297
|
+
else if (a === '--help' || a === '-h') o.help = true;
|
|
298
|
+
else rest.push(a);
|
|
299
|
+
}
|
|
300
|
+
o.asset = rest[0];
|
|
301
|
+
return o;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
function main() {
|
|
305
|
+
const o = parseArgs(process.argv.slice(2));
|
|
306
|
+
if (o.help || !o.asset) {
|
|
307
|
+
process.stdout.write(
|
|
308
|
+
'usage: smart-frames <asset> [--root DIR] [--out-dir DIR] [--frames N]\n' +
|
|
309
|
+
' [--engine auto|gemma|ffmpeg|blind] [--scene-thresh 0.3] [--scout-fps 4]\n'
|
|
310
|
+
);
|
|
311
|
+
process.exit(o.asset ? 0 : 2);
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
// App-served runs honor the persisted keyframeEngine pref; an explicit --engine /
|
|
315
|
+
// $MAUDE_SMARTFRAMES_ENGINE always wins; standalone terminal runs self-detect.
|
|
316
|
+
if (!o.engineExplicit) {
|
|
317
|
+
const pref = readEnginePref(o.root);
|
|
318
|
+
if (pref) o.engine = pref;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
const avail = detectAvailability();
|
|
322
|
+
let tier;
|
|
323
|
+
try {
|
|
324
|
+
tier = selectTier(o.engine, avail);
|
|
325
|
+
} catch (e) {
|
|
326
|
+
process.stderr.write(`smart-frames: ${e.message}\n`);
|
|
327
|
+
process.exit(3);
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
// blind floor delegates wholesale to probe-footage (which does its own, stricter
|
|
331
|
+
// content-addressed resolution) — forward the original args, don't pre-resolve.
|
|
332
|
+
if (tier === 'blind') {
|
|
333
|
+
const passthru = process.argv.slice(2).filter((a, i, arr) => {
|
|
334
|
+
// drop --engine and its value, and --scene-thresh/--scout-fps (probe doesn't know them)
|
|
335
|
+
const prev = arr[i - 1];
|
|
336
|
+
return (
|
|
337
|
+
!['--engine', '--scene-thresh', '--scout-fps'].includes(a) &&
|
|
338
|
+
!['--engine', '--scene-thresh', '--scout-fps'].includes(prev)
|
|
339
|
+
);
|
|
340
|
+
});
|
|
341
|
+
const manifest = blindProbe(passthru);
|
|
342
|
+
process.stdout.write(`${JSON.stringify(manifest)}\n`);
|
|
343
|
+
process.stderr.write('smart-frames: engine=blind (probe-footage / Chromium, even-spaced)\n');
|
|
344
|
+
return;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// ffmpeg/gemma tiers accept any readable clip (terminal power users analyze
|
|
348
|
+
// arbitrary files, not just content-addressed assets). Try the candidate
|
|
349
|
+
// locations probe-footage would, plus a raw path.
|
|
350
|
+
const clip = resolveClip(o.asset, o.root, o.designRoot || '.design');
|
|
351
|
+
if (!clip) {
|
|
352
|
+
process.stderr.write(`smart-frames: asset not found: ${o.asset}\n`);
|
|
353
|
+
process.exit(2);
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
const meta = ffprobeMeta(clip);
|
|
357
|
+
if (!meta.durationSec) {
|
|
358
|
+
process.stderr.write('smart-frames: ffprobe found no video/duration\n');
|
|
359
|
+
process.exit(4);
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
const cuts = ffmpegSceneCuts(clip, o.sceneThresh);
|
|
363
|
+
let beats = [];
|
|
364
|
+
let method = 'ffmpeg';
|
|
365
|
+
if (tier === 'gemma') {
|
|
366
|
+
const model = process.env.MAUDE_GEMMA_MODEL || 'mlx-community/gemma-4-e4b-it-4bit';
|
|
367
|
+
const scout = gemmaScout(clip, meta.durationSec, {
|
|
368
|
+
python: avail.mlxPython,
|
|
369
|
+
model,
|
|
370
|
+
fps: o.scoutFps,
|
|
371
|
+
});
|
|
372
|
+
if (scout.ok && scout.beats.length) {
|
|
373
|
+
beats = scout.beats;
|
|
374
|
+
method = 'gemma';
|
|
375
|
+
} else {
|
|
376
|
+
process.stderr.write(
|
|
377
|
+
'smart-frames: gemma scout produced no beats — falling back to ffmpeg tier\n'
|
|
378
|
+
);
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
const times = mergeTimestamps({
|
|
383
|
+
durationSec: meta.durationSec,
|
|
384
|
+
cuts,
|
|
385
|
+
beats,
|
|
386
|
+
maxFrames: o.frames,
|
|
387
|
+
});
|
|
388
|
+
|
|
389
|
+
const outDir = o.outDir || join(process.env.TMPDIR || '/tmp', `smart-frames-${Date.now()}`);
|
|
390
|
+
mkdirSync(outDir, { recursive: true });
|
|
391
|
+
const frames = [];
|
|
392
|
+
times.forEach((t, i) => {
|
|
393
|
+
const png = join(outDir, `f_${String(i + 1).padStart(2, '0')}.png`);
|
|
394
|
+
if (extractFrame(clip, t, png)) frames.push({ index: i + 1, t, png });
|
|
395
|
+
});
|
|
396
|
+
if (!frames.length) {
|
|
397
|
+
process.stderr.write('smart-frames: ffmpeg extracted no frames\n');
|
|
398
|
+
process.exit(4);
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
const manifest = {
|
|
402
|
+
asset: o.asset,
|
|
403
|
+
durationSec: meta.durationSec,
|
|
404
|
+
width: meta.width,
|
|
405
|
+
height: meta.height,
|
|
406
|
+
method,
|
|
407
|
+
sceneCuts: cuts,
|
|
408
|
+
scoutBeats: beats,
|
|
409
|
+
outDir,
|
|
410
|
+
frames,
|
|
411
|
+
};
|
|
412
|
+
process.stdout.write(`${JSON.stringify(manifest)}\n`);
|
|
413
|
+
process.stderr.write(
|
|
414
|
+
`smart-frames: engine=${method} · ${frames.length} frames · ${cuts.length} scene cuts · ${beats.length} scout beats\n`
|
|
415
|
+
);
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
// run only as a CLI, not when imported by the test
|
|
419
|
+
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) main();
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
// _smart-frames.test.mjs — the pure timestamp/tier logic behind
|
|
2
|
+
// `maude design smart-frames` (feature-scene-aware-keyframes). The ffmpeg/gemma
|
|
3
|
+
// spawns + Chromium delegate are soft-dep live gates covered by manual runs; this
|
|
4
|
+
// covers the deterministic math that decides WHICH frames get pulled.
|
|
5
|
+
|
|
6
|
+
import { describe, expect, test } from 'bun:test';
|
|
7
|
+
import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
|
|
8
|
+
import { tmpdir } from 'node:os';
|
|
9
|
+
import { join } from 'node:path';
|
|
10
|
+
import {
|
|
11
|
+
clipCandidates,
|
|
12
|
+
mergeTimestamps,
|
|
13
|
+
parseBeats,
|
|
14
|
+
parseSceneCuts,
|
|
15
|
+
readEnginePref,
|
|
16
|
+
selectTier,
|
|
17
|
+
} from './_smart-frames.mjs';
|
|
18
|
+
|
|
19
|
+
describe('parseSceneCuts', () => {
|
|
20
|
+
test('extracts pts_time values from ffmpeg showinfo stderr', () => {
|
|
21
|
+
const stderr =
|
|
22
|
+
'frame:0 pts_time:0.233333 ...\nframe:1 pts_time:6.533333 ...\nnoise pts_time:bad';
|
|
23
|
+
expect(parseSceneCuts(stderr)).toEqual([0.233333, 6.533333]);
|
|
24
|
+
});
|
|
25
|
+
test('empty on no cuts', () => {
|
|
26
|
+
expect(parseSceneCuts('nothing here')).toEqual([]);
|
|
27
|
+
});
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
describe('parseBeats', () => {
|
|
31
|
+
test('parses the TIME=<sec> format', () => {
|
|
32
|
+
const t = 'TIME=0.5 | opener\nTIME=6.9 | run play\n';
|
|
33
|
+
expect(parseBeats(t, 8).map((b) => b.t)).toEqual([0.5, 6.9]);
|
|
34
|
+
});
|
|
35
|
+
test('parses the M:SS format the small model tends to emit', () => {
|
|
36
|
+
const t = '0:00 | game action\n0:06 | interview\n1:03 | reveal\n';
|
|
37
|
+
const beats = parseBeats(t, 90);
|
|
38
|
+
expect(beats.map((b) => b.t)).toEqual([0, 6, 63]);
|
|
39
|
+
expect(beats[1].what).toBe('interview');
|
|
40
|
+
});
|
|
41
|
+
test('drops out-of-range beats', () => {
|
|
42
|
+
expect(parseBeats('TIME=99 | too late', 8)).toEqual([]);
|
|
43
|
+
});
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
describe('mergeTimestamps', () => {
|
|
47
|
+
test('always includes the true first and last frame', () => {
|
|
48
|
+
const ts = mergeTimestamps({ durationSec: 8, cuts: [], beats: [], maxFrames: 12 });
|
|
49
|
+
expect(ts[0]).toBe(0);
|
|
50
|
+
expect(ts[ts.length - 1]).toBeCloseTo(7.95, 2);
|
|
51
|
+
});
|
|
52
|
+
test('puts a frame just inside a shot at a cut (when not deduped against an endpoint)', () => {
|
|
53
|
+
const ts = mergeTimestamps({ durationSec: 8, cuts: [0.233, 6.533], beats: [], maxFrames: 20 });
|
|
54
|
+
// 6.583 = just after the 6.533 cut — clearly separated, survives dedup.
|
|
55
|
+
expect(ts.some((t) => Math.abs(t - 6.583) < 0.01)).toBe(true);
|
|
56
|
+
// 0.283 (just after the 0.233 cut) is within 0.4s of t=0, so dedup folds it into
|
|
57
|
+
// the opening frame — same moment, correct behavior.
|
|
58
|
+
expect(ts.filter((t) => t < 0.4)).toEqual([0]);
|
|
59
|
+
});
|
|
60
|
+
test('samples long shots more densely', () => {
|
|
61
|
+
// one 8s shot, no cuts → midpoints added (gap>3 → floor(8/3)=2 extra)
|
|
62
|
+
const ts = mergeTimestamps({ durationSec: 8, cuts: [], beats: [], maxFrames: 20 });
|
|
63
|
+
expect(ts.length).toBeGreaterThanOrEqual(4);
|
|
64
|
+
});
|
|
65
|
+
test('dedups within 0.4s', () => {
|
|
66
|
+
const ts = mergeTimestamps({
|
|
67
|
+
durationSec: 8,
|
|
68
|
+
cuts: [],
|
|
69
|
+
beats: [
|
|
70
|
+
{ t: 0.1, what: '' },
|
|
71
|
+
{ t: 0.2, what: '' },
|
|
72
|
+
],
|
|
73
|
+
maxFrames: 20,
|
|
74
|
+
});
|
|
75
|
+
// 0, 0.05(inside), 0.1, 0.2 all within 0.4 → collapse toward the earliest kept
|
|
76
|
+
const near0 = ts.filter((t) => t < 0.4);
|
|
77
|
+
expect(near0.length).toBe(1);
|
|
78
|
+
});
|
|
79
|
+
test('caps to maxFrames with an even spread', () => {
|
|
80
|
+
const beats = Array.from({ length: 40 }, (_, i) => ({ t: (i * 8) / 40, what: '' }));
|
|
81
|
+
const ts = mergeTimestamps({ durationSec: 8, cuts: [], beats, maxFrames: 6 });
|
|
82
|
+
expect(ts.length).toBeLessThanOrEqual(6);
|
|
83
|
+
expect(ts[0]).toBe(0);
|
|
84
|
+
});
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
describe('selectTier', () => {
|
|
88
|
+
const gemmaReady = { ffmpeg: true, mlxPython: 'python3' };
|
|
89
|
+
const ffmpegOnly = { ffmpeg: true, mlxPython: null };
|
|
90
|
+
const bare = { ffmpeg: false, mlxPython: null };
|
|
91
|
+
|
|
92
|
+
test('auto degrades gemma → ffmpeg → blind', () => {
|
|
93
|
+
expect(selectTier('auto', gemmaReady)).toBe('gemma');
|
|
94
|
+
expect(selectTier('auto', ffmpegOnly)).toBe('ffmpeg');
|
|
95
|
+
expect(selectTier('auto', bare)).toBe('blind');
|
|
96
|
+
});
|
|
97
|
+
test('explicit engine errors when its deps are missing (no silent downgrade)', () => {
|
|
98
|
+
expect(() => selectTier('gemma', ffmpegOnly)).toThrow(/mlx-vlm/);
|
|
99
|
+
expect(() => selectTier('ffmpeg', bare)).toThrow(/ffmpeg/);
|
|
100
|
+
});
|
|
101
|
+
test('blind is always allowed', () => {
|
|
102
|
+
expect(selectTier('blind', bare)).toBe('blind');
|
|
103
|
+
});
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
describe('clipCandidates', () => {
|
|
107
|
+
test('tries raw, root-relative, and designRoot/assets locations', () => {
|
|
108
|
+
const c = clipCandidates('assets/abc12345.mp4', '/repo', '.design');
|
|
109
|
+
expect(c).toContain('assets/abc12345.mp4');
|
|
110
|
+
expect(c).toContain('/repo/assets/abc12345.mp4');
|
|
111
|
+
expect(c).toContain('/repo/.design/assets/abc12345.mp4');
|
|
112
|
+
});
|
|
113
|
+
test('bare filename resolves under designRoot/assets', () => {
|
|
114
|
+
const c = clipCandidates('abc12345.mp4', '/repo', '.design');
|
|
115
|
+
expect(c).toContain('/repo/.design/assets/abc12345.mp4');
|
|
116
|
+
});
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
describe('readEnginePref', () => {
|
|
120
|
+
function repoWith(config) {
|
|
121
|
+
const root = mkdtempSync(join(tmpdir(), 'maude-sf-'));
|
|
122
|
+
mkdirSync(join(root, '.design'), { recursive: true });
|
|
123
|
+
if (config !== undefined)
|
|
124
|
+
writeFileSync(join(root, '.design', 'config.json'), JSON.stringify(config));
|
|
125
|
+
return root;
|
|
126
|
+
}
|
|
127
|
+
test('reads generation.keyframes.engine when present', () => {
|
|
128
|
+
const root = repoWith({ generation: { keyframes: { engine: 'ffmpeg' } } });
|
|
129
|
+
expect(readEnginePref(root)).toBe('ffmpeg');
|
|
130
|
+
});
|
|
131
|
+
test('null when absent / no config / no root (terminal self-detect path)', () => {
|
|
132
|
+
expect(readEnginePref(repoWith({}))).toBeNull();
|
|
133
|
+
expect(readEnginePref(repoWith(undefined))).toBeNull();
|
|
134
|
+
expect(readEnginePref(null)).toBeNull();
|
|
135
|
+
});
|
|
136
|
+
test('null on an invalid engine value', () => {
|
|
137
|
+
const root = repoWith({ generation: { keyframes: { engine: 'nonsense' } } });
|
|
138
|
+
expect(readEnginePref(root)).toBeNull();
|
|
139
|
+
});
|
|
140
|
+
});
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# smart-frames.sh — thin shim over _smart-frames.mjs; reached via
|
|
3
|
+
# `maude design smart-frames` (DDR-062), never a raw bin path. Scene-AWARE keyframe
|
|
4
|
+
# extraction for the footage-analyst's vision pass (feature-scene-aware-keyframes):
|
|
5
|
+
# picks frames at scene cuts + semantic beats + endpoints instead of a blind frame
|
|
6
|
+
# rate. Three tiers, auto-detected — gemma (opt-in) → ffmpeg (default) → blind
|
|
7
|
+
# (probe-footage / Chromium floor). See _smart-frames.mjs + skill footage-keyframes.
|
|
8
|
+
#
|
|
9
|
+
# Usage:
|
|
10
|
+
# smart-frames.sh <assets/<sha8>.<ext>> [--root <repo>] [--out-dir DIR]
|
|
11
|
+
# [--frames N] [--engine auto|gemma|ffmpeg|blind]
|
|
12
|
+
# [--scene-thresh 0.3] [--scout-fps 4]
|
|
13
|
+
#
|
|
14
|
+
# stdout on success = a JSON manifest (superset of probe-footage's).
|
|
15
|
+
# Exit: 0 ok · 2 usage · 3 forced-engine dep missing · 4 decode/extract error · 1 other.
|
|
16
|
+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
17
|
+
|
|
18
|
+
case "$1" in
|
|
19
|
+
--help|-h) sed -n '2,18p' "$0" | sed 's/^# \?//'; exit 0 ;;
|
|
20
|
+
esac
|
|
21
|
+
|
|
22
|
+
# Pure JS (no .ts imports) — prefer node like probe-footage; fall back to bun.
|
|
23
|
+
if command -v node >/dev/null 2>&1; then
|
|
24
|
+
exec node "$SCRIPT_DIR/_smart-frames.mjs" "$@"
|
|
25
|
+
elif command -v bun >/dev/null 2>&1; then
|
|
26
|
+
exec bun run "$SCRIPT_DIR/_smart-frames.mjs" "$@"
|
|
27
|
+
else
|
|
28
|
+
echo "smart-frames.sh: node or bun is required." >&2
|
|
29
|
+
exit 1
|
|
30
|
+
fi
|