@tandryukha/aidemo 0.14.1 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/AUTHORING.md +142 -20
- package/package.json +1 -1
- package/src/captions.ts +174 -30
- package/src/cli.ts +134 -7
- package/src/compose.ts +340 -36
- package/src/cursor.ts +13 -4
- package/src/ffmpeg.ts +68 -0
- package/src/frames.ts +54 -5
- package/src/import-trace.ts +20 -9
- package/src/inspect.ts +9 -0
- package/src/lint.ts +75 -3
- package/src/mcp/jobs.ts +1 -0
- package/src/mcp/server.ts +67 -3
- package/src/player.ts +317 -15
- package/src/qa.ts +295 -0
- package/src/recorder.ts +65 -7
- package/src/safe-regex.ts +36 -0
- package/src/types.ts +155 -19
package/docs/AUTHORING.md
CHANGED
|
@@ -30,13 +30,14 @@ Every operation exists on both surfaces. Agents should prefer the MCP server
|
|
|
30
30
|
| Scaffold a demo | `init_demo` (`fromUrl` drafts from a live page) | `aidemo init <name> [--from-url <url>]` |
|
|
31
31
|
| Draft from a Playwright trace / test | `import_trace` | `aidemo import-trace <trace.zip \| spec.ts> --name <name>` |
|
|
32
32
|
| Environment check | `doctor` | `aidemo doctor` |
|
|
33
|
-
| Discover selectors on a page | `inspect` (job) | `aidemo inspect <url> [--dir <dir>] [--frame name=sel]` |
|
|
33
|
+
| Discover selectors on a page | `inspect` (job) | `aidemo inspect <url> [--dir <dir>] [--frame name=sel] [--storage-state <f>] [--cookie k=v] [--no-setup]` |
|
|
34
34
|
| Dry-run the flow | `probe` (job) | `aidemo probe <dir>` |
|
|
35
|
-
| Full pipeline | `render` (job) | `aidemo render <dir>` |
|
|
35
|
+
| Full pipeline | `render` (job) | `aidemo render <dir>` (`--strict` fails the run on compose warnings) |
|
|
36
36
|
| One stage | `voice` / `record` / `captions` / `compose` (jobs) | `aidemo voice\|record\|captions\|compose <dir>` |
|
|
37
37
|
| README GIF | `gif` (job) | `aidemo gif <dir>` |
|
|
38
38
|
| Named stills (screenshot mode) | `stills` (job) | `aidemo stills <dir>` |
|
|
39
|
-
| Frames for review | `frames` (job) | `aidemo frames <dir> [--every 3] [--source raw\|take]` |
|
|
39
|
+
| Frames for review | `frames` (job) | `aidemo frames <dir> [--every 3] [--source raw\|take] [--sheet] [--columns 5]` |
|
|
40
|
+
| Post-render QA report | `qa` (job) | `aidemo qa <dir> [--lang] [--json] [--strict]` |
|
|
40
41
|
| Walkthrough export (HTML + Markdown + frames + captions) | `walkthrough` (job) | `aidemo walkthrough <dir> [--lang] [--width]` |
|
|
41
42
|
| Job progress / result | `job_status`, `job_list`, `job_cancel` | (CLI runs block in the foreground) |
|
|
42
43
|
|
|
@@ -70,6 +71,13 @@ first selector of the element you mean. It writes `logs/inspect-<ts>.json`
|
|
|
70
71
|
and a screenshot next to it. Elements below the fold are listed too (marked
|
|
71
72
|
`↓` on the CLI); a `scrollTo` on them works as usual.
|
|
72
73
|
|
|
74
|
+
`inspect` runs the storyboard's **`setup`** block by default — preflight hook,
|
|
75
|
+
`storageState`, cookies — so a gated app scans as the logged-in user instead of
|
|
76
|
+
returning the login screen's selectors; it also picks up the storyboard's
|
|
77
|
+
`video` size (so "in viewport" means what the take will see) and its `frames`
|
|
78
|
+
map. Pass `--storage-state <file>` / `--cookie "k=v;domain=host"` for a one-off
|
|
79
|
+
seed without touching the storyboard, or `--no-setup` to scan signed-out.
|
|
80
|
+
|
|
73
81
|
**Lint before you spend a take.** `lint_storyboard` (CLI: `aidemo lint <dir>`)
|
|
74
82
|
is a browser-free preflight that predicts what compose will do to each scene:
|
|
75
83
|
it estimates the action time from the ops (a `type` at human speed, a scroll's
|
|
@@ -179,8 +187,9 @@ network.
|
|
|
179
187
|
The precise contract is the JSON Schema from `get_storyboard_schema`
|
|
180
188
|
(generated from the engine's own zod schema, `src/types.ts`).
|
|
181
189
|
|
|
182
|
-
Top level: `title`, `language?`, `knownTerms?`, `targetLengthSeconds?`,
|
|
183
|
-
(default 1280x720
|
|
190
|
+
Top level: `title`, `language?`, `knownTerms?`, `targetLengthSeconds?`,
|
|
191
|
+
`video{width,height,deviceScaleFactor?}` (default 1280x720; see *Capture
|
|
192
|
+
resolution*), `frames{ name: iframeSelector }`,
|
|
184
193
|
`voice{voiceId,instructions,speed,pronounce?}` (default, scenes may override), `music?`,
|
|
185
194
|
`zoom?`, `intro?`, `outro?`, `transition?`, `hold?`, `output?`, `setup?`,
|
|
186
195
|
`attention?`, `keystrokes?`, `captions?`, `redact?`, `hide?`, `frame?`,
|
|
@@ -246,25 +255,31 @@ Cinematic keys (all opt-in; omit for the plain look):
|
|
|
246
255
|
`output.loudness` (see below).
|
|
247
256
|
- `transition: {type:"crossfade", durationMs?=400}` — **cross-dissolve every
|
|
248
257
|
scene boundary** instead of hard-cutting (see below).
|
|
249
|
-
- `hold: {mode?="freeze"|"drift", driftScale?=1.06, backoffMs?=0
|
|
258
|
+
- `hold: {mode?="freeze"|"drift", driftScale?=1.06, backoffMs?=0,
|
|
259
|
+
minDriftMs?=2000, driftInFrame?=false}` — what a
|
|
250
260
|
scene shows while the narration outlasts its actions: a frozen frame
|
|
251
261
|
(default) or a **slow Ken-Burns drift** so it never reads as a stall (see
|
|
252
262
|
below).
|
|
253
|
-
- `output: {width?, height?, fit
|
|
263
|
+
- `output: {width?, height?, fit?, background?, loudness?, trimLeadingBlank?,
|
|
264
|
+
blankOpenCapMs?}` — **render
|
|
254
265
|
at a different size/aspect** (`width`+`height`, set together) and/or **set the
|
|
255
266
|
master loudness** (`loudness`), e.g. a vertical social clip (see below).
|
|
256
267
|
- `motionBlur: {frames?=3}` — **subtle motion blur** on fast motion (cursor,
|
|
257
268
|
scroll, zoom pan); static UI stays sharp (see below).
|
|
258
|
-
- `cursor: {hidden?, hideScenes?, scale?, style?, color
|
|
259
|
-
cursor control**: hide, resize
|
|
260
|
-
post-hoc instead of
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
`
|
|
269
|
+
- `cursor: {hidden?, hideScenes?, scale?, style?, color?, outline?,
|
|
270
|
+
outlineWidth?, halo?}` — **compose-time cursor control**: hide, resize,
|
|
271
|
+
restyle (`"arrow"` | `"dot"`) or re-contrast the cursor post-hoc instead of
|
|
272
|
+
baking it (see below).
|
|
273
|
+
- `attention: {color?, clicks?, cursorClear?}` — accent color for
|
|
274
|
+
highlight/spotlight/callout overlays; `clicks:true` draws a click ring
|
|
275
|
+
wherever a click lands (needs the `cursor` block); `cursorClear:true` parks
|
|
276
|
+
the cursor off the element an attention beat is about, so the pointer never
|
|
277
|
+
sits on top of the thing you just highlighted. See *Attention*.
|
|
264
278
|
- `keystrokes: true` — show a keystroke chip ("⌘ K", "Enter") on every `press`
|
|
265
279
|
(per-action `keystrokes` overrides). See *Attention*.
|
|
266
280
|
- `captions: {position?="bottom"|"top", style?="pill"|"bar"|"none", font?,
|
|
267
|
-
size?, color?, background
|
|
281
|
+
size?, color?, background?, maxWords?, maxCueMs?, minCueMs?, maxLines?,
|
|
282
|
+
maxCharsPerLine?}` — caption strip placement and look; scenes may
|
|
268
283
|
override `position` with their own `captions`. Compose also auto-flips a cue
|
|
269
284
|
to the top while an overlay occupies the bottom band. See *Attention* and
|
|
270
285
|
*Produced look*.
|
|
@@ -273,7 +288,7 @@ Cinematic keys (all opt-in; omit for the plain look):
|
|
|
273
288
|
- `hide: [selector, …]` — hide elements at **record** time (teasers, cookie
|
|
274
289
|
bars, ad slots); scenes may add their own `hide`. See *Attention*.
|
|
275
290
|
- `frame: {padding?, background?, radius?, shadow?, chrome?="none"|"browser"|"mac",
|
|
276
|
-
safeTop?, title?, url?}` — pad the video onto a styled canvas with a rounded,
|
|
291
|
+
safeTop?, title?, url?}` (`url: "auto"` tracks the page per scene) — pad the video onto a styled canvas with a rounded,
|
|
277
292
|
shadowed window and optional browser chrome. See *Produced look*.
|
|
278
293
|
- `brand: {logo?, accent?, font?, watermark?}` — brand kit: accent for cards /
|
|
279
294
|
frame tint / attention overlays, font for cards + captions + callouts +
|
|
@@ -295,7 +310,9 @@ Cinematic keys (all opt-in; omit for the plain look):
|
|
|
295
310
|
|
|
296
311
|
Each scene: `id`, `title?` (chapter name), `narration`, `voice?`, `music?`,
|
|
297
312
|
`zoom?` (false to disable), `autoIdle?` (overrides the top-level setting),
|
|
298
|
-
`
|
|
313
|
+
`transition?` (`false` hard-cuts *into* this scene, or an object overriding the
|
|
314
|
+
top-level crossfade for that one boundary), `captions?`, `redact?`, `hide?`,
|
|
315
|
+
`actions[]`.
|
|
299
316
|
Narration may carry `{{@name}}` anchor markers (see *Narration-anchored
|
|
300
317
|
beats*); the engine strips them at load and keeps the word index on the
|
|
301
318
|
scene as `anchors` (don't author that field).
|
|
@@ -317,6 +334,13 @@ scene join is re-encoded (the default hard-cut path is a lossless stream-copy
|
|
|
317
334
|
concat). Needs ≥2 scenes; a shorter `durationMs` (250–400) reads as a snappy
|
|
318
335
|
dissolve, longer (600+) as a slow cinematic fade.
|
|
319
336
|
|
|
337
|
+
A dissolve across a **navigation** ghosts — two unrelated pages bleed through
|
|
338
|
+
each other — so compose hard-cuts any boundary where the next scene opens on a
|
|
339
|
+
load or either side is a `goto`/`back`. Override per boundary with a scene's
|
|
340
|
+
own `transition`: `"transition": false` on a scene forces a hard cut into it,
|
|
341
|
+
and `"transition": {"durationMs": 700}` overrides the top-level timing for that
|
|
342
|
+
one join. The compose log names every auto-cut boundary.
|
|
343
|
+
|
|
320
344
|
**`hold: {mode?, driftScale?, backoffMs?}`** — controls the **hold** compose
|
|
321
345
|
adds when a scene's narration runs longer than its (already x1.6-slowed)
|
|
322
346
|
recording. By default the last frame is frozen for the remainder
|
|
@@ -326,14 +350,20 @@ default 1.06 ≈ 6% over the hold) — motion that reads as intentional rather
|
|
|
326
350
|
than a stalled screen. `backoffMs` (0–2000) takes the held frame from that
|
|
327
351
|
many ms *before* the scene's end instead of the very last frame, so a hold
|
|
328
352
|
never lands on a half-drawn transition, a spinner, or a white flash at the
|
|
329
|
-
tail of a navigation.
|
|
353
|
+
tail of a navigation. A drift that is too short reads as a jitter rather than a
|
|
354
|
+
move, so drifts below `minDriftMs` (default 2000) fall back to a freeze; so
|
|
355
|
+
does a drift inside a `frame` block, where the static window chrome makes the
|
|
356
|
+
push-in obviously fake — set `driftInFrame: true` if you want it anyway. Compose
|
|
357
|
+
logs each fallback (`drift hold → freeze`). Only scenes that actually hold are affected; a scene
|
|
330
358
|
whose actions fill the narration is untouched, and a scene with no hold is
|
|
331
359
|
byte-identical. The compose log and `output/report.json` list every scene's
|
|
332
360
|
hold length and percentage; the lint predicts them before the take.
|
|
333
361
|
|
|
334
362
|
**`output: {width, height, fit?, background?}`** — reframes the finished video
|
|
335
363
|
(cards and captions already baked in) to a target size, applied as the last
|
|
336
|
-
step. `fit
|
|
364
|
+
step. `fit` defaults to `"contain"`, and a `preset`'s `fit` applies only when
|
|
365
|
+
you don't set one — so `{"preset": "youtube", "fit": "cover"}` exports
|
|
366
|
+
edge-to-edge with no letterbox bars. `fit`:
|
|
337
367
|
- `"contain"` (default) — scale to fit inside `width×height` and pad the
|
|
338
368
|
remainder with `background` (ffmpeg color syntax, e.g. `black`, `0x1a1a1a`;
|
|
339
369
|
default black). Letterbox/pillarbox bars, nothing cropped.
|
|
@@ -460,6 +490,19 @@ moves the strip to the top edge. Without a per-scene override, compose
|
|
|
460
490
|
box or a keystroke chip sitting in the bottom band, so a mark and a caption
|
|
461
491
|
never collide (`report.json` → `attention.captionsFlipped`).
|
|
462
492
|
|
|
493
|
+
**Cursor gets out of the way.** `attention: {cursorClear: true}` moves the
|
|
494
|
+
pointer off the element a highlight/spotlight/callout is about (to the nearest
|
|
495
|
+
free side, at record time) before the mark is drawn — otherwise the cursor
|
|
496
|
+
often sits exactly on the thing you just asked the viewer to look at. Pair it
|
|
497
|
+
with `hover {anchor:"edge"}` when you want a specific side.
|
|
498
|
+
|
|
499
|
+
**Late layout.** Attention rects are measured after the element's box has been
|
|
500
|
+
*stable* for a beat (bounded wait), and re-measured after the dwell: if the
|
|
501
|
+
target moved more than ~24px in the meantime (a font swap, a lazy image, an
|
|
502
|
+
async row) the take uses the settled position and records a warning on that
|
|
503
|
+
action in `timeline.json` (also printed by record). A target that scrolled out
|
|
504
|
+
of view warns too, instead of silently marking empty screen.
|
|
505
|
+
|
|
463
506
|
**Redact (compose-time blur).** `redact: [{selector, frame?, blur?=14}]` at
|
|
464
507
|
the top level and/or per scene. After every action the player re-measures
|
|
465
508
|
each selector (all matches, up to 8) and logs a span per box; compose crops,
|
|
@@ -502,6 +545,11 @@ shadowed window on a styled canvas, with captions in the padding band below.
|
|
|
502
545
|
- `radius` (default 14), `shadow` (default true).
|
|
503
546
|
- `chrome`: `"browser"` (neutral dots + address pill) or `"mac"` (traffic
|
|
504
547
|
lights). `url` (wins) or `title` fills the pill; omit both for an empty bar.
|
|
548
|
+
`"url": "auto"` **tracks the page**: each scene's pill shows the host (plus
|
|
549
|
+
path) the scene is actually on, taken from the take's `goto`s — use it for a
|
|
550
|
+
demo that moves between hosts, where one fixed address would be a lie. The
|
|
551
|
+
pill only changes at scene boundaries (it's a time-gated overlay, not a live
|
|
552
|
+
browser).
|
|
505
553
|
For a phone-sized take (`video: {width: 390, height: 844}` — record at the
|
|
506
554
|
mobile viewport) use `"iphone"` (rounded bezel + dynamic island) or
|
|
507
555
|
`"android"` (bezel + punch-hole): no bar, `radius` defaults to 40, and the
|
|
@@ -559,7 +607,31 @@ frames, the SRT/VTT captions, and `walkthrough.json`. Frames come from the
|
|
|
559
607
|
**Poster** (`output.poster: true`): also writes `output/poster.png`
|
|
560
608
|
(`poster.<lang>.png` for language variants) — the first content frame after
|
|
561
609
|
the intro card, for READMEs, social cards and `<video poster>`. The path is
|
|
562
|
-
in `report.json` as `poster`.
|
|
610
|
+
in `report.json` as `poster`. It skips past a blank opening (below), so the
|
|
611
|
+
poster isn't a white rectangle; `"poster": 4200` takes the still at that
|
|
612
|
+
millisecond instead, when you want a specific moment.
|
|
613
|
+
|
|
614
|
+
**Blank opening** (`output.trimLeadingBlank: true`): a take that starts on a
|
|
615
|
+
page still painting opens on white/blank frames. With this on, compose trims
|
|
616
|
+
the flat lead of the first scene (capped by `output.blankOpenCapMs`, default
|
|
617
|
+
1200) and re-aligns the scene's cursor, zoom and anchors to the trim. Compose
|
|
618
|
+
always *measures* the opening either way and reports it as `blankOpenMs` plus
|
|
619
|
+
a `blank-open` warning past ~400 ms; `aidemo lint` forecasts the same pitfall
|
|
620
|
+
from a first scene that opens with a `goto` and no wait.
|
|
621
|
+
|
|
622
|
+
## Capture resolution (`video.deviceScaleFactor`)
|
|
623
|
+
|
|
624
|
+
`video: {width, height}` is the **logical** viewport — the CSS px the page
|
|
625
|
+
lays out in. `deviceScaleFactor` (1–3, default 1) is how many real pixels each
|
|
626
|
+
of those gets, exactly like a Retina display: `{"width": 1280, "height": 720,
|
|
627
|
+
"deviceScaleFactor": 2}` records a 2560×1440 file of a 1280×720 page. Text and
|
|
628
|
+
UI keep their on-screen *size* but get twice the pixels, which is the fix for
|
|
629
|
+
a demo that looks soft after a zoom or a 1920 export, and for mobile-viewport
|
|
630
|
+
takes whose small type turns to mush. Cost: bigger raw footage and a slower
|
|
631
|
+
compose — 2 is enough for any delivery size; 3 is for stills. Bumping it
|
|
632
|
+
changes the recording, so it's a re-record, not a recompose. `aidemo lint`
|
|
633
|
+
warns when a viewport that small is about to be blown up on export
|
|
634
|
+
(`viewport-legibility`).
|
|
563
635
|
|
|
564
636
|
## Motion blur & cursor
|
|
565
637
|
|
|
@@ -581,6 +653,10 @@ without re-recording**:
|
|
|
581
653
|
- `hideScenes: ["s5"]` — hide the cursor only on those scenes (e.g. a
|
|
582
654
|
full-screen result or confirmation).
|
|
583
655
|
- `scale: 1.3` — resize the cursor (1 = the 24px baseline arrow).
|
|
656
|
+
- `color`, `outline` (stroke color), `outlineWidth` (0–6) and `halo: true`
|
|
657
|
+
(a soft dark glow behind it) — make the cursor readable over a brand-colored
|
|
658
|
+
or busy surface, where a plain white arrow disappears. `halo` is the
|
|
659
|
+
one-key fix; `outline` is for matching a brand palette exactly.
|
|
584
660
|
|
|
585
661
|
How it works: with a `cursor` block present, `record` leaves the take
|
|
586
662
|
**cursor-free** and logs the cursor path into `timeline.json`; `compose` draws
|
|
@@ -603,12 +679,26 @@ for UI that re-renders under the cursor) and `anchor?` (land this action on a
|
|
|
603
679
|
screen and the take doesn't get longer for it.
|
|
604
680
|
- `{op:"type", target, text, humanize?}` — human-cadence typing
|
|
605
681
|
- `{op:"press", key}` — e.g. "Enter"
|
|
606
|
-
- `{op:"click", target, followPopup?}` · `{op:"hover", target
|
|
682
|
+
- `{op:"click", target, followPopup?}` · `{op:"hover", target, anchor?,
|
|
683
|
+
offset?}` — `anchor` (`"center"` default, or `"edge"|"top"|"bottom"|"left"|
|
|
684
|
+
"right"`) and `offset:{x,y}` aim the pointer at a *part* of the element:
|
|
685
|
+
`"anchor": "top"` hovers the top edge of a row so the tooltip it opens isn't
|
|
686
|
+
covered by the cursor, `"edge"` parks just outside it.
|
|
687
|
+
|
|
607
688
|
`followPopup:true` handles a link/button that opens a **new tab**
|
|
608
689
|
(`target=_blank`, `window.open`): the tab is closed and the recorded tab
|
|
609
690
|
navigates to its URL. The take is one window — a second tab is never in the
|
|
610
691
|
video, so there is no `newTab`/`switchTab`; write the story in one tab.
|
|
611
692
|
- `{op:"back"}` — browser history back, same readiness wait as `goto`.
|
|
693
|
+
- `{op:"dialog", action?="accept"|"dismiss", promptText?, holdMs?, required?}`
|
|
694
|
+
— **arm a native dialog handler** for the *next* action in the scene.
|
|
695
|
+
`alert`/`confirm`/`window.print` and friends block the page and are invisible
|
|
696
|
+
to a recording otherwise (Playwright auto-dismisses an unhandled dialog, so
|
|
697
|
+
the take silently loses the step). Put the `dialog` action immediately
|
|
698
|
+
**before** the click that triggers it; `holdMs` waits before answering (the page
|
|
699
|
+
stays paused, so the moment reads as deliberate), `promptText`
|
|
700
|
+
answers a `window.prompt`, and `required: true` fails the scene if no dialog
|
|
701
|
+
ever appeared (the default is best-effort).
|
|
612
702
|
- `{op:"select", target, value? | label?}` — native `<select>`: the cursor
|
|
613
703
|
clicks it, the option is committed programmatically (the OS dropdown never
|
|
614
704
|
paints into a recording), `change` fires.
|
|
@@ -948,6 +1038,14 @@ Always set `"last": true` on widget targets (newest widget for this turn).
|
|
|
948
1038
|
records the wrong numbers; resuming keeps the earned footage and picks up at
|
|
949
1039
|
the scene that broke. Pair it with `setup.expectState: true` so the
|
|
950
1040
|
carried-over-state warning stays quiet on the seeded profile.
|
|
1041
|
+
**But a plain resume still *replays* the earlier scenes' actions** to rebuild
|
|
1042
|
+
state — on a state-earning flow that replay itself fails (the first scene
|
|
1043
|
+
waits for a "stranger" element the profile no longer shows). Add
|
|
1044
|
+
`--no-replay` (`record`/`render`, `{noReplay: true}`) to skip the replayed
|
|
1045
|
+
scenes' actions entirely and trust the profile as it stands; the scene you
|
|
1046
|
+
resume from must then open with its own `goto`. This is also how you record
|
|
1047
|
+
one storyboard across two real days (day 0 today, day 1 after the app's
|
|
1048
|
+
date rolls over) on one profile.
|
|
951
1049
|
The same flag is the cheap way to re-shoot
|
|
952
1050
|
a tail you changed after an approved take.
|
|
953
1051
|
|
|
@@ -1053,6 +1151,21 @@ scene's `anchors[]` shows `targetMs` / `landedMs` / `offMs`). A held scene
|
|
|
1053
1151
|
is a storyboard problem, not a compose problem: give it on-screen beats or
|
|
1054
1152
|
shorten its narration, then re-run `voice` + `compose`.
|
|
1055
1153
|
|
|
1154
|
+
Then run **`aidemo qa <dir>`** (MCP: the `qa` job) on the finished file — a
|
|
1155
|
+
browser-free, API-free audit of what actually shipped: length vs
|
|
1156
|
+
`targetLengthSeconds`, a blank opening or blank poster, master loudness (LUFS
|
|
1157
|
+
/ true peak) and whether the music bed is audible under the narration,
|
|
1158
|
+
per-scene hold percentage and compose warnings, caption cue stats, and
|
|
1159
|
+
hygiene checks (a fixed `frame.url` on a demo that visits several hosts, a
|
|
1160
|
+
non-English demo with ASCII-only chapter titles). `--json` for the structured
|
|
1161
|
+
report, `--strict` to exit non-zero on any finding.
|
|
1162
|
+
|
|
1163
|
+
`render`/`compose` also take **`--strict`** (exit non-zero when compose emitted
|
|
1164
|
+
any warning — the CI gate), and every warning that has a moment on screen
|
|
1165
|
+
leaves a PNG in `output/warnings/<name>.png` so you can look at the frame
|
|
1166
|
+
instead of guessing. For a fast visual scan, `aidemo frames <dir> --sheet`
|
|
1167
|
+
writes a single contact-sheet PNG (`--columns`, default 5) next to the frames.
|
|
1168
|
+
|
|
1056
1169
|
Then play (or frame-extract — `aidemo frames <dir>` / the `frames` job)
|
|
1057
1170
|
`output/final-demo.mp4`: cursor glides and clicks pulse, narration matches
|
|
1058
1171
|
on-screen actions, captions are readable and in sync,
|
|
@@ -1206,6 +1319,15 @@ all) using each scene's measured duration from `voice.json`; only the timing
|
|
|
1206
1319
|
measured speech rhythm). `captions` logs a one-line reminder of this whenever
|
|
1207
1320
|
the resolved language isn't English.
|
|
1208
1321
|
|
|
1322
|
+
**Cue segmentation.** Cues are broken at sentence ends first, then at clause
|
|
1323
|
+
punctuation, then before a conjunction, and only then on length — so a cue
|
|
1324
|
+
stops reading like it was cut mid-thought ("we open the" / "settings panel").
|
|
1325
|
+
Very short cues are merged forward. Tune per storyboard under `captions`:
|
|
1326
|
+
`maxWords` (12), `maxCueMs` (6000), `minCueMs` (1000), `maxLines` (2),
|
|
1327
|
+
`maxCharsPerLine` (38). `captions` logs the resulting cue stats (count, median
|
|
1328
|
+
length, how many fall under `minCueMs`), and `aidemo qa` flags a run with lots
|
|
1329
|
+
of flash cues or mid-clause splits.
|
|
1330
|
+
|
|
1209
1331
|
## Multi-language renders (one take, N languages)
|
|
1210
1332
|
|
|
1211
1333
|
Record the browser flow **once**, then ship the same demo in several languages —
|
package/package.json
CHANGED
package/src/captions.ts
CHANGED
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
VoiceManifestSchema,
|
|
12
12
|
CaptionsManifestSchema,
|
|
13
13
|
type CaptionsManifest,
|
|
14
|
+
type CaptionsConfig,
|
|
14
15
|
type Storyboard,
|
|
15
16
|
} from "./types.js";
|
|
16
17
|
import { resolveNarrationLanguage } from "./i18n.js";
|
|
@@ -38,8 +39,52 @@ interface Word {
|
|
|
38
39
|
end: number; // seconds
|
|
39
40
|
}
|
|
40
41
|
|
|
41
|
-
|
|
42
|
-
|
|
42
|
+
/**
|
|
43
|
+
* Cue-segmentation defaults (issue #48). The old 6-word / 3 s hard cap split
|
|
44
|
+
* half of all cue breaks mid-clause ("…nobody has / filled in a form, and no /
|
|
45
|
+
* account exists yet"), which is harder to read than the narration. Broadcast
|
|
46
|
+
* practice is ~2 lines x ~37 characters, 1–6 s per cue, broken at punctuation
|
|
47
|
+
* or a conjunction. Override per storyboard via the `captions` block.
|
|
48
|
+
*/
|
|
49
|
+
const DEFAULT_SEG: Required<
|
|
50
|
+
Pick<CaptionsConfig, "maxWords" | "maxCueMs" | "minCueMs" | "maxLines" | "maxCharsPerLine">
|
|
51
|
+
> = {
|
|
52
|
+
maxWords: 12,
|
|
53
|
+
maxCueMs: 6000,
|
|
54
|
+
minCueMs: 1000,
|
|
55
|
+
maxLines: 2,
|
|
56
|
+
maxCharsPerLine: 38,
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
/** Conjunctions/discourse markers that make a decent break point, by language. */
|
|
60
|
+
const CONJUNCTIONS = new Set([
|
|
61
|
+
// en
|
|
62
|
+
"and", "but", "so", "or", "then", "because", "while", "when", "which", "that", "if",
|
|
63
|
+
// et
|
|
64
|
+
"ja", "aga", "või", "siis", "kui", "sest", "mis", "ning", "kuid",
|
|
65
|
+
// de / nl / sv
|
|
66
|
+
"und", "aber", "oder", "dann", "weil", "wenn", "och", "men", "eller", "en", "maar",
|
|
67
|
+
// fr / es / it / pt
|
|
68
|
+
"et", "mais", "ou", "donc", "parce", "quand", "y", "pero", "o", "entonces", "porque",
|
|
69
|
+
"e", "ma", "oppure", "quindi", "perché", "mas", "então",
|
|
70
|
+
]);
|
|
71
|
+
|
|
72
|
+
type SegConfig = typeof DEFAULT_SEG;
|
|
73
|
+
|
|
74
|
+
function segConfig(storyboard?: Storyboard): SegConfig {
|
|
75
|
+
return { ...DEFAULT_SEG, ...cleanSeg(storyboard?.captions) };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function cleanSeg(c: CaptionsConfig | undefined): Partial<SegConfig> {
|
|
79
|
+
if (!c) return {};
|
|
80
|
+
const out: Partial<SegConfig> = {};
|
|
81
|
+
if (c.maxWords != null) out.maxWords = c.maxWords;
|
|
82
|
+
if (c.maxCueMs != null) out.maxCueMs = c.maxCueMs;
|
|
83
|
+
if (c.minCueMs != null) out.minCueMs = c.minCueMs;
|
|
84
|
+
if (c.maxLines != null) out.maxLines = c.maxLines;
|
|
85
|
+
if (c.maxCharsPerLine != null) out.maxCharsPerLine = c.maxCharsPerLine;
|
|
86
|
+
return out;
|
|
87
|
+
}
|
|
43
88
|
|
|
44
89
|
/** Whisper's `prompt` is only used to bias roughly its first ~224 tokens of
|
|
45
90
|
* context; cap what we send so a long demo's script doesn't balloon the
|
|
@@ -93,7 +138,8 @@ export async function generateCaptions(
|
|
|
93
138
|
for (let i = 0; i < total; i++) opts.onSceneComplete?.(ids[i] ?? "", i, total);
|
|
94
139
|
};
|
|
95
140
|
if (total > 0) opts.onSceneStart?.(ids[0] ?? "", 0, total);
|
|
96
|
-
const
|
|
141
|
+
const seg = segConfig(storyboard);
|
|
142
|
+
const config = captionConfig(await loadGapMs(project), seg);
|
|
97
143
|
|
|
98
144
|
const prompt = storyboard ? buildSttPrompt(storyboard) : undefined;
|
|
99
145
|
const language = opts.language ?? resolveNarrationLanguage(storyboard, project.lang);
|
|
@@ -151,7 +197,7 @@ export async function generateCaptions(
|
|
|
151
197
|
if (words.length === 0) {
|
|
152
198
|
log("no word timestamps returned; captions may be empty");
|
|
153
199
|
}
|
|
154
|
-
const cues = groupWords(words, sceneEnds);
|
|
200
|
+
const cues = groupWords(words, sceneEnds, seg);
|
|
155
201
|
await writeCaptionFiles(project, cues);
|
|
156
202
|
await writeCaptionsManifest(project, {
|
|
157
203
|
mode: "stt",
|
|
@@ -252,7 +298,8 @@ export async function generateCaptionsOffline(
|
|
|
252
298
|
await readJson(project.voiceManifestPath)
|
|
253
299
|
);
|
|
254
300
|
const durById = new Map(voice.scenes.map((s) => [s.id, s.durationMs]));
|
|
255
|
-
const
|
|
301
|
+
const seg = segConfig(storyboard);
|
|
302
|
+
const config = captionConfig(voice.gapMs, seg);
|
|
256
303
|
|
|
257
304
|
// Per-scene caption identity = narration + this scene's measured duration (+
|
|
258
305
|
// grouping config). The scene-relative word timings depend on nothing else, so
|
|
@@ -333,7 +380,7 @@ export async function generateCaptionsOffline(
|
|
|
333
380
|
cursor += voice.gapMs;
|
|
334
381
|
}
|
|
335
382
|
|
|
336
|
-
const cues = groupWords(words, sceneEnds);
|
|
383
|
+
const cues = groupWords(words, sceneEnds, seg);
|
|
337
384
|
await writeCaptionFiles(project, cues);
|
|
338
385
|
await writeCaptionsManifest(project, {
|
|
339
386
|
mode: "offline",
|
|
@@ -369,6 +416,12 @@ async function writeCaptionFiles(project: Project, cues: Cue[]): Promise<void> {
|
|
|
369
416
|
await fs.writeFile(project.captionsSrtPath, toSrt(cues));
|
|
370
417
|
await fs.writeFile(project.captionsVttPath, toVtt(cues));
|
|
371
418
|
await writeJson(project.captionsCuesPath, cues);
|
|
419
|
+
// Readability, so an agent sees it without opening the SRT (issue #48).
|
|
420
|
+
const stats = cueStats(cues);
|
|
421
|
+
log(
|
|
422
|
+
`cues: ${cues.length}, ${stats.midClausePct}% mid-clause breaks, ` +
|
|
423
|
+
`${stats.short} shorter than 1s`
|
|
424
|
+
);
|
|
372
425
|
ok(`captions → ${project.captionsSrtPath} (${cues.length} cues)`);
|
|
373
426
|
ok(`captions → ${project.captionsCuesPath}`);
|
|
374
427
|
}
|
|
@@ -381,8 +434,8 @@ async function writeCaptionFiles(project: Project, cues: Cue[]): Promise<void> {
|
|
|
381
434
|
// ---------------------------------------------------------------------------
|
|
382
435
|
|
|
383
436
|
/** Grouping parameters that affect cue segmentation — part of the cache key. */
|
|
384
|
-
function captionConfig(gapMs: number): CaptionsManifest["config"] {
|
|
385
|
-
return { gapMs, maxWords:
|
|
437
|
+
function captionConfig(gapMs: number, cfg: SegConfig = DEFAULT_SEG): CaptionsManifest["config"] {
|
|
438
|
+
return { gapMs, maxWords: cfg.maxWords, maxCueMs: cfg.maxCueMs };
|
|
386
439
|
}
|
|
387
440
|
|
|
388
441
|
/** Inter-scene gap from voice.json (0 when absent — captions need voice.json). */
|
|
@@ -451,38 +504,129 @@ async function sceneIdsFor(project: Project): Promise<string[]> {
|
|
|
451
504
|
return voice.success ? voice.data.scenes.map((s) => s.id) : [];
|
|
452
505
|
}
|
|
453
506
|
|
|
454
|
-
|
|
455
|
-
|
|
507
|
+
/** Text of a run of words, with punctuation re-attached. */
|
|
508
|
+
function cueText(ws: Word[]): string {
|
|
509
|
+
// trim each word: faster-whisper servers pad words with leading spaces
|
|
510
|
+
return ws
|
|
511
|
+
.map((w) => w.word.trim())
|
|
512
|
+
.join(" ")
|
|
513
|
+
.replace(/\s+([,.!?;:])/g, "$1")
|
|
514
|
+
.trim();
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
/** A word that ends a clause (punctuation) — the best place to break. */
|
|
518
|
+
function endsClause(w: Word): boolean {
|
|
519
|
+
return /[,;:—–]$/.test(w.word.trim());
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
function endsSentence(w: Word): boolean {
|
|
523
|
+
return /[.!?]["'”’)]?$/.test(w.word.trim());
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
/**
|
|
527
|
+
* Split a run of words into cues at the most readable boundaries: sentence
|
|
528
|
+
* ends first, then clause punctuation, then a conjunction, and only then on
|
|
529
|
+
* pure length. Never leaves a 2–3 word tail dangling when the previous
|
|
530
|
+
* boundary can absorb it.
|
|
531
|
+
*/
|
|
532
|
+
function splitRun(ws: Word[], cfg: SegConfig): Word[][] {
|
|
533
|
+
const capChars = cfg.maxLines * cfg.maxCharsPerLine;
|
|
534
|
+
const fits = (run: Word[]): boolean =>
|
|
535
|
+
run.length <= cfg.maxWords &&
|
|
536
|
+
cueText(run).length <= capChars &&
|
|
537
|
+
(run[run.length - 1].end - run[0].start) * 1000 <= cfg.maxCueMs;
|
|
538
|
+
if (ws.length === 0) return [];
|
|
539
|
+
if (fits(ws)) return [ws];
|
|
540
|
+
// The furthest index that still fits, then walk back to a real boundary.
|
|
541
|
+
let end = 1;
|
|
542
|
+
while (end < ws.length && fits(ws.slice(0, end + 1))) end++;
|
|
543
|
+
const floor = Math.max(1, Math.ceil(end * 0.45));
|
|
544
|
+
let cut = end;
|
|
545
|
+
for (let i = end - 1; i >= floor; i--) {
|
|
546
|
+
if (endsSentence(ws[i]) || endsClause(ws[i])) {
|
|
547
|
+
cut = i + 1;
|
|
548
|
+
break;
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
if (cut === end) {
|
|
552
|
+
for (let i = end - 1; i >= floor; i--) {
|
|
553
|
+
const next = ws[i + 1];
|
|
554
|
+
if (next && CONJUNCTIONS.has(next.word.trim().toLowerCase().replace(/[^\p{L}]/gu, ""))) {
|
|
555
|
+
cut = i + 1;
|
|
556
|
+
break;
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
return [ws.slice(0, cut), ...splitRun(ws.slice(cut), cfg)];
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
/** Merge a too-short cue into the neighbour it still fits in. */
|
|
564
|
+
function mergeShort(runs: Word[][], cfg: SegConfig): Word[][] {
|
|
565
|
+
const capChars = cfg.maxLines * cfg.maxCharsPerLine;
|
|
566
|
+
const room = (a: Word[], b: Word[]): boolean =>
|
|
567
|
+
a.length + b.length <= cfg.maxWords &&
|
|
568
|
+
cueText([...a, ...b]).length <= capChars &&
|
|
569
|
+
(b[b.length - 1].end - a[0].start) * 1000 <= cfg.maxCueMs;
|
|
570
|
+
const out: Word[][] = [];
|
|
571
|
+
for (const run of runs) {
|
|
572
|
+
const durMs = (run[run.length - 1].end - run[0].start) * 1000;
|
|
573
|
+
const prev = out[out.length - 1];
|
|
574
|
+
if (durMs < cfg.minCueMs && prev && room(prev, run)) {
|
|
575
|
+
out[out.length - 1] = [...prev, ...run];
|
|
576
|
+
continue;
|
|
577
|
+
}
|
|
578
|
+
out.push(run);
|
|
579
|
+
}
|
|
580
|
+
return out;
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
function groupWords(words: Word[], sceneEnds: number[], cfg: SegConfig = DEFAULT_SEG): Cue[] {
|
|
584
|
+
// Scene boundaries and sentence ends are hard breaks; inside those runs the
|
|
585
|
+
// split is clause-aware.
|
|
586
|
+
const runs: Word[][] = [];
|
|
456
587
|
let cur: Word[] = [];
|
|
457
588
|
let sceneIdx = 0;
|
|
458
|
-
const
|
|
459
|
-
if (cur.length
|
|
460
|
-
cues.push({
|
|
461
|
-
index: cues.length + 1,
|
|
462
|
-
startMs: Math.round(cur[0].start * 1000),
|
|
463
|
-
endMs: Math.round(cur[cur.length - 1].end * 1000),
|
|
464
|
-
// trim each word: faster-whisper servers pad words with leading spaces
|
|
465
|
-
text: cur.map((w) => w.word.trim()).join(" ").replace(/\s+([,.!?])/g, "$1").trim(),
|
|
466
|
-
});
|
|
589
|
+
const cut = (): void => {
|
|
590
|
+
if (cur.length) runs.push(cur);
|
|
467
591
|
cur = [];
|
|
468
592
|
};
|
|
469
593
|
for (const w of words) {
|
|
470
594
|
const startMs = w.start * 1000;
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
sceneIdx < sceneEnds.length - 1 &&
|
|
474
|
-
startMs >= sceneEnds[sceneIdx]
|
|
475
|
-
) {
|
|
476
|
-
flush();
|
|
595
|
+
while (sceneIdx < sceneEnds.length - 1 && startMs >= sceneEnds[sceneIdx]) {
|
|
596
|
+
cut();
|
|
477
597
|
sceneIdx++;
|
|
478
598
|
}
|
|
479
599
|
cur.push(w);
|
|
480
|
-
|
|
481
|
-
const endsSentence = /[.!?]$/.test(w.word.trim());
|
|
482
|
-
if (cur.length >= MAX_WORDS || durMs >= MAX_CUE_MS || endsSentence) flush();
|
|
600
|
+
if (endsSentence(w)) cut();
|
|
483
601
|
}
|
|
484
|
-
|
|
485
|
-
|
|
602
|
+
cut();
|
|
603
|
+
const split = runs.flatMap((r) => splitRun(r, cfg));
|
|
604
|
+
return mergeShort(split, cfg).map((ws, i) => ({
|
|
605
|
+
index: i + 1,
|
|
606
|
+
startMs: Math.round(ws[0].start * 1000),
|
|
607
|
+
endMs: Math.round(ws[ws.length - 1].end * 1000),
|
|
608
|
+
text: cueText(ws),
|
|
609
|
+
}));
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
/** Readability stats for the log: mid-clause breaks and too-short cues. */
|
|
613
|
+
export function cueStats(cues: Cue[], minCueMs = DEFAULT_SEG.minCueMs): {
|
|
614
|
+
short: number;
|
|
615
|
+
midClause: number;
|
|
616
|
+
midClausePct: number;
|
|
617
|
+
} {
|
|
618
|
+
let short = 0;
|
|
619
|
+
let midClause = 0;
|
|
620
|
+
cues.forEach((c, i) => {
|
|
621
|
+
if (c.endMs - c.startMs < minCueMs) short++;
|
|
622
|
+
const next = cues[i + 1];
|
|
623
|
+
if (!next) return;
|
|
624
|
+
const cleanBreak = /[.!?,;:—–]["'”’)]?$/.test(c.text.trim());
|
|
625
|
+
const startsLower = /^\p{Ll}/u.test(next.text.trim());
|
|
626
|
+
if (!cleanBreak && startsLower) midClause++;
|
|
627
|
+
});
|
|
628
|
+
const breaks = Math.max(1, cues.length - 1);
|
|
629
|
+
return { short, midClause, midClausePct: Math.round((midClause / breaks) * 100) };
|
|
486
630
|
}
|
|
487
631
|
|
|
488
632
|
function toSrt(cues: Cue[]): string {
|