@tandryukha/aidemo 0.14.1 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/docs/AUTHORING.md CHANGED
@@ -30,13 +30,14 @@ Every operation exists on both surfaces. Agents should prefer the MCP server
30
30
  | Scaffold a demo | `init_demo` (`fromUrl` drafts from a live page) | `aidemo init <name> [--from-url <url>]` |
31
31
  | Draft from a Playwright trace / test | `import_trace` | `aidemo import-trace <trace.zip \| spec.ts> --name <name>` |
32
32
  | Environment check | `doctor` | `aidemo doctor` |
33
- | Discover selectors on a page | `inspect` (job) | `aidemo inspect <url> [--dir <dir>] [--frame name=sel]` |
33
+ | Discover selectors on a page | `inspect` (job) | `aidemo inspect <url> [--dir <dir>] [--frame name=sel] [--storage-state <f>] [--cookie k=v] [--no-setup]` |
34
34
  | Dry-run the flow | `probe` (job) | `aidemo probe <dir>` |
35
- | Full pipeline | `render` (job) | `aidemo render <dir>` |
35
+ | Full pipeline | `render` (job) | `aidemo render <dir>` (`--strict` fails the run on compose warnings) |
36
36
  | One stage | `voice` / `record` / `captions` / `compose` (jobs) | `aidemo voice\|record\|captions\|compose <dir>` |
37
37
  | README GIF | `gif` (job) | `aidemo gif <dir>` |
38
38
  | Named stills (screenshot mode) | `stills` (job) | `aidemo stills <dir>` |
39
- | Frames for review | `frames` (job) | `aidemo frames <dir> [--every 3] [--source raw\|take]` |
39
+ | Frames for review | `frames` (job) | `aidemo frames <dir> [--every 3] [--source raw\|take] [--sheet] [--columns 5]` |
40
+ | Post-render QA report | `qa` (job) | `aidemo qa <dir> [--lang] [--json] [--strict]` |
40
41
  | Walkthrough export (HTML + Markdown + frames + captions) | `walkthrough` (job) | `aidemo walkthrough <dir> [--lang] [--width]` |
41
42
  | Job progress / result | `job_status`, `job_list`, `job_cancel` | (CLI runs block in the foreground) |
42
43
 
@@ -70,6 +71,13 @@ first selector of the element you mean. It writes `logs/inspect-<ts>.json`
70
71
  and a screenshot next to it. Elements below the fold are listed too (marked
71
72
  `↓` on the CLI); a `scrollTo` on them works as usual.
72
73
 
74
+ `inspect` runs the storyboard's **`setup`** block by default — preflight hook,
75
+ `storageState`, cookies — so a gated app scans as the logged-in user instead of
76
+ returning the login screen's selectors; it also picks up the storyboard's
77
+ `video` size (so "in viewport" means what the take will see) and its `frames`
78
+ map. Pass `--storage-state <file>` / `--cookie "k=v;domain=host"` for a one-off
79
+ seed without touching the storyboard, or `--no-setup` to scan signed-out.
80
+
73
81
  **Lint before you spend a take.** `lint_storyboard` (CLI: `aidemo lint <dir>`)
74
82
  is a browser-free preflight that predicts what compose will do to each scene:
75
83
  it estimates the action time from the ops (a `type` at human speed, a scroll's
@@ -179,8 +187,9 @@ network.
179
187
  The precise contract is the JSON Schema from `get_storyboard_schema`
180
188
  (generated from the engine's own zod schema, `src/types.ts`).
181
189
 
182
- Top level: `title`, `language?`, `knownTerms?`, `targetLengthSeconds?`, `video{width,height}`
183
- (default 1280x720), `frames{ name: iframeSelector }`,
190
+ Top level: `title`, `language?`, `knownTerms?`, `targetLengthSeconds?`,
191
+ `video{width,height,deviceScaleFactor?}` (default 1280x720; see *Capture
192
+ resolution*), `frames{ name: iframeSelector }`,
184
193
  `voice{voiceId,instructions,speed,pronounce?}` (default, scenes may override), `music?`,
185
194
  `zoom?`, `intro?`, `outro?`, `transition?`, `hold?`, `output?`, `setup?`,
186
195
  `attention?`, `keystrokes?`, `captions?`, `redact?`, `hide?`, `frame?`,
@@ -246,25 +255,31 @@ Cinematic keys (all opt-in; omit for the plain look):
246
255
  `output.loudness` (see below).
247
256
  - `transition: {type:"crossfade", durationMs?=400}` — **cross-dissolve every
248
257
  scene boundary** instead of hard-cutting (see below).
249
- - `hold: {mode?="freeze"|"drift", driftScale?=1.06, backoffMs?=0}` — what a
258
+ - `hold: {mode?="freeze"|"drift", driftScale?=1.06, backoffMs?=0,
259
+ minDriftMs?=2000, driftInFrame?=false}` — what a
250
260
  scene shows while the narration outlasts its actions: a frozen frame
251
261
  (default) or a **slow Ken-Burns drift** so it never reads as a stall (see
252
262
  below).
253
- - `output: {width?, height?, fit?="contain", background?, loudness?}` — **render
263
+ - `output: {width?, height?, fit?, background?, loudness?, trimLeadingBlank?,
264
+ blankOpenCapMs?}` — **render
254
265
  at a different size/aspect** (`width`+`height`, set together) and/or **set the
255
266
  master loudness** (`loudness`), e.g. a vertical social clip (see below).
256
267
  - `motionBlur: {frames?=3}` — **subtle motion blur** on fast motion (cursor,
257
268
  scroll, zoom pan); static UI stays sharp (see below).
258
- - `cursor: {hidden?, hideScenes?, scale?, style?, color?}` — **compose-time
259
- cursor control**: hide, resize or restyle (`"arrow"` | `"dot"`) the cursor
260
- post-hoc instead of baking it (see below).
261
- - `attention: {color?, clicks?}` — accent color for highlight/spotlight/callout
262
- overlays; `clicks:true` draws a click ring wherever a click lands (needs the
263
- `cursor` block). See *Attention*.
269
+ - `cursor: {hidden?, hideScenes?, scale?, style?, color?, outline?,
270
+ outlineWidth?, halo?}` — **compose-time cursor control**: hide, resize,
271
+ restyle (`"arrow"` | `"dot"`) or re-contrast the cursor post-hoc instead of
272
+ baking it (see below).
273
+ - `attention: {color?, clicks?, cursorClear?}` — accent color for
274
+ highlight/spotlight/callout overlays; `clicks:true` draws a click ring
275
+ wherever a click lands (needs the `cursor` block); `cursorClear:true` parks
276
+ the cursor off the element an attention beat is about, so the pointer never
277
+ sits on top of the thing you just highlighted. See *Attention*.
264
278
  - `keystrokes: true` — show a keystroke chip ("⌘ K", "Enter") on every `press`
265
279
  (per-action `keystrokes` overrides). See *Attention*.
266
280
  - `captions: {position?="bottom"|"top", style?="pill"|"bar"|"none", font?,
267
- size?, color?, background?}` — caption strip placement and look; scenes may
281
+ size?, color?, background?, maxWords?, maxCueMs?, minCueMs?, maxLines?,
282
+ maxCharsPerLine?}` — caption strip placement and look; scenes may
268
283
  override `position` with their own `captions`. Compose also auto-flips a cue
269
284
  to the top while an overlay occupies the bottom band. See *Attention* and
270
285
  *Produced look*.
@@ -273,7 +288,7 @@ Cinematic keys (all opt-in; omit for the plain look):
273
288
  - `hide: [selector, …]` — hide elements at **record** time (teasers, cookie
274
289
  bars, ad slots); scenes may add their own `hide`. See *Attention*.
275
290
  - `frame: {padding?, background?, radius?, shadow?, chrome?="none"|"browser"|"mac",
276
- safeTop?, title?, url?}` — pad the video onto a styled canvas with a rounded,
291
+ safeTop?, title?, url?}` (`url: "auto"` tracks the page per scene) — pad the video onto a styled canvas with a rounded,
277
292
  shadowed window and optional browser chrome. See *Produced look*.
278
293
  - `brand: {logo?, accent?, font?, watermark?}` — brand kit: accent for cards /
279
294
  frame tint / attention overlays, font for cards + captions + callouts +
@@ -295,7 +310,9 @@ Cinematic keys (all opt-in; omit for the plain look):
295
310
 
296
311
  Each scene: `id`, `title?` (chapter name), `narration`, `voice?`, `music?`,
297
312
  `zoom?` (false to disable), `autoIdle?` (overrides the top-level setting),
298
- `captions?`, `redact?`, `hide?`, `actions[]`.
313
+ `transition?` (`false` hard-cuts *into* this scene, or an object overriding the
314
+ top-level crossfade for that one boundary), `captions?`, `redact?`, `hide?`,
315
+ `actions[]`.
299
316
  Narration may carry `{{@name}}` anchor markers (see *Narration-anchored
300
317
  beats*); the engine strips them at load and keeps the word index on the
301
318
  scene as `anchors` (don't author that field).
@@ -317,6 +334,13 @@ scene join is re-encoded (the default hard-cut path is a lossless stream-copy
317
334
  concat). Needs ≥2 scenes; a shorter `durationMs` (250–400) reads as a snappy
318
335
  dissolve, longer (600+) as a slow cinematic fade.
319
336
 
337
+ A dissolve across a **navigation** ghosts — two unrelated pages bleed through
338
+ each other — so compose hard-cuts any boundary where the next scene opens on a
339
+ load or either side is a `goto`/`back`. Override per boundary with a scene's
340
+ own `transition`: `"transition": false` on a scene forces a hard cut into it,
341
+ and `"transition": {"durationMs": 700}` overrides the top-level timing for that
342
+ one join. The compose log names every auto-cut boundary.
343
+
320
344
  **`hold: {mode?, driftScale?, backoffMs?}`** — controls the **hold** compose
321
345
  adds when a scene's narration runs longer than its (already x1.6-slowed)
322
346
  recording. By default the last frame is frozen for the remainder
@@ -326,14 +350,20 @@ default 1.06 ≈ 6% over the hold) — motion that reads as intentional rather
326
350
  than a stalled screen. `backoffMs` (0–2000) takes the held frame from that
327
351
  many ms *before* the scene's end instead of the very last frame, so a hold
328
352
  never lands on a half-drawn transition, a spinner, or a white flash at the
329
- tail of a navigation. Only scenes that actually hold are affected; a scene
353
+ tail of a navigation. A drift that is too short reads as a jitter rather than a
354
+ move, so drifts below `minDriftMs` (default 2000) fall back to a freeze; so
355
+ does a drift inside a `frame` block, where the static window chrome makes the
356
+ push-in obviously fake — set `driftInFrame: true` if you want it anyway. Compose
357
+ logs each fallback (`drift hold → freeze`). Only scenes that actually hold are affected; a scene
330
358
  whose actions fill the narration is untouched, and a scene with no hold is
331
359
  byte-identical. The compose log and `output/report.json` list every scene's
332
360
  hold length and percentage; the lint predicts them before the take.
333
361
 
334
362
  **`output: {width, height, fit?, background?}`** — reframes the finished video
335
363
  (cards and captions already baked in) to a target size, applied as the last
336
- step. `fit`:
364
+ step. `fit` defaults to `"contain"`, and a `preset`'s `fit` applies only when
365
+ you don't set one — so `{"preset": "youtube", "fit": "cover"}` exports
366
+ edge-to-edge with no letterbox bars. `fit`:
337
367
  - `"contain"` (default) — scale to fit inside `width×height` and pad the
338
368
  remainder with `background` (ffmpeg color syntax, e.g. `black`, `0x1a1a1a`;
339
369
  default black). Letterbox/pillarbox bars, nothing cropped.
@@ -460,6 +490,19 @@ moves the strip to the top edge. Without a per-scene override, compose
460
490
  box or a keystroke chip sitting in the bottom band, so a mark and a caption
461
491
  never collide (`report.json` → `attention.captionsFlipped`).
462
492
 
493
+ **Cursor gets out of the way.** `attention: {cursorClear: true}` moves the
494
+ pointer off the element a highlight/spotlight/callout is about (to the nearest
495
+ free side, at record time) before the mark is drawn — otherwise the cursor
496
+ often sits exactly on the thing you just asked the viewer to look at. Pair it
497
+ with `hover {anchor:"edge"}` when you want a specific side.
498
+
499
+ **Late layout.** Attention rects are measured after the element's box has been
500
+ *stable* for a beat (bounded wait), and re-measured after the dwell: if the
501
+ target moved more than ~24px in the meantime (a font swap, a lazy image, an
502
+ async row) the take uses the settled position and records a warning on that
503
+ action in `timeline.json` (also printed by record). A target that scrolled out
504
+ of view warns too, instead of silently marking empty screen.
505
+
463
506
  **Redact (compose-time blur).** `redact: [{selector, frame?, blur?=14}]` at
464
507
  the top level and/or per scene. After every action the player re-measures
465
508
  each selector (all matches, up to 8) and logs a span per box; compose crops,
@@ -502,6 +545,11 @@ shadowed window on a styled canvas, with captions in the padding band below.
502
545
  - `radius` (default 14), `shadow` (default true).
503
546
  - `chrome`: `"browser"` (neutral dots + address pill) or `"mac"` (traffic
504
547
  lights). `url` (wins) or `title` fills the pill; omit both for an empty bar.
548
+ `"url": "auto"` **tracks the page**: each scene's pill shows the host (plus
549
+ path) the scene is actually on, taken from the take's `goto`s — use it for a
550
+ demo that moves between hosts, where one fixed address would be a lie. The
551
+ pill only changes at scene boundaries (it's a time-gated overlay, not a live
552
+ browser).
505
553
  For a phone-sized take (`video: {width: 390, height: 844}` — record at the
506
554
  mobile viewport) use `"iphone"` (rounded bezel + dynamic island) or
507
555
  `"android"` (bezel + punch-hole): no bar, `radius` defaults to 40, and the
@@ -559,7 +607,31 @@ frames, the SRT/VTT captions, and `walkthrough.json`. Frames come from the
559
607
  **Poster** (`output.poster: true`): also writes `output/poster.png`
560
608
  (`poster.<lang>.png` for language variants) — the first content frame after
561
609
  the intro card, for READMEs, social cards and `<video poster>`. The path is
562
- in `report.json` as `poster`.
610
+ in `report.json` as `poster`. It skips past a blank opening (below), so the
611
+ poster isn't a white rectangle; `"poster": 4200` takes the still at that
612
+ millisecond instead, when you want a specific moment.
613
+
614
+ **Blank opening** (`output.trimLeadingBlank: true`): a take that starts on a
615
+ page still painting opens on white/blank frames. With this on, compose trims
616
+ the flat lead of the first scene (capped by `output.blankOpenCapMs`, default
617
+ 1200) and re-aligns the scene's cursor, zoom and anchors to the trim. Compose
618
+ always *measures* the opening either way and reports it as `blankOpenMs` plus
619
+ a `blank-open` warning past ~400 ms; `aidemo lint` forecasts the same pitfall
620
+ from a first scene that opens with a `goto` and no wait.
621
+
622
+ ## Capture resolution (`video.deviceScaleFactor`)
623
+
624
+ `video: {width, height}` is the **logical** viewport — the CSS px the page
625
+ lays out in. `deviceScaleFactor` (1–3, default 1) is how many real pixels each
626
+ of those gets, exactly like a Retina display: `{"width": 1280, "height": 720,
627
+ "deviceScaleFactor": 2}` records a 2560×1440 file of a 1280×720 page. Text and
628
+ UI keep their on-screen *size* but get twice the pixels, which is the fix for
629
+ a demo that looks soft after a zoom or a 1920 export, and for mobile-viewport
630
+ takes whose small type turns to mush. Cost: bigger raw footage and a slower
631
+ compose — 2 is enough for any delivery size; 3 is for stills. Bumping it
632
+ changes the recording, so it's a re-record, not a recompose. `aidemo lint`
633
+ warns when a viewport that small is about to be blown up on export
634
+ (`viewport-legibility`).
563
635
 
564
636
  ## Motion blur & cursor
565
637
 
@@ -581,6 +653,10 @@ without re-recording**:
581
653
  - `hideScenes: ["s5"]` — hide the cursor only on those scenes (e.g. a
582
654
  full-screen result or confirmation).
583
655
  - `scale: 1.3` — resize the cursor (1 = the 24px baseline arrow).
656
+ - `color`, `outline` (stroke color), `outlineWidth` (0–6) and `halo: true`
657
+ (a soft dark glow behind it) — make the cursor readable over a brand-colored
658
+ or busy surface, where a plain white arrow disappears. `halo` is the
659
+ one-key fix; `outline` is for matching a brand palette exactly.
584
660
 
585
661
  How it works: with a `cursor` block present, `record` leaves the take
586
662
  **cursor-free** and logs the cursor path into `timeline.json`; `compose` draws
@@ -603,12 +679,26 @@ for UI that re-renders under the cursor) and `anchor?` (land this action on a
603
679
  screen and the take doesn't get longer for it.
604
680
  - `{op:"type", target, text, humanize?}` — human-cadence typing
605
681
  - `{op:"press", key}` — e.g. "Enter"
606
- - `{op:"click", target, followPopup?}` · `{op:"hover", target}` —
682
+ - `{op:"click", target, followPopup?}` · `{op:"hover", target, anchor?,
683
+ offset?}` — `anchor` (`"center"` default, or `"edge"|"top"|"bottom"|"left"|
684
+ "right"`) and `offset:{x,y}` aim the pointer at a *part* of the element:
685
+ `"anchor": "top"` hovers the top edge of a row so the tooltip it opens isn't
686
+ covered by the cursor, `"edge"` parks just outside it.
687
+
607
688
  `followPopup:true` handles a link/button that opens a **new tab**
608
689
  (`target=_blank`, `window.open`): the tab is closed and the recorded tab
609
690
  navigates to its URL. The take is one window — a second tab is never in the
610
691
  video, so there is no `newTab`/`switchTab`; write the story in one tab.
611
692
  - `{op:"back"}` — browser history back, same readiness wait as `goto`.
693
+ - `{op:"dialog", action?="accept"|"dismiss", promptText?, holdMs?, required?}`
694
+ — **arm a native dialog handler** for the *next* action in the scene.
695
+ `alert`/`confirm`/`window.print` and friends block the page and are invisible
696
+ to a recording otherwise (Playwright auto-dismisses an unhandled dialog, so
697
+ the take silently loses the step). Put the `dialog` action immediately
698
+ **before** the click that triggers it; `holdMs` waits before answering (the page
699
+ stays paused, so the moment reads as deliberate), `promptText`
700
+ answers a `window.prompt`, and `required: true` fails the scene if no dialog
701
+ ever appeared (the default is best-effort).
612
702
  - `{op:"select", target, value? | label?}` — native `<select>`: the cursor
613
703
  clicks it, the option is committed programmatically (the OS dropdown never
614
704
  paints into a recording), `change` fires.
@@ -948,6 +1038,14 @@ Always set `"last": true` on widget targets (newest widget for this turn).
948
1038
  records the wrong numbers; resuming keeps the earned footage and picks up at
949
1039
  the scene that broke. Pair it with `setup.expectState: true` so the
950
1040
  carried-over-state warning stays quiet on the seeded profile.
1041
+ **But a plain resume still *replays* the earlier scenes' actions** to rebuild
1042
+ state — on a state-earning flow that replay itself fails (the first scene
1043
+ waits for a "stranger" element the profile no longer shows). Add
1044
+ `--no-replay` (`record`/`render`, `{noReplay: true}`) to skip the replayed
1045
+ scenes' actions entirely and trust the profile as it stands; the scene you
1046
+ resume from must then open with its own `goto`. This is also how you record
1047
+ one storyboard across two real days (day 0 today, day 1 after the app's
1048
+ date rolls over) on one profile.
951
1049
  The same flag is the cheap way to re-shoot
952
1050
  a tail you changed after an approved take.
953
1051
 
@@ -1053,6 +1151,21 @@ scene's `anchors[]` shows `targetMs` / `landedMs` / `offMs`). A held scene
1053
1151
  is a storyboard problem, not a compose problem: give it on-screen beats or
1054
1152
  shorten its narration, then re-run `voice` + `compose`.
1055
1153
 
1154
+ Then run **`aidemo qa <dir>`** (MCP: the `qa` job) on the finished file — a
1155
+ browser-free, API-free audit of what actually shipped: length vs
1156
+ `targetLengthSeconds`, a blank opening or blank poster, master loudness (LUFS
1157
+ / true peak) and whether the music bed is audible under the narration,
1158
+ per-scene hold percentage and compose warnings, caption cue stats, and
1159
+ hygiene checks (a fixed `frame.url` on a demo that visits several hosts, a
1160
+ non-English demo with ASCII-only chapter titles). `--json` for the structured
1161
+ report, `--strict` to exit non-zero on any finding.
1162
+
1163
+ `render`/`compose` also take **`--strict`** (exit non-zero when compose emitted
1164
+ any warning — the CI gate), and every warning that has a moment on screen
1165
+ leaves a PNG in `output/warnings/<name>.png` so you can look at the frame
1166
+ instead of guessing. For a fast visual scan, `aidemo frames <dir> --sheet`
1167
+ writes a single contact-sheet PNG (`--columns`, default 5) next to the frames.
1168
+
1056
1169
  Then play (or frame-extract — `aidemo frames <dir>` / the `frames` job)
1057
1170
  `output/final-demo.mp4`: cursor glides and clicks pulse, narration matches
1058
1171
  on-screen actions, captions are readable and in sync,
@@ -1206,6 +1319,15 @@ all) using each scene's measured duration from `voice.json`; only the timing
1206
1319
  measured speech rhythm). `captions` logs a one-line reminder of this whenever
1207
1320
  the resolved language isn't English.
1208
1321
 
1322
+ **Cue segmentation.** Cues are broken at sentence ends first, then at clause
1323
+ punctuation, then before a conjunction, and only then on length — so a cue
1324
+ stops reading like it was cut mid-thought ("we open the" / "settings panel").
1325
+ Very short cues are merged forward. Tune per storyboard under `captions`:
1326
+ `maxWords` (12), `maxCueMs` (6000), `minCueMs` (1000), `maxLines` (2),
1327
+ `maxCharsPerLine` (38). `captions` logs the resulting cue stats (count, median
1328
+ length, how many fall under `minCueMs`), and `aidemo qa` flags a run with lots
1329
+ of flash cues or mid-clause splits.
1330
+
1209
1331
  ## Multi-language renders (one take, N languages)
1210
1332
 
1211
1333
  Record the browser flow **once**, then ship the same demo in several languages —
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@tandryukha/aidemo",
3
- "version": "0.14.1",
3
+ "version": "0.15.0",
4
4
  "license": "MIT",
5
5
  "description": "AI Demo Engine — turn a storyboard into a narrated, captioned product demo video",
6
6
  "type": "module",
package/src/captions.ts CHANGED
@@ -11,6 +11,7 @@ import {
11
11
  VoiceManifestSchema,
12
12
  CaptionsManifestSchema,
13
13
  type CaptionsManifest,
14
+ type CaptionsConfig,
14
15
  type Storyboard,
15
16
  } from "./types.js";
16
17
  import { resolveNarrationLanguage } from "./i18n.js";
@@ -38,8 +39,52 @@ interface Word {
38
39
  end: number; // seconds
39
40
  }
40
41
 
41
- const MAX_WORDS = 6;
42
- const MAX_CUE_MS = 3000;
42
+ /**
43
+ * Cue-segmentation defaults (issue #48). The old 6-word / 3 s hard cap split
44
+ * half of all cue breaks mid-clause ("…nobody has / filled in a form, and no /
45
+ * account exists yet"), which is harder to read than the narration. Broadcast
46
+ * practice is ~2 lines x ~37 characters, 1–6 s per cue, broken at punctuation
47
+ * or a conjunction. Override per storyboard via the `captions` block.
48
+ */
49
+ const DEFAULT_SEG: Required<
50
+ Pick<CaptionsConfig, "maxWords" | "maxCueMs" | "minCueMs" | "maxLines" | "maxCharsPerLine">
51
+ > = {
52
+ maxWords: 12,
53
+ maxCueMs: 6000,
54
+ minCueMs: 1000,
55
+ maxLines: 2,
56
+ maxCharsPerLine: 38,
57
+ };
58
+
59
+ /** Conjunctions/discourse markers that make a decent break point, by language. */
60
+ const CONJUNCTIONS = new Set([
61
+ // en
62
+ "and", "but", "so", "or", "then", "because", "while", "when", "which", "that", "if",
63
+ // et
64
+ "ja", "aga", "või", "siis", "kui", "sest", "mis", "ning", "kuid",
65
+ // de / nl / sv
66
+ "und", "aber", "oder", "dann", "weil", "wenn", "och", "men", "eller", "en", "maar",
67
+ // fr / es / it / pt
68
+ "et", "mais", "ou", "donc", "parce", "quand", "y", "pero", "o", "entonces", "porque",
69
+ "e", "ma", "oppure", "quindi", "perché", "mas", "então",
70
+ ]);
71
+
72
+ type SegConfig = typeof DEFAULT_SEG;
73
+
74
+ function segConfig(storyboard?: Storyboard): SegConfig {
75
+ return { ...DEFAULT_SEG, ...cleanSeg(storyboard?.captions) };
76
+ }
77
+
78
+ function cleanSeg(c: CaptionsConfig | undefined): Partial<SegConfig> {
79
+ if (!c) return {};
80
+ const out: Partial<SegConfig> = {};
81
+ if (c.maxWords != null) out.maxWords = c.maxWords;
82
+ if (c.maxCueMs != null) out.maxCueMs = c.maxCueMs;
83
+ if (c.minCueMs != null) out.minCueMs = c.minCueMs;
84
+ if (c.maxLines != null) out.maxLines = c.maxLines;
85
+ if (c.maxCharsPerLine != null) out.maxCharsPerLine = c.maxCharsPerLine;
86
+ return out;
87
+ }
43
88
 
44
89
  /** Whisper's `prompt` is only used to bias roughly its first ~224 tokens of
45
90
  * context; cap what we send so a long demo's script doesn't balloon the
@@ -93,7 +138,8 @@ export async function generateCaptions(
93
138
  for (let i = 0; i < total; i++) opts.onSceneComplete?.(ids[i] ?? "", i, total);
94
139
  };
95
140
  if (total > 0) opts.onSceneStart?.(ids[0] ?? "", 0, total);
96
- const config = captionConfig(await loadGapMs(project));
141
+ const seg = segConfig(storyboard);
142
+ const config = captionConfig(await loadGapMs(project), seg);
97
143
 
98
144
  const prompt = storyboard ? buildSttPrompt(storyboard) : undefined;
99
145
  const language = opts.language ?? resolveNarrationLanguage(storyboard, project.lang);
@@ -151,7 +197,7 @@ export async function generateCaptions(
151
197
  if (words.length === 0) {
152
198
  log("no word timestamps returned; captions may be empty");
153
199
  }
154
- const cues = groupWords(words, sceneEnds);
200
+ const cues = groupWords(words, sceneEnds, seg);
155
201
  await writeCaptionFiles(project, cues);
156
202
  await writeCaptionsManifest(project, {
157
203
  mode: "stt",
@@ -252,7 +298,8 @@ export async function generateCaptionsOffline(
252
298
  await readJson(project.voiceManifestPath)
253
299
  );
254
300
  const durById = new Map(voice.scenes.map((s) => [s.id, s.durationMs]));
255
- const config = captionConfig(voice.gapMs);
301
+ const seg = segConfig(storyboard);
302
+ const config = captionConfig(voice.gapMs, seg);
256
303
 
257
304
  // Per-scene caption identity = narration + this scene's measured duration (+
258
305
  // grouping config). The scene-relative word timings depend on nothing else, so
@@ -333,7 +380,7 @@ export async function generateCaptionsOffline(
333
380
  cursor += voice.gapMs;
334
381
  }
335
382
 
336
- const cues = groupWords(words, sceneEnds);
383
+ const cues = groupWords(words, sceneEnds, seg);
337
384
  await writeCaptionFiles(project, cues);
338
385
  await writeCaptionsManifest(project, {
339
386
  mode: "offline",
@@ -369,6 +416,12 @@ async function writeCaptionFiles(project: Project, cues: Cue[]): Promise<void> {
369
416
  await fs.writeFile(project.captionsSrtPath, toSrt(cues));
370
417
  await fs.writeFile(project.captionsVttPath, toVtt(cues));
371
418
  await writeJson(project.captionsCuesPath, cues);
419
+ // Readability, so an agent sees it without opening the SRT (issue #48).
420
+ const stats = cueStats(cues);
421
+ log(
422
+ `cues: ${cues.length}, ${stats.midClausePct}% mid-clause breaks, ` +
423
+ `${stats.short} shorter than 1s`
424
+ );
372
425
  ok(`captions → ${project.captionsSrtPath} (${cues.length} cues)`);
373
426
  ok(`captions → ${project.captionsCuesPath}`);
374
427
  }
@@ -381,8 +434,8 @@ async function writeCaptionFiles(project: Project, cues: Cue[]): Promise<void> {
381
434
  // ---------------------------------------------------------------------------
382
435
 
383
436
  /** Grouping parameters that affect cue segmentation — part of the cache key. */
384
- function captionConfig(gapMs: number): CaptionsManifest["config"] {
385
- return { gapMs, maxWords: MAX_WORDS, maxCueMs: MAX_CUE_MS };
437
+ function captionConfig(gapMs: number, cfg: SegConfig = DEFAULT_SEG): CaptionsManifest["config"] {
438
+ return { gapMs, maxWords: cfg.maxWords, maxCueMs: cfg.maxCueMs };
386
439
  }
387
440
 
388
441
  /** Inter-scene gap from voice.json (0 when absent — captions need voice.json). */
@@ -451,38 +504,129 @@ async function sceneIdsFor(project: Project): Promise<string[]> {
451
504
  return voice.success ? voice.data.scenes.map((s) => s.id) : [];
452
505
  }
453
506
 
454
- function groupWords(words: Word[], sceneEnds: number[]): Cue[] {
455
- const cues: Cue[] = [];
507
+ /** Text of a run of words, with punctuation re-attached. */
508
+ function cueText(ws: Word[]): string {
509
+ // trim each word: faster-whisper servers pad words with leading spaces
510
+ return ws
511
+ .map((w) => w.word.trim())
512
+ .join(" ")
513
+ .replace(/\s+([,.!?;:])/g, "$1")
514
+ .trim();
515
+ }
516
+
517
+ /** A word that ends a clause (punctuation) — the best place to break. */
518
+ function endsClause(w: Word): boolean {
519
+ return /[,;:—–]$/.test(w.word.trim());
520
+ }
521
+
522
+ function endsSentence(w: Word): boolean {
523
+ return /[.!?]["'”’)]?$/.test(w.word.trim());
524
+ }
525
+
526
+ /**
527
+ * Split a run of words into cues at the most readable boundaries: sentence
528
+ * ends first, then clause punctuation, then a conjunction, and only then on
529
+ * pure length. Never leaves a 2–3 word tail dangling when the previous
530
+ * boundary can absorb it.
531
+ */
532
+ function splitRun(ws: Word[], cfg: SegConfig): Word[][] {
533
+ const capChars = cfg.maxLines * cfg.maxCharsPerLine;
534
+ const fits = (run: Word[]): boolean =>
535
+ run.length <= cfg.maxWords &&
536
+ cueText(run).length <= capChars &&
537
+ (run[run.length - 1].end - run[0].start) * 1000 <= cfg.maxCueMs;
538
+ if (ws.length === 0) return [];
539
+ if (fits(ws)) return [ws];
540
+ // The furthest index that still fits, then walk back to a real boundary.
541
+ let end = 1;
542
+ while (end < ws.length && fits(ws.slice(0, end + 1))) end++;
543
+ const floor = Math.max(1, Math.ceil(end * 0.45));
544
+ let cut = end;
545
+ for (let i = end - 1; i >= floor; i--) {
546
+ if (endsSentence(ws[i]) || endsClause(ws[i])) {
547
+ cut = i + 1;
548
+ break;
549
+ }
550
+ }
551
+ if (cut === end) {
552
+ for (let i = end - 1; i >= floor; i--) {
553
+ const next = ws[i + 1];
554
+ if (next && CONJUNCTIONS.has(next.word.trim().toLowerCase().replace(/[^\p{L}]/gu, ""))) {
555
+ cut = i + 1;
556
+ break;
557
+ }
558
+ }
559
+ }
560
+ return [ws.slice(0, cut), ...splitRun(ws.slice(cut), cfg)];
561
+ }
562
+
563
+ /** Merge a too-short cue into the neighbour it still fits in. */
564
+ function mergeShort(runs: Word[][], cfg: SegConfig): Word[][] {
565
+ const capChars = cfg.maxLines * cfg.maxCharsPerLine;
566
+ const room = (a: Word[], b: Word[]): boolean =>
567
+ a.length + b.length <= cfg.maxWords &&
568
+ cueText([...a, ...b]).length <= capChars &&
569
+ (b[b.length - 1].end - a[0].start) * 1000 <= cfg.maxCueMs;
570
+ const out: Word[][] = [];
571
+ for (const run of runs) {
572
+ const durMs = (run[run.length - 1].end - run[0].start) * 1000;
573
+ const prev = out[out.length - 1];
574
+ if (durMs < cfg.minCueMs && prev && room(prev, run)) {
575
+ out[out.length - 1] = [...prev, ...run];
576
+ continue;
577
+ }
578
+ out.push(run);
579
+ }
580
+ return out;
581
+ }
582
+
583
+ function groupWords(words: Word[], sceneEnds: number[], cfg: SegConfig = DEFAULT_SEG): Cue[] {
584
+ // Scene boundaries and sentence ends are hard breaks; inside those runs the
585
+ // split is clause-aware.
586
+ const runs: Word[][] = [];
456
587
  let cur: Word[] = [];
457
588
  let sceneIdx = 0;
458
- const flush = () => {
459
- if (cur.length === 0) return;
460
- cues.push({
461
- index: cues.length + 1,
462
- startMs: Math.round(cur[0].start * 1000),
463
- endMs: Math.round(cur[cur.length - 1].end * 1000),
464
- // trim each word: faster-whisper servers pad words with leading spaces
465
- text: cur.map((w) => w.word.trim()).join(" ").replace(/\s+([,.!?])/g, "$1").trim(),
466
- });
589
+ const cut = (): void => {
590
+ if (cur.length) runs.push(cur);
467
591
  cur = [];
468
592
  };
469
593
  for (const w of words) {
470
594
  const startMs = w.start * 1000;
471
- // Crossed into a later scene? Break the cue there.
472
- while (
473
- sceneIdx < sceneEnds.length - 1 &&
474
- startMs >= sceneEnds[sceneIdx]
475
- ) {
476
- flush();
595
+ while (sceneIdx < sceneEnds.length - 1 && startMs >= sceneEnds[sceneIdx]) {
596
+ cut();
477
597
  sceneIdx++;
478
598
  }
479
599
  cur.push(w);
480
- const durMs = (w.end - cur[0].start) * 1000;
481
- const endsSentence = /[.!?]$/.test(w.word.trim());
482
- if (cur.length >= MAX_WORDS || durMs >= MAX_CUE_MS || endsSentence) flush();
600
+ if (endsSentence(w)) cut();
483
601
  }
484
- flush();
485
- return cues;
602
+ cut();
603
+ const split = runs.flatMap((r) => splitRun(r, cfg));
604
+ return mergeShort(split, cfg).map((ws, i) => ({
605
+ index: i + 1,
606
+ startMs: Math.round(ws[0].start * 1000),
607
+ endMs: Math.round(ws[ws.length - 1].end * 1000),
608
+ text: cueText(ws),
609
+ }));
610
+ }
611
+
612
+ /** Readability stats for the log: mid-clause breaks and too-short cues. */
613
+ export function cueStats(cues: Cue[], minCueMs = DEFAULT_SEG.minCueMs): {
614
+ short: number;
615
+ midClause: number;
616
+ midClausePct: number;
617
+ } {
618
+ let short = 0;
619
+ let midClause = 0;
620
+ cues.forEach((c, i) => {
621
+ if (c.endMs - c.startMs < minCueMs) short++;
622
+ const next = cues[i + 1];
623
+ if (!next) return;
624
+ const cleanBreak = /[.!?,;:—–]["'”’)]?$/.test(c.text.trim());
625
+ const startsLower = /^\p{Ll}/u.test(next.text.trim());
626
+ if (!cleanBreak && startsLower) midClause++;
627
+ });
628
+ const breaks = Math.max(1, cues.length - 1);
629
+ return { short, midClause, midClausePct: Math.round((midClause / breaks) * 100) };
486
630
  }
487
631
 
488
632
  function toSrt(cues: Cue[]): string {