@openvidstudio/mcp-server 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/dist/beats.d.ts +47 -0
  2. package/dist/beats.js +9 -0
  3. package/dist/capture.d.ts +116 -0
  4. package/dist/capture.js +148 -0
  5. package/dist/config.d.ts +58 -0
  6. package/dist/config.js +130 -0
  7. package/dist/paths.d.ts +20 -0
  8. package/dist/paths.js +57 -0
  9. package/dist/scenes/pngSize.d.ts +11 -0
  10. package/dist/scenes/pngSize.js +75 -0
  11. package/dist/scenes/templates.d.ts +42 -0
  12. package/dist/scenes/templates.js +468 -0
  13. package/dist/server.d.ts +16 -0
  14. package/dist/server.js +37 -0
  15. package/dist/stdio.d.ts +2 -0
  16. package/dist/stdio.js +33 -0
  17. package/dist/tools/captureScreenRecording.d.ts +28 -0
  18. package/dist/tools/captureScreenRecording.js +181 -0
  19. package/dist/tools/captureScreenshot.d.ts +34 -0
  20. package/dist/tools/captureScreenshot.js +148 -0
  21. package/dist/tools/contactSheet.d.ts +36 -0
  22. package/dist/tools/contactSheet.js +116 -0
  23. package/dist/tools/generateNarration.d.ts +54 -0
  24. package/dist/tools/generateNarration.js +185 -0
  25. package/dist/tools/importHiggsfieldClip.d.ts +41 -0
  26. package/dist/tools/importHiggsfieldClip.js +108 -0
  27. package/dist/tools/index.d.ts +13 -0
  28. package/dist/tools/index.js +47 -0
  29. package/dist/tools/initProject.d.ts +37 -0
  30. package/dist/tools/initProject.js +166 -0
  31. package/dist/tools/mcp.d.ts +33 -0
  32. package/dist/tools/mcp.js +36 -0
  33. package/dist/tools/planBeats.d.ts +40 -0
  34. package/dist/tools/planBeats.js +143 -0
  35. package/dist/tools/preflight.d.ts +30 -0
  36. package/dist/tools/preflight.js +170 -0
  37. package/dist/tools/qcExtractFrames.d.ts +25 -0
  38. package/dist/tools/qcExtractFrames.js +129 -0
  39. package/dist/tools/renderVideo.d.ts +43 -0
  40. package/dist/tools/renderVideo.js +120 -0
  41. package/dist/tools/scaffoldScene.d.ts +23 -0
  42. package/dist/tools/scaffoldScene.js +147 -0
  43. package/dist/tools/stitchComposition.d.ts +24 -0
  44. package/dist/tools/stitchComposition.js +230 -0
  45. package/dist/tools/validateBeats.d.ts +14 -0
  46. package/dist/tools/validateBeats.js +164 -0
  47. package/dist/tools/validateScenes.d.ts +31 -0
  48. package/dist/tools/validateScenes.js +191 -0
  49. package/dist/tools/writeBeatsFile.d.ts +20 -0
  50. package/dist/tools/writeBeatsFile.js +81 -0
  51. package/dist/util.d.ts +43 -0
  52. package/dist/util.js +163 -0
  53. package/docs/API.md +344 -0
  54. package/docs/CAPTURE.md +106 -0
  55. package/docs/HIGGSFIELD.md +118 -0
  56. package/docs/NARRATION.md +111 -0
  57. package/docs/OVERVIEW.md +112 -0
  58. package/docs/PIPELINE.md +173 -0
  59. package/docs/PLANNING.md +206 -0
  60. package/docs/SCRIPT.md +50 -0
  61. package/docs/STYLE.md +88 -0
  62. package/package.json +54 -0
  63. package/templates/default/CLAUDE.md +63 -0
  64. package/templates/default/openvidstudio-core/package.json +51 -0
  65. package/templates/default/openvidstudio-core/scripts/gen-sfx.sh +74 -0
  66. package/templates/default/openvidstudio-core/src/ChecklistPanel.tsx +97 -0
  67. package/templates/default/openvidstudio-core/src/CinematicScene.tsx +53 -0
  68. package/templates/default/openvidstudio-core/src/CodePanel.tsx +122 -0
  69. package/templates/default/openvidstudio-core/src/CursorActor.tsx +90 -0
  70. package/templates/default/openvidstudio-core/src/DeviceFrame.tsx +74 -0
  71. package/templates/default/openvidstudio-core/src/KineticText.tsx +142 -0
  72. package/templates/default/openvidstudio-core/src/RepoCta.tsx +128 -0
  73. package/templates/default/openvidstudio-core/src/TerminalReplay.tsx +263 -0
  74. package/templates/default/openvidstudio-core/src/camera.tsx +141 -0
  75. package/templates/default/openvidstudio-core/src/index.ts +60 -0
  76. package/templates/default/openvidstudio-core/src/look.tsx +90 -0
  77. package/templates/default/openvidstudio-core/src/motion.ts +50 -0
  78. package/templates/default/openvidstudio-core/src/sfx.tsx +106 -0
  79. package/templates/default/openvidstudio-core/src/tokens.ts +41 -0
  80. package/templates/default/package.json +31 -0
  81. package/templates/default/public/sfx/bell.wav +0 -0
  82. package/templates/default/public/sfx/blip.wav +0 -0
  83. package/templates/default/public/sfx/click.wav +0 -0
  84. package/templates/default/public/sfx/key_a.wav +0 -0
  85. package/templates/default/public/sfx/key_b.wav +0 -0
  86. package/templates/default/public/sfx/key_c.wav +0 -0
  87. package/templates/default/public/sfx/key_enter.wav +0 -0
  88. package/templates/default/public/sfx/music-bed.mp3 +0 -0
  89. package/templates/default/public/sfx/success.wav +0 -0
  90. package/templates/default/public/sfx/whoosh.wav +0 -0
  91. package/templates/default/remotion.config.ts +13 -0
  92. package/templates/default/src/Root.tsx +9 -0
  93. package/templates/default/src/index.ts +4 -0
  94. package/templates/default/tsconfig.json +16 -0
@@ -0,0 +1,118 @@
1
+ # HIGGSFIELD.md: the AI b-roll tier
2
+
3
+ This is the protocol for the one gap real capture can't fill: an
4
+ atmosphere/b-roll shot with no real product UI in it at all (a desk/hands
5
+ shot, an establishing texture, an abstract visual metaphor). Everything else
6
+ in this pipeline stays real by provenance, per `STYLE.md`; this tier exists
7
+ for the beats where "real" was never an option in the first place.
8
+
9
+ ## Prerequisite
10
+
11
+ Using this tier at all requires two things the calling dev, not
12
+ openvidstudio, provides:
13
+
14
+ 1. **A Higgsfield subscription.**
15
+ 2. **The Higgsfield MCP connector enabled in the dev's AI coding agent
16
+ session** (Claude Code, Cursor, or whatever else the dev is running
17
+ openvidstudio's own MCP tools alongside).
18
+
19
+ openvidstudio never holds a Higgsfield API key, never authenticates to
20
+ Higgsfield, and never bills anyone for a generation. There is no clean way
21
+ for one local MCP server to reach into a different, already-authorized MCP
22
+ connection that only the calling agent's own client session holds: MCP's
23
+ protocol gives a server exactly three ways to ask its client for help
24
+ (sampling, for an LLM completion; elicitation, for a form/URL prompt to the
25
+ user; and a roots list), and none of them is "call this other tool on this
26
+ other server for me." So the design here isn't a workaround, it's the only
27
+ architecture the protocol actually supports: the calling agent uses its own
28
+ Higgsfield connection directly, and openvidstudio only receives what that
29
+ connection already produced.
30
+
31
+ ## Config gate
32
+
33
+ `import_higgsfield_clip` (the one tool this tier adds) only appears in this
34
+ server's tool list when a project's `openvidstudio.config.json` has
35
+ `"hasHiggsfield": true`. If that field is `false`, or missing entirely
36
+ (the default for any project that hasn't set it), the tool is not
37
+ registered at all, not present-but-erroring: it simply won't show up in the
38
+ calling agent's tool list for that project.
39
+
40
+ `init_project` is what sets this field, from the dev's own answer to
41
+ `PLANNING.md`'s intake question 4 ("Higgsfield access: yes or no"). A
42
+ project that answered "no" (or never ran that question) genuinely does not
43
+ have this tool available, and any beat that would have needed it either
44
+ gets dropped or reworked as a `dom-demo`, per `PLANNING.md`'s decision tree.
45
+
46
+ ## Scope: b-roll and atmosphere only, never product UI or on-screen text
47
+
48
+ This is `STYLE.md`'s provenance rule, restated as the operational
49
+ consequence for anyone writing a Higgsfield prompt: an AI-generated clip
50
+ must never carry on-screen product text or real UI. Real product surfaces
51
+ are ground truth by provenance (DOM-rendered, screenshotted, or
52
+ screen-recorded, all captured per `CAPTURE.md`); an AI-generated clip is not
53
+ ground truth of anything, and mixing the two undermines the entire premise
54
+ of this pipeline, that everything on screen came from the real product.
55
+
56
+ Be honest about what enforces this: nothing here does, at the code level.
57
+ Neither the calling agent's own Higgsfield generation call nor
58
+ `import_higgsfield_clip`'s ingest step inspects the resulting video's
59
+ content. This is a convention enforced by doc guidance and by whoever
60
+ reviews the video before it ships, not a filter that can catch a
61
+ prompt-writer who ignores it. A "Higgsfield-text soft-guard linter" (an
62
+ automated check that flags a generated clip likely to contain readable text
63
+ or UI-like chrome) is a plausible future nice-to-have, but it is explicitly
64
+ out of this tool's v1 scope and does not exist yet.
65
+
66
+ ## Prompt-writing guidance
67
+
68
+ Write a Higgsfield prompt around the shot's actual visual purpose in the
69
+ scene, the same way you'd brief a stock-footage search, not around "the
70
+ product." A prompt like:
71
+
72
+ > a hand typing on a mechanical keyboard, warm desk lighting, shallow depth
73
+ > of field
74
+
75
+ describes the atmosphere the beat needs and gives the model nothing to
76
+ hallucinate a screen or interface out of. A prompt like "show the product"
77
+ or "someone using the app" invites the model to render its own guess at UI,
78
+ text, or branding, which is exactly what this tier must never produce.
79
+ Before writing the prompt, know which specific beat the clip is standing in
80
+ for and what mood or texture it needs to carry; write to that, not to the
81
+ feature the beat happens to sit next to.
82
+
83
+ ## What `import_higgsfield_clip` actually does
84
+
85
+ `import_higgsfield_clip` is deliberately not a wrapper with opinions about
86
+ video generation. It does not call Higgsfield's `generate_video` or
87
+ `jobs_wait` tools itself, and it does not talk to Higgsfield at all. The
88
+ calling agent does that part, over its own already-authorized Higgsfield
89
+ MCP connection, exactly the way it would for any other Higgsfield request in
90
+ the same session. Once that generation finishes, the agent has a result:
91
+ either a downloadable URL or a local file path. `import_higgsfield_clip`'s
92
+ entire job is taking that result and landing it at this project's asset
93
+ convention.
94
+
95
+ Input:
96
+
97
+ ```
98
+ {
99
+ projectRoot?: string, // defaults to the calling agent's cwd
100
+ beatId: string,
101
+ source: { type: "url"; url: string } | { type: "path"; path: string },
102
+ outPath?: string, // override; default is public/video/<beatId>.mp4
103
+ }
104
+ ```
105
+
106
+ Behavior: fetches the URL or reads the local file, then writes those bytes
107
+ to `public/video/<beatId>.mp4` under `projectRoot`, the exact same
108
+ convention `capture_screen_recording` and `scaffold_scene`'s
109
+ `higgsfield-clip` kind already use. That convention match is load-bearing:
110
+ a `higgsfield-clip` scene's `<OffthreadVideo src=...>` expects the asset at
111
+ that path regardless of which tool produced it, so a `higgsfield-clip`
112
+ scene is structurally identical to a `real-recording` scene once the asset
113
+ exists.
114
+
115
+ Returns `{ outPath, beatId }` on success. On failure (a bad or unreachable
116
+ URL, a missing local file, a failed download), it returns a structured
117
+ error the same way every other tool in this package does; it never throws
118
+ uncaught.
@@ -0,0 +1,111 @@
1
+ # NARRATION.md: voice, music and sound
2
+
3
+ Narration used to be entirely the caller's problem. It is not any more:
4
+ `generate_narration` reads `beats.json` and writes one clip per beat to the path
5
+ `stitch_composition` already looks in. This doc covers what that tool does, what it
6
+ deliberately refuses to do, and why generated voiceover usually sounds synthetic.
7
+
8
+ ## The short version
9
+
10
+ ```
11
+ generate_narration -> public/audio/vo/<beatId>.mp3
12
+ stitch_composition -> picks them up automatically
13
+ ```
14
+
15
+ ## Why generated voiceover sounds like a robot
16
+
17
+ Rarely the engine. Almost always the stretching.
18
+
19
+ Every engine reads at its own pace, so a clip rarely lands exactly on its beat. The
20
+ obvious fix is to time-stretch it to fit. A pitch-preserving stretch past roughly ten
21
+ percent is audible, and past twenty it is the drawly, processed sound people mean by
22
+ "AI voice". Building the openvidstudio demo, one beat was stretched to **0.645**, a
23
+ 35 percent slowdown, and that single clip was most of why the whole video sounded
24
+ synthetic.
25
+
26
+ So `generate_narration` clamps stretching to **0.9 to 1.12**. When a line still does
27
+ not fit, it says so and names the beat rather than mangling it, because the real fix
28
+ is one of:
29
+
30
+ - **Too short for the beat.** Add a sentence, or shorten the beat.
31
+ - **Too long for the beat.** Cut words, or lengthen the beat.
32
+
33
+ Both are edits to `beats.json`, which is yours to change. Stretching audio to hide a
34
+ mismatch trades a timing problem for an audible one.
35
+
36
+ ## Pace
37
+
38
+ `validate_beats` enforces SCRIPT.md's 2.3 to 2.9 words per second as a hard floor and
39
+ ceiling, per beat. It is a floor as well as a ceiling, so a beat can be rejected for
40
+ being too sparse, which is not the failure most people expect.
41
+
42
+ `plan_beats` gives you the word budget per beat up front, so narration is written to
43
+ a number instead of guessed and revised.
44
+
45
+ ## Engines
46
+
47
+ | Engine | Quality | Speed | Runs locally |
48
+ |---|---|---|---|
49
+ | `edge-tts` (default) | good | about a minute for five minutes of speech | no, sends text to Microsoft |
50
+ | `say` (macOS) | fair | instant | yes |
51
+ | `espeak` | poor | instant | yes |
52
+
53
+ `edge-tts` needs `pip install edge-tts`.
54
+
55
+ **If the text must not leave the machine**, use `say` on macOS, or wire up a local
56
+ neural model. A local model on CPU is slow: one measured at a real time factor of
57
+ 41.9x, meaning six minutes of compute for nine seconds of audio, or about three and
58
+ a half hours for a five minute video. Fine overnight, not fine in a loop.
59
+
60
+ **If you want it to sound genuinely good**, record it yourself. A real voice beats
61
+ every engine here, and for a launch video it is worth the hour.
62
+
63
+ ## Silence is silent
64
+
65
+ `stitch_composition` skips a missing `public/audio/vo/<beatId>.mp3` with no error and
66
+ no warning. A half generated narration renders successfully and simply plays silent
67
+ in those beats.
68
+
69
+ Two things catch it:
70
+
71
+ - `generate_narration` reports which beats have audio and which do not
72
+ - `validate_scenes` flags every beat with no narration file
73
+
74
+ Neither is optional if you care whether the video has a voice.
75
+
76
+ ## Music and sound effects
77
+
78
+ Everything ships synthesized, generated by `scripts/gen-sfx.sh` with ffmpeg. Nothing
79
+ is sampled or licensed, so **a rendered video carries no third-party audio rights**.
80
+ No attribution, no content ID, no licensing questions.
81
+
82
+ | File | Sound |
83
+ |---|---|
84
+ | `key_a/b/c.wav` | Mechanical key thocks, three variants so typing is not a metronome |
85
+ | `key_enter.wav` | Deeper thock for the end of a command |
86
+ | `click.wav` | Cursor click |
87
+ | `whoosh.wav` | Camera move or cut |
88
+ | `blip.wav` | UI reveal |
89
+ | `success.wav` | Rising two tone |
90
+ | `bell.wav` | Struck harmonic stack, long decay |
91
+ | `music-bed.mp3` | 60s ambient pad, loops |
92
+
93
+ Regenerate any of them by editing `scripts/gen-sfx.sh` and running it. The
94
+ `sfx.tsx` API does not change, so you can swap in a recorded pack later without
95
+ touching a scene.
96
+
97
+ ### Using the music bed
98
+
99
+ ```tsx
100
+ import { MusicBed } from "@openvidstudio/core";
101
+
102
+ <MusicBed durationInFrames={9000} volume={0.16} />
103
+ ```
104
+
105
+ Keep it quiet. It sits under narration, and 0.16 is already at the edge of noticeable
106
+ when a voice is speaking over it.
107
+
108
+ ### If you want real music instead
109
+
110
+ Pixabay Music and the YouTube Audio Library are both no-attribution. Drop the file at
111
+ `public/audio/music-bed.mp3` and `stitch_composition` picks it up, same as narration.
@@ -0,0 +1,112 @@
1
+ # OVERVIEW.md: what openvidstudio is
2
+
3
+ openvidstudio is a free, open-source video-generation add-on for any
4
+ software project. It's built on Remotion (React-based programmatic video)
5
+ and Playwright (real-browser automation), and it's driven entirely through
6
+ MCP tools by whatever AI coding agent you're already using (Claude Code,
7
+ Cursor, ChatGPT, or anything else that speaks MCP). There's no separate app
8
+ to open, no timeline editor, no manual keyframing.
9
+
10
+ ## The core idea
11
+
12
+ Launch and demo videos for software usually mean one of two compromises:
13
+ record a screen and cut it in a traditional editor (slow, and it looks like
14
+ a screen recording), or hand a brief to an AI video generator (fast, but
15
+ the UI, the product's actual text, and every detail a viewer would
16
+ recognize get hallucinated away).
17
+
18
+ openvidstudio does neither. It builds a cinematic camera, push-ins, drift,
19
+ shallow depth of field, an oversized cursor acting as an on-screen actor,
20
+ that moves over your product's real, DOM-rendered UI, composited from real
21
+ Playwright-captured screenshots and (v1 scope) full-viewport screen
22
+ recordings of the actual running app. Nothing on screen is invented: every
23
+ piece of product text a viewer can read came from the real product.
24
+
25
+ Where the real UI genuinely can't produce a shot (an atmosphere beat, a
26
+ desk/hands shot, some establishing texture that isn't product UI at all),
27
+ openvidstudio has an optional Higgsfield AI b-roll tier for exactly that
28
+ gap, and only that gap: an AI-generated clip never carries on-screen
29
+ product text or real UI, full stop. See `HIGGSFIELD.md` for that tier's
30
+ protocol: the calling agent generates the clip itself over its own
31
+ Higgsfield MCP connection, and openvidstudio's `import_higgsfield_clip`
32
+ tool only ingests the result into this pipeline's asset convention.
33
+
34
+ ## Why not a hosted AI-storyboard tool?
35
+
36
+ There are hosted, MCP-callable pipelines that go storyboard to AI-generated
37
+ visuals plus VO plus music plus captions in one pass (Motion.so is one real
38
+ example). Those are genuinely fast for a rough pacing/storyboard draft to
39
+ react to, and openvidstudio doesn't try to replace that use case.
40
+
41
+ But that whole model, AI-generated visuals standing in for the product,
42
+ is the opposite of why openvidstudio exists: no traditional editor, no
43
+ AI-generated footage of the product itself, pixel-perfect real UI on
44
+ screen. openvidstudio deliberately doesn't work that way. A tool like that
45
+ isn't a component of this pipeline and isn't the model to reach for when
46
+ the goal is a video that shows what a real product actually looks like.
47
+
48
+ ## The intended flow
49
+
50
+ 1. **Intake.** The calling agent asks the guided question set in
51
+ `PLANNING.md` (or, if the dev pastes a README, feature list, or other
52
+ existing material instead, folds that into the same beat candidates;
53
+ see `PLANNING.md`'s free-form-input section).
54
+ 2. **Draft.** The agent turns intake answers into a `beats.json` draft:
55
+ timing, VO, and, critically, a decided `captureMethod` for every beat,
56
+ per `PLANNING.md`'s decision tree.
57
+ 3. **Approval gate.** The agent shows the dev the full drafted
58
+ `beats.json`, not a summary, and gets explicit approval before writing
59
+ anything to disk. This is a protocol rule for the calling agent, not
60
+ something enforced by the tooling itself; see `PLANNING.md`.
61
+ 4. **Execute.** Once approved, the agent runs capture (screenshots and/or
62
+ recordings, per `CAPTURE.md`'s protocol) and stitches the composition,
63
+ following `PIPELINE.md`'s beat-to-scene-to-composition-to-QC loop and
64
+ `STYLE.md`'s binding art-direction rules throughout.
65
+
66
+ ## Doc map
67
+
68
+ | Doc | What it covers |
69
+ |---|---|
70
+ | `PIPELINE.md` | Step-by-step: beats.json, scenes, composition, QC loop, render |
71
+ | `STYLE.md` | Binding art-direction rules: camera, depth of field, lighting, cursor, motion, palette, type |
72
+ | `CAPTURE.md` | The real-capture protocol for screenshots and screen recordings: the Playwright zoom-desync bug and its fix |
73
+ | `PLANNING.md` | The guided intake to beats.json protocol, including the capture-method decision tree |
74
+ | `SCRIPT.md` | VO pacing, duration math, and the claim-obligates-visual writing rule |
75
+ | `HIGGSFIELD.md` | AI b-roll tier protocol: prerequisites, the `hasHiggsfield` config gate, provenance scope, prompt-writing guidance, and how `import_higgsfield_clip` ingests a clip the calling agent already generated |
76
+
77
+ ## Two rules that changed on the way to being a public tool
78
+
79
+ This pipeline started as a private tool built for one person's own repos.
80
+ Two of its rules only made sense in that narrower context and don't hold up
81
+ once other people's projects are the ones going through it. Both are
82
+ documented changes here, not silent drops:
83
+
84
+ - **"No GitHub-star click action in a CTA."** The private version of this
85
+ pipeline treated this as a hard QC-fail rule: it assumed every video was
86
+ for a personal, no-self-promo repo. Here it's the default `RepoCta`
87
+ behavior (no star action baked in, unless a project opts in), not
88
+ something QC fails on. This pipeline now serves other people's projects,
89
+ and a hardcoded no-self-promo stance doesn't make sense for every user
90
+ the way it did for a single person's own repos.
91
+ - **Terminal windows with a fixed title and slash-command convention.** The
92
+ private version hardcoded a specific title and command prefix. Here it's
93
+ a documented default convention a project can override, not a QC-fail
94
+ rule, for the same reason: other people's terminals show other people's
95
+ commands.
96
+
97
+ ## One refinement: the "DOM text" rule is now stated by provenance, not medium
98
+
99
+ The private version's rule was "all text is DOM text," reasoned from
100
+ medium: if it wasn't rendered as DOM inside Remotion, it wasn't trusted.
101
+ That was too strict. A genuine screen recording of the real, running
102
+ product isn't AI-hallucinated; it's ground truth, exactly like a screenshot
103
+ is. A medium-based rule would have wrongly forbidden a legitimate capture
104
+ method just because it wasn't literally re-rendered as DOM text inside the
105
+ composition.
106
+
107
+ The restated rule (full detail in `STYLE.md`): real product UI/text,
108
+ whether DOM-rendered, screenshotted, or screen-recorded, is always allowed
109
+ on screen, because it's ground truth by provenance. AI-generated clips
110
+ (Higgsfield or any future model) never carry on-screen product text or real
111
+ UI; they stay b-roll/atmosphere only. This is a refinement over the private
112
+ original, not a silent rewrite.
@@ -0,0 +1,173 @@
1
+ # PIPELINE.md: how a new video gets made
2
+
3
+ Brief in, video out. Follow in order; `STYLE.md`'s rules are binding at
4
+ every step. Beat authoring itself is driven by `PLANNING.md`'s guided
5
+ intake process: read that first, before hand-writing a `beats.json`.
6
+
7
+ ## 0. Prereqs (once per machine)
8
+ - Install dependencies for the scaffolded project (`@openvidstudio/core`
9
+ plus Remotion/React, see the project's own `package.json`).
10
+ - An SFX pack exists at `public/sfx/` (regen: `bash scripts/gen-sfx.sh`,
11
+ needs ffmpeg).
12
+
13
+ ## 1. Beats file (the single source of truth)
14
+ Create `<video>/beats.json`:
15
+ - `fps`, `title`, `beats[]`: each beat has `id`, `start` (frame),
16
+ `duration` (frames), `visual`, `vo`.
17
+ - Beats must be contiguous (`start` = previous `start + duration`). 30 fps.
18
+ - This file drives: scene sequencing, the markers/SRT export, and the VO
19
+ script table.
20
+ - **`captureMethod`**: every beat's `visual` block now carries one of
21
+ `"screenshot"` | `"recording"` | `"dom-demo"` | `"higgsfield"`, decided
22
+ per `PLANNING.md`'s decision tree during intake. At a glance:
23
+ `screenshot` is a real captured still of the running product;
24
+ `recording` is a real captured full-viewport screen recording of the
25
+ running product; `dom-demo` is a hand-authored scene built from the
26
+ project's real design tokens/copy, for a claim the real product can't
27
+ currently back with a capture; `higgsfield` is an AI-generated b-roll
28
+ clip for a shot that isn't product UI at all. `PLANNING.md` owns the
29
+ full field spec and decision tree; this file just needs every beat to
30
+ carry the field.
31
+ - Two beat arcs, pick by product complexity:
32
+ - **Simple demo** (single core feature): hook (8s), problem (11s), demo
33
+ x2-3 (12-14s each), differentiator (10s), cta (9s). 60-75s total.
34
+ - **Packed-hook, multi-feature**: one dense ~15s hero beat whose VO
35
+ states every core thing the product does in one pass (what it is,
36
+ what a user can do, how it's verified/differentiated) before any demo
37
+ beat, written so a judge or cold viewer understands the whole pitch
38
+ from audio alone in the first 15s, even muted-video-unaware. Then one
39
+ demo beat per feature claimed in the hook (so nothing gets asserted in
40
+ VO without being shown), differentiator recap, cta. Total scales with
41
+ feature count but stays under whatever hard ceiling the brief sets:
42
+ budget ~10-20s/demo beat and check the running total as you add
43
+ beats, don't let it creep past the brief's stated limit "for
44
+ completeness."
45
+ - No em dashes in any on-screen string or VO line (writing rule).
46
+ - **Claiming a feature in VO obligates showing it.** If a beat's VO
47
+ asserts something the product does, either show the real thing or say
48
+ so isn't wired yet and build a labeled `dom-demo` instead (see
49
+ "Constructed mockups" below): never let VO overclaim past what the
50
+ visual backs up. Full generalized version of this rule: `SCRIPT.md`.
51
+
52
+ ## 2. Scenes
53
+ One file per beat. Rules:
54
+ - Any beat with `captureMethod: "screenshot"` or `"recording"`: capture it
55
+ per **CAPTURE.md**, not ad hoc. That doc exists because uncorrected
56
+ captures reliably come out small, soft, or bleed in content from a
57
+ neighboring page section, both tracing to the same root cause (an
58
+ uncompensated Chrome zoom desync in the Playwright profile).
59
+ - **Screen recordings** (`captureMethod: "recording"`): composite as an
60
+ `OffthreadVideo` inside a `Layer`, the same camera-keyframe pattern as a
61
+ static `<Img>`. The camera moves over the recording exactly like it
62
+ would over a still. v1 scope is fixed full-viewport recordings only: no
63
+ post-hoc DOM-rect cropping of a moving recording (that's future work; a
64
+ moving recording can't be re-cropped to a DOM rect the way a still
65
+ screenshot can, because the rect would need to track the content frame
66
+ by frame).
67
+ - **Constructed mockups** (`captureMethod: "dom-demo"`), for a claim the
68
+ product doesn't do yet, or where a real screenshot isn't available or
69
+ would show something contradicting the claim: build it as a hand-coded
70
+ panel using the project's real design tokens and colors, reusing real
71
+ copy, real claim IDs, or real UI conventions from an actual screenshot
72
+ elsewhere in the same video where possible, so it reads as a
73
+ continuation, not an invention. For example: a fictional product's
74
+ "export your data" beat, before that feature is wired up, might reuse
75
+ the exact card style the viewer already saw in a real, captured
76
+ screenshot two beats earlier: same panel, same accent color, same copy
77
+ voice. Never screenshot a real page whose own on-screen copy contradicts
78
+ the claim you're making about it. A real settings page stating "your
79
+ data stays private, never shared" directly contradicts a beat's VO
80
+ claiming a public-sharing feature; screenshotting it for that beat would
81
+ put the opposite of the VO on screen. Where a real adjacent page
82
+ genuinely backs part of the claim, use the real screenshot for that part
83
+ and confine the constructed part to just the aspirational connection: a
84
+ real screenshot of a real results page, bridged by one constructed
85
+ arrow to a hand-built "share" panel, since the results page is real but
86
+ the sharing wiring isn't.
87
+ - Every scene = `CinematicScene` with camera keyframes. Motion never stops
88
+ (min: subtle push-in).
89
+ - Captions, title cards, dips/fades: always in the `overlay` prop
90
+ (screen-space). Never inside the rig.
91
+ - Reuse the library first: `TerminalReplay`, `CodePanel`, `ChecklistPanel`,
92
+ `BrowserFrame`, `KineticText` (`TitleSlam`/`Caption`), `CursorActor`,
93
+ `RepoCta`. Build a new component only when a beat truly needs it, then
94
+ keep it generic.
95
+ - Terminal windows: pick a title and slash-command convention and stay
96
+ consistent within a video (see `OVERVIEW.md` for why this is now a
97
+ documented default, not a hard rule).
98
+ - CTAs: `RepoCta`, no star action by default (see `OVERVIEW.md`).
99
+ - Camera keyframes derive from the terminal timeline
100
+ (`computeTerminalTimeline`) where applicable, so cuts land frame-exact on
101
+ content events.
102
+ - SFX: typing via `TerminalReplay` (built in), `Whoosh` on cuts/whips,
103
+ `Blip` on reveals, `SuccessChime` on completion moments. Real
104
+ third-party components the product actually uses (an embeddable widget,
105
+ a real chart library, etc.): npm install and render them, never fake a
106
+ screenshot.
107
+
108
+ ## 3. Composition
109
+ `<video>/<Name>Demo.tsx`: `<Series>` mapping `beats.json` ids to scene
110
+ components. Register in the project's `Root.tsx` with duration from beats.
111
+
112
+ ## 4. Preview + QC loop
113
+ - `npm run dev` for live preview while writing scenes.
114
+ - Render, then extract frames at each beat midpoint:
115
+ `ffmpeg -nostdin -ss <t> -i out/<video>.mp4 -frames:v 1 -q:v 3 out/qc/<t>.jpg -y`
116
+ - QC checklist (all must pass):
117
+ - [ ] Text crisp on a 1080p pause-frame at every zoom level
118
+ - [ ] Zero linear easing; every scene has camera motion
119
+ - [ ] Captions fully visible (screen-space) at all camera positions
120
+ - [ ] No em dashes on screen
121
+ - [ ] SFX lands on the visual events (spot-check typing + whip cuts)
122
+ - [ ] Beat durations match beats.json
123
+ - [ ] Glassmorphism on captions/cards; UI content detailed, not skeleton
124
+ - [ ] Every real-screenshot or real-recording beat: camera `scale` in
125
+ 120-250% (STYLE.md), frame centered on the stage
126
+ (`LEFT + FRAME_W/2 == 960`), no leftover dead space around a
127
+ small, unzoomed shot
128
+ - [ ] Pull a frame from the END of every real-screenshot/recording beat
129
+ specifically, not just the midpoint: a static crop that looks fine
130
+ early can push the important content (the actual payoff, not just
131
+ the headline) off the bottom of frame once the camera has zoomed
132
+ in
133
+ - [ ] Every claim the VO makes has a matching visual in the same or an
134
+ adjacent beat; if a beat is a `dom-demo`, confirm it isn't a
135
+ screenshot of a real page whose own on-screen text contradicts the
136
+ claim
137
+
138
+ ## 5. Render + exports
139
+ - `npx remotion render <composition-id> out/<video>.mp4`
140
+ - `node scripts/markers.mjs <video>` writes `out/<video>-markers.json` and
141
+ `out/<video>-vo.srt`
142
+ - Write `<video>/script.md`: VO table (timestamps from markers), delivery
143
+ notes, description draft. See `SCRIPT.md` for VO pacing/duration rules.
144
+
145
+ ## Asset conventions
146
+
147
+ Every tool in this pipeline that reads or writes a beat's media agrees on
148
+ these paths (relative to the scaffolded project root). Get an asset onto
149
+ disk at the wrong path and the tool that's supposed to consume it either
150
+ errors (scenes reference a `staticFile` that 404s at render time) or, for
151
+ VO specifically, silently omits the layer with no warning:
152
+
153
+ | Asset | Path | Written by | Read by |
154
+ |---|---|---|---|
155
+ | Screenshot | `public/images/<beatId>.png` | `capture_screenshot` | the `real-screenshot` scene `scaffold_scene` generates |
156
+ | Recording / Higgsfield clip | `public/video/<beatId>.mp4` | `capture_screen_recording` / `import_higgsfield_clip` | the `real-recording` / `higgsfield-clip` scene `scaffold_scene` generates |
157
+ | VO narration | `public/audio/vo/<beatId>.mp3` | (dev-provided, e.g. an edge-tts run) | `stitch_composition`, only if the file exists at this exact path when it runs |
158
+ | Music bed | `public/audio/music-bed.mp3` | (dev-provided) | `stitch_composition`, only if the file exists at this exact path when it runs |
159
+
160
+ **VO omission is silent.** `stitch_composition` checks
161
+ `public/audio/vo/<beatId>.mp3` for every beat and just skips the Audio
162
+ layer for any beat where the file isn't there, no error, no warning. A
163
+ video with a beat missing its VO file renders fine and plays with no
164
+ narration for that beat; the QC checklist below is the actual backstop,
165
+ not a tool error.
166
+
167
+ ## Known flakes
168
+ - Webpack "wasm-hash / Cannot read properties of undefined" on render:
169
+ `rm -rf node_modules/.cache` and retry.
170
+ - `npx remotion skills add` fails on Windows (spawn EINVAL): use
171
+ `npx -y skills@1.2.0 add remotion-dev/skills` directly.
172
+ - Run npm commands from the project's own directory (shell cwd resets
173
+ between calls in some agent harnesses).