@lalalic/markcut 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/.env.example +27 -0
  2. package/.github/user-steer.md +9 -0
  3. package/.vscode/settings.json +3 -0
  4. package/AGENTS.md +271 -0
  5. package/README.md +219 -0
  6. package/SKILL.md +209 -0
  7. package/docs/dynamic-components.md +191 -0
  8. package/docs/edit-mode.md +220 -0
  9. package/docs/json-descriptive.md +539 -0
  10. package/docs/label-mode.md +110 -0
  11. package/docs/markdown-descriptive.md +751 -0
  12. package/docs/templates.md +52 -0
  13. package/package.json +64 -0
  14. package/remotion.config.ts +5 -0
  15. package/scripts/artlist-dl.mjs +190 -0
  16. package/scripts/build-pipeline.sh +19 -0
  17. package/scripts/build-player.sh +20 -0
  18. package/src/Root.tsx +55 -0
  19. package/src/config.mjs +88 -0
  20. package/src/context/EventContext.tsx +168 -0
  21. package/src/context/index.tsx +42 -0
  22. package/src/descriptive/compiler.test.ts +1135 -0
  23. package/src/descriptive/compiler.ts +1230 -0
  24. package/src/descriptive/dsl.ts +455 -0
  25. package/src/descriptive/markdown.test.ts +866 -0
  26. package/src/descriptive/markdown.ts +674 -0
  27. package/src/descriptive/resolve.test.ts +951 -0
  28. package/src/descriptive/resolve.ts +891 -0
  29. package/src/entry.tsx +163 -0
  30. package/src/index.ts +4 -0
  31. package/src/player/browser.tsx +356 -0
  32. package/src/player/bundle/player.js +60259 -0
  33. package/src/player/bundler.mjs +269 -0
  34. package/src/player/label-server.mjs +599 -0
  35. package/src/player/pipeline.mjs +11123 -0
  36. package/src/player/pipeline.ts +117 -0
  37. package/src/player/server-shared.mjs +144 -0
  38. package/src/player/server.mjs +1006 -0
  39. package/src/render/cli-tools.ts +177 -0
  40. package/src/render/cli.mjs +628 -0
  41. package/src/schema/index.ts +259 -0
  42. package/src/types/Audio.tsx +56 -0
  43. package/src/types/Component.tsx +135 -0
  44. package/src/types/Effect.tsx +88 -0
  45. package/src/types/Folder.tsx +180 -0
  46. package/src/types/FrameSyncStyle.tsx +51 -0
  47. package/src/types/Image.tsx +51 -0
  48. package/src/types/Include.tsx +394 -0
  49. package/src/types/Map.tsx +252 -0
  50. package/src/types/Rhythm.tsx +58 -0
  51. package/src/types/Scene.tsx +42 -0
  52. package/src/types/Subtitle.tsx +218 -0
  53. package/src/types/Video.tsx +70 -0
  54. package/src/types/keyframes.ts +454 -0
  55. package/src/utils/__tests__/vtt.test.ts +129 -0
  56. package/src/utils/index.ts +168 -0
  57. package/src/utils/tween.ts +118 -0
  58. package/src/vision/cli.mjs +1187 -0
  59. package/src/vision/vision_prompts.md +67 -0
  60. package/tests/dsl.test.ts +317 -0
  61. package/tests/fixtures/audio.json +25 -0
  62. package/tests/fixtures/basic.json +27 -0
  63. package/tests/fixtures/component-all.json +38 -0
  64. package/tests/fixtures/components.json +38 -0
  65. package/tests/fixtures/effects.json +64 -0
  66. package/tests/fixtures/full.json +51 -0
  67. package/tests/fixtures/map.json +23 -0
  68. package/tests/fixtures/md/all-nodes.md +28 -0
  69. package/tests/fixtures/md/basic.md +6 -0
  70. package/tests/fixtures/md/component-imports.md +20 -0
  71. package/tests/fixtures/md/edge-cases.md +33 -0
  72. package/tests/fixtures/md/effects.md +17 -0
  73. package/tests/fixtures/md/frontmatter.md +20 -0
  74. package/tests/fixtures/md/full-feature.md +58 -0
  75. package/tests/fixtures/md/imports-block.md +19 -0
  76. package/tests/fixtures/md/include-main.md +11 -0
  77. package/tests/fixtures/md/include-sub.md +25 -0
  78. package/tests/fixtures/md/jsx-code-fence.md +21 -0
  79. package/tests/fixtures/md/map.md +11 -0
  80. package/tests/fixtures/md/nested-scenes.md +25 -0
  81. package/tests/fixtures/md/rhythm.md +17 -0
  82. package/tests/fixtures/md/scenes.md +16 -0
  83. package/tests/fixtures/md/tween.md +11 -0
  84. package/tests/fixtures/md/vars-test.md +6 -0
  85. package/tests/fixtures/scenes.json +40 -0
  86. package/tests/fixtures/subtitle.json +59 -0
  87. package/tests/fixtures/subvideo.json +59 -0
  88. package/tests/fixtures/templates/courseware.md +351 -0
  89. package/tests/fixtures/tween-visual.json +28 -0
  90. package/tests/fixtures/video-series.json +54 -0
  91. package/tests/md-descriptive.test.ts +742 -0
  92. package/tests/render.test.ts +985 -0
  93. package/tests/schema.test.ts +68 -0
  94. package/tests/server.test.ts +308 -0
  95. package/tests/utils.ts +391 -0
  96. package/tests/vitest.config.ts +18 -0
  97. package/tests/vitest.integration.config.ts +16 -0
  98. package/tsconfig.json +20 -0
package/.env.example ADDED
@@ -0,0 +1,27 @@
1
+ # ── Google Maps ──────────────────────────────────────────────────────────────
2
+ # API key for map stream type rendering.
3
+ # Get one at https://console.cloud.google.com/apis/credentials
4
+ GOOGLE_MAPS_API_KEY=your_key_here
5
+
6
+ # ── CLI templates (config.mjs) ──────────────────────────────────────────────
7
+ # All use MARKCUT_*_CLI env vars. Supported placeholders vary per CLI:
8
+ #
9
+ # {input} → media file path (image, video, audio)
10
+ # {prompt} → text prompt / instruction
11
+ # {output} → output file or directory path
12
+ #
13
+ # Shared by vision and render pipelines:
14
+ # MARKCUT_ITT_CLI= # Image-to-text — placeholders: {input} {prompt}
15
+ # MARKCUT_VTT_CLI= # Video-to-text — placeholders: {input} {prompt}
16
+ # MARKCUT_STT_CLI= # Speech-to-text — placeholders: {input} {output}
17
+ # MARKCUT_TTS_CLI= # Text-to-speech — placeholders: {input} {output}
18
+ # MARKCUT_AGENT_CLI= # General-purpose agent — placeholders: {prompt}
19
+ # Render-only:
20
+ # MARKCUT_TTI_CLI= # Text-to-image — placeholders: {input} {output}
21
+ # MARKCUT_TTV_CLI= # Text-to-video — placeholders: {input} {output}
22
+ # # (empty default → TTI + ffmpeg fallback)
23
+ # Other:
24
+ # MARKCUT_MAX_IMAGE_DIMENSION=384
25
+ # MARKCUT_MAX_VIDEO_DURATION=60
26
+ # MARKCUT_MAX_VIDEO_DIMENSION=360
27
+ # MARKCUT_VTT_SAMPLE_INTERVAL=5
@@ -0,0 +1,9 @@
1
+
2
+ [Discord 2026-07-04T21:03:40.038Z]
3
+ Discord has been disabled.
4
+
5
+ [Discord 2026-07-04T21:03:42.727Z]
6
+ Discord has been enabled.
7
+
8
+ [Discord 2026-07-11T16:22:32.656Z]
9
+ Discord has been disabled.
@@ -0,0 +1,3 @@
1
+ {
2
+ "copilot-infinite.discord.enabled": false
3
+ }
package/AGENTS.md ADDED
@@ -0,0 +1,271 @@
1
+ # markcut — AGENT.md
2
+
3
+ > Markdown-to-video engine. Describe scenes in markdown, get a rendered video with TTS narration.
4
+
5
+ ## Quick Start
6
+
7
+ ```bash
8
+ npm run render storyboard.md # render markdown → MP4 (9x16 default)
9
+ npm run preview storyboard.md --edit # live preview with auto-reload
10
+ npm test # unit + integration tests
11
+ npm run typecheck # TypeScript type check
12
+ ```
13
+
14
+ ## Input formats
15
+
16
+ | Format | CLI | Description |
17
+ |---|---|---|
18
+ | Markdown | `markcut render storyboard.md` | `## Scene` headings, `- image src:...` bullets, `script:"narration"` |
19
+ | Descriptive JSON | `markcut render video.json` | Same schema as markdown, JSON syntax |
20
+ | Compiled JSON | `markcut render tree.json` | Pre-compiled stream tree (no pipeline needed) |
21
+
22
+ See [docs/markdown-strict-descriptive.md](docs/markdown-strict-descriptive.md) for the complete markdown syntax reference.
23
+
24
+ ## Key concepts
25
+
26
+ - **Scene** — narrative unit with `name`, `script` (TTS), `instruction` (visual intent), `children`
27
+ - **Layout** — `parallel` (simultaneous), `series` (sequential), `transitionSeries` (sequential with transitions)
28
+ - **Script → TTS → STT** — `script` field generates edge-tts audio, whisper transcribes to VTT subtitles
29
+ - **Theme** — preset colors/fonts/effects: `cinematic`, `neon`, `minimal`, `corporate`
30
+ - **TTL config** — root-level `tts.cli`, `tts.voice`, `tts.rate` with per-scene `tts` overrides
31
+ - **Subtitle** — root-level VTT overlay from merged per-clip STT. Not a tree node. Supports animated caption types (`typewriter`, `fade`, `bounce`, `glowing`, etc.) via `remotion-subtitle`, HTML in cue text, and per-cue `<Sequence>` rendering for zero CPU when inactive.
32
+
33
+ ## Stream types
34
+
35
+ | Type | Purpose |
36
+ |---|---|
37
+ | `root` | Canvas config: width, height, fps, theme, subtitle, stylesheet |
38
+ | `scene` | Storyboard container: name, layout, script, children |
39
+ | `folder` | Internal container (series/parallel) |
40
+ | `image` | Still photo with fit mode |
41
+ | `video` | Video clip with startFrom/endAt trimming, playbackRate |
42
+ | `audio` | Soundtrack/SFX with foreground ducking, loop |
43
+ | `component` | External React component by componentName + props |
44
+ | `effect` | CSS keyframe animation wrapper (fadeIn, zoomIn, bounceIn, etc.) |
45
+ | `include` | Embed external video JSON (file, URL, or data URI) |
46
+
47
+ ## CLI
48
+
49
+ ```bash
50
+ markcut render <file.json|.md> [--aspect 16x9|9x16|1x1|all] [--output path]
51
+ markcut preview <file.json|.md> [--edit] [--label] [--port 3001]
52
+ ```
53
+
54
+ ## Project structure
55
+
56
+ ```
57
+ src/
58
+ ├── entry.tsx → MarkCut + DescriptiveComposition
59
+ ├── descriptive/ → Compiler, markdown parser, resolve pipeline
60
+ ├── types/ → React renderers (one per stream type)
61
+ ├── schema/ → Zod stream tree schemas
62
+ ├── themes/ → Theme presets + ThemeProvider
63
+ ├── render/cli.mjs → CLI entry
64
+ ├── render/tts.ts → TTS via CLI template + variable substitution
65
+ ├── player/pipeline.mjs → Bundled pipeline (server imports this)
66
+ ├── player/server.mjs → --edit server
67
+ ├── player/label-server.mjs→ --label server
68
+ └── tests/ → Vitest integration tests
69
+ ```
70
+
71
+ ## External component contract
72
+
73
+ Register components at render time:
74
+
75
+ ```tsx
76
+ <MarkCut
77
+ root={descriptiveJson}
78
+ compose={{ components: { AnimatedHeadline, StatCounter } }}
79
+ />
80
+ ```
81
+
82
+ In descriptive markdown:
83
+
84
+ ```md
85
+ - component componentName:StatCounter duration:3 props:{value:42}
86
+ ```
87
+
88
+ Components receive props as defined in the descriptive JSON. No eval, no JSX parsing — just `React.createElement(registry[name], props)`.
89
+ ```
90
+
91
+ ## Stream Tree Architecture
92
+
93
+ ### Core Concept
94
+
95
+ Everything is a **stream tree** — nested JSON nodes keyed by `type`.
96
+ The tree is validated by Zod and rendered by Remotion via recursive React components.
97
+
98
+ ```json
99
+ { "id":"root", "type":"root", "width":1080, "height":1920, "fps":30,
100
+ "isSeries":true, "transition":"fade", "children":[...] }
101
+ ```
102
+
103
+ ### Composition Rules
104
+
105
+ | Concept | Rule |
106
+ |---------|------|
107
+ | **Series** (`isSeries:true`) | Children play sequentially. Optional `transition` (fade/slide/wipe/flip/clockWipe) + `transitionTime` (default 0.5s). |
108
+ | **Parallel** (`isSeries:false`) | Children play simultaneously. Duration = max child duration. |
109
+ | **Background** (`isBackground:true`) | Loops for parent duration. Excluded from duration calculation. |
110
+ | **Actions** | Every leaf has `actions[{start, end, startFrom?, endAt?, loop?, volume?, style?}]` — seconds relative to parent. |
111
+ | **Scene** (`type:"scene"`) | UI storyboard card; engine treats as folder. Has `name` + `description` + `script`. |
112
+
113
+ ### Duration Calculation
114
+
115
+ Computed by `getDurationInSeconds()` in `src/utils/index.ts`:
116
+ - **Leaf nodes**: duration = last action's `end` value
117
+ - **Series**: sum of children durations minus transition overlaps
118
+ - **Parallel**: max of children durations
119
+ - **Background children**: excluded from calculation
120
+ - **Include with `src`**: treated as leaf (duration from action end)
121
+ - Sets `durationInSeconds` on every node as a side effect
122
+
123
+ ### 12 Stream Types
124
+
125
+ All types share base fields from `BaseShape`: `id`, `name`, `title`, `description`, `script`, `src`, `style`, `visible`, `isBackground`, `durationInSeconds`.
126
+
127
+ | Type | Purpose | Key Fields |
128
+ |------|---------|------------|
129
+ | `root` | Canvas container | `width`, `height`, `fps`, `isSeries`, `transition`, `stylesheet` |
130
+ | `folder` | Group children | `isSeries`, `transition`, `children[]` |
131
+ | `scene` | Storyboard node (alias for folder) | `name`, `description`, `script`, `children[]` |
132
+ | `image` | Still photo | `src`, `fit` (contain/cover/fill) |
133
+ | `video` | Video clip | `src`, `volume`, `playbackRate` |
134
+ | `audio` | Soundtrack/SFX | `src`, `volume`, `foreground` (ducks parent) |
135
+ | `subtitle` | Text overlay | `src` (text/VTT) or `cues[]`, `fontSize`, `style` |
136
+ | `component` | React component | `componentName`, `props`, `src` (remote ESM) |
137
+ | `effect` | CSS animation wrapper | `animation`, `customKeyframes`, `children[]` |
138
+ | `rhythm` | Beat-synced loop | `src`, `spots[]`, `children[]` |
139
+ | `map` | Route visualization | `waypoints[{lat,lng,label}]`, `routeColor` |
140
+ | `include` | Embed sub-tree | `src` (path/URL/data URI), `children[]` |
141
+
142
+ ## Key Implementation Details
143
+
144
+ ### Folder.tsx — Series/Parallel Engine
145
+
146
+ - `FolderLeaf` is the recursive workhorse that renders all stream types
147
+ - For `isSeries:true`: uses Remotion `<Series>` or `<TransitionSeries>` (if transition set)
148
+ - For `isSeries:false`: renders children as parallel `<Sequence>` elements
149
+ - Each child's duration comes from `child.durationInSeconds` (set by `getDurationInSeconds`)
150
+ - Transition elements are interleaved between series children
151
+ - Background children are wrapped in `<Loop>`
152
+ - Scene nodes delegate to `SceneLeaf` → `FolderLeaf` (identity)
153
+ - Effect nodes delegate to `EffectWrapper` → `FolderLeaf`
154
+
155
+ ### Video.tsx — startFrom/endAt Trimming
156
+
157
+ - Uses Remotion `<OffthreadVideo>` for CPU-efficient video decoding
158
+ - `startFrom` / `endAt` on actions trim the source video (in seconds)
159
+ - `playbackRate` is capped at 1 (`Math.min(1, ...)`) so source longer than timeline plays at normal speed and is truncated by the Sequence
160
+ - **Critical formula**: `OffthreadVideo endAt = startFrom_frames + ((endAt_source - startFrom_source) * fps / playbackRate)`
161
+ This sets `endAt` beyond the actual source end when `playbackRate < 1`, which prevents Remotion from rendering blank frames before the Sequence ends.
162
+ - `endAt` without explicit value falls back to `stream.durationInSeconds ?? end - start`
163
+
164
+ ### Rendering Pipeline
165
+
166
+ 1. `Root.tsx` reads `getInputProps()` → parses via Zod `rootSchema`
167
+ 2. `getDurationInSeconds()` stamps `durationInSeconds` on every node
168
+ 3. `Composition.durationInFrames` is set to total duration
169
+ 4. `MarkCut` wraps everything in `ComposeContext` + `ThemeProvider` + `AbsoluteFill`
170
+ 5. `FolderLeaf` recursively renders the tree
171
+
172
+ ### CLI Usage
173
+
174
+ ```bash
175
+ node src/render/cli.mjs render <file.json> [--aspect 16x9|9x16|1x1|all] [--output path]
176
+ node src/render/cli.mjs render --template <id> --data <data.json>
177
+ node src/render/cli.mjs preview <file.json> [--edit] [--label] [--port 3001]
178
+ node src/render/cli.mjs templates
179
+ ```
180
+
181
+ ### Player Server (Edit Mode)
182
+
183
+ - `--edit` flag starts a live-reload loop: edit JSON → player auto-refreshes
184
+ - `--label` flag starts a labeling UI for selecting and annotating media
185
+ - `--port` flag sets the server port (default 3001)
186
+ - Two implementations: `server.mjs` (Remotion Player) and `label-server.mjs` (label input overlay)
187
+
188
+ ## Testing
189
+
190
+ ### Test Framework
191
+ - **Vitest** with custom config in `tests/vitest.config.ts`
192
+ - Non-parallel (`pool: "forks", fileParallelism: false`)
193
+ - 600s timeout per test (integration tests render real MP4s)
194
+
195
+ ### Key Test Utilities (`tests/utils.ts`)
196
+
197
+ | Function | Purpose |
198
+ |----------|---------|
199
+ | `renderFixture(path, opts)` | Render a stream tree JSON → MP4 using `npx remotion render` |
200
+ | `renderScenes(path, opts)` | Render scene-based video.json using `Main16x9` composition |
201
+ | `getVideoInfo(path)` | Extract metadata via ffprobe (duration, dimensions, FPS, audio) |
202
+ | `extractFrame(video, time, out)` | Extract PNG frame at timestamp via ffmpeg |
203
+ | `isFrameNonBlank(path)` | Check center pixel isn't black (>10 in any RGB channel) |
204
+ | `getFrameFileSize(path)` | Get PNG file size (proxy for visual content) |
205
+ | `extractAudio(video, out)` | Extract audio track to WAV |
206
+ | `transcribeAudio(path)` | Run whisper STT on audio |
207
+
208
+ ### Test Categories (30 tests)
209
+
210
+ 1. Basic Rendering — empty composition, dimension verification
211
+ 2. Image + Subtitle — image rendering, inline/VTT/karaoke subtitles
212
+ 3. Built-in Components — AnimatedHeadline, StatCounter, GradientBackground
213
+ 4. Effects — fadeIn, bounceIn, custom keyframes
214
+ 5. Map Rendering — canvas route visualization
215
+ 6. Audio Rendering — audio track, non-silent verification
216
+ 7. Include — nested composition via include
217
+ 8. Full Feature Combination — multiple features
218
+ 9. Scene Node — scene-based rendering
219
+ 10. Audio STT — speech-to-text verification via whisper
220
+ 11. Multiple Aspect Ratios — 16x9, 9x16, 1x1
221
+ 12. Cross-Stream Types — parallel rendering, effect-wrapped children
222
+ 13. Frame-Accurate — timeline consistency, frame count
223
+ 14. Description Field — schema validation, render preservation
224
+ 15. Scene as Folder Alias — scene/folder interchangeability
225
+ 16. **Video startFrom/endAt Trimming** — video trimming + photo in series
226
+
227
+ ### Running Tests
228
+
229
+ ```bash
230
+ npm test # all tests
231
+ npm run test:integration # render integration tests only
232
+ npx vitest run tests/render.test.ts -t "Test Name" # single test
233
+ ```
234
+
235
+ ## Known Issues & Gotchas
236
+
237
+ 1. **TransitionSeries + isBackground**: `TransitionSeries` from `@remotion/transitions` rejects non-Sequence/Transition children. Don't mix `isBackground:true` children with `isSeries:true, transition:"..."` at the same level.
238
+ 2. **Subvideo internal actions**: Should use `start:0` when placed in a series parent (series handles positioning).
239
+ 3. **Video endAt formula**: Must be `startFrom_frames + ((endAt_source - startFrom_source) * fps / playbackRate)` — changing to `endAt_source * fps` breaks slow-motion playback.
240
+ 4. **`signalstats` ffmpeg filter**: Doesn't output YAVG on all systems; use file size as proxy for visual content.
241
+ 5. **Asset paths**: Local files are resolved via `staticFile()` — files go in `public/` and are referenced without the `public/` prefix.
242
+ 6. **Map rendering**: Depends on external tile availability; may produce blank frames offline.
243
+
244
+ ## 3-Level Authoring Workflow (see SKILL.md)
245
+
246
+ | Level | Format | Purpose |
247
+ |-------|--------|---------|
248
+ | **Label** | `labels.json` | Browse media, label clips, add descriptions |
249
+ | **Storyboard** | `storyboard.json` | `scene` nodes with high-level structure + script |
250
+ | **Assemble** | `video.json` | Full stream tree with all types, render to MP4 |
251
+
252
+ ### Script → TTS → STT → VTT Pipeline
253
+
254
+ Every node carries an optional `script` field for narration/dialogue text. The pipeline:
255
+ 1. **Storyboard** — agent writes `script` on any node (scene, folder, leaf) as narrative text
256
+ 2. **TTS** — walk the tree, collect all `script` values with node id + timing, generate WAV audio per segment via `edge-tts`
257
+ 3. **STT** — transcribe each TTS audio or concatenated audio to VTT via whisper
258
+ 4. **Subtitle** — attach resulting VTT as a `subtitle` child node for final rendering
259
+
260
+ This separates concerns: `script` = authoring text (human/agent-friendly), `subtitle` = rendered output (timed VTT cues).
261
+
262
+ ## Dependencies
263
+
264
+ - **Remotion 4.0.469** — core rendering engine
265
+ - **React 19** — component model
266
+ - **Zod 4** — schema validation
267
+ - **@remotion/transitions** — scene transitions (fade, slide, wipe, flip, clockWipe)
268
+ - **@vis.gl/react-google-maps** — map visualization
269
+ - **immer** — state management (player app)
270
+ - **vitest** — test runner
271
+ - **TypeScript 5.6** — strict mode with `noUncheckedIndexedAccess`
package/README.md ADDED
@@ -0,0 +1,219 @@
1
+ # markcut — Markdown-to-Video Engine
2
+
3
+ Write a storyboard in markdown, get a rendered video with TTS narration and subtitles.
4
+
5
+ ```bash
6
+ # Render a storyboard to MP4
7
+ npx markcut render storyboard.md --aspect 9x16
8
+
9
+ # Preview with live edit (edit .md file, player auto-reloads)
10
+ npx markcut preview storyboard.md --edit
11
+
12
+ # Label clips in a stream tree
13
+ npx markcut preview labels.json --label --port=3031
14
+ ```
15
+
16
+ ## How it works
17
+
18
+ ```
19
+ storyboard.md ──[parse]──▶ DescriptiveRoot ──[compile]──▶ Stream Tree ──[Remotion]──▶ MP4
20
+ │ │
21
+ resolveMediaDurations() <Sequence>
22
+ resolveScripts() → TTS <Series>
23
+ resolveSubtitles() → VTT <Img>, <OffthreadVideo>
24
+ ```
25
+
26
+ 1. **Write** a storyboard in markdown (scenes with `## headings`, media with `-` bullets)
27
+ 2. **Parse** the markdown into a descriptive tree
28
+ 3. **Resolve** media durations (ffprobe), generate TTS narration (edge-tts), transcribe to VTT (whisper)
29
+ 4. **Compile** the descriptive tree into a stream tree (the low-level Remotion JSON)
30
+ 5. **Render** with Remotion — no React code needed
31
+
32
+ ## Features
33
+
34
+ | Feature | Description |
35
+ |---|---|
36
+ | **Markdown input** | Write `## Scene` headings, `- bullet` leaves. Components via `jsx:"<Tag />"` syntax |
37
+ | **JS imports block** | `` ```js imports `` code fence with real ESM `import` statements instead of YAML |
38
+ | **Dynamic components** | Load any React component from npm, GitHub, or any URL via `import { X } from "pkg"` |
39
+ | **Tween animation** | `tween(from, to)` function calls in JSX for frame-accurate numeric animation |
40
+ | **Compiled input** | Pre-compiled stream tree for direct Remotion rendering |
41
+ | **TTS narration** | `script` field → edge-tts CLI → WAV audio. Configurable engine (mlx-audio, custom) |
42
+ | **STT subtitles** | TTS audio → whisper → VTT → root subtitle overlay with animated caption types (typewriter, fade, bounce, etc.) |
43
+ | **Tween animation** | `tween(from, to)` function calls in JSX for frame-accurate numeric animation |
44
+ | **Styling** | Inline `style` strings on any node for CSS. JSX components use inline React styles |
45
+ | **Live edit** | `--edit` watches the input file, re-runs pipeline, auto-reloads player |
46
+ | **Label mode** | `--label` interactive player with per-scene label input, saves to labels.json |
47
+ | **CLI** | `render`, `preview` commands for MP4 export and Remotion Studio |
48
+ | **Programmatic** | `MarkCut` / `DescriptiveComposition` React components for embedding |
49
+
50
+ ## Example
51
+
52
+ ```md
53
+ # video
54
+ width:1080 height:1920 fps:30 layout:series
55
+
56
+ ## Hook
57
+ layout:parallel script:"Set the mood with a beautiful landscape"
58
+ - image src:cover.jpg duration:3
59
+
60
+ ## Features
61
+ layout:transitionSeries transition:fade transitionTime:0.4 script:"Show what we built"
62
+ - component duration:6 jsx:"<DeviceMockup src='screenshot.png' />"
63
+ - component duration:4 jsx:"<StatCounter value={100} suffix='K' label='Users' />"
64
+ ```
65
+
66
+ Components are registered via frontmatter `imports:` or a `` ```js imports `` code block:
67
+
68
+ ````
69
+ ```js imports
70
+ import { DeviceMockup } from "mockup-component"
71
+ import { StatCounter } from "stat-counter"
72
+
73
+ // Inline component definition
74
+ export function Greeting({ name }) {
75
+ return <h1 style={{color: '#fff'}}>Hello {name}</h1>;
76
+ }
77
+ ```
78
+ ````
79
+
80
+ ### Dynamic Components
81
+
82
+ Any React component from npm, GitHub, or any URL can be imported and used directly in JSX expressions. The engine loads them at render time via esm.sh — no build step required.
83
+
84
+ ```markdown
85
+ - component duration:3 jsx:"<PieChart data={[{value:40,color:'#E38627'}]} />"
86
+ ```
87
+
88
+ Inline components can be defined entirely in the imports block using `export function`, making the video self-contained with no external files.
89
+
90
+ ## Docs
91
+
92
+ | Document | What it covers |
93
+ |---|---|
94
+ | [docs/json-descriptive.md](docs/json-descriptive.md) | Full JSON descriptive schema reference |
95
+ | [docs/markdown-strict-descriptive.md](docs/markdown-strict-descriptive.md) | Markdown descriptive syntax reference |
96
+ | [docs/label-mode.md](docs/label-mode.md) | Label mode player and workflow |
97
+ | [docs/edit-mode.md](docs/edit-mode.md) | Live edit mode with SSE reload |
98
+
99
+
100
+ ## Variants
101
+
102
+ Create different versions of the same video for different platforms, languages, or formats using **variant sections** in your markdown.
103
+
104
+ ### Defining Variants
105
+
106
+ Add `# variant-name` headings alongside `# video` (the base). Each variant can override any property using variant-prefixed keys:
107
+
108
+ ```markdown
109
+ # video
110
+ width:1080 height:1920 fps:30 layout:series
111
+
112
+ ## Intro
113
+ layout:parallel
114
+ - component duration:3 jsx:"<h1>Hello</h1>"
115
+ - image src:default.jpg duration:3
116
+
117
+ # zh
118
+ ## Intro
119
+ layout:parallel
120
+ - component duration:3 zh:"<h1>你好</h1>" ← bare key overrides jsx (primary content)
121
+ - image zh-src:zh-image.jpg src:default.jpg ← zh-src overrides src
122
+
123
+ # youtube
124
+ width:1920 height:1080 fps:30 ← different aspect ratio
125
+ ## Intro
126
+ layout:parallel
127
+ - component duration:3 jsx:"<h1>YouTube Hello</h1>"
128
+ ```
129
+
130
+ Two override mechanisms:
131
+ - **Variant-prefixed keys**: `zh-src` → `src`, `zh-jsx` → `jsx`, `zh-script` → `script`
132
+ - **Bare variant keys**: `zh:"<h1>你好</h1>"` replaces the node's primary content key (`jsx` for components, `src` for images/video, `script` for audio)
133
+
134
+ ### Multi-Variant Preview Server
135
+
136
+ The `preview` command accepts multiple `--variant` flags, each defining a separate compilation target. Variants are compiled **sequentially** (not in parallel) to avoid resource contention from CLI tools like whisper and ffprobe.
137
+
138
+ ```bash
139
+ # Preview 4 variants simultaneously
140
+ npx markcut preview storyboard.md \
141
+ --variant default \
142
+ --variant zh-tiktok \
143
+ --variant en-tiktok \
144
+ --variant youtube
145
+
146
+ # Each variant is served at its own URL:
147
+ # http://localhost:3001 → default
148
+ # http://localhost:3001/zh-tiktok → zh + tiktok chain
149
+ # http://localhost:3001/en-tiktok → en + tiktok chain
150
+ # http://localhost:3001/youtube → youtube
151
+ ```
152
+
153
+ Compound variant labels (like `zh-tiktok`) are split on `-` to form a **variant chain** — the parser applies `zh` overrides first, then `tiktok` overrides on top. Each variant gets its own compiled output and subtitle directory. Component bundles are stored in a shared cache (`.markcut/generated/components/`) — identical imports across variants reuse the same bundle.
154
+
155
+ ### Variant-Aware Browser
156
+
157
+ The browser player reads `window.VARIANT` from the URL path and fetches the correct compiled data via `/api/video-data?variant=<name>`. A **variant switcher bar** at the top of the player lets you jump between variants instantly.
158
+
159
+ ## `.markcut/` Directory Layout
160
+
161
+ When you run `preview` or `render`, all generated artifacts live under `.markcut/` next to the source file. With multiple variants, each gets its own subdirectory:
162
+
163
+ ```
164
+ .markcut/
165
+ generated/ ← shared, content-addressed (across all variants & files)
166
+ tts/ ← TTS narration audio (content-hash filenames)
167
+ media/ ← TTI/TTV media (content-hash filenames)
168
+ includes/ ← compiled sub-video JSON (content-hash)
169
+ components/ ← component bundles (content-hash, shared across variants)
170
+ <basename>/ ← per-source-file
171
+ default/ ← default variant artifacts
172
+ compiled.json ← compiled stream tree
173
+ subtitles.vtt
174
+ zh-tiktok/ ← variant-specific artifacts
175
+ compiled.json
176
+ subtitles.vtt
177
+ youtube/
178
+ compiled.json
179
+ subtitles.vtt
180
+ ```
181
+
182
+ - **`generated/`** — shared artifacts using content-hash filenames, so identical scripts/prompts across different variants reuse the same cached file
183
+ - **`generated/components/`** — component bundles shared across all variants and source files. Identical imports produce the same content hash, so variants with the same component registrations reuse the same bundle
184
+ - **`<basename>/<variant>/compiled.json`** — the compiled stream tree for each variant, cached to disk for fast restarts
185
+ - The server serves `.markcut/` as a document root, so components at `.markcut/generated/components/abc.js` are served as `/generated/components/abc.js`
186
+
187
+ The `.markcut/` directory is gitignored by default — it's regenerated on each run.
188
+
189
+ ## Architecture
190
+
191
+ ```
192
+ src/
193
+ ├── entry.tsx ← Library entry: MarkCut + DescriptiveComposition
194
+ ├── index.ts ← Remotion registerRoot (for studio/render CLI)
195
+ ├── Root.tsx ← Remotion Composition wrapper
196
+ ├── schema/ ← Zod schemas for all stream types
197
+ ├── types/ ← React renderers (Folder, Video, Image, Audio, etc.)
198
+ ├── descriptive/ ← Compiler, markdown parser, resolve pipeline
199
+ ├── themes/ ← Theme presets + ThemeProvider
200
+ ├── render/
201
+ │ ├── cli.mjs ← CLI entry point
202
+ │ └── tts.ts ← TTS via CLI template + variable substitution
203
+ ├── player/
204
+ │ ├── server.mjs ← --edit player server
205
+ │ ├── label-server.mjs ← --label player server
206
+ │ ├── pipeline.ts ← Pipeline entry (bundled to pipeline.mjs)
207
+ │ └── browser.tsx ← Browser player (bundled with esbuild)
208
+ ├── utils/ ← Duration calc, VTT parser, helpers
209
+ └── tests/ ← Vitest integration tests
210
+ ```
211
+
212
+ ## Scripts
213
+
214
+ ```bash
215
+ npm run render # render a JSON stream tree to MP4
216
+ npm run preview # open Remotion Studio
217
+ npm run typecheck # TypeScript type check
218
+ npm test # run unit + integration tests
219
+ ```