@lalalic/markcut 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +27 -0
- package/.github/user-steer.md +9 -0
- package/.vscode/settings.json +3 -0
- package/AGENTS.md +271 -0
- package/README.md +219 -0
- package/SKILL.md +209 -0
- package/docs/dynamic-components.md +191 -0
- package/docs/edit-mode.md +220 -0
- package/docs/json-descriptive.md +539 -0
- package/docs/label-mode.md +110 -0
- package/docs/markdown-descriptive.md +751 -0
- package/docs/templates.md +52 -0
- package/package.json +64 -0
- package/remotion.config.ts +5 -0
- package/scripts/artlist-dl.mjs +190 -0
- package/scripts/build-pipeline.sh +19 -0
- package/scripts/build-player.sh +20 -0
- package/src/Root.tsx +55 -0
- package/src/config.mjs +88 -0
- package/src/context/EventContext.tsx +168 -0
- package/src/context/index.tsx +42 -0
- package/src/descriptive/compiler.test.ts +1135 -0
- package/src/descriptive/compiler.ts +1230 -0
- package/src/descriptive/dsl.ts +455 -0
- package/src/descriptive/markdown.test.ts +866 -0
- package/src/descriptive/markdown.ts +674 -0
- package/src/descriptive/resolve.test.ts +951 -0
- package/src/descriptive/resolve.ts +891 -0
- package/src/entry.tsx +163 -0
- package/src/index.ts +4 -0
- package/src/player/browser.tsx +356 -0
- package/src/player/bundle/player.js +60259 -0
- package/src/player/bundler.mjs +269 -0
- package/src/player/label-server.mjs +599 -0
- package/src/player/pipeline.mjs +11123 -0
- package/src/player/pipeline.ts +117 -0
- package/src/player/server-shared.mjs +144 -0
- package/src/player/server.mjs +1006 -0
- package/src/render/cli-tools.ts +177 -0
- package/src/render/cli.mjs +628 -0
- package/src/schema/index.ts +259 -0
- package/src/types/Audio.tsx +56 -0
- package/src/types/Component.tsx +135 -0
- package/src/types/Effect.tsx +88 -0
- package/src/types/Folder.tsx +180 -0
- package/src/types/FrameSyncStyle.tsx +51 -0
- package/src/types/Image.tsx +51 -0
- package/src/types/Include.tsx +394 -0
- package/src/types/Map.tsx +252 -0
- package/src/types/Rhythm.tsx +58 -0
- package/src/types/Scene.tsx +42 -0
- package/src/types/Subtitle.tsx +218 -0
- package/src/types/Video.tsx +70 -0
- package/src/types/keyframes.ts +454 -0
- package/src/utils/__tests__/vtt.test.ts +129 -0
- package/src/utils/index.ts +168 -0
- package/src/utils/tween.ts +118 -0
- package/src/vision/cli.mjs +1187 -0
- package/src/vision/vision_prompts.md +67 -0
- package/tests/dsl.test.ts +317 -0
- package/tests/fixtures/audio.json +25 -0
- package/tests/fixtures/basic.json +27 -0
- package/tests/fixtures/component-all.json +38 -0
- package/tests/fixtures/components.json +38 -0
- package/tests/fixtures/effects.json +64 -0
- package/tests/fixtures/full.json +51 -0
- package/tests/fixtures/map.json +23 -0
- package/tests/fixtures/md/all-nodes.md +28 -0
- package/tests/fixtures/md/basic.md +6 -0
- package/tests/fixtures/md/component-imports.md +20 -0
- package/tests/fixtures/md/edge-cases.md +33 -0
- package/tests/fixtures/md/effects.md +17 -0
- package/tests/fixtures/md/frontmatter.md +20 -0
- package/tests/fixtures/md/full-feature.md +58 -0
- package/tests/fixtures/md/imports-block.md +19 -0
- package/tests/fixtures/md/include-main.md +11 -0
- package/tests/fixtures/md/include-sub.md +25 -0
- package/tests/fixtures/md/jsx-code-fence.md +21 -0
- package/tests/fixtures/md/map.md +11 -0
- package/tests/fixtures/md/nested-scenes.md +25 -0
- package/tests/fixtures/md/rhythm.md +17 -0
- package/tests/fixtures/md/scenes.md +16 -0
- package/tests/fixtures/md/tween.md +11 -0
- package/tests/fixtures/md/vars-test.md +6 -0
- package/tests/fixtures/scenes.json +40 -0
- package/tests/fixtures/subtitle.json +59 -0
- package/tests/fixtures/subvideo.json +59 -0
- package/tests/fixtures/templates/courseware.md +351 -0
- package/tests/fixtures/tween-visual.json +28 -0
- package/tests/fixtures/video-series.json +54 -0
- package/tests/md-descriptive.test.ts +742 -0
- package/tests/render.test.ts +985 -0
- package/tests/schema.test.ts +68 -0
- package/tests/server.test.ts +308 -0
- package/tests/utils.ts +391 -0
- package/tests/vitest.config.ts +18 -0
- package/tests/vitest.integration.config.ts +16 -0
- package/tsconfig.json +20 -0
package/.env.example
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# ── Google Maps ──────────────────────────────────────────────────────────────
|
|
2
|
+
# API key for map stream type rendering.
|
|
3
|
+
# Get one at https://console.cloud.google.com/apis/credentials
|
|
4
|
+
GOOGLE_MAPS_API_KEY=your_key_here
|
|
5
|
+
|
|
6
|
+
# ── CLI templates (config.mjs) ──────────────────────────────────────────────
|
|
7
|
+
# All use MARKCUT_*_CLI env vars. Supported placeholders vary per CLI:
|
|
8
|
+
#
|
|
9
|
+
# {input} → media file path (image, video, audio)
|
|
10
|
+
# {prompt} → text prompt / instruction
|
|
11
|
+
# {output} → output file or directory path
|
|
12
|
+
#
|
|
13
|
+
# Shared by vision and render pipelines:
|
|
14
|
+
# MARKCUT_ITT_CLI= # Image-to-text — placeholders: {input} {prompt}
|
|
15
|
+
# MARKCUT_VTT_CLI= # Video-to-text — placeholders: {input} {prompt}
|
|
16
|
+
# MARKCUT_STT_CLI= # Speech-to-text — placeholders: {input} {output}
|
|
17
|
+
# MARKCUT_TTS_CLI= # Text-to-speech — placeholders: {input} {output}
|
|
18
|
+
# MARKCUT_AGENT_CLI= # General-purpose agent — placeholders: {prompt}
|
|
19
|
+
# Render-only:
|
|
20
|
+
# MARKCUT_TTI_CLI= # Text-to-image — placeholders: {input} {output}
|
|
21
|
+
# MARKCUT_TTV_CLI= # Text-to-video — placeholders: {input} {output}
|
|
22
|
+
# # (empty default → TTI + ffmpeg fallback)
|
|
23
|
+
# Other:
|
|
24
|
+
# MARKCUT_MAX_IMAGE_DIMENSION=384
|
|
25
|
+
# MARKCUT_MAX_VIDEO_DURATION=60
|
|
26
|
+
# MARKCUT_MAX_VIDEO_DIMENSION=360
|
|
27
|
+
# MARKCUT_VTT_SAMPLE_INTERVAL=5
|
package/AGENTS.md
ADDED
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
# markcut — AGENT.md
|
|
2
|
+
|
|
3
|
+
> Markdown-to-video engine. Describe scenes in markdown, get a rendered video with TTS narration.
|
|
4
|
+
|
|
5
|
+
## Quick Start
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
npm run render storyboard.md # render markdown → MP4 (9x16 default)
|
|
9
|
+
npm run preview storyboard.md --edit # live preview with auto-reload
|
|
10
|
+
npm test # unit + integration tests
|
|
11
|
+
npm run typecheck # TypeScript type check
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
## Input formats
|
|
15
|
+
|
|
16
|
+
| Format | CLI | Description |
|
|
17
|
+
|---|---|---|
|
|
18
|
+
| Markdown | `markcut render storyboard.md` | `## Scene` headings, `- image src:...` bullets, `script:"narration"` |
|
|
19
|
+
| Descriptive JSON | `markcut render video.json` | Same schema as markdown, JSON syntax |
|
|
20
|
+
| Compiled JSON | `markcut render tree.json` | Pre-compiled stream tree (no pipeline needed) |
|
|
21
|
+
|
|
22
|
+
See [docs/markdown-strict-descriptive.md](docs/markdown-strict-descriptive.md) for the complete markdown syntax reference.
|
|
23
|
+
|
|
24
|
+
## Key concepts
|
|
25
|
+
|
|
26
|
+
- **Scene** — narrative unit with `name`, `script` (TTS), `instruction` (visual intent), `children`
|
|
27
|
+
- **Layout** — `parallel` (simultaneous), `series` (sequential), `transitionSeries` (sequential with transitions)
|
|
28
|
+
- **Script → TTS → STT** — `script` field generates edge-tts audio, whisper transcribes to VTT subtitles
|
|
29
|
+
- **Theme** — preset colors/fonts/effects: `cinematic`, `neon`, `minimal`, `corporate`
|
|
30
|
+
- **TTL config** — root-level `tts.cli`, `tts.voice`, `tts.rate` with per-scene `tts` overrides
|
|
31
|
+
- **Subtitle** — root-level VTT overlay from merged per-clip STT. Not a tree node. Supports animated caption types (`typewriter`, `fade`, `bounce`, `glowing`, etc.) via `remotion-subtitle`, HTML in cue text, and per-cue `<Sequence>` rendering for zero CPU when inactive.
|
|
32
|
+
|
|
33
|
+
## Stream types
|
|
34
|
+
|
|
35
|
+
| Type | Purpose |
|
|
36
|
+
|---|---|
|
|
37
|
+
| `root` | Canvas config: width, height, fps, theme, subtitle, stylesheet |
|
|
38
|
+
| `scene` | Storyboard container: name, layout, script, children |
|
|
39
|
+
| `folder` | Internal container (series/parallel) |
|
|
40
|
+
| `image` | Still photo with fit mode |
|
|
41
|
+
| `video` | Video clip with startFrom/endAt trimming, playbackRate |
|
|
42
|
+
| `audio` | Soundtrack/SFX with foreground ducking, loop |
|
|
43
|
+
| `component` | External React component by componentName + props |
|
|
44
|
+
| `effect` | CSS keyframe animation wrapper (fadeIn, zoomIn, bounceIn, etc.) |
|
|
45
|
+
| `include` | Embed external video JSON (file, URL, or data URI) |
|
|
46
|
+
|
|
47
|
+
## CLI
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
markcut render <file.json|.md> [--aspect 16x9|9x16|1x1|all] [--output path]
|
|
51
|
+
markcut preview <file.json|.md> [--edit] [--label] [--port 3001]
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Project structure
|
|
55
|
+
|
|
56
|
+
```
|
|
57
|
+
src/
|
|
58
|
+
├── entry.tsx → MarkCut + DescriptiveComposition
|
|
59
|
+
├── descriptive/ → Compiler, markdown parser, resolve pipeline
|
|
60
|
+
├── types/ → React renderers (one per stream type)
|
|
61
|
+
├── schema/ → Zod stream tree schemas
|
|
62
|
+
├── themes/ → Theme presets + ThemeProvider
|
|
63
|
+
├── render/cli.mjs → CLI entry
|
|
64
|
+
├── render/tts.ts → TTS via CLI template + variable substitution
|
|
65
|
+
├── player/pipeline.mjs → Bundled pipeline (server imports this)
|
|
66
|
+
├── player/server.mjs → --edit server
|
|
67
|
+
├── player/label-server.mjs→ --label server
|
|
68
|
+
└── tests/ → Vitest integration tests
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## External component contract
|
|
72
|
+
|
|
73
|
+
Register components at render time:
|
|
74
|
+
|
|
75
|
+
```tsx
|
|
76
|
+
<MarkCut
|
|
77
|
+
root={descriptiveJson}
|
|
78
|
+
compose={{ components: { AnimatedHeadline, StatCounter } }}
|
|
79
|
+
/>
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
In descriptive markdown:
|
|
83
|
+
|
|
84
|
+
```md
|
|
85
|
+
- component componentName:StatCounter duration:3 props:{value:42}
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Components receive props as defined in the descriptive JSON. No eval, no JSX parsing — just `React.createElement(registry[name], props)`.
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Stream Tree Architecture
|
|
92
|
+
|
|
93
|
+
### Core Concept
|
|
94
|
+
|
|
95
|
+
Everything is a **stream tree** — nested JSON nodes keyed by `type`.
|
|
96
|
+
The tree is validated by Zod and rendered by Remotion via recursive React components.
|
|
97
|
+
|
|
98
|
+
```json
|
|
99
|
+
{ "id":"root", "type":"root", "width":1080, "height":1920, "fps":30,
|
|
100
|
+
"isSeries":true, "transition":"fade", "children":[...] }
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
### Composition Rules
|
|
104
|
+
|
|
105
|
+
| Concept | Rule |
|
|
106
|
+
|---------|------|
|
|
107
|
+
| **Series** (`isSeries:true`) | Children play sequentially. Optional `transition` (fade/slide/wipe/flip/clockWipe) + `transitionTime` (default 0.5s). |
|
|
108
|
+
| **Parallel** (`isSeries:false`) | Children play simultaneously. Duration = max child duration. |
|
|
109
|
+
| **Background** (`isBackground:true`) | Loops for parent duration. Excluded from duration calculation. |
|
|
110
|
+
| **Actions** | Every leaf has `actions[{start, end, startFrom?, endAt?, loop?, volume?, style?}]` — seconds relative to parent. |
|
|
111
|
+
| **Scene** (`type:"scene"`) | UI storyboard card; engine treats as folder. Has `name` + `description` + `script`. |
|
|
112
|
+
|
|
113
|
+
### Duration Calculation
|
|
114
|
+
|
|
115
|
+
Computed by `getDurationInSeconds()` in `src/utils/index.ts`:
|
|
116
|
+
- **Leaf nodes**: duration = last action's `end` value
|
|
117
|
+
- **Series**: sum of children durations minus transition overlaps
|
|
118
|
+
- **Parallel**: max of children durations
|
|
119
|
+
- **Background children**: excluded from calculation
|
|
120
|
+
- **Include with `src`**: treated as leaf (duration from action end)
|
|
121
|
+
- Sets `durationInSeconds` on every node as a side effect
|
|
122
|
+
|
|
123
|
+
### 12 Stream Types
|
|
124
|
+
|
|
125
|
+
All types share base fields from `BaseShape`: `id`, `name`, `title`, `description`, `script`, `src`, `style`, `visible`, `isBackground`, `durationInSeconds`.
|
|
126
|
+
|
|
127
|
+
| Type | Purpose | Key Fields |
|
|
128
|
+
|------|---------|------------|
|
|
129
|
+
| `root` | Canvas container | `width`, `height`, `fps`, `isSeries`, `transition`, `stylesheet` |
|
|
130
|
+
| `folder` | Group children | `isSeries`, `transition`, `children[]` |
|
|
131
|
+
| `scene` | Storyboard node (alias for folder) | `name`, `description`, `script`, `children[]` |
|
|
132
|
+
| `image` | Still photo | `src`, `fit` (contain/cover/fill) |
|
|
133
|
+
| `video` | Video clip | `src`, `volume`, `playbackRate` |
|
|
134
|
+
| `audio` | Soundtrack/SFX | `src`, `volume`, `foreground` (ducks parent) |
|
|
135
|
+
| `subtitle` | Text overlay | `src` (text/VTT) or `cues[]`, `fontSize`, `style` |
|
|
136
|
+
| `component` | React component | `componentName`, `props`, `src` (remote ESM) |
|
|
137
|
+
| `effect` | CSS animation wrapper | `animation`, `customKeyframes`, `children[]` |
|
|
138
|
+
| `rhythm` | Beat-synced loop | `src`, `spots[]`, `children[]` |
|
|
139
|
+
| `map` | Route visualization | `waypoints[{lat,lng,label}]`, `routeColor` |
|
|
140
|
+
| `include` | Embed sub-tree | `src` (path/URL/data URI), `children[]` |
|
|
141
|
+
|
|
142
|
+
## Key Implementation Details
|
|
143
|
+
|
|
144
|
+
### Folder.tsx — Series/Parallel Engine
|
|
145
|
+
|
|
146
|
+
- `FolderLeaf` is the recursive workhorse that renders all stream types
|
|
147
|
+
- For `isSeries:true`: uses Remotion `<Series>` or `<TransitionSeries>` (if transition set)
|
|
148
|
+
- For `isSeries:false`: renders children as parallel `<Sequence>` elements
|
|
149
|
+
- Each child's duration comes from `child.durationInSeconds` (set by `getDurationInSeconds`)
|
|
150
|
+
- Transition elements are interleaved between series children
|
|
151
|
+
- Background children are wrapped in `<Loop>`
|
|
152
|
+
- Scene nodes delegate to `SceneLeaf` → `FolderLeaf` (identity)
|
|
153
|
+
- Effect nodes delegate to `EffectWrapper` → `FolderLeaf`
|
|
154
|
+
|
|
155
|
+
### Video.tsx — startFrom/endAt Trimming
|
|
156
|
+
|
|
157
|
+
- Uses Remotion `<OffthreadVideo>` for CPU-efficient video decoding
|
|
158
|
+
- `startFrom` / `endAt` on actions trim the source video (in seconds)
|
|
159
|
+
- `playbackRate` is capped at 1 (`Math.min(1, ...)`) so source longer than timeline plays at normal speed and is truncated by the Sequence
|
|
160
|
+
- **Critical formula**: `OffthreadVideo endAt = startFrom_frames + ((endAt_source - startFrom_source) * fps / playbackRate)`
|
|
161
|
+
This sets `endAt` beyond the actual source end when `playbackRate < 1`, which prevents Remotion from rendering blank frames before the Sequence ends.
|
|
162
|
+
- `endAt` without explicit value falls back to `stream.durationInSeconds ?? end - start`
|
|
163
|
+
|
|
164
|
+
### Rendering Pipeline
|
|
165
|
+
|
|
166
|
+
1. `Root.tsx` reads `getInputProps()` → parses via Zod `rootSchema`
|
|
167
|
+
2. `getDurationInSeconds()` stamps `durationInSeconds` on every node
|
|
168
|
+
3. `Composition.durationInFrames` is set to total duration
|
|
169
|
+
4. `MarkCut` wraps everything in `ComposeContext` + `ThemeProvider` + `AbsoluteFill`
|
|
170
|
+
5. `FolderLeaf` recursively renders the tree
|
|
171
|
+
|
|
172
|
+
### CLI Usage
|
|
173
|
+
|
|
174
|
+
```bash
|
|
175
|
+
node src/render/cli.mjs render <file.json> [--aspect 16x9|9x16|1x1|all] [--output path]
|
|
176
|
+
node src/render/cli.mjs render --template <id> --data <data.json>
|
|
177
|
+
node src/render/cli.mjs preview <file.json> [--edit] [--label] [--port 3001]
|
|
178
|
+
node src/render/cli.mjs templates
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
### Player Server (Edit Mode)
|
|
182
|
+
|
|
183
|
+
- `--edit` flag starts a live-reload loop: edit JSON → player auto-refreshes
|
|
184
|
+
- `--label` flag starts a labeling UI for selecting and annotating media
|
|
185
|
+
- `--port` flag sets the server port (default 3001)
|
|
186
|
+
- Two implementations: `server.mjs` (Remotion Player) and `label-server.mjs` (label input overlay)
|
|
187
|
+
|
|
188
|
+
## Testing
|
|
189
|
+
|
|
190
|
+
### Test Framework
|
|
191
|
+
- **Vitest** with custom config in `tests/vitest.config.ts`
|
|
192
|
+
- Non-parallel (`pool: "forks", fileParallelism: false`)
|
|
193
|
+
- 600s timeout per test (integration tests render real MP4s)
|
|
194
|
+
|
|
195
|
+
### Key Test Utilities (`tests/utils.ts`)
|
|
196
|
+
|
|
197
|
+
| Function | Purpose |
|
|
198
|
+
|----------|---------|
|
|
199
|
+
| `renderFixture(path, opts)` | Render a stream tree JSON → MP4 using `npx remotion render` |
|
|
200
|
+
| `renderScenes(path, opts)` | Render scene-based video.json using `Main16x9` composition |
|
|
201
|
+
| `getVideoInfo(path)` | Extract metadata via ffprobe (duration, dimensions, FPS, audio) |
|
|
202
|
+
| `extractFrame(video, time, out)` | Extract PNG frame at timestamp via ffmpeg |
|
|
203
|
+
| `isFrameNonBlank(path)` | Check center pixel isn't black (>10 in any RGB channel) |
|
|
204
|
+
| `getFrameFileSize(path)` | Get PNG file size (proxy for visual content) |
|
|
205
|
+
| `extractAudio(video, out)` | Extract audio track to WAV |
|
|
206
|
+
| `transcribeAudio(path)` | Run whisper STT on audio |
|
|
207
|
+
|
|
208
|
+
### Test Categories (30 tests)
|
|
209
|
+
|
|
210
|
+
1. Basic Rendering — empty composition, dimension verification
|
|
211
|
+
2. Image + Subtitle — image rendering, inline/VTT/karaoke subtitles
|
|
212
|
+
3. Built-in Components — AnimatedHeadline, StatCounter, GradientBackground
|
|
213
|
+
4. Effects — fadeIn, bounceIn, custom keyframes
|
|
214
|
+
5. Map Rendering — canvas route visualization
|
|
215
|
+
6. Audio Rendering — audio track, non-silent verification
|
|
216
|
+
7. Include — nested composition via include
|
|
217
|
+
8. Full Feature Combination — multiple features
|
|
218
|
+
9. Scene Node — scene-based rendering
|
|
219
|
+
10. Audio STT — speech-to-text verification via whisper
|
|
220
|
+
11. Multiple Aspect Ratios — 16x9, 9x16, 1x1
|
|
221
|
+
12. Cross-Stream Types — parallel rendering, effect-wrapped children
|
|
222
|
+
13. Frame-Accurate — timeline consistency, frame count
|
|
223
|
+
14. Description Field — schema validation, render preservation
|
|
224
|
+
15. Scene as Folder Alias — scene/folder interchangeability
|
|
225
|
+
16. **Video startFrom/endAt Trimming** — video trimming + photo in series
|
|
226
|
+
|
|
227
|
+
### Running Tests
|
|
228
|
+
|
|
229
|
+
```bash
|
|
230
|
+
npm test # all tests
|
|
231
|
+
npm run test:integration # render integration tests only
|
|
232
|
+
npx vitest run tests/render.test.ts -t "Test Name" # single test
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
## Known Issues & Gotchas
|
|
236
|
+
|
|
237
|
+
1. **TransitionSeries + isBackground**: `TransitionSeries` from `@remotion/transitions` rejects non-Sequence/Transition children. Don't mix `isBackground:true` children with `isSeries:true, transition:"..."` at the same level.
|
|
238
|
+
2. **Subvideo internal actions**: Should use `start:0` when placed in a series parent (series handles positioning).
|
|
239
|
+
3. **Video endAt formula**: Must be `startFrom_frames + ((endAt_source - startFrom_source) * fps / playbackRate)` — changing to `endAt_source * fps` breaks slow-motion playback.
|
|
240
|
+
4. **`signalstats` ffmpeg filter**: Doesn't output YAVG on all systems; use file size as proxy for visual content.
|
|
241
|
+
5. **Asset paths**: Local files are resolved via `staticFile()` — files go in `public/` and are referenced without the `public/` prefix.
|
|
242
|
+
6. **Map rendering**: Depends on external tile availability; may produce blank frames offline.
|
|
243
|
+
|
|
244
|
+
## 3-Level Authoring Workflow (see SKILL.md)
|
|
245
|
+
|
|
246
|
+
| Level | Format | Purpose |
|
|
247
|
+
|-------|--------|---------|
|
|
248
|
+
| **Label** | `labels.json` | Browse media, label clips, add descriptions |
|
|
249
|
+
| **Storyboard** | `storyboard.json` | `scene` nodes with high-level structure + script |
|
|
250
|
+
| **Assemble** | `video.json` | Full stream tree with all types, render to MP4 |
|
|
251
|
+
|
|
252
|
+
### Script → TTS → STT → VTT Pipeline
|
|
253
|
+
|
|
254
|
+
Every node carries an optional `script` field for narration/dialogue text. The pipeline:
|
|
255
|
+
1. **Storyboard** — agent writes `script` on any node (scene, folder, leaf) as narrative text
|
|
256
|
+
2. **TTS** — walk the tree, collect all `script` values with node id + timing, generate WAV audio per segment via `edge-tts`
|
|
257
|
+
3. **STT** — transcribe each TTS audio or concatenated audio to VTT via whisper
|
|
258
|
+
4. **Subtitle** — attach resulting VTT as a `subtitle` child node for final rendering
|
|
259
|
+
|
|
260
|
+
This separates concerns: `script` = authoring text (human/agent-friendly), `subtitle` = rendered output (timed VTT cues).
|
|
261
|
+
|
|
262
|
+
## Dependencies
|
|
263
|
+
|
|
264
|
+
- **Remotion 4.0.469** — core rendering engine
|
|
265
|
+
- **React 19** — component model
|
|
266
|
+
- **Zod 4** — schema validation
|
|
267
|
+
- **@remotion/transitions** — scene transitions (fade, slide, wipe, flip, clockWipe)
|
|
268
|
+
- **@vis.gl/react-google-maps** — map visualization
|
|
269
|
+
- **immer** — state management (player app)
|
|
270
|
+
- **vitest** — test runner
|
|
271
|
+
- **TypeScript 5.6** — strict mode with `noUncheckedIndexedAccess`
|
package/README.md
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
# markcut — Markdown-to-Video Engine
|
|
2
|
+
|
|
3
|
+
Write a storyboard in markdown, get a rendered video with TTS narration and subtitles.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
# Render a storyboard to MP4
|
|
7
|
+
npx markcut render storyboard.md --aspect 9x16
|
|
8
|
+
|
|
9
|
+
# Preview with live edit (edit .md file, player auto-reloads)
|
|
10
|
+
npx markcut preview storyboard.md --edit
|
|
11
|
+
|
|
12
|
+
# Label clips in a stream tree
|
|
13
|
+
npx markcut preview labels.json --label --port=3031
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## How it works
|
|
17
|
+
|
|
18
|
+
```
|
|
19
|
+
storyboard.md ──[parse]──▶ DescriptiveRoot ──[compile]──▶ Stream Tree ──[Remotion]──▶ MP4
|
|
20
|
+
│ │
|
|
21
|
+
resolveMediaDurations() <Sequence>
|
|
22
|
+
resolveScripts() → TTS <Series>
|
|
23
|
+
resolveSubtitles() → VTT <Img>, <OffthreadVideo>
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
1. **Write** a storyboard in markdown (scenes with `## headings`, media with `-` bullets)
|
|
27
|
+
2. **Parse** the markdown into a descriptive tree
|
|
28
|
+
3. **Resolve** media durations (ffprobe), generate TTS narration (edge-tts), transcribe to VTT (whisper)
|
|
29
|
+
4. **Compile** the descriptive tree into a stream tree (the low-level Remotion JSON)
|
|
30
|
+
5. **Render** with Remotion — no React code needed
|
|
31
|
+
|
|
32
|
+
## Features
|
|
33
|
+
|
|
34
|
+
| Feature | Description |
|
|
35
|
+
|---|---|
|
|
36
|
+
| **Markdown input** | Write `## Scene` headings, `- bullet` leaves. Components via `jsx:"<Tag />"` syntax |
|
|
37
|
+
| **JS imports block** | `` ```js imports `` code fence with real ESM `import` statements instead of YAML |
|
|
38
|
+
| **Dynamic components** | Load any React component from npm, GitHub, or any URL via `import { X } from "pkg"` |
|
|
39
|
+
| **Tween animation** | `tween(from, to)` function calls in JSX for frame-accurate numeric animation |
|
|
40
|
+
| **Compiled input** | Pre-compiled stream tree for direct Remotion rendering |
|
|
41
|
+
| **TTS narration** | `script` field → edge-tts CLI → WAV audio. Configurable engine (mlx-audio, custom) |
|
|
42
|
+
| **STT subtitles** | TTS audio → whisper → VTT → root subtitle overlay with animated caption types (typewriter, fade, bounce, etc.) |
|
|
43
|
+
| **Tween animation** | `tween(from, to)` function calls in JSX for frame-accurate numeric animation |
|
|
44
|
+
| **Styling** | Inline `style` strings on any node for CSS. JSX components use inline React styles |
|
|
45
|
+
| **Live edit** | `--edit` watches the input file, re-runs pipeline, auto-reloads player |
|
|
46
|
+
| **Label mode** | `--label` interactive player with per-scene label input, saves to labels.json |
|
|
47
|
+
| **CLI** | `render`, `preview` commands for MP4 export and Remotion Studio |
|
|
48
|
+
| **Programmatic** | `MarkCut` / `DescriptiveComposition` React components for embedding |
|
|
49
|
+
|
|
50
|
+
## Example
|
|
51
|
+
|
|
52
|
+
```md
|
|
53
|
+
# video
|
|
54
|
+
width:1080 height:1920 fps:30 layout:series
|
|
55
|
+
|
|
56
|
+
## Hook
|
|
57
|
+
layout:parallel script:"Set the mood with a beautiful landscape"
|
|
58
|
+
- image src:cover.jpg duration:3
|
|
59
|
+
|
|
60
|
+
## Features
|
|
61
|
+
layout:transitionSeries transition:fade transitionTime:0.4 script:"Show what we built"
|
|
62
|
+
- component duration:6 jsx:"<DeviceMockup src='screenshot.png' />"
|
|
63
|
+
- component duration:4 jsx:"<StatCounter value={100} suffix='K' label='Users' />"
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Components are registered via frontmatter `imports:` or a `` ```js imports `` code block:
|
|
67
|
+
|
|
68
|
+
````
|
|
69
|
+
```js imports
|
|
70
|
+
import { DeviceMockup } from "mockup-component"
|
|
71
|
+
import { StatCounter } from "stat-counter"
|
|
72
|
+
|
|
73
|
+
// Inline component definition
|
|
74
|
+
export function Greeting({ name }) {
|
|
75
|
+
return <h1 style={{color: '#fff'}}>Hello {name}</h1>;
|
|
76
|
+
}
|
|
77
|
+
```
|
|
78
|
+
````
|
|
79
|
+
|
|
80
|
+
### Dynamic Components
|
|
81
|
+
|
|
82
|
+
Any React component from npm, GitHub, or any URL can be imported and used directly in JSX expressions. The engine loads them at render time via esm.sh — no build step required.
|
|
83
|
+
|
|
84
|
+
```markdown
|
|
85
|
+
- component duration:3 jsx:"<PieChart data={[{value:40,color:'#E38627'}]} />"
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Inline components can be defined entirely in the imports block using `export function`, making the video self-contained with no external files.
|
|
89
|
+
|
|
90
|
+
## Docs
|
|
91
|
+
|
|
92
|
+
| Document | What it covers |
|
|
93
|
+
|---|---|
|
|
94
|
+
| [docs/json-descriptive.md](docs/json-descriptive.md) | Full JSON descriptive schema reference |
|
|
95
|
+
| [docs/markdown-strict-descriptive.md](docs/markdown-strict-descriptive.md) | Markdown descriptive syntax reference |
|
|
96
|
+
| [docs/label-mode.md](docs/label-mode.md) | Label mode player and workflow |
|
|
97
|
+
| [docs/edit-mode.md](docs/edit-mode.md) | Live edit mode with SSE reload |
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
## Variants
|
|
101
|
+
|
|
102
|
+
Create different versions of the same video for different platforms, languages, or formats using **variant sections** in your markdown.
|
|
103
|
+
|
|
104
|
+
### Defining Variants
|
|
105
|
+
|
|
106
|
+
Add `# variant-name` headings alongside `# video` (the base). Each variant can override any property using variant-prefixed keys:
|
|
107
|
+
|
|
108
|
+
```markdown
|
|
109
|
+
# video
|
|
110
|
+
width:1080 height:1920 fps:30 layout:series
|
|
111
|
+
|
|
112
|
+
## Intro
|
|
113
|
+
layout:parallel
|
|
114
|
+
- component duration:3 jsx:"<h1>Hello</h1>"
|
|
115
|
+
- image src:default.jpg duration:3
|
|
116
|
+
|
|
117
|
+
# zh
|
|
118
|
+
## Intro
|
|
119
|
+
layout:parallel
|
|
120
|
+
- component duration:3 zh:"<h1>你好</h1>" ← bare key overrides jsx (primary content)
|
|
121
|
+
- image zh-src:zh-image.jpg src:default.jpg ← zh-src overrides src
|
|
122
|
+
|
|
123
|
+
# youtube
|
|
124
|
+
width:1920 height:1080 fps:30 ← different aspect ratio
|
|
125
|
+
## Intro
|
|
126
|
+
layout:parallel
|
|
127
|
+
- component duration:3 jsx:"<h1>YouTube Hello</h1>"
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
Two override mechanisms:
|
|
131
|
+
- **Variant-prefixed keys**: `zh-src` → `src`, `zh-jsx` → `jsx`, `zh-script` → `script`
|
|
132
|
+
- **Bare variant keys**: `zh:"<h1>你好</h1>"` replaces the node's primary content key (`jsx` for components, `src` for images/video, `script` for audio)
|
|
133
|
+
|
|
134
|
+
### Multi-Variant Preview Server
|
|
135
|
+
|
|
136
|
+
The `preview` command accepts multiple `--variant` flags, each defining a separate compilation target. Variants are compiled **sequentially** (not in parallel) to avoid resource contention from CLI tools like whisper and ffprobe.
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
# Preview 4 variants simultaneously
|
|
140
|
+
npx markcut preview storyboard.md \
|
|
141
|
+
--variant default \
|
|
142
|
+
--variant zh-tiktok \
|
|
143
|
+
--variant en-tiktok \
|
|
144
|
+
--variant youtube
|
|
145
|
+
|
|
146
|
+
# Each variant is served at its own URL:
|
|
147
|
+
# http://localhost:3001 → default
|
|
148
|
+
# http://localhost:3001/zh-tiktok → zh + tiktok chain
|
|
149
|
+
# http://localhost:3001/en-tiktok → en + tiktok chain
|
|
150
|
+
# http://localhost:3001/youtube → youtube
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
Compound variant labels (like `zh-tiktok`) are split on `-` to form a **variant chain** — the parser applies `zh` overrides first, then `tiktok` overrides on top. Each variant gets its own compiled output and subtitle directory. Component bundles are stored in a shared cache (`.markcut/generated/components/`) — identical imports across variants reuse the same bundle.
|
|
154
|
+
|
|
155
|
+
### Variant-Aware Browser
|
|
156
|
+
|
|
157
|
+
The browser player reads `window.VARIANT` from the URL path and fetches the correct compiled data via `/api/video-data?variant=<name>`. A **variant switcher bar** at the top of the player lets you jump between variants instantly.
|
|
158
|
+
|
|
159
|
+
## `.markcut/` Directory Layout
|
|
160
|
+
|
|
161
|
+
When you run `preview` or `render`, all generated artifacts live under `.markcut/` next to the source file. With multiple variants, each gets its own subdirectory:
|
|
162
|
+
|
|
163
|
+
```
|
|
164
|
+
.markcut/
|
|
165
|
+
generated/ ← shared, content-addressed (across all variants & files)
|
|
166
|
+
tts/ ← TTS narration audio (content-hash filenames)
|
|
167
|
+
media/ ← TTI/TTV media (content-hash filenames)
|
|
168
|
+
includes/ ← compiled sub-video JSON (content-hash)
|
|
169
|
+
components/ ← component bundles (content-hash, shared across variants)
|
|
170
|
+
<basename>/ ← per-source-file
|
|
171
|
+
default/ ← default variant artifacts
|
|
172
|
+
compiled.json ← compiled stream tree
|
|
173
|
+
subtitles.vtt
|
|
174
|
+
zh-tiktok/ ← variant-specific artifacts
|
|
175
|
+
compiled.json
|
|
176
|
+
subtitles.vtt
|
|
177
|
+
youtube/
|
|
178
|
+
compiled.json
|
|
179
|
+
subtitles.vtt
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
- **`generated/`** — shared artifacts using content-hash filenames, so identical scripts/prompts across different variants reuse the same cached file
|
|
183
|
+
- **`generated/components/`** — component bundles shared across all variants and source files. Identical imports produce the same content hash, so variants with the same component registrations reuse the same bundle
|
|
184
|
+
- **`<basename>/<variant>/compiled.json`** — the compiled stream tree for each variant, cached to disk for fast restarts
|
|
185
|
+
- The server serves `.markcut/` as a document root, so components at `.markcut/generated/components/abc.js` are served as `/generated/components/abc.js`
|
|
186
|
+
|
|
187
|
+
The `.markcut/` directory is gitignored by default — it's regenerated on each run.
|
|
188
|
+
|
|
189
|
+
## Architecture
|
|
190
|
+
|
|
191
|
+
```
|
|
192
|
+
src/
|
|
193
|
+
├── entry.tsx ← Library entry: MarkCut + DescriptiveComposition
|
|
194
|
+
├── index.ts ← Remotion registerRoot (for studio/render CLI)
|
|
195
|
+
├── Root.tsx ← Remotion Composition wrapper
|
|
196
|
+
├── schema/ ← Zod schemas for all stream types
|
|
197
|
+
├── types/ ← React renderers (Folder, Video, Image, Audio, etc.)
|
|
198
|
+
├── descriptive/ ← Compiler, markdown parser, resolve pipeline
|
|
199
|
+
├── themes/ ← Theme presets + ThemeProvider
|
|
200
|
+
├── render/
|
|
201
|
+
│ ├── cli.mjs ← CLI entry point
|
|
202
|
+
│ └── tts.ts ← TTS via CLI template + variable substitution
|
|
203
|
+
├── player/
|
|
204
|
+
│ ├── server.mjs ← --edit player server
|
|
205
|
+
│ ├── label-server.mjs ← --label player server
|
|
206
|
+
│ ├── pipeline.ts ← Pipeline entry (bundled to pipeline.mjs)
|
|
207
|
+
│ └── browser.tsx ← Browser player (bundled with esbuild)
|
|
208
|
+
├── utils/ ← Duration calc, VTT parser, helpers
|
|
209
|
+
└── tests/ ← Vitest integration tests
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
## Scripts
|
|
213
|
+
|
|
214
|
+
```bash
|
|
215
|
+
npm run render # render a JSON stream tree to MP4
|
|
216
|
+
npm run preview # open Remotion Studio
|
|
217
|
+
npm run typecheck # TypeScript type check
|
|
218
|
+
npm test # run unit + integration tests
|
|
219
|
+
```
|