@lalalic/markcut 1.1.1 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/AGENTS.md +39 -15
  2. package/README.md +7 -3
  3. package/SKILL.md +26 -172
  4. package/docs/edit-mode.md +1 -1
  5. package/docs/label-mode.md +6 -4
  6. package/docs/markdown-descriptive.md +46 -1
  7. package/docs/system-prompt-edit.md +16 -0
  8. package/package.json +1 -1
  9. package/src/config.mjs +8 -3
  10. package/src/descriptive/compiler.test.ts +42 -42
  11. package/src/descriptive/compiler.ts +41 -52
  12. package/src/descriptive/markdown.ts +8 -4
  13. package/src/descriptive/resolve.test.ts +14 -7
  14. package/src/descriptive/resolve.ts +148 -20
  15. package/src/entry.tsx +8 -2
  16. package/src/player/browser.tsx +178 -54
  17. package/src/player/bundle/player.js +1168 -566
  18. package/src/player/components/EditControls.tsx +92 -0
  19. package/src/player/components/HeaderBar.tsx +60 -0
  20. package/src/player/components/LabelControls.tsx +367 -0
  21. package/src/player/components/SceneThumbnails.tsx +87 -0
  22. package/src/player/components/VariantBar.tsx +39 -0
  23. package/src/player/components/index.ts +5 -0
  24. package/src/player/pipeline.mjs +130 -66
  25. package/src/player/pipeline.ts +3 -1
  26. package/src/player/server-shared.mjs +5 -7
  27. package/src/player/server.mjs +194 -187
  28. package/src/render/cli.mjs +66 -13
  29. package/src/schema/index.ts +20 -23
  30. package/src/types/Audio.tsx +25 -33
  31. package/src/types/Component.tsx +18 -24
  32. package/src/types/Effect.tsx +31 -39
  33. package/src/types/Image.tsx +23 -30
  34. package/src/types/Include.tsx +70 -76
  35. package/src/types/Map.tsx +48 -44
  36. package/src/types/Rhythm.tsx +19 -27
  37. package/src/types/Video.tsx +40 -47
  38. package/src/utils/component-import-map.ts +93 -0
  39. package/src/utils/index.ts +23 -10
  40. package/src/vision/cli.mjs +6 -6
  41. package/templates/courseware/TEMPLATE.md +317 -0
  42. package/templates/courseware/agents/reviewer.md +84 -0
  43. package/templates/courseware/prompts/fix.md +30 -0
  44. package/templates/courseware/prompts/outline.md +53 -0
  45. package/templates/courseware/prompts/scene.md +64 -0
  46. package/tests/fixtures/audio.json +4 -2
  47. package/tests/fixtures/basic.json +2 -1
  48. package/tests/fixtures/component-all.json +4 -2
  49. package/tests/fixtures/components.json +4 -2
  50. package/tests/fixtures/effects.json +12 -6
  51. package/tests/fixtures/full.json +8 -4
  52. package/tests/fixtures/map.json +2 -1
  53. package/tests/fixtures/md/dialogue.md +12 -0
  54. package/tests/fixtures/md/edge-cases.md +9 -0
  55. package/tests/fixtures/scenes.json +4 -2
  56. package/tests/fixtures/subtitle.json +6 -3
  57. package/tests/fixtures/subvideo.json +11 -6
  58. package/tests/fixtures/templates/courseware.md +61 -4
  59. package/tests/fixtures/tween-visual.json +2 -6
  60. package/tests/fixtures/video-series.json +6 -14
  61. package/tests/md-descriptive.test.ts +170 -0
  62. package/tests/render.test.ts +32 -16
  63. package/tests/schema.test.ts +6 -3
  64. package/tests/server.test.ts +9 -6
  65. package/tests/utils.ts +4 -4
  66. package/docs/dynamic-components.md +0 -191
  67. package/docs/templates.md +0 -52
  68. package/scripts/artlist-dl.mjs +0 -190
  69. package/src/player/label-server.mjs +0 -599
package/AGENTS.md CHANGED
@@ -63,8 +63,12 @@ src/
63
63
  ├── render/cli.mjs → CLI entry
64
64
  ├── render/tts.ts → TTS via CLI template + variable substitution
65
65
  ├── player/pipeline.mjs → Bundled pipeline (server imports this)
66
- ├── player/server.mjs → --edit server
67
- ├── player/label-server.mjs--label server
66
+ ├── player/server.mjs → Unified server (--edit, --label, preview)
67
+ ├── player/components/ React control components (edit, label, variant bar)
68
+ │ ├── index.ts
69
+ │ ├── EditControls.tsx
70
+ │ ├── LabelControls.tsx
71
+ │ └── VariantBar.tsx
68
72
  └── tests/ → Vitest integration tests
69
73
  ```
70
74
 
@@ -107,22 +111,22 @@ The tree is validated by Zod and rendered by Remotion via recursive React compon
107
111
  | **Series** (`isSeries:true`) | Children play sequentially. Optional `transition` (fade/slide/wipe/flip/clockWipe) + `transitionTime` (default 0.5s). |
108
112
  | **Parallel** (`isSeries:false`) | Children play simultaneously. Duration = max child duration. |
109
113
  | **Background** (`isBackground:true`) | Loops for parent duration. Excluded from duration calculation. |
110
- | **Actions** | Every leaf has `actions[{start, end, startFrom?, endAt?, loop?, volume?, style?}]` seconds relative to parent. |
114
+ | **Timing** | Every leaf carries base fields `start`/`end` (seconds, relative to parent) + optional `startFrom`/`endAt` (source trim) + `duration` (convenience). `end` is the source of truth; `duration` normalizes to `end = start + duration` when `end` is absent. Inside series parents `start` is 0 (positioning is implicit). |
111
115
  | **Scene** (`type:"scene"`) | UI storyboard card; engine treats as folder. Has `name` + `description` + `script`. |
112
116
 
113
117
  ### Duration Calculation
114
118
 
115
119
  Computed by `getDurationInSeconds()` in `src/utils/index.ts`:
116
- - **Leaf nodes**: duration = last action's `end` value
120
+ - **Leaf nodes**: duration = `leafEnd(stream)` = `end ?? (start + duration)` (single deterministic read; `start`/`end`/`duration` live on the base node)
117
121
  - **Series**: sum of children durations minus transition overlaps
118
122
  - **Parallel**: max of children durations
119
123
  - **Background children**: excluded from calculation
120
- - **Include with `src`**: treated as leaf (duration from action end)
124
+ - **Include with `src`**: treated as leaf (duration from base `end`)
121
125
  - Sets `durationInSeconds` on every node as a side effect
122
126
 
123
127
  ### 12 Stream Types
124
128
 
125
- All types share base fields from `BaseShape`: `id`, `name`, `title`, `description`, `script`, `src`, `style`, `visible`, `isBackground`, `durationInSeconds`.
129
+ All types share base fields from `BaseShape`: `id`, `name`, `title`, `description`, `script`, `src`, `style`, `visible`, `isBackground`, `start`, `end`, `duration`, `startFrom`, `endAt`, `durationInSeconds`.
126
130
 
127
131
  | Type | Purpose | Key Fields |
128
132
  |------|---------|------------|
@@ -130,8 +134,8 @@ All types share base fields from `BaseShape`: `id`, `name`, `title`, `descriptio
130
134
  | `folder` | Group children | `isSeries`, `transition`, `children[]` |
131
135
  | `scene` | Storyboard node (alias for folder) | `name`, `description`, `script`, `children[]` |
132
136
  | `image` | Still photo | `src`, `fit` (contain/cover/fill) |
133
- | `video` | Video clip | `src`, `volume`, `playbackRate` |
134
- | `audio` | Soundtrack/SFX | `src`, `volume`, `foreground` (ducks parent) |
137
+ | `video` | Video clip | `src`, `volume`, `playbackRate`, `loop` |
138
+ | `audio` | Soundtrack/SFX | `src`, `volume`, `foreground` (ducks parent), `loop` |
135
139
  | `subtitle` | Text overlay | `src` (text/VTT) or `cues[]`, `fontSize`, `style` |
136
140
  | `component` | React component | `componentName`, `props`, `src` (remote ESM) |
137
141
  | `effect` | CSS animation wrapper | `animation`, `customKeyframes`, `children[]` |
@@ -155,7 +159,7 @@ All types share base fields from `BaseShape`: `id`, `name`, `title`, `descriptio
155
159
  ### Video.tsx — startFrom/endAt Trimming
156
160
 
157
161
  - Uses Remotion `<OffthreadVideo>` for CPU-efficient video decoding
158
- - `startFrom` / `endAt` on actions trim the source video (in seconds)
162
+ - `startFrom` / `endAt` on the base node trim the source video (in seconds)
159
163
  - `playbackRate` is capped at 1 (`Math.min(1, ...)`) so source longer than timeline plays at normal speed and is truncated by the Sequence
160
164
  - **Critical formula**: `OffthreadVideo endAt = startFrom_frames + ((endAt_source - startFrom_source) * fps / playbackRate)`
161
165
  This sets `endAt` beyond the actual source end when `playbackRate < 1`, which prevents Remotion from rendering blank frames before the Sequence ends.
@@ -183,7 +187,7 @@ node src/render/cli.mjs templates
183
187
  - `--edit` flag starts a live-reload loop: edit JSON → player auto-refreshes
184
188
  - `--label` flag starts a labeling UI for selecting and annotating media
185
189
  - `--port` flag sets the server port (default 3001)
186
- - Two implementations: `server.mjs` (Remotion Player) and `label-server.mjs` (label input overlay)
190
+ - Unified server: `server.mjs` serves all modes (preview, edit, label). React control components in `components/`.
187
191
 
188
192
  ## Testing
189
193
 
@@ -235,11 +239,13 @@ npx vitest run tests/render.test.ts -t "Test Name" # single test
235
239
  ## Known Issues & Gotchas
236
240
 
237
241
  1. **TransitionSeries + isBackground**: `TransitionSeries` from `@remotion/transitions` rejects non-Sequence/Transition children. Don't mix `isBackground:true` children with `isSeries:true, transition:"..."` at the same level.
238
- 2. **Subvideo internal actions**: Should use `start:0` when placed in a series parent (series handles positioning).
239
- 3. **Video endAt formula**: Must be `startFrom_frames + ((endAt_source - startFrom_source) * fps / playbackRate)` changing to `endAt_source * fps` breaks slow-motion playback.
240
- 4. **`signalstats` ffmpeg filter**: Doesn't output YAVG on all systems; use file size as proxy for visual content.
241
- 5. **Asset paths**: Local files are resolved via `staticFile()` files go in `public/` and are referenced without the `public/` prefix.
242
- 6. **Map rendering**: Depends on external tile availability; may produce blank frames offline.
242
+ 2. **Video endAt formula**: Must be `startFrom_frames + ((endAt_source - startFrom_source) * fps / playbackRate)` changing to `endAt_source * fps` breaks slow-motion playback.
243
+ 3. **`signalstats` ffmpeg filter**: Doesn't output YAVG on all systems; use file size as proxy for visual content.
244
+ 4. **Asset paths**: Local files are resolved via `staticFile()` files go in `public/` and are referenced without the `public/` prefix.
245
+ 5. **Map rendering**: Depends on external tile availability; may produce blank frames offline.
246
+ 6. **CLI md render needs three things preview gets for free**: (a) `parseMarkdownVariants(raw)` (returns `{base, variants}`) — `parseMarkdownDescriptive` returns the root directly; (b) bundling the ` ```js imports ``` ` block via `bundleFromEntries` and setting `root.imports` (otherwise jsx components are unregistered → slides render as unstyled black frames); (c) staging `.markcut/` assets into `public/.render-assets/` and rewriting srcs to relative paths (remotion render only serves its publicDir). All three live in `src/render/cli.mjs`; the import-map shim for shared react/remotion instances is `src/utils/component-import-map.ts` (must stay in sync with `player/browser.tsx` + `bundler.mjs getSharedExternals`).
247
+ 7. **Publishing**: verify `npx @lalalic/markcut render <md>` from a clean temp dir before/after publish — v1.1.1 shipped with cli.mjs/pipeline.mjs out of sync and md rendering was completely broken. Also beware stale npx caches (`~/.npm/_npx`) silently running old code.
248
+ 8. **Clean break from `actions[]`**: leaf timing is now flat base fields (`start`/`end`/`duration`/`startFrom`/`endAt`). Old compiled JSON carrying `actions:[{...}]` no longer parses into a duration — re-compile from markdown/descriptive JSON, or migrate `actions[0]` → flat fields (drop `id`/`volume`/`style`/`effectId`; keep `start`/`end`/`startFrom`/`endAt`/`loop`).
243
249
 
244
250
  ## 3-Level Authoring Workflow (see SKILL.md)
245
251
 
@@ -259,6 +265,24 @@ Every node carries an optional `script` field for narration/dialogue text. The p
259
265
 
260
266
  This separates concerns: `script` = authoring text (human/agent-friendly), `subtitle` = rendered output (timed VTT cues).
261
267
 
268
+ ### Multi-turn Dialogue
269
+
270
+ When `script` text contains multiple `SpeakerName: text` lines (2+), the pipeline auto-expands it into a multi-turn dialogue. Each line becomes a separate audio node with `speaker` field, wrapped in a `series` container for sequential playback.
271
+
272
+ ```md
273
+ - script "Ray: Hello everyone
274
+ Alice: Good day to you
275
+ Ray: Let's begin"
276
+ ```
277
+
278
+ Per-speaker voices are configured via `voices` on root. Each value is extra CLI flags appended to the TTS template:
279
+
280
+ ```md
281
+ voices:{"Ray":"--voice en-US-GuyNeural","Alice":"--voice en-US-JennyNeural"}
282
+ ```
283
+
284
+ The speaker name is matched against `voices` map and the extra flags are appended to the TTS CLI template, naturally supporting voice cloning or edge-tts features (rate, pitch, etc.). Subtitles include the speaker prefix (`Ray: Hello everyone`).
285
+
262
286
  ## Dependencies
263
287
 
264
288
  - **Remotion 4.0.469** — core rendering engine
package/README.md CHANGED
@@ -201,10 +201,14 @@ src/
201
201
  │ ├── cli.mjs ← CLI entry point
202
202
  │ └── tts.ts ← TTS via CLI template + variable substitution
203
203
  ├── player/
204
- │ ├── server.mjs ← --edit player server
205
- │ ├── label-server.mjs --label player server
204
+ │ ├── server.mjs ← Unified server (--edit, --label, preview)
205
+ │ ├── components/ React control components for all modes
206
+ │ │ ├── index.ts
207
+ │ │ ├── EditControls.tsx
208
+ │ │ ├── LabelControls.tsx
209
+ │ │ └── VariantBar.tsx
206
210
  │ ├── pipeline.ts ← Pipeline entry (bundled to pipeline.mjs)
207
- │ └── browser.tsx ← Browser player (bundled with esbuild)
211
+ │ └── browser.tsx ← Browser player (React, bundled to player.js)
208
212
  ├── utils/ ← Duration calc, VTT parser, helpers
209
213
  └── tests/ ← Vitest integration tests
210
214
  ```
package/SKILL.md CHANGED
@@ -1,194 +1,45 @@
1
1
  ---
2
2
  name: markcut
3
- description: >-
4
- Compose and render videos from streamline trees. CLI renders stream tree
5
- to MP4. Run via `npx markcut` — no install, no code.
3
+ description: use markdown to describe stream trees, provide CLI to render stream trees to video via `npx @lalalic/markcut`. use it to structure video scenes, and generate TTS, TTI, TTV, STT media automatically.
6
4
  ---
7
5
 
8
6
  ## Stream Tree Specs
9
7
 
10
- Everything video is a **stream tree**. see [docs/markdown-descriptive.md](docs/markdown-descriptive.md) for full details.
8
+ Everything video is a **stream tree** described with markdown. see [docs/markdown-descriptive.md](docs/markdown-descriptive.md) for full details.
11
9
 
12
- ## .markcut/ Directory Layout
13
10
 
14
- When you run `preview` or `render`, all generated artifacts live under `.markcut/` next to the source file:
11
+ ## video design utitlies
15
12
 
16
- ```
17
- .markcut/
18
- generated/ ← shared, content-addressed
19
- tts/ ← TTS audio (content-hash filenames)
20
- media/ ← TTI/TTV media (content-hash filenames)
21
- includes/ ← compiled sub-video JSON (content-hash)
22
- <basename>/ ← per-source-file artifacts
23
- components/ ← component bundles (per-file: imports differ)
24
- ```
25
-
26
- - `generated/` — shared artifacts using content-hash filenames, so identical scripts/prompts across different source files reuse the same cached file
27
- - `<basename>/components/` — per-file component bundles; each source file has its own `imports` block, so bundles live under its basename
28
- - The server serves `.markcut/` as a document root, so components at `.markcut/slides/components/abc.js` are served as `/slides/components/abc.js`
29
-
30
- ## Include (Sub-Video Embedding)
31
-
32
- The `include` node embeds an external `.md` file as a sub-video. The pipeline fully resolves it recursively:
33
-
34
- 1. **`resolveIncludes()`** (step 0 of `resolveAll`): reads the referenced `.md`, parses it, resolves its own includes/TTS/media, compiles to a stream tree, writes the compiled JSON to `.markcut/generated/includes/<hash>.json`
35
- 2. **`resolveIncludeImports()`** (server post-compile): if the sub-video has a ` ```js imports ``` ` block, bundles its components independently and writes the bundle URL into the compiled JSON
36
- 3. **`IncludeLeaf`** (render): loads the compiled JSON, dynamically imports the sub-video's component bundle, creates a nested `ComposeContext` with the sub-video's registry merged, and renders via `FolderLeaf`
37
-
38
- The sub-video's own components are isolated from the parent's — naming conflicts are resolved by "sub-video wins" priority.
39
-
40
- ```markdown
41
- # video
42
- ## Main
43
- - component duration:3 jsx:"<MainTitle />"
44
- ## Embedded
45
- - include src:./sub-video.md
46
- ## End
47
- - component duration:2 jsx:"<Outro />"
48
- ```
49
-
50
- ```markdown
51
- # sub-video
52
- - component duration:3 jsx:"<SubSlide scene='1' />"
53
-
54
- ```js imports
55
- export function SubSlide({ scene }) {
56
- return <div style={{background:'#667eea', ...}}>Scene {scene}</div>;
57
- }
58
- ```
59
- ```
60
- ---
61
-
62
- ## Video Design Best Practice (How-To)
63
-
64
- The engine supports four authoring phases. Each has a dedicated doc with full detail.
65
-
66
- ### Label — annotate clips in a stream tree
67
-
68
- Load a stream tree, preview each clip, and attach descriptive labels.
69
-
70
- ```bash
71
- npx markcut preview <flat stream tree file> --label
72
- ```
73
-
74
- Labels save to `labels.json`. See [docs/label-mode.md](docs/label-mode.md) for:
75
- - Label player UI (thumbnails, player, label input)
76
- - Labels persistence format
77
- - Full workflow diagram
13
+ ### User labels — label media with text, time ranges
14
+ - `uvx @lalalic/markcut preview --label` provides tool to label video and images with text.
78
15
 
79
16
  ### Storyboard — plan video structure with scene nodes
80
17
 
81
- Use `scene` nodes to organize your video. Each scene has `title`, `instruction`, `script` (TTS narration), `children`. Scenes can nest inside other scenes.
18
+ - Use `scene` nodes to organize your video. Scenes can nest inside other scenes.
19
+ - Use `description`, `scene.instruction`, `script`, `image|video.prompt` to structure your video content.
82
20
 
83
- ```bash
84
21
  see [docs/markdown-descriptive.md](docs/markdown-descriptive.md) for full details.
85
22
 
86
- ### 3. Assemble — render stream tree to MP4
87
-
88
- ```bash
89
- npx markcut render <stream tree file> --aspect all # MP4 output
90
- ```
91
-
92
- ---
93
-
94
- ## Template Variables
95
-
96
- `${width}`, `${height}`, `${fps}`, `${variant}` can be used in `src`,
97
- `prompt`, and `stylesheet` values. They resolve to the root config values.
23
+ ### Video Variants
24
+ - define variants for different video configurations, such as different languages, aspects, platforms
98
25
 
99
- ```markdown
100
- # video
101
- width:1920 height:1080
102
- ## Scene
103
- - image src:photo_${width}x${height}.jpg
104
- - component jsx:"<h1>${width}</h1>" ← NOT resolved (jsx excluded)
105
- ```
106
-
107
- > Only `src`, `prompt`, and `stylesheet` support template variables.
108
- > Root config keys, jsx, script, style, etc. do not.
109
-
110
- ## Frontmatter
111
-
112
- The YAML `---` block is **metadata only**. It does NOT affect video config.
113
- All configuration comes from the root config line after `# video`:
114
-
115
- ```markdown
116
- ---
117
- title: My Campaign
118
- ---
119
- # video
120
- width:640 height:480 fps:30 layout:series tts:"edge-tts --voice '...' --text '{input}' --write-media '{output}'"
121
- ```
122
-
123
- ## AI Media Generation (TTS / STT / TTI(text-to-image) / TTV(text-to-video))
124
-
125
- All four pipelines are configured via a **single CLI string** on the root config line. The user/LLM embeds every tool-specific parameter directly in the string — only `{input}` and `{output}` are substituted by the engine.
126
-
127
- Default CLI templates (used when not overridden on the root config line):
128
-
129
- | Config | Default |
130
- |--------|---------|
131
- | `tts` | `uvx edge-tts --voice "en-US-GuyNeural" --text "{input}" --write-media "{output}"` |
132
- | `stt` | `uvx --from openai-whisper whisper "{input}" --output_format vtt --output_dir "{output}"` |
133
- | `tti` | `uvx --from mflux mflux-generate-flux2 --model flux2-klein-4b --steps 5 --prompt "{input}" --output "{output}"` |
134
- | `ttv` | (uses TTI to create an image, then ffmpeg to loop it for 5s) |
135
-
136
- Override on the root config line:
137
-
138
- ```markdown
139
- # video
140
- width:1920 height:1080 tts:"uvx edge-tts --voice 'zh-CN-YunxiNeural' --text '{input}' --write-media '{output}'"
141
- ```
142
-
143
- ---
144
26
 
145
27
  ## 3. CLI
146
28
 
147
29
  ```bash
148
- npx markcut <command> [options]
149
- ```
150
-
151
- ### Commands
152
-
153
- | Command | Description |
154
- |---------|-------------|
155
- | `render <file>` | Render stream tree to MP4 |
156
- | `preview <file> --label` | Label clips in a simplified stream tree |
157
- | `preview <file> --edit` | Live editing loop (auto-reload on file change) |
158
- | `verify <file>` | Validate descriptive file + check imports |
159
- ### Options
160
-
161
- | Flag | Values | Default |
162
- |------|--------|---------|
163
- | `--aspect` | `16x9` / `9x16` / `1x1` / `all` | `16x9` |
164
- | `--output` | path | `out/video-{aspect}.mp4` |
165
- | `--port` | number | `3001` |
166
- | `--verbose` | flag | `false` (compact progress) |
167
- | `--compile` | flag | `false` (resolve only) |
168
- | `--script-output-dir` | dir | `.markcut/generated/tts/` |
169
- | `--media-output-dir` | dir | `.markcut/generated/media/` |
170
-
171
- > Default output dirs auto-resolve to `.markcut/<basename>/<type>/` relative to the source file. You only need `--script-output-dir` / `--media-output-dir` to override.
172
-
173
- ### Edit Mode for Agents
174
-
175
- ```bash
176
- # Start player in background
177
- npx markcut preview <stream tree file> --edit --port 3001 &
178
-
179
- # Player auto-opens browser. Agent edits JSON → player auto-reloads.
180
- # User clicks ✕ → server exits → agent regains control.
181
- # Browser feedback input writes to feedback.txt.
30
+ npx @lalalic/markcut <command> [options]
31
+ npx @lalalic/markcut --help # get overall information
182
32
  ```
183
33
 
184
34
  ---
185
35
 
186
36
  ## self verification
187
37
  some common issues (photo or video can't be displayed, audio missing), take below actions to verify
188
- ### preview or render mode
189
- - screenshot some key frames, and understand image to verify if the image is correct
38
+ ### preview
39
+ - take screenshot for some key frames in player, and understand image to verify intent
40
+
190
41
  ### final video
191
- - screenshot some key frames, and understand image to verify if the image is correct
42
+ - screenshot some key frames, and understand image to verify intent
192
43
  - stt the final video audio, and verify if vtt result is correct
193
44
 
194
45
 
@@ -197,13 +48,16 @@ some common issues (photo or video can't be displayed, audio missing), take belo
197
48
  | Topic | File |
198
49
  |-------|------|
199
50
  | Markdown descriptive format (primary authoring format) | [docs/markdown-descriptive.md](docs/markdown-descriptive.md) |
200
- | **Video Templates** — ready-to-use markdown for common scenarios | **[docs/templates/](docs/templates/)** |
201
- | ┣ Courseware / 课件 | [docs/templates/courseware.md](docs/templates/courseware.md) |
202
- | ┣ Product Ad / 产品广告 | [docs/templates/product-ad.md](docs/templates/product-ad.md) |
203
- | ┣ Movie Review / 影视讲解 | [docs/templates/movie-review.md](docs/templates/movie-review.md) |
204
- | ┣ Audiobook / 有声图书 | [docs/templates/audiobook.md](docs/templates/audiobook.md) |
205
- | ┣ Story Video / 故事视频 | [docs/templates/story-video.md](docs/templates/story-video.md) |
206
- | ┣ Travel Log / 旅行日志 | [docs/templates/travel-log.md](docs/templates/travel-log.md) |
207
51
  | Label system (browse, label, export labels.json) | [docs/label-mode.md](docs/label-mode.md) |
208
52
  | Player servers (label + edit mode) | [docs/edit-mode.md](docs/edit-mode.md) |
209
- | Template overview and TTI/TTV config | [docs/templates.md](docs/templates.md) |
53
+
54
+
55
+ ## common used components
56
+ - `react-markdown` — render markdown content, use plugins to extend functionality
57
+ - `remark-gfm` — support GitHub Flavored Markdown (tables, strikethrough, task lists)
58
+ - `remark-toc` — generate table of contents
59
+ - `remark-math` — support math formulas
60
+ - `rehype-katex` — render math formulas with KaTeX
61
+
62
+ - `@remotion/shapes` — render shapes like arrows, circles, rectangles, etc
63
+ - `@remotion/starburst` — render starburst animations
package/docs/edit-mode.md CHANGED
@@ -177,7 +177,7 @@ The tree description walks the JSON recursively, showing:
177
177
  - **Nesting depth** via indentation
178
178
  - **Type** and **id/name** of each node
179
179
  - **Key fields** (dimensions, fps, theme, series/parallel, transitions)
180
- - **Timing** from actions (`[startend]`)
180
+ - **Timing** from leaf span (`[start→end]`)
181
181
  - **isBackground** flag (`[bg]`)
182
182
  - **Component props** and node-specific fields (fontSize, fit, style, volume, etc.)
183
183
 
@@ -36,7 +36,7 @@ Labels are saved to the source file.
36
36
  "id": "scene-1-media",
37
37
  "type": "image",
38
38
  "src": "/media/photo.jpg",
39
- "actions": [{ "start": 0, "end": 5 }],
39
+ "start": 0, "end": 5,
40
40
  "description": "Best group shot from the campfire"
41
41
  }
42
42
  ]
@@ -49,7 +49,7 @@ Labels are saved to the source file.
49
49
  "id": "scene-2-media",
50
50
  "type": "video",
51
51
  "src": "/media/clip.mp4",
52
- "actions": [{ "start": 0, "end": 5 }],
52
+ "start": 0, "end": 5,
53
53
  "description": "Sunset timelapse — use for intro"
54
54
  }
55
55
  ]
@@ -94,12 +94,14 @@ Stream tree JSON (labels.json)
94
94
 
95
95
  ## Implementation
96
96
 
97
- Source: `src/player/label-server.mjs`
97
+ Source: `src/player/server.mjs` (unified server, use `--label` flag)
98
+ UI controls: `src/player/components/LabelControls.tsx` (React component)
98
99
 
99
100
  - Built on Node.js `http` module (no Express dependency)
100
101
  - Serves bundled `player.js` (esbuild output from `src/player/browser.tsx`)
101
102
  - Labels saved atomically to disk on each POST
102
- - Accepts stream tree JSON only (no media folder mode)
103
+ - Supports stream tree JSON, descriptive JSON, and markdown input
104
+ - React components in `src/player/components/` replace the old HTML-based `ui/` directory
103
105
 
104
106
  ## Reference
105
107
 
@@ -174,7 +174,9 @@ For compatibility, `import { Name } from "spec"` also works and produces the sam
174
174
  | `title` | display title | scene |
175
175
  | `instruction` | visual intent / style / any prompt; NOT rendered | any |
176
176
  | `script` | narration/dialogue text; TTS source; NOT rendered directly | audio | Only on audio nodes — see Narration section below |
177
- | `tts` | CLI template string; per-scene TTS override (overrides root `tts`) | root, scene |
177
+ | `speaker` | speaker name for multi-turn dialogue; set automatically by dialogue expansion | audio | Used for per-speaker voice lookup and subtitle prefix |
178
+ | `voices` | JSON object mapping speaker names to extra TTS CLI flags e.g. `{"Ray":"--voice en-US-GuyNeural"}` | root | Flags are appended to the resolved TTS CLI template |
179
+ | `tts` | CLI template string e.g. `"edge-tts --voice en-US-GuyNeural --text {input} --write-media {output}"` | root | Set once at the root level; all TTS uses this template |
178
180
  | `metadata` | arbitrary metadata string | root |
179
181
  | `stylesheet` | global CSS string; selectors use `.className` on elements | root |
180
182
  | `style` | inline CSS applied to the node's container div e.g. `"border-radius:12px"` | any |
@@ -254,6 +256,37 @@ For longer text, use an indented code fence:
254
256
 
255
257
  All three patterns produce an `audio` node with a `script` field. The pipeline's TTS resolver (`resolveScripts`) picks up these nodes, generates speech audio files, and sets the `src` field to the output path. The `script` field is consumed by the resolver and is not present in the compiled stream tree.
256
258
 
259
+ #### Multi-turn Dialogue
260
+
261
+ When a `script` contains multiple lines matching `SpeakerName: text` format, the pipeline automatically expands it into a multi-turn dialogue where each line becomes a separate audio node with its own TTS generation. This allows different speakers to have different voices.
262
+
263
+ ```md
264
+ - script "Ray: Hello everyone and welcome
265
+ Alice: Good day to you all
266
+ Ray: Let's get started"
267
+ ```
268
+
269
+ The dialogue lines play sequentially. Each line is transcribed separately, and subtitles include the speaker prefix (e.g., `Ray: Hello everyone and welcome`).
270
+
271
+ ##### Per-Speaker Voices
272
+
273
+ Configure different TTS voices for each speaker via the root `voices` config. Each value is **extra CLI flags** appended to the TTS template:
274
+
275
+ ```md
276
+ # video
277
+ voices:{"Ray":"--voice en-US-GuyNeural","Alice":"--voice en-US-JennyNeural"}
278
+ - script "Ray: Tell us about your project
279
+ Alice: I'm working on something exciting"
280
+ ```
281
+
282
+ Since values are raw CLI flags, they naturally support voice cloning, rate, pitch, and any edge-tts feature:
283
+
284
+ ```md
285
+ voices:{"Ray":"--voice en-US-GuyNeural --rate +20%","Clone":"--voice clone-xxx --pitch +5Hz"}
286
+ ```
287
+
288
+ The final TTS command for a speaker: `<root tts cli> <speaker voice flags>`.
289
+
257
290
  ### `subtitle` (root-level overlay)
258
291
 
259
292
  Subtitles are configured at the root level as a VTT overlay, not as tree nodes. Set via YAML frontmatter `subtitle:` or `root.subtitle` in JSON.
@@ -707,6 +740,18 @@ Use `markcut verify` to parse and validate a descriptive markdown file without r
707
740
  markcut verify courseware.md
708
741
  ```
709
742
 
743
+ # common used components
744
+ - `react-markdown` — render markdown content, use plugins to extend functionality
745
+ - `remark-gfm` — support GitHub Flavored Markdown (tables, strikethrough, task lists)
746
+ - `remark-toc` — generate table of contents
747
+ - `remark-math` — support math formulas
748
+ - `rehype-katex` — render math formulas with KaTeX
749
+ - `remark-mermaidjs` — render mermaid diagrams in
750
+ - `react-markdown-mermaid` — render mermaid diagrams in standalone mode (no need to install mermaid separately)
751
+
752
+ - `@remotion/shapes` — render shapes like arrows, circles, rectangles, etc
753
+ - `@remotion/starburst` — render starburst animations
754
+
710
755
  ## Example
711
756
 
712
757
  ```md
@@ -0,0 +1,16 @@
1
+ You are a video director editing a markdown-descriptive (.md) markcut video project.
2
+
3
+ The markcut engine compiles markdown descriptions into rendered videos with TTS narration, transitions, and media generation.
4
+
5
+ The file you are editing is: ${fileName}
6
+ Full path: ${filePath}
7
+
8
+ --- Markdown-Descriptive Format Reference ---
9
+ @{docs/markdown-descriptive.md}
10
+
11
+ TASKS:
12
+ 1. Read the .md file directly using your file tools to see its current content
13
+ 2. Edit the .md file directly — change content, add/remove sections, update frontmatter
14
+ 3. Save the changes to the .md file using your write/edit tools
15
+ 4. Output ONLY a one-line summary of what specific change you made
16
+ 5. Do NOT output explanations, reasoning, or extra text
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@lalalic/markcut",
3
- "version": "1.1.1",
3
+ "version": "2.0.0",
4
4
  "description": "Markdown-to-video engine. Describe scenes in markdown, get a rendered video.",
5
5
  "bin": {
6
6
  "markcut": "bin/markcut"
package/src/config.mjs CHANGED
@@ -37,6 +37,8 @@ export const DEFAULT_ITT_CLI =
37
37
  /**
38
38
  * Seconds between sampled video frames when using the default ITT-based VTT.
39
39
  */
40
+ export const GOOGLE_MAPS_API_KEY = process.env.GOOGLE_MAPS_API_KEY || "";
41
+
40
42
  export const DEFAULT_VTT_SAMPLE_INTERVAL = Number(process.env.MARKCUT_VTT_SAMPLE_INTERVAL) || 5;
41
43
 
42
44
  /**
@@ -73,11 +75,14 @@ export const DEFAULT_TTS_CLI =
73
75
 
74
76
  /**
75
77
  * Default agent CLI — general-purpose text LLM.
76
- * Used for text-only tasks like detect-scenes.
77
- * Placeholders: {prompt}=the prompt text.
78
+ * Used for text-only tasks like detect-scenes and editing.
79
+ * Placeholders:
80
+ * {systemprompt} = system prompt (role, instructions, knowledge)
81
+ * {prompt} = user prompt (current context + edit request)
82
+ * {sessionid} = unique session ID for conversation continuity
78
83
  * Override with MARKCUT_AGENT_CLI env var.
79
84
  */
80
- export const DEFAULT_AGENT_CLI = process.env.MARKCUT_AGENT_CLI || 'npx pi -p {prompt}';
85
+ export const DEFAULT_AGENT_CLI = process.env.MARKCUT_AGENT_CLI || 'npx pi --session-id {sessionid} --system-prompt {systemprompt} -p {prompt}';
81
86
 
82
87
  // ── Render-only pipeline CLI templates ─────────────────────────────────────
83
88
  // These are specific to the render pipeline (no vision pipeline equivalent).