@lalalic/markcut 1.1.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +39 -15
- package/README.md +7 -3
- package/SKILL.md +26 -172
- package/docs/edit-mode.md +1 -1
- package/docs/label-mode.md +6 -4
- package/docs/markdown-descriptive.md +46 -1
- package/docs/system-prompt-edit.md +16 -0
- package/package.json +1 -1
- package/src/config.mjs +8 -3
- package/src/descriptive/compiler.test.ts +42 -42
- package/src/descriptive/compiler.ts +41 -52
- package/src/descriptive/markdown.ts +8 -4
- package/src/descriptive/resolve.test.ts +14 -7
- package/src/descriptive/resolve.ts +148 -20
- package/src/entry.tsx +8 -2
- package/src/player/browser.tsx +178 -54
- package/src/player/bundle/player.js +1168 -566
- package/src/player/components/EditControls.tsx +92 -0
- package/src/player/components/HeaderBar.tsx +60 -0
- package/src/player/components/LabelControls.tsx +367 -0
- package/src/player/components/SceneThumbnails.tsx +87 -0
- package/src/player/components/VariantBar.tsx +39 -0
- package/src/player/components/index.ts +5 -0
- package/src/player/pipeline.mjs +130 -66
- package/src/player/pipeline.ts +3 -1
- package/src/player/server-shared.mjs +5 -7
- package/src/player/server.mjs +194 -187
- package/src/render/cli.mjs +66 -13
- package/src/schema/index.ts +20 -23
- package/src/types/Audio.tsx +25 -33
- package/src/types/Component.tsx +18 -24
- package/src/types/Effect.tsx +31 -39
- package/src/types/Image.tsx +23 -30
- package/src/types/Include.tsx +70 -76
- package/src/types/Map.tsx +48 -44
- package/src/types/Rhythm.tsx +19 -27
- package/src/types/Video.tsx +40 -47
- package/src/utils/component-import-map.ts +93 -0
- package/src/utils/index.ts +23 -10
- package/src/vision/cli.mjs +6 -6
- package/templates/courseware/TEMPLATE.md +317 -0
- package/templates/courseware/agents/reviewer.md +84 -0
- package/templates/courseware/prompts/fix.md +30 -0
- package/templates/courseware/prompts/outline.md +53 -0
- package/templates/courseware/prompts/scene.md +64 -0
- package/tests/fixtures/audio.json +4 -2
- package/tests/fixtures/basic.json +2 -1
- package/tests/fixtures/component-all.json +4 -2
- package/tests/fixtures/components.json +4 -2
- package/tests/fixtures/effects.json +12 -6
- package/tests/fixtures/full.json +8 -4
- package/tests/fixtures/map.json +2 -1
- package/tests/fixtures/md/dialogue.md +12 -0
- package/tests/fixtures/md/edge-cases.md +9 -0
- package/tests/fixtures/scenes.json +4 -2
- package/tests/fixtures/subtitle.json +6 -3
- package/tests/fixtures/subvideo.json +11 -6
- package/tests/fixtures/templates/courseware.md +61 -4
- package/tests/fixtures/tween-visual.json +2 -6
- package/tests/fixtures/video-series.json +6 -14
- package/tests/md-descriptive.test.ts +170 -0
- package/tests/render.test.ts +32 -16
- package/tests/schema.test.ts +6 -3
- package/tests/server.test.ts +9 -6
- package/tests/utils.ts +4 -4
- package/docs/dynamic-components.md +0 -191
- package/docs/templates.md +0 -52
- package/scripts/artlist-dl.mjs +0 -190
- package/src/player/label-server.mjs +0 -599
package/AGENTS.md
CHANGED
|
@@ -63,8 +63,12 @@ src/
|
|
|
63
63
|
├── render/cli.mjs → CLI entry
|
|
64
64
|
├── render/tts.ts → TTS via CLI template + variable substitution
|
|
65
65
|
├── player/pipeline.mjs → Bundled pipeline (server imports this)
|
|
66
|
-
├── player/server.mjs → --edit
|
|
67
|
-
├── player/
|
|
66
|
+
├── player/server.mjs → Unified server (--edit, --label, preview)
|
|
67
|
+
├── player/components/ → React control components (edit, label, variant bar)
|
|
68
|
+
│ ├── index.ts
|
|
69
|
+
│ ├── EditControls.tsx
|
|
70
|
+
│ ├── LabelControls.tsx
|
|
71
|
+
│ └── VariantBar.tsx
|
|
68
72
|
└── tests/ → Vitest integration tests
|
|
69
73
|
```
|
|
70
74
|
|
|
@@ -107,22 +111,22 @@ The tree is validated by Zod and rendered by Remotion via recursive React compon
|
|
|
107
111
|
| **Series** (`isSeries:true`) | Children play sequentially. Optional `transition` (fade/slide/wipe/flip/clockWipe) + `transitionTime` (default 0.5s). |
|
|
108
112
|
| **Parallel** (`isSeries:false`) | Children play simultaneously. Duration = max child duration. |
|
|
109
113
|
| **Background** (`isBackground:true`) | Loops for parent duration. Excluded from duration calculation. |
|
|
110
|
-
| **
|
|
114
|
+
| **Timing** | Every leaf carries base fields `start`/`end` (seconds, relative to parent) + optional `startFrom`/`endAt` (source trim) + `duration` (convenience). `end` is the source of truth; `duration` normalizes to `end = start + duration` when `end` is absent. Inside series parents `start` is 0 (positioning is implicit). |
|
|
111
115
|
| **Scene** (`type:"scene"`) | UI storyboard card; engine treats as folder. Has `name` + `description` + `script`. |
|
|
112
116
|
|
|
113
117
|
### Duration Calculation
|
|
114
118
|
|
|
115
119
|
Computed by `getDurationInSeconds()` in `src/utils/index.ts`:
|
|
116
|
-
- **Leaf nodes**: duration =
|
|
120
|
+
- **Leaf nodes**: duration = `leafEnd(stream)` = `end ?? (start + duration)` (single deterministic read; `start`/`end`/`duration` live on the base node)
|
|
117
121
|
- **Series**: sum of children durations minus transition overlaps
|
|
118
122
|
- **Parallel**: max of children durations
|
|
119
123
|
- **Background children**: excluded from calculation
|
|
120
|
-
- **Include with `src`**: treated as leaf (duration from
|
|
124
|
+
- **Include with `src`**: treated as leaf (duration from base `end`)
|
|
121
125
|
- Sets `durationInSeconds` on every node as a side effect
|
|
122
126
|
|
|
123
127
|
### 12 Stream Types
|
|
124
128
|
|
|
125
|
-
All types share base fields from `BaseShape`: `id`, `name`, `title`, `description`, `script`, `src`, `style`, `visible`, `isBackground`, `durationInSeconds`.
|
|
129
|
+
All types share base fields from `BaseShape`: `id`, `name`, `title`, `description`, `script`, `src`, `style`, `visible`, `isBackground`, `start`, `end`, `duration`, `startFrom`, `endAt`, `durationInSeconds`.
|
|
126
130
|
|
|
127
131
|
| Type | Purpose | Key Fields |
|
|
128
132
|
|------|---------|------------|
|
|
@@ -130,8 +134,8 @@ All types share base fields from `BaseShape`: `id`, `name`, `title`, `descriptio
|
|
|
130
134
|
| `folder` | Group children | `isSeries`, `transition`, `children[]` |
|
|
131
135
|
| `scene` | Storyboard node (alias for folder) | `name`, `description`, `script`, `children[]` |
|
|
132
136
|
| `image` | Still photo | `src`, `fit` (contain/cover/fill) |
|
|
133
|
-
| `video` | Video clip | `src`, `volume`, `playbackRate` |
|
|
134
|
-
| `audio` | Soundtrack/SFX | `src`, `volume`, `foreground` (ducks parent) |
|
|
137
|
+
| `video` | Video clip | `src`, `volume`, `playbackRate`, `loop` |
|
|
138
|
+
| `audio` | Soundtrack/SFX | `src`, `volume`, `foreground` (ducks parent), `loop` |
|
|
135
139
|
| `subtitle` | Text overlay | `src` (text/VTT) or `cues[]`, `fontSize`, `style` |
|
|
136
140
|
| `component` | React component | `componentName`, `props`, `src` (remote ESM) |
|
|
137
141
|
| `effect` | CSS animation wrapper | `animation`, `customKeyframes`, `children[]` |
|
|
@@ -155,7 +159,7 @@ All types share base fields from `BaseShape`: `id`, `name`, `title`, `descriptio
|
|
|
155
159
|
### Video.tsx — startFrom/endAt Trimming
|
|
156
160
|
|
|
157
161
|
- Uses Remotion `<OffthreadVideo>` for CPU-efficient video decoding
|
|
158
|
-
- `startFrom` / `endAt` on
|
|
162
|
+
- `startFrom` / `endAt` on the base node trim the source video (in seconds)
|
|
159
163
|
- `playbackRate` is capped at 1 (`Math.min(1, ...)`) so source longer than timeline plays at normal speed and is truncated by the Sequence
|
|
160
164
|
- **Critical formula**: `OffthreadVideo endAt = startFrom_frames + ((endAt_source - startFrom_source) * fps / playbackRate)`
|
|
161
165
|
This sets `endAt` beyond the actual source end when `playbackRate < 1`, which prevents Remotion from rendering blank frames before the Sequence ends.
|
|
@@ -183,7 +187,7 @@ node src/render/cli.mjs templates
|
|
|
183
187
|
- `--edit` flag starts a live-reload loop: edit JSON → player auto-refreshes
|
|
184
188
|
- `--label` flag starts a labeling UI for selecting and annotating media
|
|
185
189
|
- `--port` flag sets the server port (default 3001)
|
|
186
|
-
-
|
|
190
|
+
- Unified server: `server.mjs` serves all modes (preview, edit, label). React control components in `components/`.
|
|
187
191
|
|
|
188
192
|
## Testing
|
|
189
193
|
|
|
@@ -235,11 +239,13 @@ npx vitest run tests/render.test.ts -t "Test Name" # single test
|
|
|
235
239
|
## Known Issues & Gotchas
|
|
236
240
|
|
|
237
241
|
1. **TransitionSeries + isBackground**: `TransitionSeries` from `@remotion/transitions` rejects non-Sequence/Transition children. Don't mix `isBackground:true` children with `isSeries:true, transition:"..."` at the same level.
|
|
238
|
-
2. **
|
|
239
|
-
3.
|
|
240
|
-
4.
|
|
241
|
-
5. **
|
|
242
|
-
6. **
|
|
242
|
+
2. **Video endAt formula**: Must be `startFrom_frames + ((endAt_source - startFrom_source) * fps / playbackRate)` — changing to `endAt_source * fps` breaks slow-motion playback.
|
|
243
|
+
3. **`signalstats` ffmpeg filter**: Doesn't output YAVG on all systems; use file size as proxy for visual content.
|
|
244
|
+
4. **Asset paths**: Local files are resolved via `staticFile()` — files go in `public/` and are referenced without the `public/` prefix.
|
|
245
|
+
5. **Map rendering**: Depends on external tile availability; may produce blank frames offline.
|
|
246
|
+
6. **CLI md render needs three things preview gets for free**: (a) `parseMarkdownVariants(raw)` (returns `{base, variants}`) — `parseMarkdownDescriptive` returns the root directly; (b) bundling the ` ```js imports ``` ` block via `bundleFromEntries` and setting `root.imports` (otherwise jsx components are unregistered → slides render as unstyled black frames); (c) staging `.markcut/` assets into `public/.render-assets/` and rewriting srcs to relative paths (remotion render only serves its publicDir). All three live in `src/render/cli.mjs`; the import-map shim for shared react/remotion instances is `src/utils/component-import-map.ts` (must stay in sync with `player/browser.tsx` + `bundler.mjs getSharedExternals`).
|
|
247
|
+
7. **Publishing**: verify `npx @lalalic/markcut render <md>` from a clean temp dir before/after publish — v1.1.1 shipped with cli.mjs/pipeline.mjs out of sync and md rendering was completely broken. Also beware stale npx caches (`~/.npm/_npx`) silently running old code.
|
|
248
|
+
8. **Clean break from `actions[]`**: leaf timing is now flat base fields (`start`/`end`/`duration`/`startFrom`/`endAt`). Old compiled JSON carrying `actions:[{...}]` no longer parses into a duration — re-compile from markdown/descriptive JSON, or migrate `actions[0]` → flat fields (drop `id`/`volume`/`style`/`effectId`; keep `start`/`end`/`startFrom`/`endAt`/`loop`).
|
|
243
249
|
|
|
244
250
|
## 3-Level Authoring Workflow (see SKILL.md)
|
|
245
251
|
|
|
@@ -259,6 +265,24 @@ Every node carries an optional `script` field for narration/dialogue text. The p
|
|
|
259
265
|
|
|
260
266
|
This separates concerns: `script` = authoring text (human/agent-friendly), `subtitle` = rendered output (timed VTT cues).
|
|
261
267
|
|
|
268
|
+
### Multi-turn Dialogue
|
|
269
|
+
|
|
270
|
+
When `script` text contains multiple `SpeakerName: text` lines (2+), the pipeline auto-expands it into a multi-turn dialogue. Each line becomes a separate audio node with `speaker` field, wrapped in a `series` container for sequential playback.
|
|
271
|
+
|
|
272
|
+
```md
|
|
273
|
+
- script "Ray: Hello everyone
|
|
274
|
+
Alice: Good day to you
|
|
275
|
+
Ray: Let's begin"
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
Per-speaker voices are configured via `voices` on root. Each value is extra CLI flags appended to the TTS template:
|
|
279
|
+
|
|
280
|
+
```md
|
|
281
|
+
voices:{"Ray":"--voice en-US-GuyNeural","Alice":"--voice en-US-JennyNeural"}
|
|
282
|
+
```
|
|
283
|
+
|
|
284
|
+
The speaker name is matched against `voices` map and the extra flags are appended to the TTS CLI template, naturally supporting voice cloning or edge-tts features (rate, pitch, etc.). Subtitles include the speaker prefix (`Ray: Hello everyone`).
|
|
285
|
+
|
|
262
286
|
## Dependencies
|
|
263
287
|
|
|
264
288
|
- **Remotion 4.0.469** — core rendering engine
|
package/README.md
CHANGED
|
@@ -201,10 +201,14 @@ src/
|
|
|
201
201
|
│ ├── cli.mjs ← CLI entry point
|
|
202
202
|
│ └── tts.ts ← TTS via CLI template + variable substitution
|
|
203
203
|
├── player/
|
|
204
|
-
│ ├── server.mjs ← --edit
|
|
205
|
-
│ ├──
|
|
204
|
+
│ ├── server.mjs ← Unified server (--edit, --label, preview)
|
|
205
|
+
│ ├── components/ ← React control components for all modes
|
|
206
|
+
│ │ ├── index.ts
|
|
207
|
+
│ │ ├── EditControls.tsx
|
|
208
|
+
│ │ ├── LabelControls.tsx
|
|
209
|
+
│ │ └── VariantBar.tsx
|
|
206
210
|
│ ├── pipeline.ts ← Pipeline entry (bundled to pipeline.mjs)
|
|
207
|
-
│ └── browser.tsx ← Browser player (bundled
|
|
211
|
+
│ └── browser.tsx ← Browser player (React, bundled to player.js)
|
|
208
212
|
├── utils/ ← Duration calc, VTT parser, helpers
|
|
209
213
|
└── tests/ ← Vitest integration tests
|
|
210
214
|
```
|
package/SKILL.md
CHANGED
|
@@ -1,194 +1,45 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: markcut
|
|
3
|
-
description:
|
|
4
|
-
Compose and render videos from streamline trees. CLI renders stream tree
|
|
5
|
-
to MP4. Run via `npx markcut` — no install, no code.
|
|
3
|
+
description: use markdown to describe stream trees, provide CLI to render stream trees to video via `npx @lalalic/markcut`. use it to structure video scenes, and generate TTS, TTI, TTV, STT media automatically.
|
|
6
4
|
---
|
|
7
5
|
|
|
8
6
|
## Stream Tree Specs
|
|
9
7
|
|
|
10
|
-
Everything video is a **stream tree
|
|
8
|
+
Everything video is a **stream tree** described with markdown. see [docs/markdown-descriptive.md](docs/markdown-descriptive.md) for full details.
|
|
11
9
|
|
|
12
|
-
## .markcut/ Directory Layout
|
|
13
10
|
|
|
14
|
-
|
|
11
|
+
## video design utitlies
|
|
15
12
|
|
|
16
|
-
|
|
17
|
-
.
|
|
18
|
-
generated/ ← shared, content-addressed
|
|
19
|
-
tts/ ← TTS audio (content-hash filenames)
|
|
20
|
-
media/ ← TTI/TTV media (content-hash filenames)
|
|
21
|
-
includes/ ← compiled sub-video JSON (content-hash)
|
|
22
|
-
<basename>/ ← per-source-file artifacts
|
|
23
|
-
components/ ← component bundles (per-file: imports differ)
|
|
24
|
-
```
|
|
25
|
-
|
|
26
|
-
- `generated/` — shared artifacts using content-hash filenames, so identical scripts/prompts across different source files reuse the same cached file
|
|
27
|
-
- `<basename>/components/` — per-file component bundles; each source file has its own `imports` block, so bundles live under its basename
|
|
28
|
-
- The server serves `.markcut/` as a document root, so components at `.markcut/slides/components/abc.js` are served as `/slides/components/abc.js`
|
|
29
|
-
|
|
30
|
-
## Include (Sub-Video Embedding)
|
|
31
|
-
|
|
32
|
-
The `include` node embeds an external `.md` file as a sub-video. The pipeline fully resolves it recursively:
|
|
33
|
-
|
|
34
|
-
1. **`resolveIncludes()`** (step 0 of `resolveAll`): reads the referenced `.md`, parses it, resolves its own includes/TTS/media, compiles to a stream tree, writes the compiled JSON to `.markcut/generated/includes/<hash>.json`
|
|
35
|
-
2. **`resolveIncludeImports()`** (server post-compile): if the sub-video has a ` ```js imports ``` ` block, bundles its components independently and writes the bundle URL into the compiled JSON
|
|
36
|
-
3. **`IncludeLeaf`** (render): loads the compiled JSON, dynamically imports the sub-video's component bundle, creates a nested `ComposeContext` with the sub-video's registry merged, and renders via `FolderLeaf`
|
|
37
|
-
|
|
38
|
-
The sub-video's own components are isolated from the parent's — naming conflicts are resolved by "sub-video wins" priority.
|
|
39
|
-
|
|
40
|
-
```markdown
|
|
41
|
-
# video
|
|
42
|
-
## Main
|
|
43
|
-
- component duration:3 jsx:"<MainTitle />"
|
|
44
|
-
## Embedded
|
|
45
|
-
- include src:./sub-video.md
|
|
46
|
-
## End
|
|
47
|
-
- component duration:2 jsx:"<Outro />"
|
|
48
|
-
```
|
|
49
|
-
|
|
50
|
-
```markdown
|
|
51
|
-
# sub-video
|
|
52
|
-
- component duration:3 jsx:"<SubSlide scene='1' />"
|
|
53
|
-
|
|
54
|
-
```js imports
|
|
55
|
-
export function SubSlide({ scene }) {
|
|
56
|
-
return <div style={{background:'#667eea', ...}}>Scene {scene}</div>;
|
|
57
|
-
}
|
|
58
|
-
```
|
|
59
|
-
```
|
|
60
|
-
---
|
|
61
|
-
|
|
62
|
-
## Video Design Best Practice (How-To)
|
|
63
|
-
|
|
64
|
-
The engine supports four authoring phases. Each has a dedicated doc with full detail.
|
|
65
|
-
|
|
66
|
-
### Label — annotate clips in a stream tree
|
|
67
|
-
|
|
68
|
-
Load a stream tree, preview each clip, and attach descriptive labels.
|
|
69
|
-
|
|
70
|
-
```bash
|
|
71
|
-
npx markcut preview <flat stream tree file> --label
|
|
72
|
-
```
|
|
73
|
-
|
|
74
|
-
Labels save to `labels.json`. See [docs/label-mode.md](docs/label-mode.md) for:
|
|
75
|
-
- Label player UI (thumbnails, player, label input)
|
|
76
|
-
- Labels persistence format
|
|
77
|
-
- Full workflow diagram
|
|
13
|
+
### User labels — label media with text, time ranges
|
|
14
|
+
- `uvx @lalalic/markcut preview --label` provides tool to label video and images with text.
|
|
78
15
|
|
|
79
16
|
### Storyboard — plan video structure with scene nodes
|
|
80
17
|
|
|
81
|
-
Use `scene` nodes to organize your video.
|
|
18
|
+
- Use `scene` nodes to organize your video. Scenes can nest inside other scenes.
|
|
19
|
+
- Use `description`, `scene.instruction`, `script`, `image|video.prompt` to structure your video content.
|
|
82
20
|
|
|
83
|
-
```bash
|
|
84
21
|
see [docs/markdown-descriptive.md](docs/markdown-descriptive.md) for full details.
|
|
85
22
|
|
|
86
|
-
###
|
|
87
|
-
|
|
88
|
-
```bash
|
|
89
|
-
npx markcut render <stream tree file> --aspect all # MP4 output
|
|
90
|
-
```
|
|
91
|
-
|
|
92
|
-
---
|
|
93
|
-
|
|
94
|
-
## Template Variables
|
|
95
|
-
|
|
96
|
-
`${width}`, `${height}`, `${fps}`, `${variant}` can be used in `src`,
|
|
97
|
-
`prompt`, and `stylesheet` values. They resolve to the root config values.
|
|
23
|
+
### Video Variants
|
|
24
|
+
- define variants for different video configurations, such as different languages, aspects, platforms
|
|
98
25
|
|
|
99
|
-
```markdown
|
|
100
|
-
# video
|
|
101
|
-
width:1920 height:1080
|
|
102
|
-
## Scene
|
|
103
|
-
- image src:photo_${width}x${height}.jpg
|
|
104
|
-
- component jsx:"<h1>${width}</h1>" ← NOT resolved (jsx excluded)
|
|
105
|
-
```
|
|
106
|
-
|
|
107
|
-
> Only `src`, `prompt`, and `stylesheet` support template variables.
|
|
108
|
-
> Root config keys, jsx, script, style, etc. do not.
|
|
109
|
-
|
|
110
|
-
## Frontmatter
|
|
111
|
-
|
|
112
|
-
The YAML `---` block is **metadata only**. It does NOT affect video config.
|
|
113
|
-
All configuration comes from the root config line after `# video`:
|
|
114
|
-
|
|
115
|
-
```markdown
|
|
116
|
-
---
|
|
117
|
-
title: My Campaign
|
|
118
|
-
---
|
|
119
|
-
# video
|
|
120
|
-
width:640 height:480 fps:30 layout:series tts:"edge-tts --voice '...' --text '{input}' --write-media '{output}'"
|
|
121
|
-
```
|
|
122
|
-
|
|
123
|
-
## AI Media Generation (TTS / STT / TTI(text-to-image) / TTV(text-to-video))
|
|
124
|
-
|
|
125
|
-
All four pipelines are configured via a **single CLI string** on the root config line. The user/LLM embeds every tool-specific parameter directly in the string — only `{input}` and `{output}` are substituted by the engine.
|
|
126
|
-
|
|
127
|
-
Default CLI templates (used when not overridden on the root config line):
|
|
128
|
-
|
|
129
|
-
| Config | Default |
|
|
130
|
-
|--------|---------|
|
|
131
|
-
| `tts` | `uvx edge-tts --voice "en-US-GuyNeural" --text "{input}" --write-media "{output}"` |
|
|
132
|
-
| `stt` | `uvx --from openai-whisper whisper "{input}" --output_format vtt --output_dir "{output}"` |
|
|
133
|
-
| `tti` | `uvx --from mflux mflux-generate-flux2 --model flux2-klein-4b --steps 5 --prompt "{input}" --output "{output}"` |
|
|
134
|
-
| `ttv` | (uses TTI to create an image, then ffmpeg to loop it for 5s) |
|
|
135
|
-
|
|
136
|
-
Override on the root config line:
|
|
137
|
-
|
|
138
|
-
```markdown
|
|
139
|
-
# video
|
|
140
|
-
width:1920 height:1080 tts:"uvx edge-tts --voice 'zh-CN-YunxiNeural' --text '{input}' --write-media '{output}'"
|
|
141
|
-
```
|
|
142
|
-
|
|
143
|
-
---
|
|
144
26
|
|
|
145
27
|
## 3. CLI
|
|
146
28
|
|
|
147
29
|
```bash
|
|
148
|
-
npx markcut <command> [options]
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
### Commands
|
|
152
|
-
|
|
153
|
-
| Command | Description |
|
|
154
|
-
|---------|-------------|
|
|
155
|
-
| `render <file>` | Render stream tree to MP4 |
|
|
156
|
-
| `preview <file> --label` | Label clips in a simplified stream tree |
|
|
157
|
-
| `preview <file> --edit` | Live editing loop (auto-reload on file change) |
|
|
158
|
-
| `verify <file>` | Validate descriptive file + check imports |
|
|
159
|
-
### Options
|
|
160
|
-
|
|
161
|
-
| Flag | Values | Default |
|
|
162
|
-
|------|--------|---------|
|
|
163
|
-
| `--aspect` | `16x9` / `9x16` / `1x1` / `all` | `16x9` |
|
|
164
|
-
| `--output` | path | `out/video-{aspect}.mp4` |
|
|
165
|
-
| `--port` | number | `3001` |
|
|
166
|
-
| `--verbose` | flag | `false` (compact progress) |
|
|
167
|
-
| `--compile` | flag | `false` (resolve only) |
|
|
168
|
-
| `--script-output-dir` | dir | `.markcut/generated/tts/` |
|
|
169
|
-
| `--media-output-dir` | dir | `.markcut/generated/media/` |
|
|
170
|
-
|
|
171
|
-
> Default output dirs auto-resolve to `.markcut/<basename>/<type>/` relative to the source file. You only need `--script-output-dir` / `--media-output-dir` to override.
|
|
172
|
-
|
|
173
|
-
### Edit Mode for Agents
|
|
174
|
-
|
|
175
|
-
```bash
|
|
176
|
-
# Start player in background
|
|
177
|
-
npx markcut preview <stream tree file> --edit --port 3001 &
|
|
178
|
-
|
|
179
|
-
# Player auto-opens browser. Agent edits JSON → player auto-reloads.
|
|
180
|
-
# User clicks ✕ → server exits → agent regains control.
|
|
181
|
-
# Browser feedback input writes to feedback.txt.
|
|
30
|
+
npx @lalalic/markcut <command> [options]
|
|
31
|
+
npx @lalalic/markcut --help # get overall information
|
|
182
32
|
```
|
|
183
33
|
|
|
184
34
|
---
|
|
185
35
|
|
|
186
36
|
## self verification
|
|
187
37
|
some common issues (photo or video can't be displayed, audio missing), take below actions to verify
|
|
188
|
-
### preview
|
|
189
|
-
- screenshot some key frames, and understand image to verify
|
|
38
|
+
### preview
|
|
39
|
+
- take screenshot for some key frames in player, and understand image to verify intent
|
|
40
|
+
|
|
190
41
|
### final video
|
|
191
|
-
- screenshot some key frames, and understand image to verify
|
|
42
|
+
- screenshot some key frames, and understand image to verify intent
|
|
192
43
|
- stt the final video audio, and verify if vtt result is correct
|
|
193
44
|
|
|
194
45
|
|
|
@@ -197,13 +48,16 @@ some common issues (photo or video can't be displayed, audio missing), take belo
|
|
|
197
48
|
| Topic | File |
|
|
198
49
|
|-------|------|
|
|
199
50
|
| Markdown descriptive format (primary authoring format) | [docs/markdown-descriptive.md](docs/markdown-descriptive.md) |
|
|
200
|
-
| **Video Templates** — ready-to-use markdown for common scenarios | **[docs/templates/](docs/templates/)** |
|
|
201
|
-
| ┣ Courseware / 课件 | [docs/templates/courseware.md](docs/templates/courseware.md) |
|
|
202
|
-
| ┣ Product Ad / 产品广告 | [docs/templates/product-ad.md](docs/templates/product-ad.md) |
|
|
203
|
-
| ┣ Movie Review / 影视讲解 | [docs/templates/movie-review.md](docs/templates/movie-review.md) |
|
|
204
|
-
| ┣ Audiobook / 有声图书 | [docs/templates/audiobook.md](docs/templates/audiobook.md) |
|
|
205
|
-
| ┣ Story Video / 故事视频 | [docs/templates/story-video.md](docs/templates/story-video.md) |
|
|
206
|
-
| ┣ Travel Log / 旅行日志 | [docs/templates/travel-log.md](docs/templates/travel-log.md) |
|
|
207
51
|
| Label system (browse, label, export labels.json) | [docs/label-mode.md](docs/label-mode.md) |
|
|
208
52
|
| Player servers (label + edit mode) | [docs/edit-mode.md](docs/edit-mode.md) |
|
|
209
|
-
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
## common used components
|
|
56
|
+
- `react-markdown` — render markdown content, use plugins to extend functionality
|
|
57
|
+
- `remark-gfm` — support GitHub Flavored Markdown (tables, strikethrough, task lists)
|
|
58
|
+
- `remark-toc` — generate table of contents
|
|
59
|
+
- `remark-math` — support math formulas
|
|
60
|
+
- `rehype-katex` — render math formulas with KaTeX
|
|
61
|
+
|
|
62
|
+
- `@remotion/shapes` — render shapes like arrows, circles, rectangles, etc
|
|
63
|
+
- `@remotion/starburst` — render starburst animations
|
package/docs/edit-mode.md
CHANGED
|
@@ -177,7 +177,7 @@ The tree description walks the JSON recursively, showing:
|
|
|
177
177
|
- **Nesting depth** via indentation
|
|
178
178
|
- **Type** and **id/name** of each node
|
|
179
179
|
- **Key fields** (dimensions, fps, theme, series/parallel, transitions)
|
|
180
|
-
- **Timing** from
|
|
180
|
+
- **Timing** from leaf span (`[start→end]`)
|
|
181
181
|
- **isBackground** flag (`[bg]`)
|
|
182
182
|
- **Component props** and node-specific fields (fontSize, fit, style, volume, etc.)
|
|
183
183
|
|
package/docs/label-mode.md
CHANGED
|
@@ -36,7 +36,7 @@ Labels are saved to the source file.
|
|
|
36
36
|
"id": "scene-1-media",
|
|
37
37
|
"type": "image",
|
|
38
38
|
"src": "/media/photo.jpg",
|
|
39
|
-
"
|
|
39
|
+
"start": 0, "end": 5,
|
|
40
40
|
"description": "Best group shot from the campfire"
|
|
41
41
|
}
|
|
42
42
|
]
|
|
@@ -49,7 +49,7 @@ Labels are saved to the source file.
|
|
|
49
49
|
"id": "scene-2-media",
|
|
50
50
|
"type": "video",
|
|
51
51
|
"src": "/media/clip.mp4",
|
|
52
|
-
"
|
|
52
|
+
"start": 0, "end": 5,
|
|
53
53
|
"description": "Sunset timelapse — use for intro"
|
|
54
54
|
}
|
|
55
55
|
]
|
|
@@ -94,12 +94,14 @@ Stream tree JSON (labels.json)
|
|
|
94
94
|
|
|
95
95
|
## Implementation
|
|
96
96
|
|
|
97
|
-
Source: `src/player/
|
|
97
|
+
Source: `src/player/server.mjs` (unified server, use `--label` flag)
|
|
98
|
+
UI controls: `src/player/components/LabelControls.tsx` (React component)
|
|
98
99
|
|
|
99
100
|
- Built on Node.js `http` module (no Express dependency)
|
|
100
101
|
- Serves bundled `player.js` (esbuild output from `src/player/browser.tsx`)
|
|
101
102
|
- Labels saved atomically to disk on each POST
|
|
102
|
-
-
|
|
103
|
+
- Supports stream tree JSON, descriptive JSON, and markdown input
|
|
104
|
+
- React components in `src/player/components/` replace the old HTML-based `ui/` directory
|
|
103
105
|
|
|
104
106
|
## Reference
|
|
105
107
|
|
|
@@ -174,7 +174,9 @@ For compatibility, `import { Name } from "spec"` also works and produces the sam
|
|
|
174
174
|
| `title` | display title | scene |
|
|
175
175
|
| `instruction` | visual intent / style / any prompt; NOT rendered | any |
|
|
176
176
|
| `script` | narration/dialogue text; TTS source; NOT rendered directly | audio | Only on audio nodes — see Narration section below |
|
|
177
|
-
| `
|
|
177
|
+
| `speaker` | speaker name for multi-turn dialogue; set automatically by dialogue expansion | audio | Used for per-speaker voice lookup and subtitle prefix |
|
|
178
|
+
| `voices` | JSON object mapping speaker names to extra TTS CLI flags e.g. `{"Ray":"--voice en-US-GuyNeural"}` | root | Flags are appended to the resolved TTS CLI template |
|
|
179
|
+
| `tts` | CLI template string e.g. `"edge-tts --voice en-US-GuyNeural --text {input} --write-media {output}"` | root | Set once at the root level; all TTS uses this template |
|
|
178
180
|
| `metadata` | arbitrary metadata string | root |
|
|
179
181
|
| `stylesheet` | global CSS string; selectors use `.className` on elements | root |
|
|
180
182
|
| `style` | inline CSS applied to the node's container div e.g. `"border-radius:12px"` | any |
|
|
@@ -254,6 +256,37 @@ For longer text, use an indented code fence:
|
|
|
254
256
|
|
|
255
257
|
All three patterns produce an `audio` node with a `script` field. The pipeline's TTS resolver (`resolveScripts`) picks up these nodes, generates speech audio files, and sets the `src` field to the output path. The `script` field is consumed by the resolver and is not present in the compiled stream tree.
|
|
256
258
|
|
|
259
|
+
#### Multi-turn Dialogue
|
|
260
|
+
|
|
261
|
+
When a `script` contains multiple lines matching `SpeakerName: text` format, the pipeline automatically expands it into a multi-turn dialogue where each line becomes a separate audio node with its own TTS generation. This allows different speakers to have different voices.
|
|
262
|
+
|
|
263
|
+
```md
|
|
264
|
+
- script "Ray: Hello everyone and welcome
|
|
265
|
+
Alice: Good day to you all
|
|
266
|
+
Ray: Let's get started"
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
The dialogue lines play sequentially. Each line is transcribed separately, and subtitles include the speaker prefix (e.g., `Ray: Hello everyone and welcome`).
|
|
270
|
+
|
|
271
|
+
##### Per-Speaker Voices
|
|
272
|
+
|
|
273
|
+
Configure different TTS voices for each speaker via the root `voices` config. Each value is **extra CLI flags** appended to the TTS template:
|
|
274
|
+
|
|
275
|
+
```md
|
|
276
|
+
# video
|
|
277
|
+
voices:{"Ray":"--voice en-US-GuyNeural","Alice":"--voice en-US-JennyNeural"}
|
|
278
|
+
- script "Ray: Tell us about your project
|
|
279
|
+
Alice: I'm working on something exciting"
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
Since values are raw CLI flags, they naturally support voice cloning, rate, pitch, and any edge-tts feature:
|
|
283
|
+
|
|
284
|
+
```md
|
|
285
|
+
voices:{"Ray":"--voice en-US-GuyNeural --rate +20%","Clone":"--voice clone-xxx --pitch +5Hz"}
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
The final TTS command for a speaker: `<root tts cli> <speaker voice flags>`.
|
|
289
|
+
|
|
257
290
|
### `subtitle` (root-level overlay)
|
|
258
291
|
|
|
259
292
|
Subtitles are configured at the root level as a VTT overlay, not as tree nodes. Set via YAML frontmatter `subtitle:` or `root.subtitle` in JSON.
|
|
@@ -707,6 +740,18 @@ Use `markcut verify` to parse and validate a descriptive markdown file without r
|
|
|
707
740
|
markcut verify courseware.md
|
|
708
741
|
```
|
|
709
742
|
|
|
743
|
+
# common used components
|
|
744
|
+
- `react-markdown` — render markdown content, use plugins to extend functionality
|
|
745
|
+
- `remark-gfm` — support GitHub Flavored Markdown (tables, strikethrough, task lists)
|
|
746
|
+
- `remark-toc` — generate table of contents
|
|
747
|
+
- `remark-math` — support math formulas
|
|
748
|
+
- `rehype-katex` — render math formulas with KaTeX
|
|
749
|
+
- `remark-mermaidjs` — render mermaid diagrams in
|
|
750
|
+
- `react-markdown-mermaid` — render mermaid diagrams in standalone mode (no need to install mermaid separately)
|
|
751
|
+
|
|
752
|
+
- `@remotion/shapes` — render shapes like arrows, circles, rectangles, etc
|
|
753
|
+
- `@remotion/starburst` — render starburst animations
|
|
754
|
+
|
|
710
755
|
## Example
|
|
711
756
|
|
|
712
757
|
```md
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
You are a video director editing a markdown-descriptive (.md) markcut video project.
|
|
2
|
+
|
|
3
|
+
The markcut engine compiles markdown descriptions into rendered videos with TTS narration, transitions, and media generation.
|
|
4
|
+
|
|
5
|
+
The file you are editing is: ${fileName}
|
|
6
|
+
Full path: ${filePath}
|
|
7
|
+
|
|
8
|
+
--- Markdown-Descriptive Format Reference ---
|
|
9
|
+
@{docs/markdown-descriptive.md}
|
|
10
|
+
|
|
11
|
+
TASKS:
|
|
12
|
+
1. Read the .md file directly using your file tools to see its current content
|
|
13
|
+
2. Edit the .md file directly — change content, add/remove sections, update frontmatter
|
|
14
|
+
3. Save the changes to the .md file using your write/edit tools
|
|
15
|
+
4. Output ONLY a one-line summary of what specific change you made
|
|
16
|
+
5. Do NOT output explanations, reasoning, or extra text
|
package/package.json
CHANGED
package/src/config.mjs
CHANGED
|
@@ -37,6 +37,8 @@ export const DEFAULT_ITT_CLI =
|
|
|
37
37
|
/**
|
|
38
38
|
* Seconds between sampled video frames when using the default ITT-based VTT.
|
|
39
39
|
*/
|
|
40
|
+
export const GOOGLE_MAPS_API_KEY = process.env.GOOGLE_MAPS_API_KEY || "";
|
|
41
|
+
|
|
40
42
|
export const DEFAULT_VTT_SAMPLE_INTERVAL = Number(process.env.MARKCUT_VTT_SAMPLE_INTERVAL) || 5;
|
|
41
43
|
|
|
42
44
|
/**
|
|
@@ -73,11 +75,14 @@ export const DEFAULT_TTS_CLI =
|
|
|
73
75
|
|
|
74
76
|
/**
|
|
75
77
|
* Default agent CLI — general-purpose text LLM.
|
|
76
|
-
* Used for text-only tasks like detect-scenes.
|
|
77
|
-
* Placeholders:
|
|
78
|
+
* Used for text-only tasks like detect-scenes and editing.
|
|
79
|
+
* Placeholders:
|
|
80
|
+
* {systemprompt} = system prompt (role, instructions, knowledge)
|
|
81
|
+
* {prompt} = user prompt (current context + edit request)
|
|
82
|
+
* {sessionid} = unique session ID for conversation continuity
|
|
78
83
|
* Override with MARKCUT_AGENT_CLI env var.
|
|
79
84
|
*/
|
|
80
|
-
export const DEFAULT_AGENT_CLI = process.env.MARKCUT_AGENT_CLI || 'npx pi -p {prompt}';
|
|
85
|
+
export const DEFAULT_AGENT_CLI = process.env.MARKCUT_AGENT_CLI || 'npx pi --session-id {sessionid} --system-prompt {systemprompt} -p {prompt}';
|
|
81
86
|
|
|
82
87
|
// ── Render-only pipeline CLI templates ─────────────────────────────────────
|
|
83
88
|
// These are specific to the render pipeline (no vision pipeline equivalent).
|