@lalalic/markcut 2.2.8 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -1
- package/skills/markcut/SKILL.md +11 -9
- package/skills/markcut/docs/markdown-descriptive.md +1 -1
- package/src/config.mjs +60 -63
- package/src/descriptive/resolve.ts +3 -2
- package/src/player/pipeline.mjs +64 -21
- package/src/render/cli.mjs +42 -69
- package/src/vision/cli.mjs +21 -45
- package/skills/markcut/docs/edit-mode.md +0 -220
- package/skills/markcut/docs/label-mode.md +0 -112
- package/skills/markcut/docs/template.md +0 -212
- package/skills/markcut/docs/templates/courseware/TEMPLATE.md +0 -526
- package/skills/markcut/docs/templates/courseware/agents/reviewer.md +0 -84
- package/skills/markcut/docs/templates/courseware/prompts/fix.md +0 -30
- package/skills/markcut/docs/templates/courseware/prompts/outline.md +0 -58
- package/skills/markcut/docs/templates/courseware/prompts/scene.md +0 -70
- package/skills/markcut/docs/templates/deep-dive/TEMPLATE.md +0 -354
- package/skills/markcut/docs/templates/deep-dive/agents/researcher.md +0 -69
- package/skills/markcut/docs/templates/deep-dive/agents/reviewer.md +0 -108
- package/skills/markcut/docs/templates/deep-dive/prompts/fix.md +0 -28
- package/skills/markcut/docs/templates/deep-dive/prompts/outline.md +0 -102
- package/skills/markcut/docs/templates/deep-dive/prompts/script.md +0 -64
- package/skills/markcut/docs/templates/illustrated-book/TEMPLATE.md +0 -356
- package/skills/markcut/docs/templates/illustrated-book/agents/reviewer.md +0 -115
- package/skills/markcut/docs/templates/illustrated-book/prompts/fix.md +0 -29
- package/skills/markcut/docs/templates/illustrated-book/prompts/illustration.md +0 -35
- package/skills/markcut/docs/templates/illustrated-book/prompts/story.md +0 -69
- package/skills/markcut/docs/templates/short-film/TEMPLATE.md +0 -385
- package/skills/markcut/docs/templates/short-film/agents/reviewer.md +0 -115
- package/skills/markcut/docs/templates/short-film/prompts/fix.md +0 -29
- package/skills/markcut/docs/templates/short-film/prompts/screenplay.md +0 -101
- package/skills/markcut/docs/templates/short-film/prompts/storyboard.md +0 -54
- package/skills/markcut/docs/templates/short-video/TEMPLATE.md +0 -301
- package/skills/markcut/docs/templates/short-video/agents/reviewer.md +0 -94
- package/skills/markcut/docs/templates/short-video/prompts/fix.md +0 -27
- package/skills/markcut/docs/templates/short-video/prompts/script.md +0 -65
- package/skills/markcut/docs/templates/vlog/TEMPLATE.md +0 -326
- package/skills/markcut/docs/templates/vlog/agents/reviewer.md +0 -89
- package/skills/markcut/docs/templates/vlog/agents/story-writer.md +0 -100
- package/skills/markcut/docs/templates/vlog/prompts/bgm-select.md +0 -38
- package/skills/markcut/docs/templates/vlog/prompts/group-clips.md +0 -93
- package/skills/markcut/docs/templates/vlog/prompts/local-context.md +0 -59
- package/skills/markcut/docs/templates/vlog/prompts/outline.md +0 -56
- package/skills/markcut/docs/templates/vlog/prompts/review-fix.md +0 -27
- package/skills/markcut/docs/templates/vlog/prompts/storyboard.md +0 -41
- package/src/render/cli.test.ts +0 -13
package/src/vision/cli.mjs
CHANGED
|
@@ -20,11 +20,7 @@
|
|
|
20
20
|
* f) Save complete metadata.json with metadata + userHint + perception
|
|
21
21
|
*
|
|
22
22
|
* Options:
|
|
23
|
-
* --label
|
|
24
|
-
* --agent <tmpl> Custom text LLM CLI for detect-scenes ({prompt})
|
|
25
|
-
* --itt <tmpl> Custom ITT CLI template with {input}, {prompt}
|
|
26
|
-
* --vtt <tmpl> Custom VTT CLI template with {input}, {prompt}
|
|
27
|
-
* --stt <tmpl> Custom STT CLI template with {input}, {output}
|
|
23
|
+
* --label Open label preview → then continue with full AI pipeline
|
|
28
24
|
* --prompts-file <path> Path to prompts markdown file (default: vision_prompts.md)
|
|
29
25
|
* --vtt-sample-interval <n> Sample one video frame every N seconds (default: 5)
|
|
30
26
|
* --context "text" Background context about people/places (injected into prompts)
|
|
@@ -32,7 +28,6 @@
|
|
|
32
28
|
* --skip-stt Skip speech-to-text for videos
|
|
33
29
|
* --dry-run Show what would be processed without running AI
|
|
34
30
|
* --show-prompts Print the prompts file and exit
|
|
35
|
-
* --show-clis Print the default ITT/VTT/STT CLI templates
|
|
36
31
|
* --help Show this help
|
|
37
32
|
*
|
|
38
33
|
* Prompt overrides:
|
|
@@ -716,13 +711,13 @@ function mergeLabelsIntoMetadata(folder, metadata) {
|
|
|
716
711
|
|
|
717
712
|
// ── Perception helpers ────────────────────────────────────────────────────
|
|
718
713
|
|
|
719
|
-
function analyzeImage(imagePath, normPath, prompts,
|
|
714
|
+
function analyzeImage(imagePath, normPath, prompts, context = "", userHint = "", userHints = null) {
|
|
720
715
|
let ctxParts = [];
|
|
721
716
|
if (context) ctxParts.push(`Context: ${context}`);
|
|
722
717
|
if (userHint) ctxParts.push(`User hint: ${userHint}`);
|
|
723
718
|
const ctx = ctxParts.length > 0 ? ctxParts.join("\n") + "\n\n" : "";
|
|
724
719
|
const prompt = ctx + getPrompt(prompts, "image-perception");
|
|
725
|
-
const raw = runITT(normPath, prompt,
|
|
720
|
+
const raw = runITT(normPath, prompt, DEFAULT_ITT_CLI);
|
|
726
721
|
const parsed = looseJSONParse(raw);
|
|
727
722
|
const rawText = stripThinkBlocks(extractRawText(raw));
|
|
728
723
|
return normalizePerception({
|
|
@@ -790,7 +785,7 @@ function buildMergedCues(userHint, cues, sceneChangesMs, totalDurationMs) {
|
|
|
790
785
|
return merged;
|
|
791
786
|
}
|
|
792
787
|
|
|
793
|
-
function analyzeVideo(videoPath, normInfo, normDir, prompts,
|
|
788
|
+
function analyzeVideo(videoPath, normInfo, normDir, prompts, context = "", userHint = "", sampleInterval = DEFAULT_VTT_SAMPLE_INTERVAL, userHints = null) {
|
|
794
789
|
let ctxParts = [];
|
|
795
790
|
if (context) ctxParts.push(`Context: ${context}`);
|
|
796
791
|
if (userHint && typeof userHint === "string") ctxParts.push(`User hint: ${userHint}`);
|
|
@@ -800,13 +795,13 @@ function analyzeVideo(videoPath, normInfo, normDir, prompts, vttCli, sttCli, con
|
|
|
800
795
|
// 1. VTT → overall video description
|
|
801
796
|
emitInfo(` Running video understanding...`);
|
|
802
797
|
const descPrompt = ctx + getPrompt(prompts, "video-perception");
|
|
803
|
-
const descRaw = runVTT(normInfo.path, descPrompt,
|
|
798
|
+
const descRaw = runVTT(normInfo.path, descPrompt, DEFAULT_VTT_CLI, DEFAULT_ITT_CLI, sampleInterval);
|
|
804
799
|
const descText = stripThinkBlocks(extractRawText(descRaw));
|
|
805
800
|
perception.desc = descText.slice(0, 500) || looseJSONParse(descRaw)?.desc || descRaw.slice(0, 500);
|
|
806
801
|
|
|
807
802
|
// 2. STT → VTT subtitle
|
|
808
803
|
emitInfo(` Running speech-to-text...`);
|
|
809
|
-
perception.subtitle = runSTT(videoPath, normDir,
|
|
804
|
+
perception.subtitle = runSTT(videoPath, normDir, DEFAULT_STT_CLI);
|
|
810
805
|
|
|
811
806
|
// 3. Build merged cue timeline from VTT + user hints + ffprobe
|
|
812
807
|
emitInfo(` Building merged segment boundaries...`);
|
|
@@ -843,7 +838,7 @@ function analyzeVideo(videoPath, normInfo, normDir, prompts, vttCli, sttCli, con
|
|
|
843
838
|
if (!existsSync(dummyInput)) {
|
|
844
839
|
writeFileSync(dummyInput, Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", "base64"));
|
|
845
840
|
}
|
|
846
|
-
const segRaw = runAgent(dummyInput, `${segPrompt}\n\nInput:\n${segInput}`,
|
|
841
|
+
const segRaw = runAgent(dummyInput, `${segPrompt}\n\nInput:\n${segInput}`, DEFAULT_AGENT_CLI || DEFAULT_ITT_CLI);
|
|
847
842
|
const segParsed = looseJSONParse(segRaw);
|
|
848
843
|
|
|
849
844
|
if (segParsed && typeof segParsed === "object" && Object.keys(segParsed).length > 0) {
|
|
@@ -903,7 +898,7 @@ function analyzeVideo(videoPath, normInfo, normDir, prompts, vttCli, sttCli, con
|
|
|
903
898
|
} else { clipPath = clipOut; }
|
|
904
899
|
|
|
905
900
|
// Use runVTT for video clips (extracts frames, then runs ITT)
|
|
906
|
-
const segRawVis = runVTT(clipPath, visSegPrompt,
|
|
901
|
+
const segRawVis = runVTT(clipPath, visSegPrompt, DEFAULT_VTT_CLI, DEFAULT_ITT_CLI, sampleInterval);
|
|
907
902
|
const segVisText = extractRawText(segRawVis);
|
|
908
903
|
seg.vision = stripThinkBlocks(looseJSONParse(segRawVis)?.desc || segVisText.slice(0, 300) || segRawVis.slice(0, 300));
|
|
909
904
|
try { if (clipPath !== normInfo.path) rmSync(clipPath, { force: true }); } catch {}
|
|
@@ -917,7 +912,7 @@ function analyzeVideo(videoPath, normInfo, normDir, prompts, vttCli, sttCli, con
|
|
|
917
912
|
|
|
918
913
|
// ── Full pipeline runner (step 2: label → normalize → percept → segments) ──
|
|
919
914
|
|
|
920
|
-
async function runFullPipeline(folder, prompts,
|
|
915
|
+
async function runFullPipeline(folder, prompts, context, pickSet, vttSampleInterval, skipSTT, dryRun) {
|
|
921
916
|
const metadataPath = join(folder, "metadata.json");
|
|
922
917
|
|
|
923
918
|
// ── Ensure metadata exists ──────────────────────────────────────────
|
|
@@ -987,12 +982,12 @@ async function runFullPipeline(folder, prompts, ittCli, vttCli, sttCli, context,
|
|
|
987
982
|
return;
|
|
988
983
|
}
|
|
989
984
|
|
|
990
|
-
await runNormalizeAndPercept(folder, metadataPath, prompts,
|
|
985
|
+
await runNormalizeAndPercept(folder, metadataPath, prompts, context, pickSet, vttSampleInterval, skipSTT);
|
|
991
986
|
}
|
|
992
987
|
|
|
993
988
|
// ── Normalize + Percept (internal step) ───────────────────────────────────
|
|
994
989
|
|
|
995
|
-
async function runNormalizeAndPercept(folder, metadataPath, prompts,
|
|
990
|
+
async function runNormalizeAndPercept(folder, metadataPath, prompts, context, pickSet, vttSampleInterval, skipSTT) {
|
|
996
991
|
const { results, cacheMap } = loadMetadata(folder);
|
|
997
992
|
const cache = { ...cacheMap };
|
|
998
993
|
const normDir = join(folder, ".normalized");
|
|
@@ -1022,14 +1017,14 @@ async function runNormalizeAndPercept(folder, metadataPath, prompts, ittCli, vtt
|
|
|
1022
1017
|
let cacheKey = "";
|
|
1023
1018
|
const ctx = context ? `Context: ${context}\n\n` : "";
|
|
1024
1019
|
const imgPrompt = (userHint ? `User hint: ${userHint}\n\n` : "") + ctx + getPrompt(prompts, "image-perception");
|
|
1025
|
-
const imgInput =
|
|
1026
|
-
const imgCmd = substituteTemplate(
|
|
1020
|
+
const imgInput = `@${shQuote(normPath)}`;
|
|
1021
|
+
const imgCmd = substituteTemplate(DEFAULT_ITT_CLI, { input: imgInput, prompt: imgPrompt });
|
|
1027
1022
|
cacheKey = perceptionCacheKey(imgPath, imgCmd, "image");
|
|
1028
1023
|
if (cache[cacheKey]) {
|
|
1029
1024
|
perception = cache[cacheKey];
|
|
1030
1025
|
emitInfo(` (cached)`);
|
|
1031
1026
|
} else {
|
|
1032
|
-
perception = analyzeImage(imgPath, normPath, prompts,
|
|
1027
|
+
perception = analyzeImage(imgPath, normPath, prompts, context, userHint, userHints);
|
|
1033
1028
|
if (perception.desc) cache[cacheKey] = perception;
|
|
1034
1029
|
emitInfo(` ${perception.desc?.slice(0, 80)}...`);
|
|
1035
1030
|
}
|
|
@@ -1057,27 +1052,25 @@ async function runNormalizeAndPercept(folder, metadataPath, prompts, ittCli, vtt
|
|
|
1057
1052
|
emitInfo(`\n🎬 Video: ${base}${userHint ? ` (hint: "${userHint}")` : ""}`);
|
|
1058
1053
|
const vidPath = join(folder, fileName);
|
|
1059
1054
|
const normInfo = normalizeVideo(vidPath, normDir, meta.duration);
|
|
1060
|
-
const stt = skipSTT ? null : sttCli;
|
|
1061
|
-
|
|
1062
1055
|
let perception = {};
|
|
1063
1056
|
let cacheKey = "";
|
|
1064
1057
|
const ctxV = context ? `Context: ${context}\n\n` : "";
|
|
1065
1058
|
const vidPrompt = (userHint ? `User hint: ${userHint}\n\n` : "") + ctxV + getPrompt(prompts, "video-perception");
|
|
1066
1059
|
let vttActualCmd;
|
|
1067
|
-
if (
|
|
1068
|
-
vttActualCmd = substituteTemplate(
|
|
1060
|
+
if (DEFAULT_VTT_CLI) {
|
|
1061
|
+
vttActualCmd = substituteTemplate(DEFAULT_VTT_CLI, { input: shQuote(normInfo.path), prompt: vidPrompt });
|
|
1069
1062
|
} else {
|
|
1070
1063
|
const dur = meta.duration || 0;
|
|
1071
1064
|
const n = Math.max(5, Math.min(10, Math.ceil(dur / vttSampleInterval)));
|
|
1072
1065
|
const framePaths = Array.from({ length: n }, (_, i) => `${basename(normInfo.path, extname(normInfo.path))}_${String(i + 1).padStart(3, "0")}.jpg`);
|
|
1073
|
-
vttActualCmd = substituteTemplate(
|
|
1066
|
+
vttActualCmd = substituteTemplate(DEFAULT_ITT_CLI, { input: framePaths.map(p => `@${p}`).join(" "), prompt: vidPrompt });
|
|
1074
1067
|
}
|
|
1075
1068
|
cacheKey = perceptionCacheKey(vidPath, vttActualCmd, "video");
|
|
1076
1069
|
if (cache[cacheKey]) {
|
|
1077
1070
|
perception = cache[cacheKey];
|
|
1078
1071
|
emitInfo(` (cached)`);
|
|
1079
1072
|
} else {
|
|
1080
|
-
perception = analyzeVideo(vidPath, normInfo, normDir, prompts,
|
|
1073
|
+
perception = analyzeVideo(vidPath, normInfo, normDir, prompts, context, userHint, vttSampleInterval, userHints);
|
|
1081
1074
|
if (perception.desc) cache[cacheKey] = perception;
|
|
1082
1075
|
}
|
|
1083
1076
|
|
|
@@ -1105,10 +1098,6 @@ Usage:
|
|
|
1105
1098
|
|
|
1106
1099
|
Options:
|
|
1107
1100
|
--label Open label preview → then continue with full AI pipeline
|
|
1108
|
-
--agent <template> Custom text LLM CLI for detect-scenes ({prompt})
|
|
1109
|
-
--itt <template> Custom ITT CLI template with {input}, {prompt}
|
|
1110
|
-
--vtt <template> Custom VTT CLI template with {input}, {prompt}
|
|
1111
|
-
--stt <template> Custom STT CLI template with {input}, {output}
|
|
1112
1101
|
--prompts-file <path> Path to prompts markdown file (default: vision_prompts.md)
|
|
1113
1102
|
--vtt-sample-interval <n> Sample one video frame every N seconds (default: 5)
|
|
1114
1103
|
--context "text" Background context about people/places (injected into prompts)
|
|
@@ -1116,7 +1105,6 @@ Options:
|
|
|
1116
1105
|
--skip-stt Skip speech-to-text for videos
|
|
1117
1106
|
--dry-run Show what would be processed without running AI
|
|
1118
1107
|
--show-prompts Print the prompts file and exit
|
|
1119
|
-
--show-clis Print the default ITT/VTT/STT CLI templates
|
|
1120
1108
|
--help Show this help
|
|
1121
1109
|
|
|
1122
1110
|
Prompt overrides:
|
|
@@ -1126,10 +1114,6 @@ Prompt overrides:
|
|
|
1126
1114
|
|
|
1127
1115
|
export async function main(args) {
|
|
1128
1116
|
let folder = "";
|
|
1129
|
-
let agentCli = null;
|
|
1130
|
-
let ittCli = null;
|
|
1131
|
-
let vttCli = null;
|
|
1132
|
-
let sttCli = null;
|
|
1133
1117
|
let promptsFile = PROMPTS_FILE;
|
|
1134
1118
|
let context = "";
|
|
1135
1119
|
let vttSampleInterval = DEFAULT_VTT_SAMPLE_INTERVAL;
|
|
@@ -1147,20 +1131,12 @@ export async function main(args) {
|
|
|
1147
1131
|
const flag = args[i++];
|
|
1148
1132
|
if (flag === "--help") { printUsage(); return; }
|
|
1149
1133
|
else if (flag === "--label") { doLabel = true; }
|
|
1150
|
-
|
|
1151
|
-
else if (flag === "--itt" && args[i]) { ittCli = args[i++]; }
|
|
1152
|
-
else if (flag === "--vtt" && args[i]) { vttCli = args[i++]; }
|
|
1153
|
-
else if (flag === "--stt" && args[i]) { sttCli = args[i++]; }
|
|
1134
|
+
|
|
1154
1135
|
else if (flag === "--prompts-file" && args[i]) { promptsFile = resolve(args[i++]); }
|
|
1155
1136
|
else if (flag === "--vtt-sample-interval" && args[i]) { vttSampleInterval = parseInt(args[i++], 10) || DEFAULT_VTT_SAMPLE_INTERVAL; }
|
|
1156
1137
|
else if (flag === "--context" && args[i]) { context = args[i++]; }
|
|
1157
1138
|
else if (flag === "--show-prompts") { console.log(readFileSync(promptsFile, "utf-8")); return; }
|
|
1158
|
-
|
|
1159
|
-
console.log(`DEFAULT_ITT_CLI:\n${DEFAULT_ITT_CLI}\n`);
|
|
1160
|
-
console.log(`DEFAULT_VTT_CLI: (empty — uses ITT via frame extraction, sample every ${DEFAULT_VTT_SAMPLE_INTERVAL}s)\n`);
|
|
1161
|
-
console.log(`DEFAULT_STT_CLI:\n${DEFAULT_STT_CLI}`);
|
|
1162
|
-
return;
|
|
1163
|
-
}
|
|
1139
|
+
|
|
1164
1140
|
else if (flag === "--skip-stt") { skipSTT = true; }
|
|
1165
1141
|
else if (flag === "--pick" && args[i]) { for (const f of args[i++].split(",")) pickSet.add(f.trim()); }
|
|
1166
1142
|
else if (flag === "--dry-run") { dryRun = true; }
|
|
@@ -1175,7 +1151,7 @@ export async function main(args) {
|
|
|
1175
1151
|
for (const [name, value] of promptOverrides) prompts.set(name, value);
|
|
1176
1152
|
|
|
1177
1153
|
if (doLabel) {
|
|
1178
|
-
await runFullPipeline(folder, prompts,
|
|
1154
|
+
await runFullPipeline(folder, prompts, context, pickSet, vttSampleInterval, skipSTT, dryRun);
|
|
1179
1155
|
} else {
|
|
1180
1156
|
await runMetadataMode(folder, pickSet, dryRun);
|
|
1181
1157
|
}
|
|
@@ -1,220 +0,0 @@
|
|
|
1
|
-
# Watch Mode
|
|
2
|
-
|
|
3
|
-
> `preview <file.md> --edit` — live editing loop with AI-assisted JSON editing.
|
|
4
|
-
|
|
5
|
-
Edit mode gives you a browser player that **automatically reloads** when the source file changes, plus an **edit input** that calls an AI coding agent (pi) to make changes on your behalf.
|
|
6
|
-
|
|
7
|
-
Generated artifacts (TTS audio, compiled includes, component bundles) are
|
|
8
|
-
stored under `.markcut/` next to the source file — see [SKILL.md](../SKILL.md) for the full layout.
|
|
9
|
-
|
|
10
|
-
## Quickstart
|
|
11
|
-
|
|
12
|
-
```bash
|
|
13
|
-
cd remotion-engine
|
|
14
|
-
npm run preview -- sample-visual.json --edit
|
|
15
|
-
# → Browser opens, terminal blocks until you close it
|
|
16
|
-
```
|
|
17
|
-
|
|
18
|
-
## Architecture
|
|
19
|
-
|
|
20
|
-
```
|
|
21
|
-
┌─────────────┐ HTTP ┌──────────────────┐ spawn ┌──────────┐
|
|
22
|
-
│ Browser │◄──────────────►│ player/server │◄────────────►│ pi CLI │
|
|
23
|
-
│ (Chrome) │ SSE │ .mjs (Node) │ one-shot │ (agent) │
|
|
24
|
-
└──────┬──────┘ └────────┬─────────┘ └──────────┘
|
|
25
|
-
│ │
|
|
26
|
-
│ POST /api/edit │ fs.watchFile
|
|
27
|
-
│ {text: "make text bigger"} │ (poll every 500ms)
|
|
28
|
-
│ │
|
|
29
|
-
│ GET /api/video-data ▼
|
|
30
|
-
│ ← stream tree JSON ┌──────────────┐
|
|
31
|
-
│ │ stream tree │
|
|
32
|
-
│ EventSource │ .json (file) │
|
|
33
|
-
│ /api/events └──────────────┘
|
|
34
|
-
│ ← {type:"reload"} ▲
|
|
35
|
-
│ │
|
|
36
|
-
│ window.dispatchEvent │ pi edits
|
|
37
|
-
│ ("refresh-player") │ this file
|
|
38
|
-
│ │
|
|
39
|
-
```
|
|
40
|
-
|
|
41
|
-
## File: `src/player/server.mjs`
|
|
42
|
-
|
|
43
|
-
### Server startup (`cli.mjs` → `server.mjs`)
|
|
44
|
-
|
|
45
|
-
```
|
|
46
|
-
node src/render/cli.mjs preview file.json --edit --port 3031
|
|
47
|
-
```
|
|
48
|
-
|
|
49
|
-
1. `cli.mjs` spawns `server.mjs` with `--edit` flag
|
|
50
|
-
2. `server.mjs` parses `file.json` to build a recursive tree description
|
|
51
|
-
3. Starts `fs.watchFile(file.json, {interval:500})` — polls every 500ms for changes
|
|
52
|
-
4. Starts HTTP server on specified port
|
|
53
|
-
5. Auto-opens browser via `open http://localhost:PORT`
|
|
54
|
-
|
|
55
|
-
### Page served
|
|
56
|
-
|
|
57
|
-
The HTML page loads two scripts:
|
|
58
|
-
|
|
59
|
-
| Script | Source | Role |
|
|
60
|
-
|--------|--------|------|
|
|
61
|
-
| `player.js` | Bundled esbuild output from `src/player/browser.tsx` | Renders the stream tree using `@remotion/player` with `MarkCut` |
|
|
62
|
-
| Inline script | Generated by `getHtml()` in `server.mjs` | Watch-mode UI controls (close button, edit input, SSE) |
|
|
63
|
-
|
|
64
|
-
### Edit flow
|
|
65
|
-
|
|
66
|
-
```
|
|
67
|
-
Browser Server pi CLI
|
|
68
|
-
│ │ │
|
|
69
|
-
│── POST /api/edit ────────────► │ │
|
|
70
|
-
│ {text: "make text bigger"} │ │
|
|
71
|
-
│ │── spawn("pi", ["-p", prompt])│
|
|
72
|
-
│ │ prompt includes: │
|
|
73
|
-
│ │ • full tree description │
|
|
74
|
-
│ │ (recursive, any depth) │
|
|
75
|
-
│ │ • edit history │
|
|
76
|
-
│ │ • knowledge (all stream │
|
|
77
|
-
│ │ types, components, │
|
|
78
|
-
│ │ schema, styling guide) │
|
|
79
|
-
│ │ • edit request │
|
|
80
|
-
│ │ │
|
|
81
|
-
│ │ pi can edit ANY field on │
|
|
82
|
-
│ │ ANY node in the tree │
|
|
83
|
-
│ │ ├── edits file.json
|
|
84
|
-
│ │ └── exits
|
|
85
|
-
│ │◄── exit code 0 │
|
|
86
|
-
│◄── {done:true, output:"..."} ──┤ │
|
|
87
|
-
│ │ │
|
|
88
|
-
│ suppressReload = true │ │
|
|
89
|
-
│ header shows "Changed text..." │ │
|
|
90
|
-
│ │ │
|
|
91
|
-
│ (fs.watchFile fires SSE │ │
|
|
92
|
-
│ but suppressReload=true │ │
|
|
93
|
-
│ so it's ignored) │ │
|
|
94
|
-
│ │ │
|
|
95
|
-
│ after 4s: │ │
|
|
96
|
-
│ dispatchEvent("refresh-player")│ │
|
|
97
|
-
│ → player re-fetches /api/data │ │
|
|
98
|
-
│ → re-renders with new JSON │ │
|
|
99
|
-
```
|
|
100
|
-
|
|
101
|
-
### Key design decisions
|
|
102
|
-
|
|
103
|
-
**Why in-place refresh instead of `location.reload()`?**
|
|
104
|
-
- Timeline playback continues uninterrupted
|
|
105
|
-
- Status text persists until next edit
|
|
106
|
-
- No page flash
|
|
107
|
-
|
|
108
|
-
**Why `suppressReload`?**
|
|
109
|
-
- SSE fires ~500ms after pi edits the file
|
|
110
|
-
- API response arrives at the same time
|
|
111
|
-
- `suppressReload` flag ignores the SSE event during edit
|
|
112
|
-
- The client reloads itself after showing the summary
|
|
113
|
-
|
|
114
|
-
## File: `src/player/browser.tsx`
|
|
115
|
-
|
|
116
|
-
The bundled player app:
|
|
117
|
-
|
|
118
|
-
1. Fetches `/api/video-data` on mount
|
|
119
|
-
2. Renders the stream tree using `@remotion/player` with `MarkCut`
|
|
120
|
-
3. Listens for `"refresh-player"` custom DOM event
|
|
121
|
-
4. On refresh event: re-fetches data, re-renders in-place
|
|
122
|
-
|
|
123
|
-
## File: `src/render/cli.mjs`
|
|
124
|
-
|
|
125
|
-
The CLI routes `--edit` flag to the player server:
|
|
126
|
-
|
|
127
|
-
```javascript
|
|
128
|
-
if (args.edit) {
|
|
129
|
-
const serverArgs = ["src/player/server.mjs", file, "--edit", `--port=${port}`];
|
|
130
|
-
const child = spawn("node", serverArgs, { cwd: ROOT, stdio: "inherit" });
|
|
131
|
-
// ...blocks until server exits...
|
|
132
|
-
}
|
|
133
|
-
```
|
|
134
|
-
|
|
135
|
-
## API Endpoints
|
|
136
|
-
|
|
137
|
-
| Endpoint | Method | Description |
|
|
138
|
-
|----------|--------|-------------|
|
|
139
|
-
| `/api/video-data` | GET | Returns the raw stream tree JSON file |
|
|
140
|
-
| `/api/scenes` | GET | Returns parsed scenes array with timing |
|
|
141
|
-
| `/api/video-info` | GET | Returns scenes + totalDuration + mode flags |
|
|
142
|
-
| `/api/edit` | POST | Accepts `{text}` → spawns pi to edit the JSON → file watcher triggers → SSE reload |
|
|
143
|
-
| `/api/shutdown` | POST | Kills the server process, unblocking the agent |
|
|
144
|
-
| `/api/events` | GET | SSE stream — sends `{type:"reload"}` when file changes |
|
|
145
|
-
| `/api/labels` | GET/POST | (label mode only) Save/load scene labels |
|
|
146
|
-
|
|
147
|
-
## The pi prompt
|
|
148
|
-
|
|
149
|
-
Each edit call sends a structured prompt with a **recursive tree description** instead of a flat scene list:
|
|
150
|
-
|
|
151
|
-
```
|
|
152
|
-
You are editing sample-visual.json, a Remotion stream tree JSON.
|
|
153
|
-
|
|
154
|
-
The stream tree (indentation shows nesting; timing in seconds):
|
|
155
|
-
root "root" (1080×1920, 30fps, series, transition:fade)
|
|
156
|
-
folder "scene-1" (parallel)
|
|
157
|
-
component GradientBackground {type:"radial",animated:true,noise:true} [05s, bg]
|
|
158
|
-
component AnimatedHeadline {"text":"Hello Remotion",...} [0.54.5s]
|
|
159
|
-
folder "scene-2" (parallel)
|
|
160
|
-
component GradientBackground {type:"linear",animated:true} [05s, bg]
|
|
161
|
-
subtitle "This text is styled with CSS" fontSize:52 [0.54.5s]
|
|
162
|
-
...
|
|
163
|
-
|
|
164
|
-
Previous edits:
|
|
165
|
-
1. make scene 1 headline text orange
|
|
166
|
-
|
|
167
|
-
Edit request: make scene 1 headline text bigger
|
|
168
|
-
|
|
169
|
-
--- Knowledge ---
|
|
170
|
-
You can edit ANY field on ANY node in the JSON...
|
|
171
|
-
Stream types: root, folder, video, audio, image, subtitle, component, effect, rhythm, map
|
|
172
|
-
...
|
|
173
|
-
IMPORTANT: Read the full existing JSON file before editing. Only edit the JSON file. You can change, add, or remove any field on any node. Output ONLY a one-line summary of what specific change you made.
|
|
174
|
-
```
|
|
175
|
-
|
|
176
|
-
The tree description walks the JSON recursively, showing:
|
|
177
|
-
- **Nesting depth** via indentation
|
|
178
|
-
- **Type** and **id/name** of each node
|
|
179
|
-
- **Key fields** (dimensions, fps, theme, series/parallel, transitions)
|
|
180
|
-
- **Timing** from leaf span (`[start→end]`)
|
|
181
|
-
- **isBackground** flag (`[bg]`)
|
|
182
|
-
- **Component props** and node-specific fields (fontSize, fit, style, volume, etc.)
|
|
183
|
-
|
|
184
|
-
This means pi can edit **any aspect of the JSON** ÔÇö not just scene props. Nested folders, effects, rhythms, maps, actions, styles, themes, global stylesheets, audio config ÔÇö everything is visible and editable.
|
|
185
|
-
|
|
186
|
-
## Session context
|
|
187
|
-
|
|
188
|
-
The server maintains `editHistory[]` — every edit is appended. On each call, previous edits are included so pi knows what was done before:
|
|
189
|
-
|
|
190
|
-
```
|
|
191
|
-
Previous edits:
|
|
192
|
-
1. make scene 1 headline text orange
|
|
193
|
-
2. change background from radial to linear gradient
|
|
194
|
-
3. add 3D angle to device mockup in scene 4
|
|
195
|
-
```
|
|
196
|
-
|
|
197
|
-
This gives pi enough context to make coherent changes across multiple edit rounds.
|
|
198
|
-
|
|
199
|
-
## Close / tab-close
|
|
200
|
-
|
|
201
|
-
| Action | What happens |
|
|
202
|
-
|--------|-------------|
|
|
203
|
-
| Click **✕** | `sendBeacon("/api/shutdown")` → server exits → agent unblocks |
|
|
204
|
-
| Close **tab** | SSE connection drops → 3s grace → server exits |
|
|
205
|
-
| Close **browser** | Same as tab close |
|
|
206
|
-
|
|
207
|
-
## Label mode (`--label`)
|
|
208
|
-
|
|
209
|
-
`preview <file.json> --label` starts the same player but with a **label input** instead of the edit input. Labels are saved to `labels.json` next to the source file. No AI editing, no file watching.
|
|
210
|
-
|
|
211
|
-
## Build
|
|
212
|
-
|
|
213
|
-
The browser bundle is built with esbuild:
|
|
214
|
-
|
|
215
|
-
```bash
|
|
216
|
-
bash scripts/build-player.sh
|
|
217
|
-
# → src/player/bundle/player.js
|
|
218
|
-
```
|
|
219
|
-
|
|
220
|
-
The bundle includes React, @remotion/player, MarkCut, all 13 built-in components, themes, templates, and the duration utility. It is served as a static file by the player server.
|
|
@@ -1,112 +0,0 @@
|
|
|
1
|
-
# Label System
|
|
2
|
-
|
|
3
|
-
Interactive media labeling before understanding clips in json stream tree.
|
|
4
|
-
The stream tree follow rules, see [docs/json-descriptive.md](docs/json-descriptive.md) for full details .
|
|
5
|
-
|
|
6
|
-
## Overview
|
|
7
|
-
|
|
8
|
-
The label player lets you load a stream tree, browse its clips, and attach descriptive labels to each clip's media.
|
|
9
|
-
The output is an updated `labels.json` file with `description` fields populated.
|
|
10
|
-
|
|
11
|
-
## Usage
|
|
12
|
-
|
|
13
|
-
```bash
|
|
14
|
-
# Label from a stream tree JSON or labels.json
|
|
15
|
-
npx markcut preview labels.json --label
|
|
16
|
-
|
|
17
|
-
# Specify port
|
|
18
|
-
npx markcut preview labels.json --label --port 3031
|
|
19
|
-
```
|
|
20
|
-
|
|
21
|
-
### Labels Persistence
|
|
22
|
-
The stream tree follow rules, see [docs/json-descriptive.md](docs/json-descriptive.md) for full details .
|
|
23
|
-
Labels are saved to the source file.
|
|
24
|
-
|
|
25
|
-
```json
|
|
26
|
-
{
|
|
27
|
-
"root": {
|
|
28
|
-
"id": "root",
|
|
29
|
-
"type": "root",
|
|
30
|
-
"children": [
|
|
31
|
-
{
|
|
32
|
-
"id": "scene-1",
|
|
33
|
-
"type": "folder",
|
|
34
|
-
"children": [
|
|
35
|
-
{
|
|
36
|
-
"id": "scene-1-media",
|
|
37
|
-
"type": "image",
|
|
38
|
-
"src": "/media/photo.jpg",
|
|
39
|
-
"start": 0, "end": 5,
|
|
40
|
-
"description": "Best group shot from the campfire"
|
|
41
|
-
}
|
|
42
|
-
]
|
|
43
|
-
},
|
|
44
|
-
{
|
|
45
|
-
"id": "scene-2",
|
|
46
|
-
"type": "folder",
|
|
47
|
-
"children": [
|
|
48
|
-
{
|
|
49
|
-
"id": "scene-2-media",
|
|
50
|
-
"type": "video",
|
|
51
|
-
"src": "/media/clip.mp4",
|
|
52
|
-
"start": 0, "end": 5,
|
|
53
|
-
"description": "Sunset timelapse — use for intro"
|
|
54
|
-
}
|
|
55
|
-
]
|
|
56
|
-
}
|
|
57
|
-
]
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
```
|
|
61
|
-
|
|
62
|
-
The `description` field on each media leaf node holds the label text.
|
|
63
|
-
|
|
64
|
-
## Workflow
|
|
65
|
-
|
|
66
|
-
```
|
|
67
|
-
Stream tree JSON (labels.json)
|
|
68
|
-
│
|
|
69
|
-
▼
|
|
70
|
-
─── preview --label ───→ Browse & label clips
|
|
71
|
-
│ │
|
|
72
|
-
│ ▼
|
|
73
|
-
│ Updated labels.json
|
|
74
|
-
│ │
|
|
75
|
-
▼ ▼
|
|
76
|
-
Use in Storyboard ──────→ Assemble → Render
|
|
77
|
-
```
|
|
78
|
-
|
|
79
|
-
1. **Prepare** a stream tree JSON with scenes
|
|
80
|
-
2. **Label** with `preview --label` to add descriptions to each scene
|
|
81
|
-
3. **Export** labels are saved alongside the source file
|
|
82
|
-
4. **Use** the labeled scenes in your storyboard or assembly workflow
|
|
83
|
-
|
|
84
|
-
## API Endpoints
|
|
85
|
-
|
|
86
|
-
| Endpoint | Method | Description |
|
|
87
|
-
|----------|--------|-------------|
|
|
88
|
-
| `/api/video-data` | GET | Returns the raw stream tree JSON |
|
|
89
|
-
| `/api/video-info` | GET | Returns scenes array with timing + totalDuration |
|
|
90
|
-
| `/api/labels` | GET | Returns the current stream tree with descriptions |
|
|
91
|
-
| `/api/labels` | POST | Saves updated descriptions ({descriptions: {index: "text"}}) |
|
|
92
|
-
| `/api/shutdown` | POST | Kills the server |
|
|
93
|
-
| `/api/events` | GET | SSE events (reload on file change) |
|
|
94
|
-
|
|
95
|
-
## Implementation
|
|
96
|
-
|
|
97
|
-
Source: `src/player/server.mjs` (unified server, use `--label` flag)
|
|
98
|
-
UI controls: `src/player/components/LabelControls.tsx` (React component)
|
|
99
|
-
|
|
100
|
-
- Built on Node.js `http` module (no Express dependency)
|
|
101
|
-
- Serves bundled `player.js` (esbuild output from `src/player/browser.tsx`)
|
|
102
|
-
- Labels saved atomically to disk on each POST
|
|
103
|
-
- Supports stream tree JSON, descriptive JSON, and markdown input
|
|
104
|
-
- React components in `src/player/components/` replace the old HTML-based `ui/` directory
|
|
105
|
-
|
|
106
|
-
## Reference
|
|
107
|
-
|
|
108
|
-
| Topic | Link |
|
|
109
|
-
|-------|------|
|
|
110
|
-
| Storyboard (video planning) | [storyboard.md](storyboard.md) |
|
|
111
|
-
| Stream tree types | `SKILL.md` |
|
|
112
|
-
| Dynamic components | [dynamic-components.md](dynamic-components.md) |
|