@lalalic/markcut 2.2.8 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/package.json +2 -1
  2. package/skills/markcut/SKILL.md +11 -9
  3. package/skills/markcut/docs/markdown-descriptive.md +1 -1
  4. package/src/config.mjs +60 -63
  5. package/src/descriptive/resolve.ts +3 -2
  6. package/src/player/pipeline.mjs +64 -21
  7. package/src/render/cli.mjs +42 -69
  8. package/src/vision/cli.mjs +21 -45
  9. package/skills/markcut/docs/edit-mode.md +0 -220
  10. package/skills/markcut/docs/label-mode.md +0 -112
  11. package/skills/markcut/docs/template.md +0 -212
  12. package/skills/markcut/docs/templates/courseware/TEMPLATE.md +0 -526
  13. package/skills/markcut/docs/templates/courseware/agents/reviewer.md +0 -84
  14. package/skills/markcut/docs/templates/courseware/prompts/fix.md +0 -30
  15. package/skills/markcut/docs/templates/courseware/prompts/outline.md +0 -58
  16. package/skills/markcut/docs/templates/courseware/prompts/scene.md +0 -70
  17. package/skills/markcut/docs/templates/deep-dive/TEMPLATE.md +0 -354
  18. package/skills/markcut/docs/templates/deep-dive/agents/researcher.md +0 -69
  19. package/skills/markcut/docs/templates/deep-dive/agents/reviewer.md +0 -108
  20. package/skills/markcut/docs/templates/deep-dive/prompts/fix.md +0 -28
  21. package/skills/markcut/docs/templates/deep-dive/prompts/outline.md +0 -102
  22. package/skills/markcut/docs/templates/deep-dive/prompts/script.md +0 -64
  23. package/skills/markcut/docs/templates/illustrated-book/TEMPLATE.md +0 -356
  24. package/skills/markcut/docs/templates/illustrated-book/agents/reviewer.md +0 -115
  25. package/skills/markcut/docs/templates/illustrated-book/prompts/fix.md +0 -29
  26. package/skills/markcut/docs/templates/illustrated-book/prompts/illustration.md +0 -35
  27. package/skills/markcut/docs/templates/illustrated-book/prompts/story.md +0 -69
  28. package/skills/markcut/docs/templates/short-film/TEMPLATE.md +0 -385
  29. package/skills/markcut/docs/templates/short-film/agents/reviewer.md +0 -115
  30. package/skills/markcut/docs/templates/short-film/prompts/fix.md +0 -29
  31. package/skills/markcut/docs/templates/short-film/prompts/screenplay.md +0 -101
  32. package/skills/markcut/docs/templates/short-film/prompts/storyboard.md +0 -54
  33. package/skills/markcut/docs/templates/short-video/TEMPLATE.md +0 -301
  34. package/skills/markcut/docs/templates/short-video/agents/reviewer.md +0 -94
  35. package/skills/markcut/docs/templates/short-video/prompts/fix.md +0 -27
  36. package/skills/markcut/docs/templates/short-video/prompts/script.md +0 -65
  37. package/skills/markcut/docs/templates/vlog/TEMPLATE.md +0 -326
  38. package/skills/markcut/docs/templates/vlog/agents/reviewer.md +0 -89
  39. package/skills/markcut/docs/templates/vlog/agents/story-writer.md +0 -100
  40. package/skills/markcut/docs/templates/vlog/prompts/bgm-select.md +0 -38
  41. package/skills/markcut/docs/templates/vlog/prompts/group-clips.md +0 -93
  42. package/skills/markcut/docs/templates/vlog/prompts/local-context.md +0 -59
  43. package/skills/markcut/docs/templates/vlog/prompts/outline.md +0 -56
  44. package/skills/markcut/docs/templates/vlog/prompts/review-fix.md +0 -27
  45. package/skills/markcut/docs/templates/vlog/prompts/storyboard.md +0 -41
  46. package/src/render/cli.test.ts +0 -13
@@ -20,11 +20,7 @@
20
20
  * f) Save complete metadata.json with metadata + userHint + perception
21
21
  *
22
22
  * Options:
23
- * --label Run full pipeline: preview → label normalize percept → segments
24
- * --agent <tmpl> Custom text LLM CLI for detect-scenes ({prompt})
25
- * --itt <tmpl> Custom ITT CLI template with {input}, {prompt}
26
- * --vtt <tmpl> Custom VTT CLI template with {input}, {prompt}
27
- * --stt <tmpl> Custom STT CLI template with {input}, {output}
23
+ * --label Open label preview → then continue with full AI pipeline
28
24
  * --prompts-file <path> Path to prompts markdown file (default: vision_prompts.md)
29
25
  * --vtt-sample-interval <n> Sample one video frame every N seconds (default: 5)
30
26
  * --context "text" Background context about people/places (injected into prompts)
@@ -32,7 +28,6 @@
32
28
  * --skip-stt Skip speech-to-text for videos
33
29
  * --dry-run Show what would be processed without running AI
34
30
  * --show-prompts Print the prompts file and exit
35
- * --show-clis Print the default ITT/VTT/STT CLI templates
36
31
  * --help Show this help
37
32
  *
38
33
  * Prompt overrides:
@@ -716,13 +711,13 @@ function mergeLabelsIntoMetadata(folder, metadata) {
716
711
 
717
712
  // ── Perception helpers ────────────────────────────────────────────────────
718
713
 
719
- function analyzeImage(imagePath, normPath, prompts, ittCli, context = "", userHint = "", userHints = null) {
714
+ function analyzeImage(imagePath, normPath, prompts, context = "", userHint = "", userHints = null) {
720
715
  let ctxParts = [];
721
716
  if (context) ctxParts.push(`Context: ${context}`);
722
717
  if (userHint) ctxParts.push(`User hint: ${userHint}`);
723
718
  const ctx = ctxParts.length > 0 ? ctxParts.join("\n") + "\n\n" : "";
724
719
  const prompt = ctx + getPrompt(prompts, "image-perception");
725
- const raw = runITT(normPath, prompt, ittCli);
720
+ const raw = runITT(normPath, prompt, DEFAULT_ITT_CLI);
726
721
  const parsed = looseJSONParse(raw);
727
722
  const rawText = stripThinkBlocks(extractRawText(raw));
728
723
  return normalizePerception({
@@ -790,7 +785,7 @@ function buildMergedCues(userHint, cues, sceneChangesMs, totalDurationMs) {
790
785
  return merged;
791
786
  }
792
787
 
793
- function analyzeVideo(videoPath, normInfo, normDir, prompts, vttCli, sttCli, context = "", userHint = "", ittCli = null, sampleInterval = DEFAULT_VTT_SAMPLE_INTERVAL, agentCli = null, userHints = null) {
788
+ function analyzeVideo(videoPath, normInfo, normDir, prompts, context = "", userHint = "", sampleInterval = DEFAULT_VTT_SAMPLE_INTERVAL, userHints = null) {
794
789
  let ctxParts = [];
795
790
  if (context) ctxParts.push(`Context: ${context}`);
796
791
  if (userHint && typeof userHint === "string") ctxParts.push(`User hint: ${userHint}`);
@@ -800,13 +795,13 @@ function analyzeVideo(videoPath, normInfo, normDir, prompts, vttCli, sttCli, con
800
795
  // 1. VTT → overall video description
801
796
  emitInfo(` Running video understanding...`);
802
797
  const descPrompt = ctx + getPrompt(prompts, "video-perception");
803
- const descRaw = runVTT(normInfo.path, descPrompt, vttCli, ittCli, sampleInterval);
798
+ const descRaw = runVTT(normInfo.path, descPrompt, DEFAULT_VTT_CLI, DEFAULT_ITT_CLI, sampleInterval);
804
799
  const descText = stripThinkBlocks(extractRawText(descRaw));
805
800
  perception.desc = descText.slice(0, 500) || looseJSONParse(descRaw)?.desc || descRaw.slice(0, 500);
806
801
 
807
802
  // 2. STT → VTT subtitle
808
803
  emitInfo(` Running speech-to-text...`);
809
- perception.subtitle = runSTT(videoPath, normDir, sttCli);
804
+ perception.subtitle = runSTT(videoPath, normDir, DEFAULT_STT_CLI);
810
805
 
811
806
  // 3. Build merged cue timeline from VTT + user hints + ffprobe
812
807
  emitInfo(` Building merged segment boundaries...`);
@@ -843,7 +838,7 @@ function analyzeVideo(videoPath, normInfo, normDir, prompts, vttCli, sttCli, con
843
838
  if (!existsSync(dummyInput)) {
844
839
  writeFileSync(dummyInput, Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", "base64"));
845
840
  }
846
- const segRaw = runAgent(dummyInput, `${segPrompt}\n\nInput:\n${segInput}`, agentCli || ittCli);
841
+ const segRaw = runAgent(dummyInput, `${segPrompt}\n\nInput:\n${segInput}`, DEFAULT_AGENT_CLI || DEFAULT_ITT_CLI);
847
842
  const segParsed = looseJSONParse(segRaw);
848
843
 
849
844
  if (segParsed && typeof segParsed === "object" && Object.keys(segParsed).length > 0) {
@@ -903,7 +898,7 @@ function analyzeVideo(videoPath, normInfo, normDir, prompts, vttCli, sttCli, con
903
898
  } else { clipPath = clipOut; }
904
899
 
905
900
  // Use runVTT for video clips (extracts frames, then runs ITT)
906
- const segRawVis = runVTT(clipPath, visSegPrompt, vttCli, ittCli, sampleInterval);
901
+ const segRawVis = runVTT(clipPath, visSegPrompt, DEFAULT_VTT_CLI, DEFAULT_ITT_CLI, sampleInterval);
907
902
  const segVisText = extractRawText(segRawVis);
908
903
  seg.vision = stripThinkBlocks(looseJSONParse(segRawVis)?.desc || segVisText.slice(0, 300) || segRawVis.slice(0, 300));
909
904
  try { if (clipPath !== normInfo.path) rmSync(clipPath, { force: true }); } catch {}
@@ -917,7 +912,7 @@ function analyzeVideo(videoPath, normInfo, normDir, prompts, vttCli, sttCli, con
917
912
 
918
913
  // ── Full pipeline runner (step 2: label → normalize → percept → segments) ──
919
914
 
920
- async function runFullPipeline(folder, prompts, ittCli, vttCli, sttCli, context, pickSet, vttSampleInterval, skipSTT, dryRun, agentCli = null) {
915
+ async function runFullPipeline(folder, prompts, context, pickSet, vttSampleInterval, skipSTT, dryRun) {
921
916
  const metadataPath = join(folder, "metadata.json");
922
917
 
923
918
  // ── Ensure metadata exists ──────────────────────────────────────────
@@ -987,12 +982,12 @@ async function runFullPipeline(folder, prompts, ittCli, vttCli, sttCli, context,
987
982
  return;
988
983
  }
989
984
 
990
- await runNormalizeAndPercept(folder, metadataPath, prompts, ittCli, vttCli, sttCli, context, pickSet, vttSampleInterval, skipSTT, agentCli);
985
+ await runNormalizeAndPercept(folder, metadataPath, prompts, context, pickSet, vttSampleInterval, skipSTT);
991
986
  }
992
987
 
993
988
  // ── Normalize + Percept (internal step) ───────────────────────────────────
994
989
 
995
- async function runNormalizeAndPercept(folder, metadataPath, prompts, ittCli, vttCli, sttCli, context, pickSet, vttSampleInterval, skipSTT, agentCli = null) {
990
+ async function runNormalizeAndPercept(folder, metadataPath, prompts, context, pickSet, vttSampleInterval, skipSTT) {
996
991
  const { results, cacheMap } = loadMetadata(folder);
997
992
  const cache = { ...cacheMap };
998
993
  const normDir = join(folder, ".normalized");
@@ -1022,14 +1017,14 @@ async function runNormalizeAndPercept(folder, metadataPath, prompts, ittCli, vtt
1022
1017
  let cacheKey = "";
1023
1018
  const ctx = context ? `Context: ${context}\n\n` : "";
1024
1019
  const imgPrompt = (userHint ? `User hint: ${userHint}\n\n` : "") + ctx + getPrompt(prompts, "image-perception");
1025
- const imgInput = `${ittCli ? "" : "@"}${shQuote(normPath)}`;
1026
- const imgCmd = substituteTemplate(ittCli || DEFAULT_ITT_CLI, { input: imgInput, prompt: imgPrompt });
1020
+ const imgInput = `@${shQuote(normPath)}`;
1021
+ const imgCmd = substituteTemplate(DEFAULT_ITT_CLI, { input: imgInput, prompt: imgPrompt });
1027
1022
  cacheKey = perceptionCacheKey(imgPath, imgCmd, "image");
1028
1023
  if (cache[cacheKey]) {
1029
1024
  perception = cache[cacheKey];
1030
1025
  emitInfo(` (cached)`);
1031
1026
  } else {
1032
- perception = analyzeImage(imgPath, normPath, prompts, ittCli, context, userHint, userHints);
1027
+ perception = analyzeImage(imgPath, normPath, prompts, context, userHint, userHints);
1033
1028
  if (perception.desc) cache[cacheKey] = perception;
1034
1029
  emitInfo(` ${perception.desc?.slice(0, 80)}...`);
1035
1030
  }
@@ -1057,27 +1052,25 @@ async function runNormalizeAndPercept(folder, metadataPath, prompts, ittCli, vtt
1057
1052
  emitInfo(`\n🎬 Video: ${base}${userHint ? ` (hint: "${userHint}")` : ""}`);
1058
1053
  const vidPath = join(folder, fileName);
1059
1054
  const normInfo = normalizeVideo(vidPath, normDir, meta.duration);
1060
- const stt = skipSTT ? null : sttCli;
1061
-
1062
1055
  let perception = {};
1063
1056
  let cacheKey = "";
1064
1057
  const ctxV = context ? `Context: ${context}\n\n` : "";
1065
1058
  const vidPrompt = (userHint ? `User hint: ${userHint}\n\n` : "") + ctxV + getPrompt(prompts, "video-perception");
1066
1059
  let vttActualCmd;
1067
- if (vttCli) {
1068
- vttActualCmd = substituteTemplate(vttCli, { input: shQuote(normInfo.path), prompt: vidPrompt });
1060
+ if (DEFAULT_VTT_CLI) {
1061
+ vttActualCmd = substituteTemplate(DEFAULT_VTT_CLI, { input: shQuote(normInfo.path), prompt: vidPrompt });
1069
1062
  } else {
1070
1063
  const dur = meta.duration || 0;
1071
1064
  const n = Math.max(5, Math.min(10, Math.ceil(dur / vttSampleInterval)));
1072
1065
  const framePaths = Array.from({ length: n }, (_, i) => `${basename(normInfo.path, extname(normInfo.path))}_${String(i + 1).padStart(3, "0")}.jpg`);
1073
- vttActualCmd = substituteTemplate(ittCli || DEFAULT_ITT_CLI, { input: framePaths.map(p => `@${p}`).join(" "), prompt: vidPrompt });
1066
+ vttActualCmd = substituteTemplate(DEFAULT_ITT_CLI, { input: framePaths.map(p => `@${p}`).join(" "), prompt: vidPrompt });
1074
1067
  }
1075
1068
  cacheKey = perceptionCacheKey(vidPath, vttActualCmd, "video");
1076
1069
  if (cache[cacheKey]) {
1077
1070
  perception = cache[cacheKey];
1078
1071
  emitInfo(` (cached)`);
1079
1072
  } else {
1080
- perception = analyzeVideo(vidPath, normInfo, normDir, prompts, vttCli, stt, context, userHint, ittCli, vttSampleInterval, agentCli, userHints);
1073
+ perception = analyzeVideo(vidPath, normInfo, normDir, prompts, context, userHint, vttSampleInterval, userHints);
1081
1074
  if (perception.desc) cache[cacheKey] = perception;
1082
1075
  }
1083
1076
 
@@ -1105,10 +1098,6 @@ Usage:
1105
1098
 
1106
1099
  Options:
1107
1100
  --label Open label preview → then continue with full AI pipeline
1108
- --agent <template> Custom text LLM CLI for detect-scenes ({prompt})
1109
- --itt <template> Custom ITT CLI template with {input}, {prompt}
1110
- --vtt <template> Custom VTT CLI template with {input}, {prompt}
1111
- --stt <template> Custom STT CLI template with {input}, {output}
1112
1101
  --prompts-file <path> Path to prompts markdown file (default: vision_prompts.md)
1113
1102
  --vtt-sample-interval <n> Sample one video frame every N seconds (default: 5)
1114
1103
  --context "text" Background context about people/places (injected into prompts)
@@ -1116,7 +1105,6 @@ Options:
1116
1105
  --skip-stt Skip speech-to-text for videos
1117
1106
  --dry-run Show what would be processed without running AI
1118
1107
  --show-prompts Print the prompts file and exit
1119
- --show-clis Print the default ITT/VTT/STT CLI templates
1120
1108
  --help Show this help
1121
1109
 
1122
1110
  Prompt overrides:
@@ -1126,10 +1114,6 @@ Prompt overrides:
1126
1114
 
1127
1115
  export async function main(args) {
1128
1116
  let folder = "";
1129
- let agentCli = null;
1130
- let ittCli = null;
1131
- let vttCli = null;
1132
- let sttCli = null;
1133
1117
  let promptsFile = PROMPTS_FILE;
1134
1118
  let context = "";
1135
1119
  let vttSampleInterval = DEFAULT_VTT_SAMPLE_INTERVAL;
@@ -1147,20 +1131,12 @@ export async function main(args) {
1147
1131
  const flag = args[i++];
1148
1132
  if (flag === "--help") { printUsage(); return; }
1149
1133
  else if (flag === "--label") { doLabel = true; }
1150
- else if (flag === "--agent" && args[i]) { agentCli = args[i++]; }
1151
- else if (flag === "--itt" && args[i]) { ittCli = args[i++]; }
1152
- else if (flag === "--vtt" && args[i]) { vttCli = args[i++]; }
1153
- else if (flag === "--stt" && args[i]) { sttCli = args[i++]; }
1134
+
1154
1135
  else if (flag === "--prompts-file" && args[i]) { promptsFile = resolve(args[i++]); }
1155
1136
  else if (flag === "--vtt-sample-interval" && args[i]) { vttSampleInterval = parseInt(args[i++], 10) || DEFAULT_VTT_SAMPLE_INTERVAL; }
1156
1137
  else if (flag === "--context" && args[i]) { context = args[i++]; }
1157
1138
  else if (flag === "--show-prompts") { console.log(readFileSync(promptsFile, "utf-8")); return; }
1158
- else if (flag === "--show-clis") {
1159
- console.log(`DEFAULT_ITT_CLI:\n${DEFAULT_ITT_CLI}\n`);
1160
- console.log(`DEFAULT_VTT_CLI: (empty — uses ITT via frame extraction, sample every ${DEFAULT_VTT_SAMPLE_INTERVAL}s)\n`);
1161
- console.log(`DEFAULT_STT_CLI:\n${DEFAULT_STT_CLI}`);
1162
- return;
1163
- }
1139
+
1164
1140
  else if (flag === "--skip-stt") { skipSTT = true; }
1165
1141
  else if (flag === "--pick" && args[i]) { for (const f of args[i++].split(",")) pickSet.add(f.trim()); }
1166
1142
  else if (flag === "--dry-run") { dryRun = true; }
@@ -1175,7 +1151,7 @@ export async function main(args) {
1175
1151
  for (const [name, value] of promptOverrides) prompts.set(name, value);
1176
1152
 
1177
1153
  if (doLabel) {
1178
- await runFullPipeline(folder, prompts, ittCli, vttCli, sttCli, context, pickSet, vttSampleInterval, skipSTT, dryRun, agentCli);
1154
+ await runFullPipeline(folder, prompts, context, pickSet, vttSampleInterval, skipSTT, dryRun);
1179
1155
  } else {
1180
1156
  await runMetadataMode(folder, pickSet, dryRun);
1181
1157
  }
@@ -1,220 +0,0 @@
1
- # Watch Mode
2
-
3
- > `preview <file.md> --edit` — live editing loop with AI-assisted JSON editing.
4
-
5
- Edit mode gives you a browser player that **automatically reloads** when the source file changes, plus an **edit input** that calls an AI coding agent (pi) to make changes on your behalf.
6
-
7
- Generated artifacts (TTS audio, compiled includes, component bundles) are
8
- stored under `.markcut/` next to the source file — see [SKILL.md](../SKILL.md) for the full layout.
9
-
10
- ## Quickstart
11
-
12
- ```bash
13
- cd remotion-engine
14
- npm run preview -- sample-visual.json --edit
15
- # → Browser opens, terminal blocks until you close it
16
- ```
17
-
18
- ## Architecture
19
-
20
- ```
21
- ┌─────────────┐ HTTP ┌──────────────────┐ spawn ┌──────────┐
22
- │ Browser │◄──────────────►│ player/server │◄────────────►│ pi CLI │
23
- │ (Chrome) │ SSE │ .mjs (Node) │ one-shot │ (agent) │
24
- └──────┬──────┘ └────────┬─────────┘ └──────────┘
25
- │ │
26
- │ POST /api/edit │ fs.watchFile
27
- │ {text: "make text bigger"} │ (poll every 500ms)
28
- │ │
29
- │ GET /api/video-data ▼
30
- │ ← stream tree JSON ┌──────────────┐
31
- │ │ stream tree │
32
- │ EventSource │ .json (file) │
33
- │ /api/events └──────────────┘
34
- │ ← {type:"reload"} ▲
35
- │ │
36
- │ window.dispatchEvent │ pi edits
37
- │ ("refresh-player") │ this file
38
- │ │
39
- ```
40
-
41
- ## File: `src/player/server.mjs`
42
-
43
- ### Server startup (`cli.mjs` → `server.mjs`)
44
-
45
- ```
46
- node src/render/cli.mjs preview file.json --edit --port 3031
47
- ```
48
-
49
- 1. `cli.mjs` spawns `server.mjs` with `--edit` flag
50
- 2. `server.mjs` parses `file.json` to build a recursive tree description
51
- 3. Starts `fs.watchFile(file.json, {interval:500})` — polls every 500ms for changes
52
- 4. Starts HTTP server on specified port
53
- 5. Auto-opens browser via `open http://localhost:PORT`
54
-
55
- ### Page served
56
-
57
- The HTML page loads two scripts:
58
-
59
- | Script | Source | Role |
60
- |--------|--------|------|
61
- | `player.js` | Bundled esbuild output from `src/player/browser.tsx` | Renders the stream tree using `@remotion/player` with `MarkCut` |
62
- | Inline script | Generated by `getHtml()` in `server.mjs` | Watch-mode UI controls (close button, edit input, SSE) |
63
-
64
- ### Edit flow
65
-
66
- ```
67
- Browser Server pi CLI
68
- │ │ │
69
- │── POST /api/edit ────────────► │ │
70
- │ {text: "make text bigger"} │ │
71
- │ │── spawn("pi", ["-p", prompt])│
72
- │ │ prompt includes: │
73
- │ │ • full tree description │
74
- │ │ (recursive, any depth) │
75
- │ │ • edit history │
76
- │ │ • knowledge (all stream │
77
- │ │ types, components, │
78
- │ │ schema, styling guide) │
79
- │ │ • edit request │
80
- │ │ │
81
- │ │ pi can edit ANY field on │
82
- │ │ ANY node in the tree │
83
- │ │ ├── edits file.json
84
- │ │ └── exits
85
- │ │◄── exit code 0 │
86
- │◄── {done:true, output:"..."} ──┤ │
87
- │ │ │
88
- │ suppressReload = true │ │
89
- │ header shows "Changed text..." │ │
90
- │ │ │
91
- │ (fs.watchFile fires SSE │ │
92
- │ but suppressReload=true │ │
93
- │ so it's ignored) │ │
94
- │ │ │
95
- │ after 4s: │ │
96
- │ dispatchEvent("refresh-player")│ │
97
- │ → player re-fetches /api/data │ │
98
- │ → re-renders with new JSON │ │
99
- ```
100
-
101
- ### Key design decisions
102
-
103
- **Why in-place refresh instead of `location.reload()`?**
104
- - Timeline playback continues uninterrupted
105
- - Status text persists until next edit
106
- - No page flash
107
-
108
- **Why `suppressReload`?**
109
- - SSE fires ~500ms after pi edits the file
110
- - API response arrives at the same time
111
- - `suppressReload` flag ignores the SSE event during edit
112
- - The client reloads itself after showing the summary
113
-
114
- ## File: `src/player/browser.tsx`
115
-
116
- The bundled player app:
117
-
118
- 1. Fetches `/api/video-data` on mount
119
- 2. Renders the stream tree using `@remotion/player` with `MarkCut`
120
- 3. Listens for `"refresh-player"` custom DOM event
121
- 4. On refresh event: re-fetches data, re-renders in-place
122
-
123
- ## File: `src/render/cli.mjs`
124
-
125
- The CLI routes `--edit` flag to the player server:
126
-
127
- ```javascript
128
- if (args.edit) {
129
- const serverArgs = ["src/player/server.mjs", file, "--edit", `--port=${port}`];
130
- const child = spawn("node", serverArgs, { cwd: ROOT, stdio: "inherit" });
131
- // ...blocks until server exits...
132
- }
133
- ```
134
-
135
- ## API Endpoints
136
-
137
- | Endpoint | Method | Description |
138
- |----------|--------|-------------|
139
- | `/api/video-data` | GET | Returns the raw stream tree JSON file |
140
- | `/api/scenes` | GET | Returns parsed scenes array with timing |
141
- | `/api/video-info` | GET | Returns scenes + totalDuration + mode flags |
142
- | `/api/edit` | POST | Accepts `{text}` → spawns pi to edit the JSON → file watcher triggers → SSE reload |
143
- | `/api/shutdown` | POST | Kills the server process, unblocking the agent |
144
- | `/api/events` | GET | SSE stream — sends `{type:"reload"}` when file changes |
145
- | `/api/labels` | GET/POST | (label mode only) Save/load scene labels |
146
-
147
- ## The pi prompt
148
-
149
- Each edit call sends a structured prompt with a **recursive tree description** instead of a flat scene list:
150
-
151
- ```
152
- You are editing sample-visual.json, a Remotion stream tree JSON.
153
-
154
- The stream tree (indentation shows nesting; timing in seconds):
155
- root "root" (1080×1920, 30fps, series, transition:fade)
156
- folder "scene-1" (parallel)
157
- component GradientBackground {type:"radial",animated:true,noise:true} [05s, bg]
158
- component AnimatedHeadline {"text":"Hello Remotion",...} [0.54.5s]
159
- folder "scene-2" (parallel)
160
- component GradientBackground {type:"linear",animated:true} [05s, bg]
161
- subtitle "This text is styled with CSS" fontSize:52 [0.54.5s]
162
- ...
163
-
164
- Previous edits:
165
- 1. make scene 1 headline text orange
166
-
167
- Edit request: make scene 1 headline text bigger
168
-
169
- --- Knowledge ---
170
- You can edit ANY field on ANY node in the JSON...
171
- Stream types: root, folder, video, audio, image, subtitle, component, effect, rhythm, map
172
- ...
173
- IMPORTANT: Read the full existing JSON file before editing. Only edit the JSON file. You can change, add, or remove any field on any node. Output ONLY a one-line summary of what specific change you made.
174
- ```
175
-
176
- The tree description walks the JSON recursively, showing:
177
- - **Nesting depth** via indentation
178
- - **Type** and **id/name** of each node
179
- - **Key fields** (dimensions, fps, theme, series/parallel, transitions)
180
- - **Timing** from leaf span (`[start→end]`)
181
- - **isBackground** flag (`[bg]`)
182
- - **Component props** and node-specific fields (fontSize, fit, style, volume, etc.)
183
-
184
- This means pi can edit **any aspect of the JSON** ÔÇö not just scene props. Nested folders, effects, rhythms, maps, actions, styles, themes, global stylesheets, audio config ÔÇö everything is visible and editable.
185
-
186
- ## Session context
187
-
188
- The server maintains `editHistory[]` — every edit is appended. On each call, previous edits are included so pi knows what was done before:
189
-
190
- ```
191
- Previous edits:
192
- 1. make scene 1 headline text orange
193
- 2. change background from radial to linear gradient
194
- 3. add 3D angle to device mockup in scene 4
195
- ```
196
-
197
- This gives pi enough context to make coherent changes across multiple edit rounds.
198
-
199
- ## Close / tab-close
200
-
201
- | Action | What happens |
202
- |--------|-------------|
203
- | Click **✕** | `sendBeacon("/api/shutdown")` → server exits → agent unblocks |
204
- | Close **tab** | SSE connection drops → 3s grace → server exits |
205
- | Close **browser** | Same as tab close |
206
-
207
- ## Label mode (`--label`)
208
-
209
- `preview <file.json> --label` starts the same player but with a **label input** instead of the edit input. Labels are saved to `labels.json` next to the source file. No AI editing, no file watching.
210
-
211
- ## Build
212
-
213
- The browser bundle is built with esbuild:
214
-
215
- ```bash
216
- bash scripts/build-player.sh
217
- # → src/player/bundle/player.js
218
- ```
219
-
220
- The bundle includes React, @remotion/player, MarkCut, all 13 built-in components, themes, templates, and the duration utility. It is served as a static file by the player server.
@@ -1,112 +0,0 @@
1
- # Label System
2
-
3
- Interactive media labeling before understanding clips in json stream tree.
4
- The stream tree follow rules, see [docs/json-descriptive.md](docs/json-descriptive.md) for full details .
5
-
6
- ## Overview
7
-
8
- The label player lets you load a stream tree, browse its clips, and attach descriptive labels to each clip's media.
9
- The output is an updated `labels.json` file with `description` fields populated.
10
-
11
- ## Usage
12
-
13
- ```bash
14
- # Label from a stream tree JSON or labels.json
15
- npx markcut preview labels.json --label
16
-
17
- # Specify port
18
- npx markcut preview labels.json --label --port 3031
19
- ```
20
-
21
- ### Labels Persistence
22
- The stream tree follow rules, see [docs/json-descriptive.md](docs/json-descriptive.md) for full details .
23
- Labels are saved to the source file.
24
-
25
- ```json
26
- {
27
- "root": {
28
- "id": "root",
29
- "type": "root",
30
- "children": [
31
- {
32
- "id": "scene-1",
33
- "type": "folder",
34
- "children": [
35
- {
36
- "id": "scene-1-media",
37
- "type": "image",
38
- "src": "/media/photo.jpg",
39
- "start": 0, "end": 5,
40
- "description": "Best group shot from the campfire"
41
- }
42
- ]
43
- },
44
- {
45
- "id": "scene-2",
46
- "type": "folder",
47
- "children": [
48
- {
49
- "id": "scene-2-media",
50
- "type": "video",
51
- "src": "/media/clip.mp4",
52
- "start": 0, "end": 5,
53
- "description": "Sunset timelapse — use for intro"
54
- }
55
- ]
56
- }
57
- ]
58
- }
59
- }
60
- ```
61
-
62
- The `description` field on each media leaf node holds the label text.
63
-
64
- ## Workflow
65
-
66
- ```
67
- Stream tree JSON (labels.json)
68
-
69
-
70
- ─── preview --label ───→ Browse & label clips
71
- │ │
72
- │ ▼
73
- │ Updated labels.json
74
- │ │
75
- ▼ ▼
76
- Use in Storyboard ──────→ Assemble → Render
77
- ```
78
-
79
- 1. **Prepare** a stream tree JSON with scenes
80
- 2. **Label** with `preview --label` to add descriptions to each scene
81
- 3. **Export** labels are saved alongside the source file
82
- 4. **Use** the labeled scenes in your storyboard or assembly workflow
83
-
84
- ## API Endpoints
85
-
86
- | Endpoint | Method | Description |
87
- |----------|--------|-------------|
88
- | `/api/video-data` | GET | Returns the raw stream tree JSON |
89
- | `/api/video-info` | GET | Returns scenes array with timing + totalDuration |
90
- | `/api/labels` | GET | Returns the current stream tree with descriptions |
91
- | `/api/labels` | POST | Saves updated descriptions ({descriptions: {index: "text"}}) |
92
- | `/api/shutdown` | POST | Kills the server |
93
- | `/api/events` | GET | SSE events (reload on file change) |
94
-
95
- ## Implementation
96
-
97
- Source: `src/player/server.mjs` (unified server, use `--label` flag)
98
- UI controls: `src/player/components/LabelControls.tsx` (React component)
99
-
100
- - Built on Node.js `http` module (no Express dependency)
101
- - Serves bundled `player.js` (esbuild output from `src/player/browser.tsx`)
102
- - Labels saved atomically to disk on each POST
103
- - Supports stream tree JSON, descriptive JSON, and markdown input
104
- - React components in `src/player/components/` replace the old HTML-based `ui/` directory
105
-
106
- ## Reference
107
-
108
- | Topic | Link |
109
- |-------|------|
110
- | Storyboard (video planning) | [storyboard.md](storyboard.md) |
111
- | Stream tree types | `SKILL.md` |
112
- | Dynamic components | [dynamic-components.md](dynamic-components.md) |