@lalalic/markcut 2.5.1 → 2.6.0

package/README.md

@@ -215,6 +215,95 @@ src/
 ```bash
 npm run render      # render a JSON stream tree to MP4
 npm run preview     # open Remotion Studio
+npm run vision      # analyze images/videos in a folder (metadata + AI perception)
 npm run typecheck   # TypeScript type check
 npm test            # run unit + integration tests
 ```
+
+## Vision Pipeline
+
+```bash
+# Full pipeline: extract metadata → normalize → AI perception (images & videos)
+npx markcut vision <folder>
+
+# With interactive labeling step before AI
+npx markcut vision <folder> --label
+
+# Provide context about people/places
+npx markcut vision <folder> --instruct "the girl is Maggie, the man in red is me"
+```
+
+```mermaid
+flowchart LR
+    subgraph Input[Input]
+        A[Media Folder<br/>images + videos]
+    end
+
+    subgraph Metadata["1. Metadata Extraction"]
+        B[exiftool / ffprobe]
+        C[metadata.json<br/>width, height, created,<br/>GPS location, duration]
+    end
+
+    subgraph Normalize["2. Normalization"]
+        D[ffmpeg resize<br/>images → 384px<br/>videos → 360p]
+    end
+
+    subgraph Label["Optional: Labeling"]
+        E[Build preview tree<br/>with seed]
+        F[Label preview server<br/>user annotates scenes]
+        G[Merge labels → userHints<br/>into metadata.json]
+    end
+
+    subgraph Image["3. Image Perception"]
+        H[ITT: image-to-text<br/>mlx-vlm model]
+        I[Perception saved:<br/>description]
+    end
+
+    subgraph Video["4. Video Understanding"]
+        J[VTT: video-to-text<br/>mlx-vlm model]
+        K[STT: speech-to-text<br/>whisper]
+        L[VTT→ overall desc]
+        M[STT → VTT subtitles]
+        N[Build merged segment<br/>boundaries]
+        O[LLM merges boundaries<br/>into final segments]
+        P[Per-segment vision<br/>clip extraction + ITT]
+    end
+
+    subgraph Output[Output]
+        Q[metadata.json<br/>with perception desc,<br/>subtitles & segments]
+    end
+
+    A --> B --> C --> D
+    D --> Image
+    D --> Video
+    C -.->|--label| E --> F --> G --> D
+    Image --> H --> I --> Q
+    Video --> J --> L
+    Video --> K --> M
+    L --> N
+    M --> N
+    N --> O --> P --> Q
+
+    style Input fill:#1a1a2e,stroke:#16213e,color:#fff
+    style Metadata fill:#0f3460,stroke:#16213e,color:#fff
+    style Normalize fill:#533483,stroke:#16213e,color:#fff
+    style Label fill:#e94560,stroke:#16213e,color:#fff
+    style Image fill:#0f3460,stroke:#16213e,color:#fff
+    style Video fill:#0f3460,stroke:#16213e,color:#fff
+    style Output fill:#1a1a2e,stroke:#16213e,color:#fff
+```
+
+### Vision CLI Options
+
+| Option | Description |
+|---|---|
+| `--label` | Add interactive labeling step before AI pipeline (opens browser preview) |
+| `--instruct "text"` | Background context about people/places (injected into AI prompts) |
+| `--prompts-file <path>` | Custom prompts markdown file (default: `src/vision/vision_prompts.md`) |
+| `--vtt-sample-interval <n>` | Sample one video frame every N seconds for VTT (default: 5) |
+| `--pick <files>` | Comma-separated filenames to process (skip others) |
+| `--skip-stt` | Skip speech-to-text for videos |
+| `--dry-run` | Show what would be processed without running AI |
+| `--show-prompts` | Print the prompts template file and exit |
+
+## Architecture

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lalalic/markcut",
-  "version": "2.5.1",
+  "version": "2.6.0",
   "description": "Markdown-to-video engine. Describe scenes in markdown, get a rendered video.",
   "bin": {
     "markcut": "bin/markcut"

package/skills/markcut/SKILL.md

@@ -45,7 +45,7 @@ npx @lalalic/markcut --show-clis # get command specific information
 npx @lalalic/markcut preview <file> # assemble and preview the video with a local server, and chat to edit the video and auto refresh
 npx @lalalic/markcut render <file> # render the video to mp4
 npx @lalalic/markcut vision <folder> # vision understanding medias in folder
-npx @lalalic/markcut vision <folder> --label # full pipeline vision understanding medias in folder, and provide ui to label the medias with text, time ranges
+npx @lalalic/markcut vision <folder> --label # an extra step to provide UI to label the medias with text, time ranges
 
 ```
 

package/skills/markcut/docs/markdown-descriptive.md

@@ -16,7 +16,7 @@ This section defines the markcut Markdown Descriptive grammar. An agent should r
 │   ───...───
 ├─ # video                    ← Root node (REQUIRED, heading level 1)
 │   width:1080 height:1920 fps:30 layout:series
-│   subtitle:{src:"captions.vtt",type:"Typewriter"}
+│   subtitle:{type:"Typewriter"}
 │   ~~~css stylesheet           ← root.stylesheet
 │   .bg { background: #000; }
 │   ~~~
@@ -223,7 +223,7 @@ The line after `# video` holds canvas-level config as space-separated `key:value
 ```markdown
 # video
 width:1920 height:1080 fps:30 layout:series transition:fade transitionTime:1.2
-subtitle:{src:"captions.vtt",type:"Typewriter",fontSize:48}
+subtitle:{type:"Typewriter",fontSize:48}
 ```
 
 Supported root keys: `width`, `height`, `fps`, `layout`, `tts`, `stt`, `tti`, `ttv`, `transition`, `transitionTime`, `instruction`, `metadata`, `stylesheet`, `subtitle`, `voices`.

package/src/descriptive/resolve.ts

@@ -944,10 +944,10 @@ export async function resolveAll(
     try {
       const raw = readFileSync(options.sourcePath, "utf-8");
       if (options.sourcePath.endsWith(".md")) {
-        // Insert seed: <N> after the first line (# video) or after existing header
+        // Insert seed: <N> right after the `# video` line, before any config.
         const lines = raw.split("\n");
-        const headerEnd = lines.findIndex((l) => l.trim() === "" || l.startsWith("##"));
-        const insertAt = headerEnd > 1 ? headerEnd : Math.min(1, lines.length);
+        const videoIdx = lines.findIndex((l) => l.trim().startsWith("# video"));
+        const insertAt = videoIdx >= 0 ? videoIdx + 1 : 1;
         lines.splice(insertAt, 0, `seed:${autoSeed}`);
         writeFileSync(options.sourcePath, lines.join("\n"), "utf-8");
       } else if (options.sourcePath.endsWith(".json")) {

package/src/render/cli.mjs

@@ -72,11 +72,11 @@ Commands:
     --output <path>                     Output path (default: out/video.mp4)
     --verbose                           Show full per-frame progress (default: compact)
 
-  vision <folder>                      Extract metadata into metadata.json
-    --label                            Full pipeline: preview → label → normalize → percept → segments
+  vision <folder>                      Full pipeline: extract → normalize → percept → segments
+    --label                            Add interactive labeling step before AI pipeline
+    --instruct "text"                   Background context about people/places (injected into prompts)
     --prompts-file <path>              Path to prompts markdown file (default: vision_prompts.md)
     --vtt-sample-interval <n>          Sample one video frame every N seconds (default: 5)
-    --context "text"                   Background context about people/places (injected into prompts)
     --skip-stt                         Skip speech-to-text for videos
     --dry-run                          Show what would be processed without running AI
     --show-prompts                     Print the prompts file and exit

package/src/vision/cli.mjs

@@ -648,9 +648,12 @@ function buildPreviewTree(folder, metadata) {
   }
   entries.sort((a, b) => a.created.localeCompare(b.created));
 
+  // Random seed for TTI/TTV generation.
+  const seed = Math.floor(Math.random() * 0xFFFFFFFF);
+
   return {
     id: "root", type: "root", width: 1080, height: 1920, fps: 30,
-    isSeries: true, transition: "fade", transitionTime: 0.5,
+    seed, isSeries: true, transition: "fade", transitionTime: 0.5,
     children: entries.map((e) => ({
       id: e.name, type: "folder", isSeries: false,
       children: [{
@@ -1093,14 +1096,14 @@ function printUsage() {
 markcut vision — Analyze images and videos in a folder for video generation
 
 Usage:
-  markcut vision <folder>                Extract metadata into metadata.json
-  markcut vision <folder> --label        Full pipeline: preview → label → normalize → percept → segments
+  markcut vision <folder>                Full pipeline: extract → normalize → percept → segments
+  markcut vision <folder> --label        Same as default, plus interactive labeling step before AI
 
 Options:
-  --label              Open label preview → then continue with full AI pipeline
+  --label              Open interactive labeling preview before running AI pipeline
+  --instruct "text"    Background context about people/places (injected into prompts)
   --prompts-file <path> Path to prompts markdown file (default: vision_prompts.md)
   --vtt-sample-interval <n> Sample one video frame every N seconds (default: 5)
-  --context "text"     Background context about people/places (injected into prompts)
   --pick <files>       Comma-separated filenames to process
   --skip-stt           Skip speech-to-text for videos
   --dry-run            Show what would be processed without running AI
@@ -1134,9 +1137,8 @@ export async function main(args) {
 
     else if (flag === "--prompts-file" && args[i]) { promptsFile = resolve(args[i++]); }
     else if (flag === "--vtt-sample-interval" && args[i]) { vttSampleInterval = parseInt(args[i++], 10) || DEFAULT_VTT_SAMPLE_INTERVAL; }
-    else if (flag === "--context" && args[i]) { context = args[i++]; }
+    else if (flag === "--instruct" && args[i]) { context = args[i++]; }
     else if (flag === "--show-prompts") { console.log(readFileSync(promptsFile, "utf-8")); return; }
-
     else if (flag === "--skip-stt") { skipSTT = true; }
     else if (flag === "--pick" && args[i]) { for (const f of args[i++].split(",")) pickSet.add(f.trim()); }
     else if (flag === "--dry-run") { dryRun = true; }
@@ -1151,12 +1153,45 @@ export async function main(args) {
   for (const [name, value] of promptOverrides) prompts.set(name, value);
 
   if (doLabel) {
+    // Full pipeline with interactive labeling
     await runFullPipeline(folder, prompts, context, pickSet, vttSampleInterval, skipSTT, dryRun);
   } else {
-    await runMetadataMode(folder, pickSet, dryRun);
+    // Default: full pipeline without labeling (extract → normalize → percept → segments)
+    const metadataPath = join(folder, "metadata.json");
+    if (!existsSync(metadataPath)) {
+      emitInfo(`\n📋 Extracting metadata...`);
+      await runMetadataMode(folder, pickSet, false);
+    }
+    if (!existsSync(metadataPath) || Object.keys(JSON.parse(readFileSync(metadataPath, "utf-8"))).length === 0) {
+      emitError("No media files found — nothing to process.");
+      return;
+    }
+    emitInfo(`\n🔧 Running full pipeline (normalize → perceive)...`);
+    await runNormalizeAndPercept(folder, metadataPath, prompts, context, pickSet, vttSampleInterval, skipSTT);
   }
 }
 
+// ═════════════════════════════════════════════════════════════════════════
+// Exported for testing
+// ═════════════════════════════════════════════════════════════════════════
+
+export {
+  scanMedia,
+  buildPreviewTree,
+  mergeLabelsIntoMetadata,
+  buildMergedCues,
+  parseVTT,
+  loadPrompts,
+  substituteTemplate,
+  shQuote,
+  normalizePerception,
+  extractRawText,
+  stripThinkBlocks,
+  looseJSONParse,
+  formatTime,
+  timeToSeconds,
+};
+
 // Direct execution
 if (process.argv[1] && process.argv[1].includes("src/vision/cli.mjs")) {
   main(process.argv).catch((e) => { emitError(e.message); process.exit(1); });

package/tests/vision.test.ts

@@ -0,0 +1,207 @@
+/**
+ * Integration tests for the vision CLI (`markcut vision`).
+ *
+ * Tests the actual workflow end-to-end:
+ *   1. Metadata extraction (exiftool/ffprobe)
+ *   2. Media normalization (ffmpeg resize)
+ *   3. Image perception (ITT via mlx-vlm)
+ *   4. Video perception (VTT + STT + segment analysis)
+ */
+import { describe, expect, it, beforeAll, afterAll } from "vitest";
+import { existsSync, mkdirSync, writeFileSync, readFileSync, rmSync, readdirSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { execSync } from "node:child_process";
+
+const __dirname = resolve(fileURLToPath(import.meta.url), "..");
+const ROOT = resolve(__dirname, "..");
+
+// ── Test Fixture Setup ────────────────────────────────────────────────────
+
+const TEST_ROOT = resolve(ROOT, "tests", "tmp", "vision-" + Date.now());
+const IMAGE_DIR = join(TEST_ROOT, "images");
+const VIDEO_ROOT = TEST_ROOT + "-video";
+const VIDEO_DIR = join(VIDEO_ROOT, "videos");
+
+/** Create a small test PNG (solid red, 64x64) using ffmpeg. */
+function createTestImage(outDir: string): string {
+  mkdirSync(outDir, { recursive: true });
+  const outPath = join(outDir, "test-photo.png");
+  execSync(
+    `ffmpeg -y -f lavfi -i color=c=red:s=64x64:d=1 -frames:v 1 ${outPath}`,
+    { stdio: "pipe" },
+  );
+  return outPath;
+}
+
+/** Create a short test video (colored bars + sine tone, 3s) using ffmpeg. */
+function createTestVideo(outDir: string): string {
+  mkdirSync(outDir, { recursive: true });
+  const outPath = join(outDir, "test-clip.mp4");
+  execSync(
+    `ffmpeg -y -f lavfi -i testsrc=s=320x240:d=3:r=15 ` +
+    `-f lavfi -i sine=frequency=440:duration=3 ` +
+    `-c:v libx264 -preset ultrafast -crf 40 -c:a aac -shortest ${outPath}`,
+    { stdio: "pipe", timeout: 30_000 },
+  );
+  return outPath;
+}
+
+// ── Image Pipeline ───────────────────────────────────────────────────────
+
+describe("image pipeline", () => {
+  let imgPath: string;
+
+  beforeAll(() => {
+    imgPath = createTestImage(IMAGE_DIR);
+    expect(existsSync(imgPath)).toBe(true);
+  });
+
+  afterAll(() => {
+    rmSync(TEST_ROOT, { recursive: true, force: true });
+  });
+
+  it("1. extracts metadata from image", async () => {
+    const { main } = await import("../src/vision/cli.mjs");
+    await main(["node", "cli.mjs", "vision", IMAGE_DIR]);
+
+    const metaPath = join(IMAGE_DIR, "metadata.json");
+    expect(existsSync(metaPath)).toBe(true);
+
+    const metadata = JSON.parse(readFileSync(metaPath, "utf-8"));
+    expect(Object.keys(metadata).length).toBeGreaterThan(0);
+
+    const key = Object.keys(metadata)[0]!;
+    const entry = metadata[key];
+    expect(entry).toHaveProperty("width");
+    expect(entry).toHaveProperty("height");
+    expect(entry).toHaveProperty("created");
+    expect(entry.width).toBeGreaterThan(0);
+    expect(entry.height).toBeGreaterThan(0);
+  });
+
+  it("2. normalizes image", () => {
+    const normDir = join(IMAGE_DIR, ".normalized");
+    expect(existsSync(normDir)).toBe(true);
+
+    const normFiles = readdirSync(normDir).filter(f => f.endsWith(".jpg"));
+    expect(normFiles.length).toBeGreaterThan(0);
+
+    // Verify normalized file exists and has valid dimensions
+    const normPath = join(normDir, normFiles[0]!);
+    expect(existsSync(normPath)).toBe(true);
+    const probe = JSON.parse(
+      execSync(`ffprobe -v quiet -print_format json -show_streams ${normPath}`, { stdio: "pipe" }).toString(),
+    );
+    const stream = probe.streams?.[0];
+    expect(stream).toBeDefined();
+    expect(stream.width).toBeLessThanOrEqual(384);
+    expect(stream.height).toBeLessThanOrEqual(384);
+  });
+
+  it("3. runs image perception (ITT) and saves to metadata", () => {
+    const metaPath = join(IMAGE_DIR, "metadata.json");
+    const metadata = JSON.parse(readFileSync(metaPath, "utf-8"));
+
+    const key = Object.keys(metadata)[0]!;
+    const entry = metadata[key];
+
+    expect(entry).toHaveProperty("perception");
+    expect(entry.perception).toHaveProperty("desc");
+    expect(entry.perception.desc).toBeTruthy();
+    expect(typeof entry.perception.desc).toBe("string");
+    expect(entry.perception.desc.length).toBeGreaterThan(10);
+  });
+});
+
+// ── Video Pipeline ───────────────────────────────────────────────────────
+
+describe("video pipeline", () => {
+  let vidPath: string;
+
+  beforeAll(() => {
+    vidPath = createTestVideo(VIDEO_DIR);
+    expect(existsSync(vidPath)).toBe(true);
+  }, 30_000);
+
+  afterAll(() => {
+    rmSync(VIDEO_ROOT, { recursive: true, force: true });
+  });
+
+  it("1. extracts metadata from video", async () => {
+    const { main } = await import("../src/vision/cli.mjs");
+    await main(["node", "cli.mjs", "vision", VIDEO_DIR]);
+
+    const metaPath = join(VIDEO_DIR, "metadata.json");
+    expect(existsSync(metaPath)).toBe(true);
+
+    const metadata = JSON.parse(readFileSync(metaPath, "utf-8"));
+    expect(Object.keys(metadata).length).toBeGreaterThan(0);
+
+    const key = Object.keys(metadata)[0]!;
+    const entry = metadata[key];
+    expect(entry).toHaveProperty("width");
+    expect(entry).toHaveProperty("height");
+    expect(entry).toHaveProperty("created");
+    expect(entry).toHaveProperty("duration");
+    expect(entry.duration).toBeGreaterThan(0);
+  }, 60_000);
+
+  it("2. normalizes video", () => {
+    const normDir = join(VIDEO_DIR, ".normalized");
+    expect(existsSync(normDir)).toBe(true);
+
+    const normFiles = readdirSync(normDir).filter(f => f.endsWith(".mp4"));
+    expect(normFiles.length).toBeGreaterThan(0);
+
+    const normPath = join(normDir, normFiles[0]!);
+    expect(existsSync(normPath)).toBe(true);
+    const probe = JSON.parse(
+      execSync(`ffprobe -v quiet -print_format json -show_streams ${normPath}`, { stdio: "pipe" }).toString(),
+    );
+    const stream = probe.streams?.find((s: any) => s.codec_type === "video");
+    expect(stream).toBeDefined();
+    expect(stream.width).toBeLessThanOrEqual(360);
+    expect(stream.height).toBeLessThanOrEqual(360);
+  });
+
+  it("3. runs video perception (VTT) and saves description", () => {
+    const metaPath = join(VIDEO_DIR, "metadata.json");
+    const metadata = JSON.parse(readFileSync(metaPath, "utf-8"));
+
+    const key = Object.keys(metadata)[0]!;
+    const entry = metadata[key];
+
+    expect(entry).toHaveProperty("perception");
+    expect(entry.perception).toHaveProperty("desc");
+    expect(entry.perception.desc).toBeTruthy();
+    expect(typeof entry.perception.desc).toBe("string");
+    expect(entry.perception.desc.length).toBeGreaterThan(10);
+  });
+
+  it("4. runs STT and produces subtitle file", () => {
+    const metaPath = join(VIDEO_DIR, "metadata.json");
+    const metadata = JSON.parse(readFileSync(metaPath, "utf-8"));
+
+    const key = Object.keys(metadata)[0]!;
+    const entry = metadata[key];
+
+    // STT may be null if no speech detected (test video has sine tone, no speech)
+    // But the subtitle field should exist in the perception object
+    expect(entry.perception).toHaveProperty("subtitle");
+  });
+
+  it("5. produces segment analysis", () => {
+    const metaPath = join(VIDEO_DIR, "metadata.json");
+    const metadata = JSON.parse(readFileSync(metaPath, "utf-8"));
+
+    const key = Object.keys(metadata)[0]!;
+    const entry = metadata[key];
+
+    // Video segments - may be single segment if no boundaries detected
+    expect(entry.perception).toHaveProperty("segments");
+    expect(typeof entry.perception.segments).toBe("object");
+    // Should have at least one segment (even if it's the full video)
+    expect(Object.keys(entry.perception.segments).length).toBeGreaterThan(0);
+  });
+});