@mevdragon/vidfarm-devcli 0.3.5 → 0.3.6

package/SKILL.director.md

@@ -306,11 +306,23 @@ For iterating on a composition on disk (e.g. Claude Code editing HTML directly),
 
 ```bash
 npx -y @mevdragon/vidfarm-devcli <template_id>
-# → downloads composition.html/json + manifest.json into .vidfarm/<template_id>/
+# → downloads composition.html/json + manifest.json + video-context.json into .vidfarm/<template_id>/
+# → downloads the videos + per-scene frames for local analysis (see below)
 # → starts dev-serve on http://localhost:4321
 # → prints https://vidfarm.cc/editor/<template_id>?fork=<forkId>&dev=http%3A%2F%2Flocalhost%3A4321
 ```
 
+### Local media for analysis (`sources.json`, videos, frames)
+
+So a coding agent can *see* what it is editing — not just read URLs — `vidfarm edit` also drops the actual media next to the composition:
+
+- **`original.mp4`** — the raw source video, before any processing.
+- **`source.mp4`** — the video the composition timeline actually renders from. This differs from `original.mp4` only when the fork was ghostcut/subtitle-removed; otherwise `sources.json` marks them `same_as_decomposed: true` and only `source.mp4` is written.
+- **`frames/<scene>.png`** — one still per timeline scene, sampled at the scene midpoint from `source.mp4`. Open these as images to analyze what each scene shows. Requires `ffmpeg` (bundled via `ffmpeg-static`); if unavailable the videos still download and frames are skipped.
+- **`sources.json`** — the manifest tying it together: each video's role/url/local path/byte size, the total duration, and per-scene `{ slug, start, duration, label, description, transcript_excerpt, frame }`.
+
+This is best-effort and never blocks the editor session. Re-run with `--refetch` to refresh the media after a new decompose or ghostcut. For deeper text grounding (full transcript, per-scene visual descriptions) read `video-context.json` — the media files complement it with the pixels.
+
 Open the printed URL in your browser. An orange **LOCAL DEV MODE** banner pins to the top with an Exit button. All fork API fetches now route to your local dir instead of the deployed API; disk edits push an SSE reload event and the editor re-fetches automatically.
 
 Flags: `--port`, `--dir`, `--host`, `--fork`, `--api-key` (or set `VIDFARM_API_KEY`), `--share`, `--refetch`. For a fork that already has composition files on disk (or that you've populated manually), use `vidfarm-devcli dev-serve --dir ./my-fork --port 4321` to skip the fetch step.

package/dist/src/cli.js

@@ -1,7 +1,10 @@
 #!/usr/bin/env node
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { createWriteStream, existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
 import path from "node:path";
 import { parseArgs } from "node:util";
+import { spawnSync } from "node:child_process";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
 // vidfarm-devcli — thin CLI that pairs a local composition directory with the
 // hosted Trackpad Editor at https://vidfarm.cc/editor/. Point it at a template
 // id, and it: resolves the user's fork, downloads the current composition,
@@ -107,6 +110,17 @@ async function runEditCommand(argv) {
         shareToken,
         refetch: parsed.values.refetch
     });
+    // Give local coding agents (Claude Code / Codex / opencode) the actual media
+    // to look at: the original source video, the composition's decomposed source
+    // (which may differ after ghostcut subtitle-removal), one still frame per
+    // scene, and a sources.json manifest tying it all together. Best-effort —
+    // never blocks the editor session if a download or frame extraction fails.
+    try {
+        await fetchVideoAssets({ dir: rootDir, apiKey, shareToken, refetch: parsed.values.refetch });
+    }
+    catch (error) {
+        console.warn(`[vidfarm] video assets skipped: ${error instanceof Error ? error.message : String(error)}`);
+    }
     const devSource = `http://localhost:${port}`;
     const editorUrl = `${host}/editor/${encodeURIComponent(templateId)}?fork=${encodeURIComponent(forkId)}&dev=${encodeURIComponent(devSource)}`;
     printEditorBanner({ templateId, forkId, dir: rootDir, port, editorUrl });
@@ -302,6 +316,232 @@ async function fetchCompositionFiles(input) {
         console.log(`[vidfarm] fetched ${filename} (${body.length} bytes)`);
     }
 }
+async function fetchVideoAssets(input) {
+    const refs = readSourceRefs(input.dir);
+    if (!refs.compositionSourceUrl && !refs.originalSourceUrl) {
+        console.log("[vidfarm] no source video URL found in composition — skipping video assets");
+        return;
+    }
+    // "decomposed video in composition" = the source the composition currently
+    // renders from (post-ghostcut if applicable). "original" = the raw source the
+    // decompose ran against, before any processing. They coincide when the fork
+    // was never ghostcut/subtitle-removed.
+    const compositionUrl = refs.compositionSourceUrl ?? refs.originalSourceUrl;
+    const originalUrl = refs.originalSourceUrl ?? refs.compositionSourceUrl;
+    const sameSource = compositionUrl === originalUrl;
+    const headers = buildAuthHeaders(input);
+    const decomposedName = "source.mp4";
+    const originalName = "original.mp4";
+    const decomposed = await downloadVideo({
+        url: compositionUrl,
+        target: path.join(input.dir, decomposedName),
+        headers,
+        refetch: input.refetch,
+        label: "decomposed source"
+    });
+    let original = decomposed && sameSource ? { ...decomposed, path: decomposedName } : null;
+    if (!sameSource) {
+        original = await downloadVideo({
+            url: originalUrl,
+            target: path.join(input.dir, originalName),
+            headers,
+            refetch: input.refetch,
+            label: "original source"
+        });
+    }
+    // One representative still per scene, taken from the decomposed source (what
+    // the composition actually renders). Falls back to the original if that's the
+    // only video that came down.
+    const frameBase = decomposed ? decomposedName : original ? originalName : null;
+    const frames = frameBase
+        ? await extractSceneFrames({
+            dir: input.dir,
+            videoName: frameBase,
+            scenes: refs.scenes,
+            durationSeconds: refs.durationSeconds,
+            refetch: input.refetch
+        })
+        : new Map();
+    const manifest = {
+        generated_by: "vidfarm-devcli",
+        note: "Local media references for coding agents. `original.mp4` is the raw source video; " +
+            "`source.mp4` is the video the composition timeline renders from (may differ from the " +
+            "original after ghostcut subtitle-removal). Scene entries map timeline segments to a still " +
+            "frame you can open as an image. Run `--refetch` to refresh.",
+        videos: {
+            decomposed: {
+                role: "video the composition timeline renders from (decomposed source)",
+                url: compositionUrl,
+                path: decomposed ? decomposedName : null,
+                bytes: decomposed?.bytes ?? null
+            },
+            original: {
+                role: "raw original source video (before any processing)",
+                url: originalUrl,
+                path: original ? original.path : null,
+                bytes: original?.bytes ?? null,
+                same_as_decomposed: sameSource
+            }
+        },
+        duration_seconds: refs.durationSeconds,
+        scenes: refs.scenes.map((scene, index) => ({
+            slug: scene.slug,
+            start: scene.start,
+            duration: scene.duration,
+            label: scene.label,
+            description: scene.description,
+            transcript_excerpt: scene.transcript_excerpt,
+            frame: frames.get(index) ?? null,
+            frame_source: frameBase
+        }))
+    };
+    writeFileSync(path.join(input.dir, "sources.json"), JSON.stringify(manifest, null, 2));
+    console.log(`[vidfarm] wrote sources.json (${refs.scenes.length} scenes, ${frames.size} frames)`);
+}
+// Derive the source URLs and scene list from the files already on disk
+// (composition.html + video-context.json) — no extra network round-trips.
+function readSourceRefs(dir) {
+    let compositionSourceUrl = null;
+    const htmlPath = path.join(dir, "composition.html");
+    if (existsSync(htmlPath)) {
+        const html = readFileSync(htmlPath, "utf8");
+        compositionSourceUrl =
+            matchAttr(html, "data-source-video") ??
+                matchPlaybackSourceSrc(html) ??
+                null;
+    }
+    let originalSourceUrl = null;
+    let scenes = [];
+    let durationSeconds = null;
+    const ctxPath = path.join(dir, "video-context.json");
+    if (existsSync(ctxPath)) {
+        try {
+            const ctx = JSON.parse(readFileSync(ctxPath, "utf8"));
+            if (typeof ctx.source_url === "string" && ctx.source_url)
+                originalSourceUrl = ctx.source_url;
+            if (typeof ctx.duration_seconds === "number")
+                durationSeconds = ctx.duration_seconds;
+            if (Array.isArray(ctx.scenes)) {
+                scenes = ctx.scenes
+                    .map((raw) => {
+                    if (!raw || typeof raw !== "object")
+                        return null;
+                    const s = raw;
+                    return {
+                        slug: typeof s.slug === "string" ? s.slug : null,
+                        start: Number(s.start ?? 0),
+                        duration: Number(s.duration ?? 0),
+                        label: typeof s.label === "string" ? s.label : null,
+                        description: typeof s.description === "string" ? s.description : "",
+                        transcript_excerpt: typeof s.transcript_excerpt === "string" ? s.transcript_excerpt : ""
+                    };
+                })
+                    .filter((s) => s !== null);
+            }
+        }
+        catch {
+            // video-context.json absent/status:none/malformed → fall back to composition source only
+        }
+    }
+    return { compositionSourceUrl, originalSourceUrl, scenes, durationSeconds };
+}
+function matchAttr(html, attr) {
+    const match = html.match(new RegExp(`${attr}="([^"]+)"`));
+    return match ? decodeHtmlEntities(match[1]) : null;
+}
+function matchPlaybackSourceSrc(html) {
+    const match = html.match(/data-vf-playback-source="true"[^>]*\ssrc="([^"]+)"/) ??
+        html.match(/<video[^>]*\ssrc="([^"]+)"/);
+    return match ? decodeHtmlEntities(match[1]) : null;
+}
+function decodeHtmlEntities(value) {
+    return value
+        .replace(/&amp;/g, "&")
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'")
+        .replace(/&lt;/g, "<")
+        .replace(/&gt;/g, ">");
+}
+async function downloadVideo(input) {
+    const name = path.basename(input.target);
+    if (existsSync(input.target) && !input.refetch) {
+        console.log(`[vidfarm] keep local ${name} (use --refetch to overwrite)`);
+        return { path: name, bytes: safeSize(input.target) };
+    }
+    try {
+        const res = await fetch(input.url, { headers: input.headers });
+        if (!res.ok || !res.body) {
+            console.warn(`[vidfarm] skip ${name} (${input.label}): ${res.status} ${res.statusText}`);
+            return null;
+        }
+        await pipeline(Readable.fromWeb(res.body), createWriteStream(input.target));
+        const bytes = safeSize(input.target);
+        console.log(`[vidfarm] fetched ${name} (${input.label}, ${formatBytes(bytes)})`);
+        return { path: name, bytes };
+    }
+    catch (error) {
+        console.warn(`[vidfarm] skip ${name} (${input.label}): ${error instanceof Error ? error.message : String(error)}`);
+        return null;
+    }
+}
+async function extractSceneFrames(input) {
+    const out = new Map();
+    if (input.scenes.length === 0)
+        return out;
+    const ffmpeg = await resolveFfmpegPath();
+    if (!ffmpeg) {
+        console.log("[vidfarm] ffmpeg unavailable — skipping scene frames (videos still downloaded)");
+        return out;
+    }
+    const framesDir = path.join(input.dir, "frames");
+    mkdirSync(framesDir, { recursive: true });
+    const videoPath = path.join(input.dir, input.videoName);
+    input.scenes.forEach((scene, index) => {
+        const slug = scene.slug || `scene-${index + 1}`;
+        const rel = path.join("frames", `${slug}.png`);
+        const target = path.join(input.dir, rel);
+        if (existsSync(target) && !input.refetch) {
+            out.set(index, rel);
+            return;
+        }
+        // Sample the scene midpoint, clamped inside the clip.
+        const mid = scene.start + Math.max(0, scene.duration) / 2;
+        const seek = Number.isFinite(mid) && mid > 0 ? mid : Math.max(0, scene.start);
+        const result = spawnSync(ffmpeg, ["-y", "-ss", seek.toFixed(3), "-i", videoPath, "-frames:v", "1", "-q:v", "3", target], { stdio: "ignore" });
+        if (result.status === 0 && existsSync(target)) {
+            out.set(index, rel);
+        }
+    });
+    console.log(`[vidfarm] extracted ${out.size}/${input.scenes.length} scene frames from ${input.videoName}`);
+    return out;
+}
+async function resolveFfmpegPath() {
+    // ffmpeg-static is already a dependency; import lazily so the common path
+    // (no frame extraction) never pays for it, and degrade gracefully if absent.
+    try {
+        const mod = (await import("ffmpeg-static"));
+        const resolved = typeof mod === "string" ? mod : mod.default;
+        return typeof resolved === "string" && resolved.length > 0 && existsSync(resolved) ? resolved : null;
+    }
+    catch {
+        return null;
+    }
+}
+function safeSize(target) {
+    try {
+        return statSync(target).size;
+    }
+    catch {
+        return 0;
+    }
+}
+function formatBytes(bytes) {
+    if (bytes >= 1024 * 1024)
+        return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+    if (bytes >= 1024)
+        return `${(bytes / 1024).toFixed(1)} KB`;
+    return `${bytes} bytes`;
+}
 function buildAuthHeaders(input) {
     const headers = { accept: "text/html,application/json" };
     // The server reads exactly these header names (see requireAuth and
@@ -328,6 +568,8 @@ function printEditorBanner(input) {
     console.log(`  local dir   ${input.dir}`);
     console.log(`  dev-serve   http://localhost:${input.port}`);
     console.log("");
+    console.log(`  ${dim}Agent media: original.mp4, source.mp4, frames/*.png, sources.json${reset}`);
+    console.log("");
     console.log(`  ${bold}Open in browser:${reset}`);
     console.log(`  ${cyan}${input.editorUrl}${reset}`);
     console.log(line);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mevdragon/vidfarm-devcli",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "description": "Local bridge for the Vidfarm Trackpad Editor. Point it at a template id, edit composition.html on disk (Claude Code, Codex, etc.), preview live in the browser at https://vidfarm.cc/editor/.",
   "type": "module",
   "bin": {