@mevdragon/vidfarm-devcli 0.7.1 → 0.8.0

package/dist/src/cli.js

@@ -9,6 +9,7 @@ import { spawnSync } from "node:child_process";
 import { Readable } from "node:stream";
 import { pipeline } from "node:stream/promises";
 import { computeCompositionGaps, insertMediaLayer, inspectComposition, replaceLayerWithMedia } from "./devcli/composition-edit.js";
+import { runClipsCommand } from "./devcli/clips.js";
 // vidfarm-devcli — command-line bridge for the Vidfarm video studio. The
 // `serve` command boots the FULL editor locally (single origin, disk-backed
 // records + storage) so power users edit compositions on disk while a browser
@@ -128,6 +129,17 @@ Generate AI media and drop it on the timeline (for local coding agents):
   remove-video-captions <forkId>   Read the two video sources:        → GET  .../compositions/:forkId/remove-video-captions
                             original vs decomposed (caption-free),
                             non-billing (alias: ghostcut)
+
+Clip curation (the third library — mine long-form video into a reusable clip store):
+  clips scan <video-path>   Detect scenes → tag + transcribe (Gemini)    LOCAL: ffmpeg + BYOK Gemini
+                            → embed → persist to ~/.vidfarm/clips.db.     (cloud: POST /clips/scan)
+    --tier flash|flash-lite   Gemini tier (default flash-lite, cheapest)
+    --dry-run                 Show the estimated cost and stop (over $1 needs --yes/confirm)
+  clips list                List the local clip library                  → GET  /clips
+  clips search "<text>"     Hybrid structured + semantic search          → POST /clips/search
+  clips preset list|run <name>|save <name> --from-query '<...>'          → /clips/presets
+  clips export <ids...> --to <dir>   Copy clip mp4s + metadata out
+                            (run 'vidfarm clips' for the full clips help)
   versions <forkId>         List immutable version snapshots             → GET  .../compositions/:forkId/versions
   snapshot <forkId>         Save the working state as a new version      → POST .../compositions/:forkId/versions
   render <forkId>           Render the fork to MP4 (--wait to poll)      → POST .../compositions/:forkId/render
@@ -351,6 +363,9 @@ async function main() {
         case "save-file":
             await runPutFileCommand(rest);
             return;
+        case "clips":
+            await runClipsCommand(rest);
+            return;
         default:
             if (!command.startsWith("-")) {
                 // Positional template id → default to booting the local editor.
@@ -803,7 +818,10 @@ async function fetchCompositionFiles(input) {
     // can ground edits in what the video actually says and shows.
     // cast.json carries the identified recurring people/characters plus their
     // reference-still URLs so agents can reuse "the same person" in new gens.
-    const files = ["composition.html", "composition.json", "manifest.json", "video-context.json", "cast.json"];
+    // scene-annotations.json carries the detected content format + per-scene
+    // literal + viral DNA an agent needs to REPLACE a scene from the clip library
+    // or RE-CREATE it from scratch (recreation_prompt + recreation_notes).
+    const files = ["composition.html", "composition.json", "manifest.json", "video-context.json", "cast.json", "scene-annotations.json"];
     for (const filename of files) {
         const target = path.join(input.dir, filename);
         if (existsSync(target) && !input.refetch) {
@@ -1305,6 +1323,43 @@ async function runDecomposeCommand(argv) {
     if (!ctx.json && result.json?.ghostcut_pending) {
         console.log(`${DIM}Subtitle removal (GhostCut) is running in the background — poll \`vidfarm api POST /api/v1/compositions/${forkId}/remove-video-captions-poll\` or GET .../remove-video-captions until done.${RESET}`);
     }
+    // The scene-annotation pass (per-scene clip-match + recreation DNA + content
+    // format) is seeded "pending" by decompose and runs as its own bounded vision
+    // call — kept out of the decompose request so that stays under the origin
+    // timeout. Drive it to completion here so CLI/agent users get annotations
+    // without needing the editor. Each poll is one call; usually one round
+    // finishes it. Best-effort: transient failures leave it pending for another
+    // client (or a later `vidfarm pull`) to finish.
+    if (!ctx.json)
+        process.stdout.write(`${DIM}Annotating scenes (clip-match + recreation DNA)… ${RESET}`);
+    let annotationStatus = "pending";
+    for (let attempt = 0; attempt < 20; attempt += 1) {
+        let poll;
+        try {
+            poll = await apiRequest({
+                method: "POST",
+                host: ctx.host,
+                path: `/api/v1/compositions/${encodeURIComponent(forkId)}/scene-annotations-poll`,
+                auth: ctx.auth,
+                body: {}
+            });
+        }
+        catch {
+            break;
+        }
+        annotationStatus = typeof poll.json?.status === "string" ? poll.json.status : "none";
+        if (annotationStatus === "none" || annotationStatus === "done" || annotationStatus === "failed")
+            break;
+        await sleep(3000);
+    }
+    if (!ctx.json) {
+        if (annotationStatus === "done")
+            console.log(`${GREEN}done${RESET}`);
+        else if (annotationStatus === "failed")
+            console.log(`${RED}failed${RESET} ${DIM}(run \`vidfarm api POST /api/v1/compositions/${forkId}/scene-annotations-poll\` to retry)${RESET}`);
+        else
+            console.log(`${DIM}still running — poll .../scene-annotations-poll until done${RESET}`);
+    }
     emitResult(result, ctx.json);
 }
 async function runRemoveVideoCaptionsCommand(argv) {
@@ -1827,8 +1882,9 @@ async function runPullCommand(argv) {
     const info = inspectComposition(readFileSync(htmlPath, "utf8"));
     const hasContext = existsSync(path.join(dir, "video-context.json"));
     const hasCast = existsSync(path.join(dir, "cast.json"));
+    const hasAnnotations = existsSync(path.join(dir, "scene-annotations.json"));
     if (ctx.json) {
-        printJson({ ok: true, fork_id: forkId, dir, has_video_context: hasContext, has_cast: hasCast, ...info });
+        printJson({ ok: true, fork_id: forkId, dir, has_video_context: hasContext, has_cast: hasCast, has_scene_annotations: hasAnnotations, ...info });
         return;
     }
     console.log(`${GREEN}Pulled ${forkId} → ${dir}${RESET}`);
@@ -1839,7 +1895,7 @@ async function runPullCommand(argv) {
         const flags = [layer.is_timeline_proxy ? "proxy" : null, layer.is_full_canvas ? "full-canvas" : null].filter(Boolean).join(",");
         console.log(`  ${layer.key}  ${DIM}${layer.kind ?? "?"} ${layer.start}-${(layer.start + layer.duration).toFixed(2)}s track${layer.track}${flags ? ` [${flags}]` : ""}${layer.slug ? ` slug=${layer.slug}` : ""}${RESET}`);
     }
-    console.log(`${DIM}Grounding: ${hasContext ? "video-context.json ✓" : "video-context.json ✗ (run `vidfarm decompose`)"}, ${hasCast ? "cast.json ✓" : "cast.json ✗"}.${RESET}`);
+    console.log(`${DIM}Grounding: ${hasContext ? "video-context.json ✓" : "video-context.json ✗ (run `vidfarm decompose`)"}, ${hasCast ? "cast.json ✓" : "cast.json ✗"}, ${hasAnnotations ? "scene-annotations.json ✓" : "scene-annotations.json ✗"}.${RESET}`);
     console.log(`${DIM}Next: vidfarm generate video --prompt "..." --aspect-ratio ${info.aspect_ratio ?? "9:16"} --place ${dir} --at <gap start>|--replace <layer_key>${RESET}`);
 }
 async function runVisibilityCommand(argv) {

package/dist/src/config.js

@@ -108,6 +108,12 @@ const schema = z.object({
     VIDFARM_RATE_LIMITS_TABLE_NAME: z.string().optional(),
     VIDFARM_SERVERLESS_MAIN_TABLE_NAME: z.string().optional(),
     VIDFARM_JOB_STATE_MACHINE_ARN: z.string().optional(),
+    // Clip-curation scan pipeline (Step Functions). Optional: when unset, the
+    // /clips/scan route records the scan request but doesn't dispatch the machine.
+    VIDFARM_CLIP_SCAN_STATE_MACHINE_ARN: z.string().optional(),
+    VIDFARM_CLIP_VECTOR_BACKEND: z.enum(["dynamo", "s3vectors"]).default("dynamo"),
+    VIDFARM_CLIP_VECTORS_BUCKET: z.string().optional(),
+    VIDFARM_CLIP_VECTORS_INDEX: z.string().optional(),
     SENTRY_DSN: z.string().optional(),
     SENTRY_ENVIRONMENT: z.string().optional(),
     SENTRY_RELEASE: z.string().optional(),

package/dist/src/devcli/clip-store.js

@@ -0,0 +1,335 @@
+// Local clip library for the devcli. SQLite (better-sqlite3) holds clip
+// metadata + presets + source-video bookkeeping; embeddings live in a
+// sqlite-vec vec0 virtual table for KNN. Clip files/thumbnails are written to
+// disk. Layout (handoff Target 1):
+//   ~/.vidfarm/clips.db          metadata + vectors
+//   ~/.vidfarm/clips/<id>.mp4    clip files
+//   ~/.vidfarm/thumbs/<id>.jpg   thumbnails
+// Override the base dir with VIDFARM_HOME.
+import Database from "better-sqlite3";
+import * as sqliteVec from "sqlite-vec";
+import { existsSync, mkdirSync, rmSync } from "node:fs";
+import path from "node:path";
+import { homedir } from "node:os";
+import { createId } from "../lib/ids.js";
+import { BUILTIN_PRESETS, EMBEDDING_DIM, cosineSimilarity, matchesCriteria, rankHits, scoreClip, structuredMatchRatio } from "../services/clip-curation/index.js";
+export function resolveClipStorePaths(baseOverride) {
+    const base = baseOverride
+        ? path.resolve(baseOverride)
+        : process.env.VIDFARM_HOME
+            ? path.resolve(process.env.VIDFARM_HOME)
+            : path.join(homedir(), ".vidfarm");
+    return {
+        base,
+        db: path.join(base, "clips.db"),
+        clipsDir: path.join(base, "clips"),
+        thumbsDir: path.join(base, "thumbs"),
+        workDir: path.join(base, "scan-tmp")
+    };
+}
+export class ClipStore {
+    paths;
+    db;
+    dim;
+    constructor(baseOverride) {
+        this.paths = resolveClipStorePaths(baseOverride);
+        mkdirSync(this.paths.base, { recursive: true });
+        mkdirSync(this.paths.clipsDir, { recursive: true });
+        mkdirSync(this.paths.thumbsDir, { recursive: true });
+        this.db = new Database(this.paths.db);
+        this.db.pragma("journal_mode = WAL");
+        sqliteVec.load(this.db);
+        this.dim = this.initSchema();
+        this.seedBuiltinPresets();
+    }
+    /**
+     * Create the base tables, resolve the embedding dimension (pinned in `meta`
+     * so it stays stable across runs even if the env default changes), then
+     * create the vec table at that dimension. Returns the resolved dim.
+     */
+    initSchema() {
+        this.migrate();
+        const existing = this.db
+            .prepare("SELECT value FROM meta WHERE key = 'embedding_dim'")
+            .get();
+        const dim = existing ? Number(existing.value) || EMBEDDING_DIM : EMBEDDING_DIM;
+        this.db.prepare("INSERT OR IGNORE INTO meta(key, value) VALUES ('embedding_dim', ?)").run(String(dim));
+        this.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS vec_clips USING vec0(clip_id TEXT PRIMARY KEY, embedding float[${dim}]);`);
+        return dim;
+    }
+    migrate() {
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
+      CREATE TABLE IF NOT EXISTS clips (
+        clip_id TEXT PRIMARY KEY,
+        source_video_id TEXT NOT NULL,
+        source_filename TEXT NOT NULL,
+        start_time_sec REAL NOT NULL,
+        end_time_sec REAL NOT NULL,
+        duration_sec REAL NOT NULL,
+        file_path TEXT NOT NULL,
+        thumbnail_path TEXT NOT NULL,
+        tags_json TEXT NOT NULL,
+        description TEXT NOT NULL DEFAULT '',
+        taxonomy_version TEXT NOT NULL,
+        tier TEXT NOT NULL,
+        user_notes TEXT,
+        created_at TEXT NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_clips_source ON clips(source_video_id);
+      CREATE INDEX IF NOT EXISTS idx_clips_created ON clips(created_at);
+      CREATE TABLE IF NOT EXISTS presets (
+        preset_id TEXT PRIMARY KEY,
+        name TEXT NOT NULL,
+        description TEXT,
+        criteria_json TEXT NOT NULL,
+        builtin INTEGER NOT NULL DEFAULT 0,
+        created_at TEXT NOT NULL
+      );
+      CREATE TABLE IF NOT EXISTS source_videos (
+        source_video_id TEXT PRIMARY KEY,
+        filename TEXT NOT NULL,
+        source_path TEXT,
+        duration_sec REAL,
+        clip_count INTEGER NOT NULL DEFAULT 0,
+        tier TEXT,
+        scanned_at TEXT NOT NULL
+      );
+    `);
+    }
+    seedBuiltinPresets() {
+        const stmt = this.db.prepare("INSERT OR REPLACE INTO presets(preset_id, name, description, criteria_json, builtin, created_at) VALUES (?, ?, ?, ?, 1, ?)");
+        for (const p of BUILTIN_PRESETS) {
+            stmt.run(p.preset_id, p.name, p.description ?? null, JSON.stringify(p.criteria), p.created_at);
+        }
+    }
+    // ── Clips ──────────────────────────────────────────────────────────────
+    insertClip(clip) {
+        const tx = this.db.transaction((c) => {
+            this.db
+                .prepare(`INSERT OR REPLACE INTO clips
+            (clip_id, source_video_id, source_filename, start_time_sec, end_time_sec, duration_sec,
+             file_path, thumbnail_path, tags_json, description, taxonomy_version, tier, user_notes, created_at)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
+                .run(c.clip_id, c.source_video_id, c.source_filename, c.start_time_sec, c.end_time_sec, c.duration_sec, c.file_path, c.thumbnail_path, JSON.stringify(c.tags), c.description ?? "", c.taxonomy_version, c.tier, c.user_notes ?? null, c.created_at);
+            if (c.embedding && c.embedding.length === this.dim) {
+                this.db.prepare("DELETE FROM vec_clips WHERE clip_id = ?").run(c.clip_id);
+                this.db
+                    .prepare("INSERT INTO vec_clips(clip_id, embedding) VALUES (?, ?)")
+                    .run(c.clip_id, JSON.stringify(c.embedding));
+            }
+        });
+        tx(clip);
+    }
+    getClip(clipId, withEmbedding = false) {
+        const row = this.db.prepare("SELECT * FROM clips WHERE clip_id = ?").get(clipId);
+        if (!row)
+            return null;
+        const clip = rowToClip(row);
+        if (withEmbedding)
+            this.attachEmbeddings([clip]);
+        return clip;
+    }
+    /** All clips (metadata only), newest first, optionally scoped to a source video. */
+    listClips(opts = {}) {
+        let sql = "SELECT * FROM clips";
+        const args = [];
+        if (opts.sourceVideoId) {
+            sql += " WHERE source_video_id = ?";
+            args.push(opts.sourceVideoId);
+        }
+        sql += " ORDER BY created_at DESC";
+        if (opts.limit) {
+            sql += " LIMIT ?";
+            args.push(opts.limit);
+        }
+        return this.db.prepare(sql).all(...args).map(rowToClip);
+    }
+    deleteClip(clipId, removeFiles = true) {
+        const clip = this.getClip(clipId);
+        if (!clip)
+            return false;
+        this.db.prepare("DELETE FROM clips WHERE clip_id = ?").run(clipId);
+        this.db.prepare("DELETE FROM vec_clips WHERE clip_id = ?").run(clipId);
+        if (removeFiles) {
+            for (const f of [path.join(this.paths.clipsDir, `${clipId}.mp4`), path.join(this.paths.thumbsDir, `${clipId}.jpg`)]) {
+                try {
+                    if (existsSync(f))
+                        rmSync(f);
+                }
+                catch {
+                    /* best effort */
+                }
+            }
+        }
+        return true;
+    }
+    /** Load embeddings from the vec table into the given clips (in place). */
+    attachEmbeddings(clips) {
+        if (clips.length === 0)
+            return;
+        const stmt = this.db.prepare("SELECT vec_to_json(embedding) AS e FROM vec_clips WHERE clip_id = ?");
+        for (const clip of clips) {
+            const row = stmt.get(clip.clip_id);
+            if (row?.e) {
+                try {
+                    clip.embedding = JSON.parse(row.e);
+                }
+                catch {
+                    /* skip */
+                }
+            }
+        }
+    }
+    /** sqlite-vec KNN: nearest clip ids to a (normalized) query vector. */
+    knnClipIds(queryEmbedding, k) {
+        if (queryEmbedding.length !== this.dim)
+            return [];
+        const rows = this.db
+            .prepare("SELECT clip_id FROM vec_clips WHERE embedding MATCH ? AND k = ? ORDER BY distance")
+            .all(JSON.stringify(queryEmbedding), k);
+        return rows.map((r) => r.clip_id);
+    }
+    /**
+     * Two-stage search (handoff): structured filter FIRST, then semantic re-rank
+     * the survivors by embedding similarity. If nothing matches structurally but
+     * we have a query vector, relax to a sqlite-vec KNN semantic pass.
+     */
+    search(input) {
+        const limit = input.limit ?? 20;
+        const all = this.listClips();
+        const structured = all.filter((c) => matchesCriteria(c, input.criteria));
+        if (structured.length > 0) {
+            if (input.queryEmbedding)
+                this.attachEmbeddings(structured);
+            const hits = structured.map((c) => scoreClip(c, input.criteria, input.queryEmbedding, false));
+            return rankHits(hits, limit);
+        }
+        // Nothing passed the hard filter. Relax to semantic-only if we can.
+        if (input.queryEmbedding) {
+            const ids = this.knnClipIds(input.queryEmbedding, Math.max(limit * 5, 50));
+            const byId = new Map(all.map((c) => [c.clip_id, c]));
+            const pool = ids.map((id) => byId.get(id)).filter((c) => Boolean(c));
+            this.attachEmbeddings(pool);
+            const hits = pool.map((c) => scoreClip(c, input.criteria, input.queryEmbedding, true));
+            return rankHits(hits, limit);
+        }
+        return [];
+    }
+    // ── Source videos ──────────────────────────────────────────────────────
+    recordSourceVideo(input) {
+        this.db
+            .prepare(`INSERT OR REPLACE INTO source_videos
+          (source_video_id, filename, source_path, duration_sec, clip_count, tier, scanned_at)
+         VALUES (?, ?, ?, ?, ?, ?, ?)`)
+            .run(input.source_video_id, input.filename, input.source_path ?? null, input.duration_sec ?? null, input.clip_count, input.tier ?? null, new Date().toISOString());
+    }
+    listSourceVideos() {
+        return this.db.prepare("SELECT * FROM source_videos ORDER BY scanned_at DESC").all();
+    }
+    // ── Presets ────────────────────────────────────────────────────────────
+    listPresets() {
+        const rows = this.db
+            .prepare("SELECT * FROM presets ORDER BY builtin DESC, name ASC")
+            .all();
+        return rows.map((r) => ({
+            preset_id: r.preset_id,
+            name: r.name,
+            description: r.description ?? undefined,
+            criteria: safeParse(r.criteria_json, {}),
+            builtin: r.builtin === 1,
+            created_at: r.created_at
+        }));
+    }
+    findPreset(nameOrId) {
+        const needle = nameOrId.trim().toLowerCase();
+        return this.listPresets().find((p) => p.preset_id.toLowerCase() === needle || p.name.toLowerCase() === needle);
+    }
+    savePreset(input) {
+        const preset = {
+            preset_id: createId("preset"),
+            name: input.name,
+            description: input.description,
+            criteria: input.criteria,
+            builtin: false,
+            created_at: new Date().toISOString()
+        };
+        this.db
+            .prepare("INSERT INTO presets(preset_id, name, description, criteria_json, builtin, created_at) VALUES (?, ?, ?, ?, 0, ?)")
+            .run(preset.preset_id, preset.name, preset.description ?? null, JSON.stringify(preset.criteria), preset.created_at);
+        return preset;
+    }
+    // ── Embedding-model bookkeeping ──────────────────────────────────────────
+    // Vectors from different embedding models aren't comparable, so a library
+    // should stay on one. Remember the first one seen; callers warn on mismatch.
+    getLibraryEmbeddingModel() {
+        const row = this.db.prepare("SELECT value FROM meta WHERE key = 'embedding_model'").get();
+        return row?.value ?? null;
+    }
+    rememberEmbeddingModel(model) {
+        this.db.prepare("INSERT OR IGNORE INTO meta(key, value) VALUES ('embedding_model', ?)").run(model);
+    }
+    // ── Stats ──────────────────────────────────────────────────────────────
+    stats() {
+        const n = (sql) => this.db.prepare(sql).get().n;
+        return {
+            clips: n("SELECT count(*) AS n FROM clips"),
+            sources: n("SELECT count(*) AS n FROM source_videos"),
+            presets: n("SELECT count(*) AS n FROM presets")
+        };
+    }
+    /** Local scoring helper reused by callers that already have a candidate pool. */
+    scoreAgainst(clips, criteria, queryEmbedding) {
+        return clips.map((c) => {
+            const relaxed = !matchesCriteria(c, criteria);
+            return {
+                clip: c,
+                similarity: queryEmbedding ? cosineSimilarity(c.embedding, queryEmbedding) : undefined,
+                score: relaxed ? structuredMatchRatio(c, criteria) : 1
+            };
+        });
+    }
+    close() {
+        this.db.close();
+    }
+}
+function rowToClip(row) {
+    return {
+        clip_id: row.clip_id,
+        source_video_id: row.source_video_id,
+        source_filename: row.source_filename,
+        start_time_sec: row.start_time_sec,
+        end_time_sec: row.end_time_sec,
+        duration_sec: row.duration_sec,
+        file_path: row.file_path,
+        thumbnail_path: row.thumbnail_path,
+        tags: safeParse(row.tags_json, emptyTags()),
+        description: row.description ?? "",
+        taxonomy_version: row.taxonomy_version,
+        tier: row.tier ?? "flash-lite",
+        user_notes: row.user_notes ?? undefined,
+        created_at: row.created_at
+    };
+}
+function emptyTags() {
+    return {
+        subject: [],
+        action: [],
+        emotion: [],
+        setting: [],
+        composition: [],
+        motion: [],
+        energy: "medium",
+        audio_type: [],
+        transcript: ""
+    };
+}
+function safeParse(json, fallback) {
+    try {
+        return JSON.parse(json);
+    }
+    catch {
+        return fallback;
+    }
+}
+//# sourceMappingURL=clip-store.js.map