@mevdragon/vidfarm-devcli 0.9.0 → 0.10.0

package/dist/src/services/clip-curation/local-agent.js

@@ -0,0 +1,247 @@
+// Local-agent model client — devcli-only. Runs scene tagging, prompt-guided
+// keep/drop/merge, and NL→criteria through the user's LOCAL agent CLI
+// subscription (claude or codex) instead of a provider API key, so `vidfarm
+// clips scan` works out of the box on a machine that has Claude Code or Codex
+// installed: local ffmpeg for compute, local agent for evaluation. Cloud is the
+// explicit backup (devcli `--cloud`), never the default. Embeddings cannot ride
+// on a CLI agent — pair with a gemini/openai key (embedding-only) or the
+// library stays structured-searchable, the same degraded mode as
+// openrouter-without-embed-key. NEVER constructed in Lambda.
+import { spawn } from "node:child_process";
+import { spawnSync } from "node:child_process";
+import { ACTIVE_TAXONOMY_VERSION, getTaxonomy, sanitizeTagValues } from "./taxonomy.js";
+import { buildQueryPrompt, buildTaggingPrompt, ClipEmbeddingClient } from "./gemini.js";
+/** Find a usable local agent CLI on PATH. Returns null when none is installed. */
+export function detectLocalAgent(preferred) {
+    const candidates = preferred === "claude" ? ["claude"]
+        : preferred === "codex" ? ["codex"]
+            : ["claude", "codex"];
+    for (const kind of candidates) {
+        const found = spawnSync("which", [kind], { encoding: "utf8" });
+        const bin = (found.stdout || "").trim();
+        if (found.status === 0 && bin)
+            return { kind, bin };
+    }
+    return null;
+}
+export class LocalAgentClipClient {
+    provider = "local-agent";
+    tier;
+    taxonomyVersion;
+    tagModelId;
+    embeddingProvider;
+    embeddingModelId;
+    kind;
+    bin;
+    embedder;
+    timeoutMs;
+    constructor(opts = {}) {
+        const detected = opts.bin
+            ? { kind: (opts.agent === "codex" ? "codex" : "claude"), bin: opts.bin }
+            : detectLocalAgent(opts.agent === "auto" ? undefined : opts.agent);
+        if (!detected) {
+            throw new Error("No local agent CLI found. Install Claude Code (`claude`) or Codex (`codex`), pass a provider key (--gemini-key / GEMINI_API_KEY), or run with --cloud.");
+        }
+        this.kind = detected.kind;
+        this.bin = detected.bin;
+        this.tagModelId = `${this.kind}-cli`;
+        this.tier = opts.tier ?? "flash-lite";
+        this.taxonomyVersion = opts.taxonomyVersion ?? ACTIVE_TAXONOMY_VERSION;
+        this.embedder = opts.embedding ? new ClipEmbeddingClient(opts.embedding) : null;
+        this.embeddingProvider = this.embedder ? this.embedder.provider : null;
+        this.embeddingModelId = this.embedder ? this.embedder.modelId : null;
+        this.timeoutMs = Math.max(30_000, opts.timeoutMs ?? 300_000);
+    }
+    get canEmbed() {
+        return this.embedder !== null;
+    }
+    /** Tag one scene: the agent Reads the keyframe files itself, then emits the tag JSON. */
+    async tagScene(input) {
+        // No audio path — CLI agents can't transcribe the mp3; transcript stays
+        // whatever the caller supplied (usually empty on local-agent scans).
+        const base = buildTaggingPrompt(input.transcript, this.taxonomyVersion, false, input.guidancePrompt);
+        const prompt = [
+            base,
+            "",
+            `The scene's keyframe image files are on disk — open and inspect EVERY one of these ${input.keyframePaths.length} file(s) before answering:`,
+            ...input.keyframePaths.map((p) => `- ${p}`),
+            "",
+            "Respond with ONLY the JSON object (no prose, no markdown fence) with keys:",
+            `subject, action, emotion, setting, composition, motion (string arrays), energy (string), audio_type (string array), transcript (string), description (string).`
+        ].join("\n");
+        try {
+            const raw = await this.runAgent(prompt);
+            const parsed = extractJsonObject(raw);
+            if (!parsed)
+                throw new Error("local agent returned no JSON");
+            const withDefaults = {
+                subject: [], action: [], emotion: [], setting: [], composition: [], motion: [],
+                energy: "medium", audio_type: [], transcript: "", description: "",
+                ...parsed
+            };
+            const sanitized = sanitizeTagValues(withDefaults, this.taxonomyVersion);
+            const tags = {
+                ...sanitized,
+                transcript: (input.transcript || String(withDefaults.transcript ?? "")).trim()
+            };
+            return { tags, description: String(withDefaults.description ?? "").trim() };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            // eslint-disable-next-line no-console
+            console.warn(`[clips] tag failed (local-agent/${this.kind}) @${input.scene.start_time_sec.toFixed(1)}s: ${message}`);
+            return {
+                tags: {
+                    subject: [], action: [], emotion: [], setting: [], composition: [], motion: [],
+                    energy: "medium",
+                    audio_type: input.transcript ? ["dialogue"] : ["silence"],
+                    transcript: input.transcript || ""
+                },
+                description: input.transcript ? input.transcript.slice(0, 120) : ""
+            };
+        }
+    }
+    /** Keep/drop/merge for prompt-guided segmentation via the local agent. */
+    async decideScenes(input) {
+        const prompt = [
+            input.prompt,
+            "",
+            `The candidate scenes' keyframe image files are on disk, in timeline order — open and inspect EVERY one of these ${input.imagePaths.length} file(s) before deciding:`,
+            ...input.imagePaths.map((p, i) => `- scene ${i}: ${p}`),
+            "",
+            `Respond with ONLY a JSON object (no prose, no markdown fence): {"decisions":[{"keep":true|false,"merge_with_previous":true|false}, …]} with exactly ${input.imagePaths.length} entries in scene order.`
+        ].join("\n");
+        try {
+            const raw = await this.runAgent(prompt);
+            const parsed = extractJsonObject(raw);
+            if (!parsed || !Array.isArray(parsed.decisions))
+                return null;
+            return parsed.decisions.map((d) => ({
+                keep: Boolean(d?.keep),
+                merge_with_previous: Boolean(d?.merge_with_previous)
+            }));
+        }
+        catch (error) {
+            if (process.env.VIDFARM_DEBUG_LOGS === "true") {
+                // eslint-disable-next-line no-console
+                console.warn(`[clips] local-agent refine failed: ${error instanceof Error ? error.message : String(error)}`);
+            }
+            return null;
+        }
+    }
+    async embedDocuments(texts) {
+        return this.embedder ? this.embedder.embed(texts, "RETRIEVAL_DOCUMENT") : [];
+    }
+    async embedQuery(text) {
+        if (!this.embedder)
+            return undefined;
+        const [vec] = await this.embedder.embed([text], "RETRIEVAL_QUERY");
+        return vec;
+    }
+    async embedQueries(texts) {
+        return this.embedder ? this.embedder.embed(texts, "RETRIEVAL_QUERY") : [];
+    }
+    /** NL → structured criteria (text-only, so a clean fit for a CLI agent). */
+    async naturalLanguageToCriteria(query) {
+        const prompt = [
+            buildQueryPrompt(query, this.taxonomyVersion),
+            "",
+            "Respond with ONLY the JSON object (no prose, no markdown fence)."
+        ].join("\n");
+        try {
+            const raw = await this.runAgent(prompt);
+            const parsed = extractJsonObject(raw);
+            if (!parsed)
+                return { semantic_text: query };
+            return { ...sanitizeCriteria(parsed, this.taxonomyVersion), semantic_text: query };
+        }
+        catch {
+            return { semantic_text: query };
+        }
+    }
+    // ── CLI plumbing ──────────────────────────────────────────────────────────
+    runAgent(prompt) {
+        const args = this.kind === "claude"
+            ? [
+                "-p", prompt,
+                "--output-format", "text",
+                // Reading N keyframes = N tool turns; leave headroom.
+                "--max-turns", "50",
+                "--allowedTools", "Read",
+                ...(process.env.VIDFARM_LOCAL_AGENT_MODEL ? ["--model", process.env.VIDFARM_LOCAL_AGENT_MODEL] : [])
+            ]
+            : [
+                "exec",
+                "--skip-git-repo-check",
+                ...(process.env.VIDFARM_LOCAL_AGENT_MODEL ? ["--model", process.env.VIDFARM_LOCAL_AGENT_MODEL] : []),
+                prompt
+            ];
+        return new Promise((resolve, reject) => {
+            const child = spawn(this.bin, args, { stdio: ["ignore", "pipe", "pipe"] });
+            let stdout = "";
+            let stderr = "";
+            const timer = setTimeout(() => {
+                child.kill("SIGKILL");
+                reject(new Error(`${this.kind} CLI timed out after ${Math.round(this.timeoutMs / 1000)}s`));
+            }, this.timeoutMs);
+            timer.unref?.();
+            child.stdout.on("data", (d) => (stdout += d.toString()));
+            child.stderr.on("data", (d) => (stderr += d.toString()));
+            child.on("error", (error) => {
+                clearTimeout(timer);
+                reject(error);
+            });
+            child.on("close", (code) => {
+                clearTimeout(timer);
+                if (code === 0)
+                    resolve(stdout);
+                else
+                    reject(new Error(`${this.kind} CLI exited ${code}: ${(stderr || stdout).slice(0, 400)}`));
+            });
+        });
+    }
+}
+/** Pull the first parseable JSON object out of agent output (fenced block, or outermost braces). */
+export function extractJsonObject(text) {
+    const trimmed = (text ?? "").trim();
+    if (!trimmed)
+        return null;
+    const fenced = /```(?:json)?\s*([\s\S]*?)```/.exec(trimmed);
+    for (const candidate of [fenced?.[1], trimmed]) {
+        if (!candidate)
+            continue;
+        const start = candidate.indexOf("{");
+        const end = candidate.lastIndexOf("}");
+        if (start < 0 || end <= start)
+            continue;
+        try {
+            return JSON.parse(candidate.slice(start, end + 1));
+        }
+        catch {
+            /* try the next candidate */
+        }
+    }
+    return null;
+}
+/** Constrain agent-produced criteria to the controlled vocabulary + known fields. */
+function sanitizeCriteria(raw, version) {
+    const cats = getTaxonomy(version).categories;
+    const out = {};
+    const arrayFields = ["subject", "action", "emotion", "setting", "composition", "motion", "audio_type", "energy"];
+    for (const field of arrayFields) {
+        const allowed = new Set(cats[field] ?? []);
+        const values = Array.isArray(raw[field]) ? raw[field].map(String).filter((v) => allowed.has(v)) : [];
+        if (values.length)
+            out[field] = values;
+    }
+    if (Number.isFinite(Number(raw.min_duration_sec)))
+        out.min_duration_sec = Number(raw.min_duration_sec);
+    if (Number.isFinite(Number(raw.max_duration_sec)))
+        out.max_duration_sec = Number(raw.max_duration_sec);
+    if (typeof raw.has_dialogue === "boolean")
+        out.has_dialogue = raw.has_dialogue;
+    if (typeof raw.has_subject === "boolean")
+        out.has_subject = raw.has_subject;
+    return out;
+}
+//# sourceMappingURL=local-agent.js.map

package/dist/src/services/clip-curation/refine.js

@@ -6,10 +6,7 @@
 // then rebuild the scene list from those decisions. Falls back to the raw scenes
 // on any mismatch/error so a scan never breaks.
 import { mkdirSync, rmSync } from "node:fs";
-import { readFile } from "node:fs/promises";
 import path from "node:path";
-import { generateObject } from "ai";
-import { z } from "zod";
 import { extractKeyframes } from "./ffmpeg.js";
 // Sending one small keyframe per scene; cap the montage so a single call stays
 // affordable/latency-bounded. Longer videos keep their raw segmentation (logged).
@@ -61,7 +58,6 @@ export async function refineScenesWithGuidance(input) {
     }
 }
 async function askForDecisions(client, guidancePrompt, scenes, frames) {
-    const images = await Promise.all(frames.map((f) => readFile(f.path)));
     const listing = scenes
         .map((s, i) => `Scene ${i}: ${s.start_time_sec.toFixed(1)}s–${s.end_time_sec.toFixed(1)}s (${s.duration_sec.toFixed(1)}s)`)
         .join("\n");
@@ -76,32 +72,14 @@ async function askForDecisions(client, guidancePrompt, scenes, frames) {
         "",
         `Return exactly ${scenes.length} decisions, in the same order as the scenes. Be selective — dropping off-topic scenes and merging fragmented ones is the point.`
     ].join("\n");
-    // No fixed-length constraint — Gemini's structured-output schema translation
-    // rejects minItems==maxItems; we align by index instead (rebuildScenes).
-    const schema = z.object({
-        decisions: z.array(z.object({ keep: z.boolean(), merge_with_previous: z.boolean() }))
+    // The client owns the model call (API providers via structured output, the
+    // local devcli agent via CLI) — refine only builds the prompt and rebuilds
+    // the scene list from the decisions.
+    return client.decideScenes({
+        prompt,
+        imagePaths: frames.map((f) => f.path),
+        maxOutputTokens: Math.max(1000, scenes.length * 40)
     });
-    try {
-        const { object } = await generateObject({
-            model: client.taggingModel,
-            schema,
-            maxOutputTokens: Math.max(1000, scenes.length * 40),
-            messages: [
-                {
-                    role: "user",
-                    content: [{ type: "text", text: prompt }, ...images.map((image) => ({ type: "image", image }))]
-                }
-            ]
-        });
-        return object.decisions;
-    }
-    catch (error) {
-        if (process.env.VIDFARM_DEBUG_LOGS === "true") {
-            // eslint-disable-next-line no-console
-            console.warn(`[clips] refine call failed: ${error instanceof Error ? error.message : String(error)}`);
-        }
-        return null;
-    }
 }
 function rebuildScenes(scenes, decisions) {
     const out = [];

package/dist/src/services/clip-curation/scan.js

@@ -9,6 +9,7 @@ import { createIdV7 } from "../../lib/ids.js";
 import { ACTIVE_TAXONOMY_VERSION } from "./taxonomy.js";
 import { buildClipEmbeddingText } from "./gemini.js";
 import { detectScenes, extractClip, extractKeyframes, extractSceneAudio, extractThumbnail, probeVideo } from "./ffmpeg.js";
+import { cropFilterFor, fitScenesToDurationBand, normalizeWindows } from "./hunt.js";
 import { refineScenesWithGuidance } from "./refine.js";
 /** Turn one scene into a Clip. Throws only on unrecoverable extraction failure. */
 export async function processSceneToClip(input) {
@@ -18,12 +19,14 @@ export async function processSceneToClip(input) {
     mkdirSync(ctx.clipsDir, { recursive: true });
     mkdirSync(ctx.thumbsDir, { recursive: true });
     const clipId = createIdV7("clip");
-    // 1. Keyframes for visual tagging (small, low-res).
+    const cropFilter = ctx.crop ? cropFilterFor(ctx.crop.aspect, ctx.crop.focus) : null;
+    // 1. Keyframes for visual tagging (small, low-res), framed like the output.
     const keyframePaths = await extractKeyframes({
         videoPath,
         scene,
         outDir: framesDir,
-        count: ctx.framesPerScene ?? 2
+        count: ctx.framesPerScene ?? 2,
+        cropFilter
     });
     // 2. Per-scene audio for ASR (one combined tag+transcribe call).
     let audioPath;
@@ -43,13 +46,22 @@ export async function processSceneToClip(input) {
         scene,
         guidancePrompt: ctx.guidancePrompt
     });
-    // 4. Cut the clip + thumbnail.
+    // 4. Cut the clip + thumbnail (cropped to the target aspect when requested).
     const clipLocal = path.join(ctx.clipsDir, `${clipId}.mp4`);
     const thumbLocal = path.join(ctx.thumbsDir, `${clipId}.jpg`);
-    const clipOk = await extractClip({ videoPath, scene, outPath: clipLocal, reencode: ctx.reencodeClips !== false });
-    if (!clipOk)
-        throw new Error(`ffmpeg failed to extract clip for scene @${scene.start_time_sec}s`);
-    await extractThumbnail({ videoPath, scene, outPath: thumbLocal });
+    let extractDetail = "";
+    const clipOk = await extractClip({
+        videoPath,
+        scene,
+        outPath: clipLocal,
+        reencode: ctx.reencodeClips !== false,
+        cropFilter,
+        onError: (detail) => (extractDetail = detail)
+    });
+    if (!clipOk) {
+        throw new Error(`ffmpeg failed to extract clip for scene @${scene.start_time_sec}s${extractDetail ? ` — ${extractDetail}` : ""}`);
+    }
+    await extractThumbnail({ videoPath, scene, outPath: thumbLocal, cropFilter });
     // 5. Embed the document text for semantic search (skipped when the client can't embed).
     let embedding;
     if (ctx.client.canEmbed) {
@@ -72,6 +84,7 @@ export async function processSceneToClip(input) {
         tier: ctx.client.tier,
         provider: ctx.client.provider,
         embedding_model: ctx.client.embeddingModelId ?? undefined,
+        aspect: cropFilter && ctx.crop ? ctx.crop.aspect : undefined,
         owner_id: input.ownerId,
         created_at: new Date().toISOString()
     };
@@ -83,11 +96,23 @@ export async function processSceneToClip(input) {
  */
 export async function scanVideo(opts) {
     const probe = opts.probe ?? (await probeVideo(opts.videoPath));
+    const windows = normalizeWindows(opts.windows, probe.duration_sec);
+    const band = opts.durationBand ?? null;
+    // Fit to the soft duration band BEFORE refine: the model then judges
+    // clip-sized units (fewer, more meaningful keyframes) instead of raw cuts.
+    // Callers passing pre-detected scenes (devcli estimate pass) must pre-fit
+    // them — the fit is intentionally NOT re-run on supplied scenes since
+    // fitting is not idempotent (a fitted 20s scene could re-merge toward the
+    // target on a second pass).
     let scenes = opts.scenes ??
-        (await detectScenes(opts.videoPath, {
+        fitScenesToDurationBand(await detectScenes(opts.videoPath, {
             durationSec: probe.duration_sec,
+            windows,
+            // A duration band lifts the scene-split ceiling so raw cuts can merge
+            // up to the requested clip length instead of being pre-chopped at 8s.
+            ...(band ? { maxSceneSec: band.max_sec } : {}),
             ...opts.sceneOptions
-        }));
+        }), band);
     // Prompt-guided segmentation: let the guidance reshape which scenes become
     // clips (keep/drop/merge) before we spend tagging tokens on them.
     if (opts.ctx.guidancePrompt?.trim() && opts.ctx.refineSegmentation !== false) {
@@ -98,7 +123,9 @@ export async function scanVideo(opts) {
             guidancePrompt: opts.ctx.guidancePrompt,
             workDir: opts.ctx.workDir
         });
-        scenes = refined.scenes;
+        // Refine merges can exceed the band max — re-enforce it (fit never merges
+        // across the gaps refine's drops created, only contiguous spans).
+        scenes = fitScenesToDurationBand(refined.scenes, band);
         opts.onRefine?.(refined);
     }
     const sourceVideoId = opts.sourceVideoId ?? createIdV7("srcvid");

package/dist/src/services/serverless-records.js

@@ -804,6 +804,11 @@ class ServerlessRecordsServiceImpl {
     }
     async createUserTemporaryFile(input) {
         const timestamp = nowIso();
+        const expiresAt = input.expiresAt ?? timestamp;
+        // Native DynamoDB TTL (top-level expires_at_epoch_seconds) so temp-file
+        // records auto-expire even if no list sweep ever runs; the S3 object is
+        // separately expired by the tag-scoped bucket lifecycle rule.
+        const expiresAtEpochSeconds = Math.floor(Date.parse(expiresAt) / 1000);
         return this.putRecord("user_temporary_file", {
             id: input.id ?? createId("tmp"),
             customerId: input.customerId,
@@ -812,9 +817,9 @@ class ServerlessRecordsServiceImpl {
             sizeBytes: input.sizeBytes ?? 0,
             storageKey: input.storageKey,
             folderPath: normalizeFolder(input.folderPath),
-            expiresAt: input.expiresAt ?? timestamp,
+            expiresAt,
             createdAt: input.createdAt ?? timestamp
-        });
+        }, Number.isFinite(expiresAtEpochSeconds) ? { expiresAtEpochSeconds } : undefined);
     }
     async listUserTemporaryFiles(customerId) {
         return this.listByCustomer("user_temporary_file", customerId);
@@ -1196,6 +1201,8 @@ class ServerlessRecordsServiceImpl {
             entity_type: entityType(collection),
             gsi1pk,
             gsi1sk: `${String(data.updatedAt)}#${id}`,
+            // Top-level so the table's native TTL (timeToLiveAttribute) can act on it.
+            ...(options?.expiresAtEpochSeconds ? { expires_at_epoch_seconds: options.expiresAtEpochSeconds } : {}),
             data
         };
         // Private templates stay out of the global FEED partition (sparse gsi2):

package/dist/src/services/storage.js

@@ -1,6 +1,6 @@
 import { mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import path from "node:path";
-import { DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
+import { DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand, PutObjectTaggingCommand, S3Client } from "@aws-sdk/client-s3";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 import { config } from "../config.js";
 function isS3NotFoundError(error) {
@@ -107,6 +107,21 @@ export class StorageService {
         writeFileSync(filePath, value);
         return { key, url: this.getPublicUrl(key) };
     }
+    /**
+     * Tag an existing S3 object. Used to mark temp-folder uploads with
+     * `vidfarm-temp=1` AFTER the client's presigned PUT lands (server-side, so
+     * no client header contract), which the bucket's tag-scoped lifecycle rule
+     * expires after 30 days. No-op on the local filesystem driver.
+     */
+    async putObjectTags(key, tags) {
+        if (!this.s3 || !config.AWS_S3_BUCKET)
+            return;
+        await this.s3.send(new PutObjectTaggingCommand({
+            Bucket: config.AWS_S3_BUCKET,
+            Key: key,
+            Tagging: { TagSet: Object.entries(tags).map(([Key, Value]) => ({ Key, Value })) }
+        }));
+    }
     // Returns a directly-fetchable URL for a storage key when the key is
     // stored under a prefix the bucket resource policy grants public read to.
     // Callers get a real S3 URL that browsers/CDNs/GhostCut/etc. can hit

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mevdragon/vidfarm-devcli",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "description": "Local bridge for the Vidfarm Trackpad Editor. `vidfarm serve <template_id>` boots the FULL editor on localhost (disk-backed records/storage, free in-process render); edit composition.html on disk (Claude Code, Codex, etc.) and the browser live-morphs it.",
   "type": "module",
   "bin": {