@mevdragon/vidfarm-devcli 0.9.0 → 0.11.0

package/dist/src/services/clip-curation/local-agent.js

@@ -0,0 +1,247 @@
+// Local-agent model client — devcli-only. Runs scene tagging, prompt-guided
+// keep/drop/merge, and NL→criteria through the user's LOCAL agent CLI
+// subscription (claude or codex) instead of a provider API key, so `vidfarm
+// clips scan` works out of the box on a machine that has Claude Code or Codex
+// installed: local ffmpeg for compute, local agent for evaluation. Cloud is the
+// explicit backup (devcli `--cloud`), never the default. Embeddings cannot ride
+// on a CLI agent — pair with a gemini/openai key (embedding-only) or the
+// library stays structured-searchable, the same degraded mode as
+// openrouter-without-embed-key. NEVER constructed in Lambda.
+import { spawn } from "node:child_process";
+import { spawnSync } from "node:child_process";
+import { ACTIVE_TAXONOMY_VERSION, getTaxonomy, sanitizeTagValues } from "./taxonomy.js";
+import { buildQueryPrompt, buildTaggingPrompt, ClipEmbeddingClient } from "./gemini.js";
+/** Find a usable local agent CLI on PATH. Returns null when none is installed. */
+export function detectLocalAgent(preferred) {
+    const candidates = preferred === "claude" ? ["claude"]
+        : preferred === "codex" ? ["codex"]
+            : ["claude", "codex"];
+    for (const kind of candidates) {
+        const found = spawnSync("which", [kind], { encoding: "utf8" });
+        const bin = (found.stdout || "").trim();
+        if (found.status === 0 && bin)
+            return { kind, bin };
+    }
+    return null;
+}
+export class LocalAgentClipClient {
+    provider = "local-agent";
+    tier;
+    taxonomyVersion;
+    tagModelId;
+    embeddingProvider;
+    embeddingModelId;
+    kind;
+    bin;
+    embedder;
+    timeoutMs;
+    constructor(opts = {}) {
+        const detected = opts.bin
+            ? { kind: (opts.agent === "codex" ? "codex" : "claude"), bin: opts.bin }
+            : detectLocalAgent(opts.agent === "auto" ? undefined : opts.agent);
+        if (!detected) {
+            throw new Error("No local agent CLI found. Install Claude Code (`claude`) or Codex (`codex`), pass a provider key (--gemini-key / GEMINI_API_KEY), or run with --cloud.");
+        }
+        this.kind = detected.kind;
+        this.bin = detected.bin;
+        this.tagModelId = `${this.kind}-cli`;
+        this.tier = opts.tier ?? "flash-lite";
+        this.taxonomyVersion = opts.taxonomyVersion ?? ACTIVE_TAXONOMY_VERSION;
+        this.embedder = opts.embedding ? new ClipEmbeddingClient(opts.embedding) : null;
+        this.embeddingProvider = this.embedder ? this.embedder.provider : null;
+        this.embeddingModelId = this.embedder ? this.embedder.modelId : null;
+        this.timeoutMs = Math.max(30_000, opts.timeoutMs ?? 300_000);
+    }
+    get canEmbed() {
+        return this.embedder !== null;
+    }
+    /** Tag one scene: the agent Reads the keyframe files itself, then emits the tag JSON. */
+    async tagScene(input) {
+        // No audio path — CLI agents can't transcribe the mp3; transcript stays
+        // whatever the caller supplied (usually empty on local-agent scans).
+        const base = buildTaggingPrompt(input.transcript, this.taxonomyVersion, false, input.guidancePrompt);
+        const prompt = [
+            base,
+            "",
+            `The scene's keyframe image files are on disk — open and inspect EVERY one of these ${input.keyframePaths.length} file(s) before answering:`,
+            ...input.keyframePaths.map((p) => `- ${p}`),
+            "",
+            "Respond with ONLY the JSON object (no prose, no markdown fence) with keys:",
+            `subject, action, emotion, setting, composition, motion (string arrays), energy (string), audio_type (string array), transcript (string), description (string).`
+        ].join("\n");
+        try {
+            const raw = await this.runAgent(prompt);
+            const parsed = extractJsonObject(raw);
+            if (!parsed)
+                throw new Error("local agent returned no JSON");
+            const withDefaults = {
+                subject: [], action: [], emotion: [], setting: [], composition: [], motion: [],
+                energy: "medium", audio_type: [], transcript: "", description: "",
+                ...parsed
+            };
+            const sanitized = sanitizeTagValues(withDefaults, this.taxonomyVersion);
+            const tags = {
+                ...sanitized,
+                transcript: (input.transcript || String(withDefaults.transcript ?? "")).trim()
+            };
+            return { tags, description: String(withDefaults.description ?? "").trim() };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            // eslint-disable-next-line no-console
+            console.warn(`[clips] tag failed (local-agent/${this.kind}) @${input.scene.start_time_sec.toFixed(1)}s: ${message}`);
+            return {
+                tags: {
+                    subject: [], action: [], emotion: [], setting: [], composition: [], motion: [],
+                    energy: "medium",
+                    audio_type: input.transcript ? ["dialogue"] : ["silence"],
+                    transcript: input.transcript || ""
+                },
+                description: input.transcript ? input.transcript.slice(0, 120) : ""
+            };
+        }
+    }
+    /** Keep/drop/merge for prompt-guided segmentation via the local agent. */
+    async decideScenes(input) {
+        const prompt = [
+            input.prompt,
+            "",
+            `The candidate scenes' keyframe image files are on disk, in timeline order — open and inspect EVERY one of these ${input.imagePaths.length} file(s) before deciding:`,
+            ...input.imagePaths.map((p, i) => `- scene ${i}: ${p}`),
+            "",
+            `Respond with ONLY a JSON object (no prose, no markdown fence): {"decisions":[{"keep":true|false,"merge_with_previous":true|false}, …]} with exactly ${input.imagePaths.length} entries in scene order.`
+        ].join("\n");
+        try {
+            const raw = await this.runAgent(prompt);
+            const parsed = extractJsonObject(raw);
+            if (!parsed || !Array.isArray(parsed.decisions))
+                return null;
+            return parsed.decisions.map((d) => ({
+                keep: Boolean(d?.keep),
+                merge_with_previous: Boolean(d?.merge_with_previous)
+            }));
+        }
+        catch (error) {
+            if (process.env.VIDFARM_DEBUG_LOGS === "true") {
+                // eslint-disable-next-line no-console
+                console.warn(`[clips] local-agent refine failed: ${error instanceof Error ? error.message : String(error)}`);
+            }
+            return null;
+        }
+    }
+    async embedDocuments(texts) {
+        return this.embedder ? this.embedder.embed(texts, "RETRIEVAL_DOCUMENT") : [];
+    }
+    async embedQuery(text) {
+        if (!this.embedder)
+            return undefined;
+        const [vec] = await this.embedder.embed([text], "RETRIEVAL_QUERY");
+        return vec;
+    }
+    async embedQueries(texts) {
+        return this.embedder ? this.embedder.embed(texts, "RETRIEVAL_QUERY") : [];
+    }
+    /** NL → structured criteria (text-only, so a clean fit for a CLI agent). */
+    async naturalLanguageToCriteria(query) {
+        const prompt = [
+            buildQueryPrompt(query, this.taxonomyVersion),
+            "",
+            "Respond with ONLY the JSON object (no prose, no markdown fence)."
+        ].join("\n");
+        try {
+            const raw = await this.runAgent(prompt);
+            const parsed = extractJsonObject(raw);
+            if (!parsed)
+                return { semantic_text: query };
+            return { ...sanitizeCriteria(parsed, this.taxonomyVersion), semantic_text: query };
+        }
+        catch {
+            return { semantic_text: query };
+        }
+    }
+    // ── CLI plumbing ──────────────────────────────────────────────────────────
+    runAgent(prompt) {
+        const args = this.kind === "claude"
+            ? [
+                "-p", prompt,
+                "--output-format", "text",
+                // Reading N keyframes = N tool turns; leave headroom.
+                "--max-turns", "50",
+                "--allowedTools", "Read",
+                ...(process.env.VIDFARM_LOCAL_AGENT_MODEL ? ["--model", process.env.VIDFARM_LOCAL_AGENT_MODEL] : [])
+            ]
+            : [
+                "exec",
+                "--skip-git-repo-check",
+                ...(process.env.VIDFARM_LOCAL_AGENT_MODEL ? ["--model", process.env.VIDFARM_LOCAL_AGENT_MODEL] : []),
+                prompt
+            ];
+        return new Promise((resolve, reject) => {
+            const child = spawn(this.bin, args, { stdio: ["ignore", "pipe", "pipe"] });
+            let stdout = "";
+            let stderr = "";
+            const timer = setTimeout(() => {
+                child.kill("SIGKILL");
+                reject(new Error(`${this.kind} CLI timed out after ${Math.round(this.timeoutMs / 1000)}s`));
+            }, this.timeoutMs);
+            timer.unref?.();
+            child.stdout.on("data", (d) => (stdout += d.toString()));
+            child.stderr.on("data", (d) => (stderr += d.toString()));
+            child.on("error", (error) => {
+                clearTimeout(timer);
+                reject(error);
+            });
+            child.on("close", (code) => {
+                clearTimeout(timer);
+                if (code === 0)
+                    resolve(stdout);
+                else
+                    reject(new Error(`${this.kind} CLI exited ${code}: ${(stderr || stdout).slice(0, 400)}`));
+            });
+        });
+    }
+}
+/** Pull the first parseable JSON object out of agent output (fenced block, or outermost braces). */
+export function extractJsonObject(text) {
+    const trimmed = (text ?? "").trim();
+    if (!trimmed)
+        return null;
+    const fenced = /```(?:json)?\s*([\s\S]*?)```/.exec(trimmed);
+    for (const candidate of [fenced?.[1], trimmed]) {
+        if (!candidate)
+            continue;
+        const start = candidate.indexOf("{");
+        const end = candidate.lastIndexOf("}");
+        if (start < 0 || end <= start)
+            continue;
+        try {
+            return JSON.parse(candidate.slice(start, end + 1));
+        }
+        catch {
+            /* try the next candidate */
+        }
+    }
+    return null;
+}
+/** Constrain agent-produced criteria to the controlled vocabulary + known fields. */
+function sanitizeCriteria(raw, version) {
+    const cats = getTaxonomy(version).categories;
+    const out = {};
+    const arrayFields = ["subject", "action", "emotion", "setting", "composition", "motion", "audio_type", "energy"];
+    for (const field of arrayFields) {
+        const allowed = new Set(cats[field] ?? []);
+        const values = Array.isArray(raw[field]) ? raw[field].map(String).filter((v) => allowed.has(v)) : [];
+        if (values.length)
+            out[field] = values;
+    }
+    if (Number.isFinite(Number(raw.min_duration_sec)))
+        out.min_duration_sec = Number(raw.min_duration_sec);
+    if (Number.isFinite(Number(raw.max_duration_sec)))
+        out.max_duration_sec = Number(raw.max_duration_sec);
+    if (typeof raw.has_dialogue === "boolean")
+        out.has_dialogue = raw.has_dialogue;
+    if (typeof raw.has_subject === "boolean")
+        out.has_subject = raw.has_subject;
+    return out;
+}
+//# sourceMappingURL=local-agent.js.map

package/dist/src/services/clip-curation/refine.js

@@ -6,10 +6,7 @@
 // then rebuild the scene list from those decisions. Falls back to the raw scenes
 // on any mismatch/error so a scan never breaks.
 import { mkdirSync, rmSync } from "node:fs";
-import { readFile } from "node:fs/promises";
 import path from "node:path";
-import { generateObject } from "ai";
-import { z } from "zod";
 import { extractKeyframes } from "./ffmpeg.js";
 // Sending one small keyframe per scene; cap the montage so a single call stays
 // affordable/latency-bounded. Longer videos keep their raw segmentation (logged).
@@ -61,7 +58,6 @@ export async function refineScenesWithGuidance(input) {
     }
 }
 async function askForDecisions(client, guidancePrompt, scenes, frames) {
-    const images = await Promise.all(frames.map((f) => readFile(f.path)));
     const listing = scenes
         .map((s, i) => `Scene ${i}: ${s.start_time_sec.toFixed(1)}s–${s.end_time_sec.toFixed(1)}s (${s.duration_sec.toFixed(1)}s)`)
         .join("\n");
@@ -76,32 +72,14 @@ async function askForDecisions(client, guidancePrompt, scenes, frames) {
         "",
         `Return exactly ${scenes.length} decisions, in the same order as the scenes. Be selective — dropping off-topic scenes and merging fragmented ones is the point.`
     ].join("\n");
-    // No fixed-length constraint — Gemini's structured-output schema translation
-    // rejects minItems==maxItems; we align by index instead (rebuildScenes).
-    const schema = z.object({
-        decisions: z.array(z.object({ keep: z.boolean(), merge_with_previous: z.boolean() }))
+    // The client owns the model call (API providers via structured output, the
+    // local devcli agent via CLI) — refine only builds the prompt and rebuilds
+    // the scene list from the decisions.
+    return client.decideScenes({
+        prompt,
+        imagePaths: frames.map((f) => f.path),
+        maxOutputTokens: Math.max(1000, scenes.length * 40)
     });
-    try {
-        const { object } = await generateObject({
-            model: client.taggingModel,
-            schema,
-            maxOutputTokens: Math.max(1000, scenes.length * 40),
-            messages: [
-                {
-                    role: "user",
-                    content: [{ type: "text", text: prompt }, ...images.map((image) => ({ type: "image", image }))]
-                }
-            ]
-        });
-        return object.decisions;
-    }
-    catch (error) {
-        if (process.env.VIDFARM_DEBUG_LOGS === "true") {
-            // eslint-disable-next-line no-console
-            console.warn(`[clips] refine call failed: ${error instanceof Error ? error.message : String(error)}`);
-        }
-        return null;
-    }
 }
 function rebuildScenes(scenes, decisions) {
     const out = [];

package/dist/src/services/clip-curation/scan.js

@@ -9,6 +9,7 @@ import { createIdV7 } from "../../lib/ids.js";
 import { ACTIVE_TAXONOMY_VERSION } from "./taxonomy.js";
 import { buildClipEmbeddingText } from "./gemini.js";
 import { detectScenes, extractClip, extractKeyframes, extractSceneAudio, extractThumbnail, probeVideo } from "./ffmpeg.js";
+import { cropFilterFor, fitScenesToDurationBand, normalizeWindows } from "./hunt.js";
 import { refineScenesWithGuidance } from "./refine.js";
 /** Turn one scene into a Clip. Throws only on unrecoverable extraction failure. */
 export async function processSceneToClip(input) {
@@ -18,12 +19,14 @@ export async function processSceneToClip(input) {
     mkdirSync(ctx.clipsDir, { recursive: true });
     mkdirSync(ctx.thumbsDir, { recursive: true });
     const clipId = createIdV7("clip");
-    // 1. Keyframes for visual tagging (small, low-res).
+    const cropFilter = ctx.crop ? cropFilterFor(ctx.crop.aspect, ctx.crop.focus) : null;
+    // 1. Keyframes for visual tagging (small, low-res), framed like the output.
     const keyframePaths = await extractKeyframes({
         videoPath,
         scene,
         outDir: framesDir,
-        count: ctx.framesPerScene ?? 2
+        count: ctx.framesPerScene ?? 2,
+        cropFilter
     });
     // 2. Per-scene audio for ASR (one combined tag+transcribe call).
     let audioPath;
@@ -43,13 +46,22 @@ export async function processSceneToClip(input) {
         scene,
         guidancePrompt: ctx.guidancePrompt
     });
-    // 4. Cut the clip + thumbnail.
+    // 4. Cut the clip + thumbnail (cropped to the target aspect when requested).
     const clipLocal = path.join(ctx.clipsDir, `${clipId}.mp4`);
     const thumbLocal = path.join(ctx.thumbsDir, `${clipId}.jpg`);
-    const clipOk = await extractClip({ videoPath, scene, outPath: clipLocal, reencode: ctx.reencodeClips !== false });
-    if (!clipOk)
-        throw new Error(`ffmpeg failed to extract clip for scene @${scene.start_time_sec}s`);
-    await extractThumbnail({ videoPath, scene, outPath: thumbLocal });
+    let extractDetail = "";
+    const clipOk = await extractClip({
+        videoPath,
+        scene,
+        outPath: clipLocal,
+        reencode: ctx.reencodeClips !== false,
+        cropFilter,
+        onError: (detail) => (extractDetail = detail)
+    });
+    if (!clipOk) {
+        throw new Error(`ffmpeg failed to extract clip for scene @${scene.start_time_sec}s${extractDetail ? ` — ${extractDetail}` : ""}`);
+    }
+    await extractThumbnail({ videoPath, scene, outPath: thumbLocal, cropFilter });
     // 5. Embed the document text for semantic search (skipped when the client can't embed).
     let embedding;
     if (ctx.client.canEmbed) {
@@ -72,6 +84,7 @@ export async function processSceneToClip(input) {
         tier: ctx.client.tier,
         provider: ctx.client.provider,
         embedding_model: ctx.client.embeddingModelId ?? undefined,
+        aspect: cropFilter && ctx.crop ? ctx.crop.aspect : undefined,
         owner_id: input.ownerId,
         created_at: new Date().toISOString()
     };
@@ -83,11 +96,23 @@ export async function processSceneToClip(input) {
  */
 export async function scanVideo(opts) {
     const probe = opts.probe ?? (await probeVideo(opts.videoPath));
+    const windows = normalizeWindows(opts.windows, probe.duration_sec);
+    const band = opts.durationBand ?? null;
+    // Fit to the soft duration band BEFORE refine: the model then judges
+    // clip-sized units (fewer, more meaningful keyframes) instead of raw cuts.
+    // Callers passing pre-detected scenes (devcli estimate pass) must pre-fit
+    // them — the fit is intentionally NOT re-run on supplied scenes since
+    // fitting is not idempotent (a fitted 20s scene could re-merge toward the
+    // target on a second pass).
     let scenes = opts.scenes ??
-        (await detectScenes(opts.videoPath, {
+        fitScenesToDurationBand(await detectScenes(opts.videoPath, {
             durationSec: probe.duration_sec,
+            windows,
+            // A duration band lifts the scene-split ceiling so raw cuts can merge
+            // up to the requested clip length instead of being pre-chopped at 8s.
+            ...(band ? { maxSceneSec: band.max_sec } : {}),
             ...opts.sceneOptions
-        }));
+        }), band);
     // Prompt-guided segmentation: let the guidance reshape which scenes become
     // clips (keep/drop/merge) before we spend tagging tokens on them.
     if (opts.ctx.guidancePrompt?.trim() && opts.ctx.refineSegmentation !== false) {
@@ -98,7 +123,9 @@ export async function scanVideo(opts) {
             guidancePrompt: opts.ctx.guidancePrompt,
             workDir: opts.ctx.workDir
         });
-        scenes = refined.scenes;
+        // Refine merges can exceed the band max — re-enforce it (fit never merges
+        // across the gaps refine's drops created, only contiguous spans).
+        scenes = fitScenesToDurationBand(refined.scenes, band);
         opts.onRefine?.(refined);
     }
     const sourceVideoId = opts.sourceVideoId ?? createIdV7("srcvid");

package/dist/src/services/provider-errors.js

@@ -0,0 +1,74 @@
+// Provider error taxonomy + HTTP-response classification, shared by every
+// surface that talks to an AI provider directly: ProviderService (cloud, leased
+// BYOK keys) and the speech module (also driven key-less from the devcli with a
+// local env key). Lives outside providers.ts so speech.ts can import it without
+// a circular dependency.
+export class ProviderKeyUnavailableError extends Error {
+}
+export class ProviderAuthError extends Error {
+}
+export class ProviderQuotaExceededError extends Error {
+}
+export class ProviderWaitExceededError extends Error {
+}
+export class ProviderLeaseTimeoutError extends Error {
+    retryAfterSeconds;
+    constructor(message, retryAfterSeconds = 5) {
+        super(message);
+        this.retryAfterSeconds = retryAfterSeconds;
+    }
+}
+export class ProviderRateLimitError extends Error {
+    retryAfterSeconds;
+    constructor(message, retryAfterSeconds = 60) {
+        super(message);
+        this.retryAfterSeconds = retryAfterSeconds;
+    }
+}
+export class ProviderTransientError extends Error {
+    retryAfterSeconds;
+    constructor(message, retryAfterSeconds = 5) {
+        super(message);
+        this.retryAfterSeconds = retryAfterSeconds;
+    }
+}
+export function parseRetryAfterSeconds(value) {
+    if (!value) {
+        return 60;
+    }
+    const numeric = Number(value);
+    if (Number.isFinite(numeric) && numeric > 0) {
+        return Math.max(1, Math.min(Math.ceil(numeric), 60 * 60));
+    }
+    const parsedDate = Date.parse(value);
+    if (Number.isFinite(parsedDate)) {
+        return Math.max(1, Math.min(Math.ceil((parsedDate - Date.now()) / 1000), 60 * 60));
+    }
+    return 60;
+}
+export function isHardQuotaError(details) {
+    const normalized = details.toLowerCase();
+    return [
+        "insufficient_quota",
+        "quota_exceeded",
+        "exceeded your current quota",
+        "billing",
+        "insufficient credits",
+        "not enough credits",
+        "credit balance",
+        "prepaid credits",
+        "resource_exhausted"
+    ].some((pattern) => normalized.includes(pattern));
+}
+export function conciseProviderDetails(details) {
+    const compact = details.replace(/\s+/g, " ").trim();
+    return compact ? `: ${compact.slice(0, 500)}` : "";
+}
+export async function rateLimitOrQuotaError(provider, response) {
+    const details = await response.text();
+    if (isHardQuotaError(details)) {
+        return new ProviderQuotaExceededError(`${provider} quota or billing limit was reached${conciseProviderDetails(details)}`);
+    }
+    return new ProviderRateLimitError(`${provider} rate limited${conciseProviderDetails(details)}`, parseRetryAfterSeconds(response.headers.get("retry-after")));
+}
+//# sourceMappingURL=provider-errors.js.map