@mevdragon/vidfarm-devcli 0.9.0 → 0.11.0

package/dist/src/services/clip-curation/gemini.js

@@ -38,6 +38,34 @@ export const EMBEDDING_MODELS = {
 // a library should stay on one embedding model (callers store embedding_model
 // and warn on mismatch).
 export const EMBEDDING_DIM = Number(process.env.VIDFARM_CLIP_EMBEDDING_DIM ?? 768);
+/**
+ * Standalone embedding surface — also composed by LocalAgentClipClient so a
+ * local-agent scan can still build a semantically searchable library when the
+ * user has a gemini/openai key lying around for embeddings only.
+ */
+export class ClipEmbeddingClient {
+    provider;
+    modelId;
+    model;
+    constructor(config) {
+        this.provider = config.provider;
+        this.modelId = EMBEDDING_MODELS[config.provider];
+        this.model = buildEmbeddingModel(config.provider, config.apiKey, this.modelId);
+    }
+    async embed(texts, taskType) {
+        if (texts.length === 0)
+            return [];
+        const providerOptions = this.provider === "gemini"
+            ? { google: { taskType, outputDimensionality: EMBEDDING_DIM } }
+            : { openai: { dimensions: EMBEDDING_DIM } };
+        const { embeddings } = await embedMany({
+            model: this.model,
+            values: texts.map((t) => (t.trim() ? t.trim() : " ")),
+            providerOptions
+        });
+        return embeddings.map((e) => normalize(e));
+    }
+}
 export class ClipModelClient {
     provider;
     tier;
@@ -46,7 +74,7 @@ export class ClipModelClient {
     embeddingProvider;
     embeddingModelId;
     tagModel;
-    embedModel;
+    embedder;
     constructor(opts) {
         if (!opts.apiKey)
             throw new Error(`A ${opts.provider ?? "gemini"} API key is required.`);
@@ -57,19 +85,19 @@ export class ClipModelClient {
         this.tagModel = buildLanguageModel(this.provider, opts.apiKey, this.tagModelId, opts.publicBaseUrl);
         const embedding = resolveEmbeddingConfig(this.provider, opts.apiKey, opts.embedding);
         if (embedding) {
+            this.embedder = new ClipEmbeddingClient(embedding);
             this.embeddingProvider = embedding.provider;
-            this.embeddingModelId = EMBEDDING_MODELS[embedding.provider];
-            this.embedModel = buildEmbeddingModel(embedding.provider, embedding.apiKey, this.embeddingModelId);
+            this.embeddingModelId = this.embedder.modelId;
         }
         else {
             this.embeddingProvider = null;
             this.embeddingModelId = null;
-            this.embedModel = null;
+            this.embedder = null;
         }
     }
     /** True when this client can produce embeddings (semantic search available). */
     get canEmbed() {
-        return this.embedModel !== null;
+        return this.embedder !== null;
     }
     /** The underlying tagging LanguageModel — used by the guided-segmentation pass. */
     get taggingModel() {
@@ -160,29 +188,50 @@ export class ClipModelClient {
         }
         return { type: "image", image };
     }
+    /**
+     * Keep/drop/merge decisions for prompt-guided segmentation — one multimodal
+     * call over one keyframe per candidate scene. Null on failure so the scan
+     * falls back to raw scenes.
+     */
+    async decideScenes(input) {
+        // No fixed-length constraint — Gemini's structured-output schema translation
+        // rejects minItems==maxItems; callers align by index instead.
+        const schema = z.object({
+            decisions: z.array(z.object({ keep: z.boolean(), merge_with_previous: z.boolean() }))
+        });
+        try {
+            const images = await Promise.all(input.imagePaths.map((p) => readFile(p)));
+            const { object } = await generateObject({
+                model: this.tagModel,
+                schema,
+                maxOutputTokens: input.maxOutputTokens,
+                messages: [
+                    {
+                        role: "user",
+                        content: [{ type: "text", text: input.prompt }, ...images.map((image) => ({ type: "image", image }))]
+                    }
+                ]
+            });
+            return object.decisions;
+        }
+        catch (error) {
+            if (process.env.VIDFARM_DEBUG_LOGS === "true") {
+                // eslint-disable-next-line no-console
+                console.warn(`[clips] refine call failed: ${error instanceof Error ? error.message : String(error)}`);
+            }
+            return null;
+        }
+    }
     async embedDocuments(texts) {
-        return this.embed(texts, "RETRIEVAL_DOCUMENT");
+        return this.embedder ? this.embedder.embed(texts, "RETRIEVAL_DOCUMENT") : [];
     }
     async embedQuery(text) {
-        const [vec] = await this.embed([text], "RETRIEVAL_QUERY");
+        const [vec] = await (this.embedder ? this.embedder.embed([text], "RETRIEVAL_QUERY") : Promise.resolve([]));
         return vec;
     }
     /** Batch-embed many retrieval queries (e.g. one per scene annotation) in a single call. */
     async embedQueries(texts) {
-        return this.embed(texts, "RETRIEVAL_QUERY");
-    }
-    async embed(texts, taskType) {
-        if (texts.length === 0 || !this.embedModel)
-            return [];
-        const providerOptions = this.embeddingProvider === "gemini"
-            ? { google: { taskType, outputDimensionality: EMBEDDING_DIM } }
-            : { openai: { dimensions: EMBEDDING_DIM } };
-        const { embeddings } = await embedMany({
-            model: this.embedModel,
-            values: texts.map((t) => (t.trim() ? t.trim() : " ")),
-            providerOptions
-        });
-        return embeddings.map((e) => normalize(e));
+        return this.embedder ? this.embedder.embed(texts, "RETRIEVAL_QUERY") : [];
     }
     /** Convert a natural-language query into a structured SplicingCriteria (categoricals constrained to the taxonomy). */
     async naturalLanguageToCriteria(query) {
@@ -271,7 +320,7 @@ export function buildClipEmbeddingText(tags, description) {
         parts.push(tagBits.join(", "));
     return parts.join("\n").trim() || "video clip";
 }
-function buildTaggingPrompt(transcript, version, hasAudio = false, guidancePrompt) {
+export function buildTaggingPrompt(transcript, version, hasAudio = false, guidancePrompt) {
     const speech = transcript?.trim()
         ? `The scene's spoken audio (ASR transcript) is: "${transcript.trim()}"`
         : hasAudio
@@ -299,7 +348,7 @@ function buildTaggingPrompt(transcript, version, hasAudio = false, guidancePromp
         "Keep the whole response under ~200 tokens."
     ].join("\n");
 }
-function buildQueryPrompt(query, version) {
+export function buildQueryPrompt(query, version) {
     return [
         "Convert this clip-search request into a structured filter.",
         `Request: "${query}"`,

package/dist/src/services/clip-curation/hunt.js

@@ -0,0 +1,332 @@
+// Clip-hunt spec — the user-facing controls for turning long-form video into
+// short-form clips: time WINDOWS ("only hunt between 12:30 and 15:45"), a soft
+// target DURATION band ("30 sec clips" → 20–40s), an output ASPECT crop
+// ("vertical video of people holding food to their face"), and an avoid-text
+// constraint ("without captions / no on-screen text" — a scene-SELECTION filter,
+// NEVER GhostCut on the long-form source). Shared by devcli (local), the Hono
+// API, and the cloud scan Lambda so every surface parses and normalizes
+// identically. All helpers are pure/deterministic — no AI calls here.
+// ── Duration band ───────────────────────────────────────────────────────────
+/**
+ * Map a spoken target ("10 sec", "30 sec", "60 sec", "120 sec") to a soft
+ * range. Anchors from product: 10 → 5–20, 30 → 20–40; the same formula extends
+ * smoothly (60 → 40–80, 120 → 80–160): ± target/3, with a minimum slack of
+ * −5s / +10s so short targets keep a usable band.
+ */
+export function resolveDurationBand(targetSec) {
+    if (!Number.isFinite(targetSec) || targetSec <= 0)
+        return null;
+    const target = Math.max(3, Math.min(600, targetSec));
+    const minSec = Math.max(2, round1(target - Math.max(target / 3, 5)));
+    const maxSec = round1(target + Math.max(target / 3, 10));
+    return { min_sec: minSec, max_sec: maxSec, target_sec: round1(target) };
+}
+// ── Timecodes & windows ─────────────────────────────────────────────────────
+/** "1:02:03(.5)" | "12:30" | "95" | "95.5" → seconds (null when unparseable). */
+export function parseTimecode(text) {
+    const trimmed = (text ?? "").trim();
+    if (!trimmed)
+        return null;
+    const parts = trimmed.split(":");
+    if (parts.length > 3)
+        return null;
+    let seconds = 0;
+    for (const part of parts) {
+        if (!/^\d+(\.\d+)?$/.test(part))
+            return null;
+        seconds = seconds * 60 + Number(part);
+    }
+    return Number.isFinite(seconds) ? seconds : null;
+}
+/**
+ * Parse "12:30-15:45, 20:00-22:10" (also "–" and "to" separators, or
+ * {start,end} objects / [start,end] tuples) into windows. Invalid entries are
+ * dropped rather than failing the scan.
+ */
+export function parseTimeRanges(input) {
+    const out = [];
+    const push = (start, end) => {
+        if (start == null || end == null)
+            return;
+        if (!Number.isFinite(start) || !Number.isFinite(end) || end <= start)
+            return;
+        out.push({ start_sec: round3(Math.max(0, start)), end_sec: round3(end) });
+    };
+    const parseOne = (value) => {
+        if (typeof value === "string") {
+            const m = value.trim().match(/^(.+?)\s*(?:-|–|—|\bto\b)\s*(.+)$/i);
+            if (m)
+                push(parseTimecode(m[1]), parseTimecode(m[2]));
+            return;
+        }
+        if (Array.isArray(value) && value.length === 2) {
+            push(toSeconds(value[0]), toSeconds(value[1]));
+            return;
+        }
+        if (value && typeof value === "object") {
+            const rec = value;
+            push(toSeconds(rec.start_sec ?? rec.start ?? rec.from), toSeconds(rec.end_sec ?? rec.end ?? rec.to));
+        }
+    };
+    if (typeof input === "string") {
+        for (const piece of input.split(/[,;]+/))
+            parseOne(piece);
+    }
+    else if (Array.isArray(input)) {
+        for (const piece of input)
+            parseOne(piece);
+    }
+    else if (input != null) {
+        parseOne(input);
+    }
+    return out;
+}
+function toSeconds(value) {
+    if (typeof value === "number")
+        return Number.isFinite(value) ? value : null;
+    if (typeof value === "string")
+        return parseTimecode(value);
+    return null;
+}
+/** Clamp to [0, durationSec], drop empties, sort, and merge overlaps. */
+export function normalizeWindows(windows, durationSec) {
+    if (!windows?.length)
+        return [];
+    const clamped = windows
+        .map((w) => ({
+        start_sec: Math.max(0, w.start_sec),
+        end_sec: durationSec && durationSec > 0 ? Math.min(w.end_sec, durationSec) : w.end_sec
+    }))
+        .filter((w) => Number.isFinite(w.start_sec) && Number.isFinite(w.end_sec) && w.end_sec - w.start_sec > 0.25)
+        .sort((a, b) => a.start_sec - b.start_sec);
+    const merged = [];
+    for (const w of clamped) {
+        const prev = merged[merged.length - 1];
+        if (prev && w.start_sec <= prev.end_sec + 0.05) {
+            prev.end_sec = Math.max(prev.end_sec, w.end_sec);
+        }
+        else {
+            merged.push({ ...w });
+        }
+    }
+    return merged;
+}
+/** The duration the scan actually processes — the cost-estimate input. */
+export function effectiveHuntDurationSec(fullDurationSec, windows) {
+    const normalized = normalizeWindows(windows, fullDurationSec);
+    if (!normalized.length)
+        return fullDurationSec;
+    return round3(normalized.reduce((acc, w) => acc + (w.end_sec - w.start_sec), 0));
+}
+// ── Aspect / crop ───────────────────────────────────────────────────────────
+const ASPECT_SYNONYMS = {
+    "9:16": "9:16",
+    "16:9": "16:9",
+    "4:3": "4:3",
+    "1:1": "1:1",
+    vertical: "9:16",
+    portrait: "9:16",
+    shorts: "9:16",
+    reels: "9:16",
+    tiktok: "9:16",
+    horizontal: "16:9",
+    landscape: "16:9",
+    widescreen: "16:9",
+    square: "1:1",
+    original: "original"
+};
+export function normalizeAspect(value) {
+    if (typeof value !== "string")
+        return null;
+    return ASPECT_SYNONYMS[value.trim().toLowerCase()] ?? null;
+}
+export function normalizeCropFocus(value) {
+    return value === "top" || value === "bottom" || value === "left" || value === "right" ? value : "center";
+}
+/**
+ * ffmpeg -vf crop expression for the target aspect: full-height (or full-width)
+ * centre crop with a focus bias — "smart centre" framing; the content guidance
+ * drives WHICH scenes are kept, this drives how each frame is cropped. Even
+ * output dims keep libx264 happy. Null = no crop.
+ */
+export function cropFilterFor(aspect, focus = "center") {
+    if (!aspect || aspect === "original")
+        return null;
+    const [aw, ah] = aspect.split(":").map(Number);
+    if (!aw || !ah)
+        return null;
+    const ratio = (aw / ah).toFixed(6);
+    const xf = focus === "left" ? "0.15" : focus === "right" ? "0.85" : "0.5";
+    // "top" biases toward faces/upper third when cropping vertically.
+    const yf = focus === "top" ? "0.2" : focus === "bottom" ? "0.8" : "0.5";
+    const w = `2*floor(min(iw\\,ih*${ratio})/2)`;
+    const h = `2*floor(min(ih\\,iw/${ratio})/2)`;
+    return `crop=w=${w}:h=${h}:x=(iw-ow)*${xf}:y=(ih-oh)*${yf}`;
+}
+// ── Scene fitting ───────────────────────────────────────────────────────────
+/**
+ * Fit detected scenes to a soft duration band: greedily merge CONTIGUOUS
+ * scenes toward the target (never across dropped gaps — post-refine lists have
+ * holes and merging across them would re-include dropped footage), and split
+ * over-long spans into ≤ max chunks. Safe to run both before refine (fewer,
+ * clip-sized units to judge) and after (re-enforce max on refine merges).
+ */
+export function fitScenesToDurationBand(scenes, band) {
+    if (!band || scenes.length === 0)
+        return scenes;
+    const EPS = 0.05;
+    const spans = [];
+    let cur = null;
+    for (const s of scenes) {
+        if (!cur) {
+            cur = { start: s.start_time_sec, end: s.end_time_sec };
+            continue;
+        }
+        const contiguous = Math.abs(s.start_time_sec - cur.end) <= EPS;
+        const curDur = cur.end - cur.start;
+        const nextDur = s.end_time_sec - s.start_time_sec;
+        const wantMore = curDur < band.min_sec || (curDur < band.target_sec && curDur + nextDur <= band.max_sec);
+        if (contiguous && wantMore) {
+            cur.end = s.end_time_sec;
+            continue;
+        }
+        spans.push(cur);
+        cur = { start: s.start_time_sec, end: s.end_time_sec };
+    }
+    if (cur)
+        spans.push(cur);
+    const fitted = [];
+    for (const span of spans) {
+        const dur = span.end - span.start;
+        if (dur > band.max_sec) {
+            const chunks = Math.ceil(dur / band.max_sec);
+            const step = dur / chunks;
+            for (let i = 0; i < chunks; i++) {
+                const start = span.start + i * step;
+                const end = i === chunks - 1 ? span.end : start + step;
+                fitted.push(makeScene(fitted.length, start, end));
+            }
+        }
+        else {
+            fitted.push(makeScene(fitted.length, span.start, span.end));
+        }
+    }
+    // Orphan slivers (isolated scenes far below the band) are noise when the user
+    // asked for e.g. 30s clips — drop them unless they are all we have.
+    const solid = fitted.filter((s) => s.duration_sec >= band.min_sec / 2);
+    const kept = solid.length > 0 ? solid : fitted;
+    return kept.map((s, i) => ({ ...s, index: i }));
+}
+// ── Guidance composition ────────────────────────────────────────────────────
+export const AVOID_TEXT_GUIDANCE = "Only keep scenes with NO burned-in captions, subtitles, watermark text, or on-screen text overlays; drop any scene whose frame shows readable overlay text.";
+/**
+ * Compose the effective guidance prompt for the refine (keep/drop/merge) and
+ * tagging passes: the user's content intent plus the avoid-text constraint.
+ * avoid_text is deliberately a SELECTION filter — never caption removal
+ * (GhostCut) on the long-form source.
+ */
+export function buildEffectiveGuidance(input) {
+    const parts = [];
+    if (input.contentPrompt?.trim())
+        parts.push(input.contentPrompt.trim());
+    if (input.avoidText)
+        parts.push(AVOID_TEXT_GUIDANCE);
+    return parts.join("\n");
+}
+/**
+ * Deterministically extract structured hints from a free-form hunt prompt:
+ * "please only hunt for clips between 12:30 and 15:45", "short 10 sec clips",
+ * "vertical video … without captions". Structured request fields always take
+ * precedence; this only fills gaps. The prompt itself stays intact as content
+ * guidance.
+ */
+export function parseClipHuntPrompt(prompt) {
+    const text = (prompt ?? "").trim();
+    const result = { windows: [], target_duration_sec: null, aspect: null, avoid_text: false };
+    if (!text)
+        return result;
+    const tc = "\\d{1,2}:\\d{2}(?::\\d{2})?(?:\\.\\d+)?";
+    // "between 12:30 and 15:45" / "from 12:30 to 15:45" / bare "12:30-15:45"
+    const rangeRe = new RegExp(`(?:between|from)?\\s*(${tc})\\s*(?:-|–|—|to|and|until)\\s*(${tc})`, "gi");
+    let m;
+    while ((m = rangeRe.exec(text)) !== null) {
+        const start = parseTimecode(m[1]);
+        const end = parseTimecode(m[2]);
+        if (start != null && end != null && end > start) {
+            result.windows.push({ start_sec: round3(start), end_sec: round3(end) });
+        }
+    }
+    const secMatch = /(\d{1,3}(?:\.\d+)?)\s*-?\s*(?:sec(?:ond)?s?)\b/i.exec(text);
+    const minMatch = /(\d{1,2}(?:\.\d+)?)\s*-?\s*(?:min(?:ute)?s?)\b(?!\s*(?:mark|point))/i.exec(text);
+    if (secMatch) {
+        result.target_duration_sec = Number(secMatch[1]);
+    }
+    else if (minMatch && /clip/i.test(text)) {
+        // "2 minute clips" — only treat minutes as a duration when clips are the topic.
+        result.target_duration_sec = Number(minMatch[1]) * 60;
+    }
+    const aspectToken = /(\d{1,2}:\d{1,2})\s*(?:video|clip|ratio|aspect|crop)/i.exec(text)?.[1];
+    result.aspect =
+        normalizeAspect(aspectToken) ??
+            (/(vertical|portrait)\b/i.test(text) ? "9:16"
+                : /(horizontal|landscape|widescreen)\b/i.test(text) ? "16:9"
+                    : /\bsquare\b/i.test(text) ? "1:1"
+                        : /\b4:3\b/.test(text) ? "4:3"
+                            : null);
+    result.avoid_text =
+        /(without|no)\s+(?:any\s+)?(captions?|subtitles?|(?:on[- ]?screen\s+)?text)/i.test(text) ||
+            /\b(caption|text)-free\b/i.test(text) ||
+            /no\s+text\s+on\s+screen/i.test(text);
+    return result;
+}
+// ── Wire normalization (API / SFN / tool payloads) ──────────────────────────
+/**
+ * Normalize an untrusted hunt-spec-shaped payload (HTTP body, tool args, SFN
+ * input) into a clean ClipHuntSpec. Unknown/invalid fields drop out silently.
+ */
+export function normalizeHuntSpec(raw) {
+    if (!raw || typeof raw !== "object")
+        return {};
+    const rec = raw;
+    const spec = {};
+    const windows = parseTimeRanges(rec.windows ?? rec.ranges ?? rec.time_ranges);
+    if (windows.length)
+        spec.windows = windows;
+    const band = rec.duration_band;
+    if (band && typeof band === "object" && Number(band.min_sec) > 0 && Number(band.max_sec) > Number(band.min_sec)) {
+        spec.duration_band = {
+            min_sec: Number(band.min_sec),
+            max_sec: Number(band.max_sec),
+            target_sec: Number(band.target_sec) > 0 ? Number(band.target_sec) : (Number(band.min_sec) + Number(band.max_sec)) / 2
+        };
+    }
+    else {
+        const target = Number(rec.target_duration_sec ?? rec.duration_sec ?? rec.clip_duration_sec);
+        const resolved = resolveDurationBand(target);
+        if (resolved)
+            spec.duration_band = resolved;
+    }
+    const aspect = normalizeAspect(rec.aspect ?? rec.aspect_ratio ?? rec.orientation);
+    if (aspect)
+        spec.aspect = aspect;
+    if (rec.crop_focus != null)
+        spec.crop_focus = normalizeCropFocus(rec.crop_focus);
+    if (rec.avoid_text === true || rec.no_text === true || rec.no_captions === true)
+        spec.avoid_text = true;
+    if (typeof rec.source_url === "string" && rec.source_url.trim())
+        spec.source_url = rec.source_url.trim();
+    return spec;
+}
+// ── misc ────────────────────────────────────────────────────────────────────
+function makeScene(index, start, end) {
+    const s = Math.max(0, start);
+    const e = Math.max(s + 0.01, end);
+    return { index, start_time_sec: round3(s), end_time_sec: round3(e), duration_sec: round3(e - s) };
+}
+function round1(n) {
+    return Math.round(n * 10) / 10;
+}
+function round3(n) {
+    return Math.round(n * 1000) / 1000;
+}
+//# sourceMappingURL=hunt.js.map

package/dist/src/services/clip-curation/index.js

@@ -4,7 +4,9 @@
 export * from "./types.js";
 export { ACTIVE_TAXONOMY_VERSION, MULTI_VALUE_CATEGORIES, getTaxonomy, sanitizeTagValues, taxonomyForPrompt } from "./taxonomy.js";
 export { BUILTIN_PRESETS, findBuiltinPreset } from "./presets.js";
-export { ClipModelClient, ClipGeminiClient, EMBEDDING_DIM, EMBEDDING_MODEL, EMBEDDING_MODELS, TAG_MODEL_BY_TIER, buildClipEmbeddingText, normalize } from "./gemini.js";
+export { ClipModelClient, ClipGeminiClient, ClipEmbeddingClient, EMBEDDING_DIM, EMBEDDING_MODEL, EMBEDDING_MODELS, TAG_MODEL_BY_TIER, buildClipEmbeddingText, buildQueryPrompt, buildTaggingPrompt, normalize } from "./gemini.js";
+export { LocalAgentClipClient, detectLocalAgent, extractJsonObject } from "./local-agent.js";
+export { AVOID_TEXT_GUIDANCE, buildEffectiveGuidance, cropFilterFor, effectiveHuntDurationSec, fitScenesToDurationBand, normalizeAspect, normalizeCropFocus, normalizeHuntSpec, normalizeWindows, parseClipHuntPrompt, parseTimecode, parseTimeRanges, resolveDurationBand } from "./hunt.js";
 export { detectScenes, extractClip, extractKeyframes, extractSceneAudio, extractThumbnail, hasFfmpeg, probeVideo, resolveFfmpeg, resolveFfprobe } from "./ffmpeg.js";
 export { cosineSimilarity, matchesCriteria, rankHits, scoreClip, searchClips, structuredMatchRatio } from "./query.js";
 export { estimateScanCostFromDuration, estimateScanCostFromScenes, formatCostEstimate } from "./cost.js";