@officexapp/vidfarm-devcli 0.21.10

package/dist/src/services/clip-curation/gemini.js

@@ -0,0 +1,418 @@
+// Model client shared by devcli + cloud. Multi-provider: the TAGGING model
+// (vision + optional audio ASR) is swappable across gemini / openai / openrouter
+// so users pick their cost/quality tradeoff; EMBEDDINGS ride on a resolvable
+// embedding provider (gemini or openai — OpenRouter has no /embeddings) so a
+// library stays in ONE vector space. BYOK: the caller passes provider keys.
+//
+// One place for the prompt templates so every provider produces byte-identical
+// clip metadata (same taxonomy, same schema).
+import { readFile } from "node:fs/promises";
+import { embedMany, generateObject } from "ai";
+import { createGoogleGenerativeAI } from "@ai-sdk/google";
+import { createOpenAI } from "@ai-sdk/openai";
+import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
+import { z } from "zod";
+import { ACTIVE_TAXONOMY_VERSION, getTaxonomy, sanitizeTagValues, taxonomyForPrompt } from "./taxonomy.js";
+// Tag model per provider/tier. Overridable via env so pinned models can move
+// forward without a code change. Handoff: gemini flash-lite (cheap, low media_resolution).
+function resolveTagModel(provider, tier) {
+    switch (provider) {
+        case "openai":
+            return process.env.VIDFARM_CLIP_TAG_MODEL_OPENAI ?? "gpt-5.4-nano";
+        case "openrouter":
+            return process.env.VIDFARM_CLIP_TAG_MODEL_OPENROUTER ?? "google/gemini-2.5-flash";
+        case "nvidia":
+            return process.env.VIDFARM_CLIP_TAG_MODEL_NVIDIA ?? "minimaxai/minimax-m3";
+        case "gemini":
+        default:
+            return tier === "flash"
+                ? process.env.VIDFARM_CLIP_TAG_MODEL_FLASH ?? "gemini-2.5-flash"
+                : process.env.VIDFARM_CLIP_TAG_MODEL_LITE ?? "gemini-2.5-flash-lite";
+    }
+}
+export const EMBEDDING_MODELS = {
+    gemini: process.env.VIDFARM_CLIP_EMBEDDING_MODEL ?? "gemini-embedding-001",
+    openai: process.env.VIDFARM_CLIP_EMBEDDING_MODEL_OPENAI ?? "text-embedding-3-small"
+};
+// gemini-embedding-001 and text-embedding-3-small both support reducing to 768
+// dims, so every provider's vectors fit the same fixed-width index. Cosine is
+// dimension-agnostic; still, vectors from DIFFERENT models are NOT comparable —
+// a library should stay on one embedding model (callers store embedding_model
+// and warn on mismatch).
+export const EMBEDDING_DIM = Number(process.env.VIDFARM_CLIP_EMBEDDING_DIM ?? 768);
+/**
+ * Standalone embedding surface — also composed by LocalAgentClipClient so a
+ * local-agent scan can still build a semantically searchable library when the
+ * user has a gemini/openai key lying around for embeddings only.
+ */
+export class ClipEmbeddingClient {
+    provider;
+    modelId;
+    model;
+    constructor(config) {
+        this.provider = config.provider;
+        this.modelId = EMBEDDING_MODELS[config.provider];
+        this.model = buildEmbeddingModel(config.provider, config.apiKey, this.modelId);
+    }
+    async embed(texts, taskType) {
+        if (texts.length === 0)
+            return [];
+        const providerOptions = this.provider === "gemini"
+            ? { google: { taskType, outputDimensionality: EMBEDDING_DIM } }
+            : { openai: { dimensions: EMBEDDING_DIM } };
+        const { embeddings } = await embedMany({
+            model: this.model,
+            values: texts.map((t) => (t.trim() ? t.trim() : " ")),
+            providerOptions
+        });
+        return embeddings.map((e) => normalize(e));
+    }
+}
+export class ClipModelClient {
+    provider;
+    tier;
+    taxonomyVersion;
+    tagModelId;
+    embeddingProvider;
+    embeddingModelId;
+    tagModel;
+    embedder;
+    constructor(opts) {
+        if (!opts.apiKey)
+            throw new Error(`A ${opts.provider ?? "gemini"} API key is required.`);
+        this.provider = opts.provider ?? "gemini";
+        this.tier = opts.tier ?? "flash-lite";
+        this.taxonomyVersion = opts.taxonomyVersion ?? ACTIVE_TAXONOMY_VERSION;
+        this.tagModelId = resolveTagModel(this.provider, this.tier);
+        this.tagModel = buildLanguageModel(this.provider, opts.apiKey, this.tagModelId, opts.publicBaseUrl);
+        const embedding = resolveEmbeddingConfig(this.provider, opts.apiKey, opts.embedding);
+        if (embedding) {
+            this.embedder = new ClipEmbeddingClient(embedding);
+            this.embeddingProvider = embedding.provider;
+            this.embeddingModelId = this.embedder.modelId;
+        }
+        else {
+            this.embeddingProvider = null;
+            this.embeddingModelId = null;
+            this.embedder = null;
+        }
+    }
+    /** True when this client can produce embeddings (semantic search available). */
+    get canEmbed() {
+        return this.embedder !== null;
+    }
+    /** The underlying tagging LanguageModel — used by the guided-segmentation pass. */
+    get taggingModel() {
+        return this.tagModel;
+    }
+    tagSchema() {
+        const cats = getTaxonomy(this.taxonomyVersion).categories;
+        const enumArray = (values) => z.array(z.enum(values)).default([]);
+        return z.object({
+            content_type: enumArray(cats.content_type),
+            subject: enumArray(cats.subject),
+            action: enumArray(cats.action),
+            emotion: enumArray(cats.emotion),
+            setting: enumArray(cats.setting),
+            composition: enumArray(cats.composition),
+            motion: enumArray(cats.motion),
+            energy: z.enum(cats.energy),
+            audio_type: enumArray(cats.audio_type),
+            transcript: z.string().default(""),
+            description: z.string()
+        });
+    }
+    /**
+     * Tag one scene from its keyframes (+ audio on Gemini, which transcribes in
+     * the same call). OpenAI/OpenRouter get frames only — their chat models don't
+     * take raw audio, so transcript stays empty unless supplied. Degrades to an
+     * empty-but-valid tag set on error so one bad scene never kills a scan.
+     */
+    async tagScene(input) {
+        const supportsAudio = this.provider === "gemini";
+        const hasAudio = supportsAudio && Boolean(input.audioPath);
+        const prompt = buildTaggingPrompt(input.transcript, this.taxonomyVersion, hasAudio, input.guidancePrompt);
+        const images = await Promise.all(input.keyframePaths.map((p) => readFile(p)));
+        const audio = hasAudio && input.audioPath ? await readFile(input.audioPath).catch(() => null) : null;
+        try {
+            const { object } = await generateObject({
+                model: this.tagModel,
+                schema: this.tagSchema(),
+                maxOutputTokens: this.provider === "openai" ? 1500 : 500,
+                providerOptions: this.taggingProviderOptions(),
+                messages: [
+                    {
+                        role: "user",
+                        content: [
+                            { type: "text", text: prompt },
+                            ...images.map((image) => this.imagePart(image)),
+                            ...(audio ? [{ type: "file", data: audio, mediaType: "audio/mpeg" }] : [])
+                        ]
+                    }
+                ]
+            });
+            const sanitized = sanitizeTagValues(object, this.taxonomyVersion);
+            const tags = {
+                ...sanitized,
+                transcript: (input.transcript || object.transcript || "").trim()
+            };
+            return { tags, description: String(object.description ?? "").trim() };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            // eslint-disable-next-line no-console
+            console.warn(`[clips] tag failed (${this.provider}) @${input.scene.start_time_sec.toFixed(1)}s: ${message}`);
+            return {
+                tags: {
+                    content_type: [],
+                    subject: [],
+                    action: [],
+                    emotion: [],
+                    setting: [],
+                    composition: [],
+                    motion: [],
+                    energy: "medium",
+                    audio_type: input.transcript ? ["dialogue"] : ["silence"],
+                    transcript: input.transcript || ""
+                },
+                description: input.transcript ? input.transcript.slice(0, 120) : ""
+            };
+        }
+    }
+    taggingProviderOptions() {
+        if (this.provider === "gemini") {
+            return { google: { mediaResolution: "MEDIA_RESOLUTION_LOW", structuredOutputs: true } };
+        }
+        return undefined;
+    }
+    imagePart(image) {
+        // Low image detail on OpenAI keeps the token cost near the Gemini low-res target.
+        if (this.provider === "openai") {
+            return { type: "image", image, providerOptions: { openai: { imageDetail: "low" } } };
+        }
+        return { type: "image", image };
+    }
+    /**
+     * Keep/drop/merge decisions for prompt-guided segmentation — one multimodal
+     * call over one keyframe per candidate scene. Null on failure so the scan
+     * falls back to raw scenes.
+     */
+    async decideScenes(input) {
+        // No fixed-length constraint — Gemini's structured-output schema translation
+        // rejects minItems==maxItems; callers align by index instead.
+        const schema = z.object({
+            decisions: z.array(z.object({ keep: z.boolean(), merge_with_previous: z.boolean() }))
+        });
+        try {
+            const images = await Promise.all(input.imagePaths.map((p) => readFile(p)));
+            const { object } = await generateObject({
+                model: this.tagModel,
+                schema,
+                maxOutputTokens: input.maxOutputTokens,
+                messages: [
+                    {
+                        role: "user",
+                        content: [{ type: "text", text: input.prompt }, ...images.map((image) => ({ type: "image", image }))]
+                    }
+                ]
+            });
+            return object.decisions;
+        }
+        catch (error) {
+            // Surface the provider's own failure with the model id so a scan that dies
+            // here reads as an AI-provider limitation (e.g. a text-only model that
+            // can't take image keyframes, or one that can't return a structured JSON
+            // schema) rather than a Vidfarm bug. The caller (refineScenesWithGuidance)
+            // decides whether this is fatal.
+            const message = error instanceof Error ? error.message : String(error);
+            // The nvidia "lite" key is a text-only model, so it can never run the
+            // vision + structured-output request AI clip hunting needs — call that
+            // out with a concrete next step. Other providers may just be a transient
+            // failure, so stay neutral and pass their raw error through.
+            const hint = this.provider === "nvidia"
+                ? ` This model is text-only and can't do the vision + structured-output request AI clip hunting needs — use a Gemini or OpenAI key for clip hunting.`
+                : "";
+            throw new Error(`Your ${this.provider} AI provider (model ${this.tagModelId}) failed the guided clip-selection step. ` +
+                `This is an error from the AI provider, not Vidfarm.${hint} Provider error: ${message}`);
+        }
+    }
+    async embedDocuments(texts) {
+        return this.embedder ? this.embedder.embed(texts, "RETRIEVAL_DOCUMENT") : [];
+    }
+    async embedQuery(text) {
+        const [vec] = await (this.embedder ? this.embedder.embed([text], "RETRIEVAL_QUERY") : Promise.resolve([]));
+        return vec;
+    }
+    /** Batch-embed many retrieval queries (e.g. one per scene annotation) in a single call. */
+    async embedQueries(texts) {
+        return this.embedder ? this.embedder.embed(texts, "RETRIEVAL_QUERY") : [];
+    }
+    /** Convert a natural-language query into a structured SplicingCriteria (categoricals constrained to the taxonomy). */
+    async naturalLanguageToCriteria(query) {
+        const cats = getTaxonomy(this.taxonomyVersion).categories;
+        const opt = (values) => z.array(z.enum(values)).optional();
+        const schema = z.object({
+            content_type: opt(cats.content_type),
+            subject: opt(cats.subject),
+            action: opt(cats.action),
+            emotion: opt(cats.emotion),
+            setting: opt(cats.setting),
+            composition: opt(cats.composition),
+            motion: opt(cats.motion),
+            audio_type: opt(cats.audio_type),
+            energy: z.array(z.enum(cats.energy)).optional(),
+            min_duration_sec: z.number().optional(),
+            max_duration_sec: z.number().optional(),
+            has_dialogue: z.boolean().optional(),
+            has_subject: z.boolean().optional()
+        });
+        try {
+            const { object } = await generateObject({
+                model: this.tagModel,
+                schema,
+                maxOutputTokens: this.provider === "openai" ? 1200 : 300,
+                prompt: buildQueryPrompt(query, this.taxonomyVersion)
+            });
+            return { ...pruneUndefined(object), semantic_text: query };
+        }
+        catch {
+            return { semantic_text: query };
+        }
+    }
+}
+/** Back-compat alias — the client used to be Gemini-only. */
+export const ClipGeminiClient = ClipModelClient;
+function buildLanguageModel(provider, apiKey, modelId, publicBaseUrl) {
+    switch (provider) {
+        case "openai":
+            return createOpenAI({ apiKey })(modelId);
+        case "openrouter":
+            return createOpenAICompatible({
+                name: "openrouter",
+                apiKey,
+                baseURL: "https://openrouter.ai/api/v1",
+                headers: { "HTTP-Referer": publicBaseUrl ?? "https://vidfarm.cc", "X-Title": "Vidfarm Clips" }
+            }).chatModel(modelId);
+        case "nvidia":
+            // Lite, clipping-first key — OpenAI-compatible multimodal chat (frames in,
+            // no audio). No embeddings API (handled by resolveEmbeddingConfig).
+            return createOpenAICompatible({
+                name: "nvidia",
+                apiKey,
+                baseURL: "https://integrate.api.nvidia.com/v1"
+            }).chatModel(modelId);
+        case "gemini":
+        default:
+            return createGoogleGenerativeAI({ apiKey })(modelId);
+    }
+}
+function buildEmbeddingModel(provider, apiKey, modelId) {
+    return provider === "openai"
+        ? createOpenAI({ apiKey }).textEmbeddingModel(modelId)
+        : createGoogleGenerativeAI({ apiKey }).textEmbeddingModel(modelId);
+}
+function resolveEmbeddingConfig(provider, apiKey, explicit) {
+    if (explicit === null)
+        return null; // caller explicitly disabled embeddings
+    if (explicit)
+        return explicit;
+    if (provider === "gemini")
+        return { provider: "gemini", apiKey };
+    if (provider === "openai")
+        return { provider: "openai", apiKey };
+    return null; // openrouter: no embeddings unless an explicit config is given
+}
+/** The document text embedded for a clip: description + transcript + salient tags + private owner notes. */
+export function buildClipEmbeddingText(tags, description, userNotes) {
+    const parts = [];
+    if (description)
+        parts.push(description);
+    if (tags.transcript)
+        parts.push(`Transcript: ${tags.transcript}`);
+    const tagBits = [
+        ...(tags.content_type ?? []),
+        ...tags.subject,
+        ...tags.action,
+        ...tags.emotion,
+        ...tags.setting,
+        ...tags.composition,
+        ...tags.motion,
+        ...tags.audio_type,
+        `energy:${tags.energy}`
+    ];
+    if (tagBits.length)
+        parts.push(tagBits.join(", "));
+    if (userNotes?.trim())
+        parts.push(`Owner notes: ${userNotes.trim()}`);
+    return parts.join("\n").trim() || "video clip";
+}
+export function buildTaggingPrompt(transcript, version, hasAudio = false, guidancePrompt) {
+    const speech = transcript?.trim()
+        ? `The scene's spoken audio (ASR transcript) is: "${transcript.trim()}"`
+        : hasAudio
+            ? `An audio clip of the scene is attached — transcribe any speech into the transcript field (use "" if there is no speech).`
+            : `There is no clear speech in this scene (no dialogue).`;
+    const guidance = guidancePrompt?.trim()
+        ? `\nThe user is curating clips for this goal: "${guidancePrompt.trim()}". Bias the description and tags toward what matters for that goal (stay accurate — never invent tags not visible in the scene).\n`
+        : "";
+    return [
+        "You are tagging one short video scene for a searchable clip library.",
+        "You are given 1-3 keyframes from the scene and its audio transcript.",
+        speech,
+        guidance,
+        "",
+        "Return JSON matching the schema. Use ONLY values from this controlled vocabulary:",
+        taxonomyForPrompt(version),
+        "",
+        "Rules:",
+        "- `content_type` is the single most important tag: what KIND of shot is this? Pick the 1-2 best from talking_head (a person speaking to camera), b_roll (supporting/cutaway footage with no one addressing the camera), product_shot (a product is the focus), screen_recording (a screen/app/UI capture), demo (showing how something works), reaction (someone reacting), interview (two+ people in conversation), establishing (a wide scene-setting shot), lifestyle (ambient real-life moment), text_graphic (text card / motion graphic). Always set at least one when you can tell.",
+        "- Pick only tags you are confident about from the keyframes/audio. Omit a category (empty list) rather than guess.",
+        "- If there is no clear person/subject, use subject: [\"no_subject\"].",
+        "- If there is no speech, set audio_type appropriately (e.g. music/ambient/silence) and leave transcript empty.",
+        "- `energy` is a single value: low | medium | high.",
+        "- `transcript` should echo the spoken words verbatim (or \"\" if none).",
+        "- `description`: one short factual sentence (<= 20 words) describing what happens — this drives semantic search, so be concrete.",
+        "Keep the whole response under ~200 tokens."
+    ].join("\n");
+}
+export function buildQueryPrompt(query, version) {
+    return [
+        "Convert this clip-search request into a structured filter.",
+        `Request: "${query}"`,
+        "",
+        "Use ONLY values from this controlled vocabulary (omit a field if the request doesn't constrain it):",
+        taxonomyForPrompt(version),
+        "",
+        "Only include fields the request clearly implies. Prefer fewer, high-precision constraints.",
+        "Set has_dialogue/has_subject only when the request is explicit about speech or a subject.",
+        "Durations are in seconds."
+    ].join("\n");
+}
+function pruneUndefined(obj) {
+    const out = {};
+    for (const [k, v] of Object.entries(obj)) {
+        if (v === undefined)
+            continue;
+        if (Array.isArray(v) && v.length === 0)
+            continue;
+        out[k] = v;
+    }
+    return out;
+}
+/** L2-normalize a vector so cosine == dot product (and reduced-dim embeddings compare correctly). */
+export function normalize(vec) {
+    let sum = 0;
+    for (const v of vec)
+        sum += v * v;
+    const norm = Math.sqrt(sum);
+    if (norm === 0)
+        return vec;
+    return vec.map((v) => v / norm);
+}
+// Legacy export kept for callers that referenced the old constant.
+export const TAG_MODEL_BY_TIER = {
+    "flash-lite": resolveTagModel("gemini", "flash-lite"),
+    flash: resolveTagModel("gemini", "flash")
+};
+export const EMBEDDING_MODEL = EMBEDDING_MODELS.gemini;
+//# sourceMappingURL=gemini.js.map