@mevdragon/vidfarm-devcli 0.7.1 → 0.8.0

package/dist/src/services/clip-search.js

@@ -0,0 +1,47 @@
+// Cloud clip search — the two-stage flow (structured filter → semantic re-rank)
+// wired over DynamoDB records + the configured vector index. Same result shape
+// as the devcli store so the API mirrors the CLI (success criterion #4).
+import { matchesCriteria, rankHits, scoreClip, searchClips } from "./clip-curation/index.js";
+import { clipRecords } from "./clip-records.js";
+import { resolveClipVectorIndex } from "./clip-vectors.js";
+const vectorIndex = resolveClipVectorIndex((owner) => clipRecords.listClipsWithEmbeddings(owner));
+export async function searchUserClips(owner, input) {
+    const limit = input.limit ?? 20;
+    const all = await clipRecords.listClips(owner, { limit: 5000 });
+    const structured = all.filter((c) => matchesCriteria(c, input.criteria));
+    if (structured.length > 0) {
+        // Semantic re-rank the survivors (embeddings already ride on the records).
+        const hits = structured.map((c) => scoreClip(c, input.criteria, input.queryEmbedding, false));
+        return rankHits(hits, limit);
+    }
+    // No structured match — relax to a semantic-only pass via the vector index.
+    if (input.queryEmbedding) {
+        const knn = await vectorIndex.query(owner, input.queryEmbedding, Math.max(limit * 5, 50));
+        const byId = new Map(all.map((c) => [c.clip_id, c]));
+        const pool = knn.map((h) => byId.get(h.clip_id)).filter((c) => Boolean(c));
+        const hits = pool.map((c) => scoreClip(c, input.criteria, input.queryEmbedding, true));
+        return rankHits(hits, limit);
+    }
+    return [];
+}
+export { vectorIndex };
+/**
+ * Match many scene annotations against the owner's clip library at once. Loads
+ * the library ONCE (embeddings ride on the records) and runs each scene's
+ * structured filter + semantic re-rank in memory — so a whole decomposed video
+ * costs one library read plus M cheap in-memory ranks.
+ */
+export async function matchScenesToClips(owner, queries) {
+    const all = await clipRecords.listClips(owner, { limit: 5000 });
+    return queries.map((q) => ({
+        id: q.id,
+        meta: q.meta,
+        matches: searchClips(all, {
+            criteria: q.criteria,
+            queryEmbedding: q.queryEmbedding,
+            limit: q.limit ?? 5,
+            relaxWhenEmpty: true
+        })
+    }));
+}
+//# sourceMappingURL=clip-search.js.map

package/dist/src/services/clip-vectors.js

@@ -0,0 +1,125 @@
+// Vector index abstraction for cloud clip search. Two backends behind one
+// interface so the pipeline is portable and cloud infra cost stays deferrable:
+//
+//  • "dynamo" (default) — embeddings live on the clip records in DynamoDB;
+//    search brute-forces cosine in memory. Zero extra infra, works with the
+//    local-dynamo shim in `serve` mode. Fine to thousands of clips per user.
+//  • "s3vectors" — Amazon S3 Vectors index (handoff target). Best-effort:
+//    dynamically imports @aws-sdk/client-s3vectors and errors clearly if the
+//    dep/config is absent, so selecting it is opt-in.
+//
+// Both return the same {clip_id, similarity} shape; the caller applies the
+// structured filter + final scoring with the shared query builder.
+import { cosineSimilarity } from "./clip-curation/index.js";
+import { config } from "../config.js";
+/** Default backend: brute-force cosine over the owner's clip embeddings in DynamoDB. */
+export class DynamoVectorIndex {
+    loadEmbeddings;
+    backend = "dynamo";
+    constructor(loadEmbeddings) {
+        this.loadEmbeddings = loadEmbeddings;
+    }
+    async upsert() {
+        /* embedding is stored on the clip record itself */
+    }
+    async delete() {
+        /* removed with the clip record */
+    }
+    async query(owner, queryEmbedding, k) {
+        const clips = await this.loadEmbeddings(owner);
+        return clips
+            .map((c) => ({ clip_id: c.clip_id, similarity: cosineSimilarity(c.embedding, queryEmbedding) }))
+            .sort((a, b) => b.similarity - a.similarity)
+            .slice(0, k);
+    }
+}
+/**
+ * Amazon S3 Vectors backend. Vectors are keyed by clip_id within a per-owner
+ * partition; metadata carries source_video_id for filtering. Requires
+ * VIDFARM_CLIP_VECTORS_BUCKET + VIDFARM_CLIP_VECTORS_INDEX and the
+ * @aws-sdk/client-s3vectors package.
+ */
+export class S3VectorsIndex {
+    bucket;
+    indexName;
+    backend = "s3vectors";
+    clientPromise = null;
+    constructor(bucket, indexName) {
+        this.bucket = bucket;
+        this.indexName = indexName;
+    }
+    async client() {
+        if (!this.clientPromise) {
+            this.clientPromise = import("@aws-sdk/client-s3vectors").catch(() => {
+                throw new Error("S3 Vectors backend selected but @aws-sdk/client-s3vectors is not installed. `npm i @aws-sdk/client-s3vectors` or set VIDFARM_CLIP_VECTOR_BACKEND=dynamo.");
+            });
+        }
+        const mod = await this.clientPromise;
+        return { mod, client: new mod.S3VectorsClient({}) };
+    }
+    async upsert(owner, clip) {
+        if (!clip.embedding?.length)
+            return;
+        const { mod, client } = await this.client();
+        await client.send(new mod.PutVectorsCommand({
+            vectorBucketName: this.bucket,
+            indexName: this.indexName,
+            vectors: [
+                {
+                    key: vectorKey(owner, clip.clip_id),
+                    data: { float32: clip.embedding },
+                    metadata: { owner, source_video_id: clip.source_video_id, clip_id: clip.clip_id }
+                }
+            ]
+        }));
+    }
+    async delete(owner, clipId) {
+        const { mod, client } = await this.client();
+        await client.send(new mod.DeleteVectorsCommand({ vectorBucketName: this.bucket, indexName: this.indexName, keys: [vectorKey(owner, clipId)] }));
+    }
+    async query(owner, queryEmbedding, k) {
+        const { mod, client } = await this.client();
+        const res = await client.send(new mod.QueryVectorsCommand({
+            vectorBucketName: this.bucket,
+            indexName: this.indexName,
+            topK: k,
+            queryVector: { float32: queryEmbedding },
+            filter: { owner },
+            returnMetadata: true,
+            returnDistance: true
+        }));
+        const vectors = res.vectors ?? [];
+        return vectors.map((v) => ({
+            clip_id: v.metadata?.clip_id ?? stripVectorKey(v.key),
+            // S3 Vectors returns cosine DISTANCE (0..2); convert to similarity.
+            similarity: 1 - Number(v.distance ?? 0)
+        }));
+    }
+}
+function vectorKey(owner, clipId) {
+    return `${owner}/${clipId}`;
+}
+function stripVectorKey(key) {
+    const idx = key.lastIndexOf("/");
+    return idx >= 0 ? key.slice(idx + 1) : key;
+}
+let cached = null;
+/** Resolve the configured vector index (default: dynamo brute-force). */
+export function resolveClipVectorIndex(loadEmbeddings) {
+    if (cached)
+        return cached;
+    const backend = (process.env.VIDFARM_CLIP_VECTOR_BACKEND ?? "dynamo").toLowerCase();
+    if (backend === "s3vectors") {
+        const bucket = process.env.VIDFARM_CLIP_VECTORS_BUCKET;
+        const index = process.env.VIDFARM_CLIP_VECTORS_INDEX;
+        if (bucket && index) {
+            cached = new S3VectorsIndex(bucket, index);
+            return cached;
+        }
+        // Missing config → fall back to the always-available dynamo backend.
+    }
+    cached = new DynamoVectorIndex(loadEmbeddings);
+    void config; // config imported for parity with sibling services / future use
+    return cached;
+}
+//# sourceMappingURL=clip-vectors.js.map

package/dist/src/services/hyperframes.js

@@ -15,6 +15,12 @@ import { config } from "../config.js";
 import { devErrorFields, devLog } from "../lib/dev-log.js";
 import { KEN_BURNS_CSS, KEN_BURNS_STYLE_MARKER, buildHyperframeCompositionHtml } from "../hyperframes/composition.js";
 import { applyMarkupUsd } from "./billing-pricing.js";
+// The scene-annotation pass reuses the clip library's controlled vocabulary and
+// embedding-text recipe verbatim so a source scene's clip-match query is
+// symmetric with how a candidate clip was tagged/embedded. This is the ONE
+// coupling to the (parallel) clip-curation work — a plain leaf import; the
+// retrieval logic itself (search over these criteria) is merged in later.
+import { buildClipEmbeddingText, sanitizeTagValues, taxonomyForPrompt } from "./clip-curation/index.js";
 let stackCache = null;
 const SMART_DECOMPOSE_TEMP_PREFIX = "vidfarm-smart-decompose-";
 const SMART_DECOMPOSE_TEMP_MAX_AGE_MS = 24 * 60 * 60 * 1000;
@@ -477,6 +483,37 @@ if (ffmpegBin && !process.env.HYPERFRAMES_FFMPEG_PATH?.trim()) {
 if (ffprobeBin && !process.env.HYPERFRAMES_FFPROBE_PATH?.trim()) {
     process.env.HYPERFRAMES_FFPROBE_PATH = ffprobeBin;
 }
+// The ARCHETYPE / production style of a source video. Detected once per
+// decompose and used two ways: (a) stamped onto the template metadata so the
+// catalog + clip matcher can filter by format, and (b) to CONDITION the
+// per-scene clip annotations, because what counts as a scene's "literal DNA"
+// differs by format — an AI fruit drama is defined by its stylized character +
+// action, a UGC trend by the creator's face/gesture/authenticity, an
+// educational series by the board/diagram/screen, live action by real-world
+// subjects + cinematography. Controlled + extendable; ambiguous videos fall back
+// to "other".
+export const CONTENT_FORMATS = [
+    "ai_fruit_drama", // AI-generated surreal food/object dramas ("fruit drama")
+    "ai_generative_skit", // other AI-generated character skits / dramas
+    "ugc_trend", // phone-shot creator-to-camera trend/challenge
+    "talking_head", // single speaker addressing camera (advice/opinion)
+    "educational_series", // explainer / how-to / lecture / listicle
+    "live_action", // real-world filmed scenes, actors, cinematography
+    "product_demo", // hands-on showcase / unboxing / review
+    "street_interview", // vox-pop / man-on-the-street
+    "reaction_commentary", // reacting to / narrating other footage
+    "cinematic_broll", // moody establishing / atmosphere / montage
+    "screen_recording", // app/screen capture, tutorials, gameplay
+    "animation_motion_graphics", // 2D/3D animation, kinetic typography
+    "other"
+];
+export const DEFAULT_CONTENT_FORMAT = {
+    format: "other",
+    label: "Unclassified",
+    confidence: 0,
+    rationale: "",
+    secondary: []
+};
 const RUN_BINARY_MAX_BUFFER_BYTES = 10 * 1024 * 1024;
 function runBinary(bin, args) {
     return new Promise((resolve, reject) => {
@@ -1003,6 +1040,311 @@ async function runPlacementSurfacePass(input) {
     const attempt = await callVisionWithFallback({ prompt, frames: input.frames, keys: input.keys });
     return normalizePlacementSurfaces(attempt.result);
 }
+// ---------------------------------------------------------------------------
+// Scene-annotation pass — clip-replacement DNA.
+//
+// A dependent vision pass (runs AFTER scenes are finalized, so annotations line
+// up 1:1 by scene slug) that (1) classifies the whole video's content FORMAT and
+// (2) for each scene emits the literal + viral DNA needed to find a REPLACEMENT
+// clip from the clip library. It reuses the sparse frames already sampled for
+// the main pass (no extra ffmpeg) and the clip library's own controlled tag
+// vocabulary + embedding recipe, so a scene's clip-match query is symmetric with
+// how a candidate clip was tagged/embedded. Non-fatal: callers wrap in .catch().
+// ---------------------------------------------------------------------------
+// Vocabulary for a scene's role in the retention arc. Constrained in the prompt,
+// clamped here; free-form-ish (unknown → "context") so it never blocks output.
+const SCENE_ROLE_VOCAB = [
+    "hook", "build", "reveal", "payoff", "transition", "cta", "proof", "context", "punchline", "loop"
+];
+function asPlainRecord(value) {
+    return value && typeof value === "object" && !Array.isArray(value) ? value : {};
+}
+function readAnnotationString(value) {
+    return typeof value === "string" ? value.trim() : "";
+}
+function readAnnotationStringList(value, max) {
+    if (!Array.isArray(value))
+        return [];
+    const out = [];
+    for (const v of value) {
+        const s = typeof v === "string" ? v.trim() : "";
+        if (s && !out.includes(s))
+            out.push(s);
+        if (out.length >= max)
+            break;
+    }
+    return out;
+}
+function humanizeContentFormat(format) {
+    return format
+        .split("_")
+        .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
+        .join(" ");
+}
+function normalizeContentFormat(raw) {
+    const rec = asPlainRecord(raw);
+    const slug = normalizeSlug(readAnnotationString(rec.format));
+    const format = CONTENT_FORMATS.includes(slug)
+        ? slug
+        : "other";
+    const secondaryRaw = Array.isArray(rec.secondary) ? rec.secondary : [];
+    const secondary = Array.from(new Set(secondaryRaw
+        .map((s) => normalizeSlug(readAnnotationString(s)))
+        .filter((s) => CONTENT_FORMATS.includes(s) && s !== format))).slice(0, 3);
+    const conf = Number(rec.confidence);
+    const label = readAnnotationString(rec.label);
+    return {
+        format,
+        label: label || humanizeContentFormat(format),
+        confidence: Number.isFinite(conf) ? Math.max(0, Math.min(1, conf)) : 0,
+        rationale: readAnnotationString(rec.rationale),
+        secondary
+    };
+}
+function normalizeSceneRole(value) {
+    const slug = normalizeSlug(readAnnotationString(value));
+    return SCENE_ROLE_VOCAB.includes(slug) ? slug : "context";
+}
+function normalizeReplaceability(value) {
+    const s = readAnnotationString(value).toLowerCase();
+    return s === "high" || s === "low" ? s : "medium";
+}
+// Deterministic per-scene spoken text pulled from the transcript segments that
+// overlap the scene window — more reliable than asking the vision model to
+// re-transcribe, and keeps `tags.transcript` grounded in the audio pass.
+function transcriptForScene(transcript, scene) {
+    if (!transcript || !Array.isArray(transcript.segments))
+        return "";
+    const end = scene.start + scene.duration;
+    return transcript.segments
+        .filter((seg) => seg.end > scene.start && seg.start < end)
+        .map((seg) => seg.text.trim())
+        .filter((t) => t.length > 0)
+        .join(" ")
+        .trim();
+}
+// Compose the retrieval query text. Leads with the clip-doc recipe (description +
+// tags) so it lands near literally-similar clip embeddings, then appends the
+// viral purpose so the vector also reflects the beat's INTENT — a replacement
+// must match both axes.
+function buildSceneEmbeddingText(input) {
+    const literalDoc = buildClipEmbeddingText(input.tags, input.literal);
+    const purpose = [
+        input.scene_role ? `Role: ${input.scene_role}` : "",
+        input.narrative_purpose,
+        input.why_it_works
+    ].filter((p) => p && p.length > 0).join(". ");
+    return purpose ? `${literalDoc}\nPurpose: ${purpose}` : literalDoc;
+}
+// Turn a scene's literal tags into a ready-to-run SplicingCriteria for the clip
+// library. Tag constraints are the high-precision structured filter; the
+// duration floor lets candidates be trimmed to fit; has_subject/has_dialogue are
+// derived from the tags. semantic_text carries the embedding query for re-rank.
+function buildSceneMatchCriteria(tags, scene, semanticText) {
+    const criteria = { semantic_text: semanticText };
+    if (tags.subject.length)
+        criteria.subject = tags.subject;
+    if (tags.action.length)
+        criteria.action = tags.action;
+    if (tags.emotion.length)
+        criteria.emotion = tags.emotion;
+    if (tags.setting.length)
+        criteria.setting = tags.setting;
+    if (tags.composition.length)
+        criteria.composition = tags.composition;
+    if (tags.motion.length)
+        criteria.motion = tags.motion;
+    if (tags.audio_type.length)
+        criteria.audio_type = tags.audio_type;
+    if (tags.energy)
+        criteria.energy = [tags.energy];
+    if (Number.isFinite(scene.duration) && scene.duration > 0) {
+        criteria.min_duration_sec = Number(Math.max(0, scene.duration * 0.6).toFixed(2));
+    }
+    const hasRealSubject = tags.subject.some((s) => s !== "no_subject");
+    if (hasRealSubject)
+        criteria.has_subject = true;
+    else if (tags.subject.includes("no_subject"))
+        criteria.has_subject = false;
+    if (tags.audio_type.includes("dialogue"))
+        criteria.has_dialogue = true;
+    return criteria;
+}
+function buildSceneAnnotationPrompt(input) {
+    const frameListing = input.frameTimestamps.length > 0
+        ? `\nFRAMES SUPPLIED (in order, each captioned with its capture timestamp):\n${input.frameTimestamps.map((t, i) => `  frame ${i + 1} @ ${t.toFixed(2)}s`).join("\n")}\n`
+        : "";
+    const sceneListing = input.scenes.map((s) => (`  - scene_slug="${s.slug}" @ ${s.start.toFixed(2)}s for ${s.duration.toFixed(2)}s — ${s.label}: ${s.description || "(no description)"}${s.viral_note ? ` [viral note: ${s.viral_note}]` : ""}`)).join("\n");
+    const dna = input.viralDna;
+    const dnaBlock = `Video-level viral DNA (context for each scene's PURPOSE):\n  hook: ${dna.hook || "—"}\n  retention: ${dna.retention || "—"}\n  payoff: ${dna.payoff || "—"}\n  preserve: ${(dna.preserve || []).join("; ") || "—"}\n  avoid: ${(dna.avoid || []).join("; ") || "—"}`;
+    const userGuidance = input.userPrompt && input.userPrompt.trim().length > 0
+        ? `\nUSER GUIDANCE (highest priority):\n"""\n${input.userPrompt.trim()}\n"""\n`
+        : "";
+    return `You are Vidfarm's scene annotator. For EACH scene of this short-form vertical video, write an annotation rich enough to do TWO things: (A) find a REPLACEMENT clip from a searchable clip library — a different piece of footage that could stand in for this scene without breaking the video; and (B) let an AI agent RE-CREATE this scene from scratch with an image/video generator when no library clip fits.
+
+Both goals hang on capturing the same TWO axes:
+  1) LITERAL DNA — what is physically on screen (subject, action, setting, shot, motion, mood). A candidate or generated clip must look and feel like this.
+  2) VIRAL / NARRATIVE DNA — the ROLE this beat plays in why the video works (the hook, the reveal, the payoff, a transition...). A replacement or regeneration must serve the same purpose in the story, or the swap kills retention.
+${userGuidance}
+FIRST classify the whole video's FORMAT (its production archetype) — what counts as "literal DNA" differs by format. Pick the single best "format" and list any close seconds in "secondary". Allowed formats:
+  ${CONTENT_FORMATS.join(", ")}
+Format guidance: ai_fruit_drama / ai_generative_skit = AI-generated surreal character dramas (e.g. talking fruit); ugc_trend = phone-shot creator-to-camera trend; talking_head = single speaker giving advice; educational_series = explainer / how-to / listicle; live_action = real filmed scenes / actors; product_demo = hands-on showcase / unboxing; street_interview = man-on-the-street; reaction_commentary = reacting to other footage; cinematic_broll = moody montage / atmosphere; screen_recording = app / screen capture; animation_motion_graphics = animation / kinetic text.
+
+Video duration: ${input.durationSeconds.toFixed(2)}s. One-line summary: ${input.summary || "(none)"}.
+${dnaBlock}
+${frameListing}
+SCENES to annotate (return EXACTLY one annotation per scene_slug below, in this order):
+${sceneListing}
+
+For the LITERAL "tags", use ONLY values from this controlled vocabulary — it is the SAME vocabulary the clip library is tagged with, so your tags must line up with how a candidate clip would be tagged:
+${taxonomyForPrompt()}
+
+Return strictly valid JSON matching this schema:
+{
+  "content_format": { "format": string, "label": string, "confidence": number, "rationale": string, "secondary": [string, ...] },
+  "scenes": [{
+    "scene_slug": string,
+    "literal": string,
+    "tags": { "subject": [string], "action": [string], "emotion": [string], "setting": [string], "composition": [string], "motion": [string], "energy": "low"|"medium"|"high", "audio_type": [string], "transcript": string },
+    "scene_role": string,
+    "narrative_purpose": string,
+    "why_it_works": string,
+    "must_preserve": [string, ...],
+    "can_vary": [string, ...],
+    "replaceability": "high"|"medium"|"low",
+    "replaceability_reason": string,
+    "recreation_prompt": string,
+    "recreation_notes": [string, ...]
+  }]
+}
+
+RULES:
+- Output one scene object per scene_slug listed above, using the EXACT scene_slug strings. Do not invent, merge, split, or reorder scenes.
+- "literal": one concrete, factual sentence describing exactly what is on screen in this scene (subject + action + setting + shot). This is the primary match key — be specific and visual, not interpretive.
+- "tags": pick only vocabulary values you are confident about from the frames; omit a category (empty list) rather than guess. "energy" is a single value. "transcript" = words spoken in this scene verbatim, or "" if none.
+- "scene_role": one of ${SCENE_ROLE_VOCAB.join(", ")} — the function this beat serves in the retention arc.
+- "narrative_purpose" / "why_it_works": the viral DNA — WHY this beat exists and how it holds attention. A replacement clip must reproduce this effect, not just the look.
+- "must_preserve": 1-4 things a replacement MUST keep (e.g. "subject facing camera", "surprised reaction on the beat drop", "product clearly visible"). "can_vary": 1-4 things free to change (specific person, exact background, color grade).
+- "replaceability": how safely this scene can be swapped from generic library clips. "low" for scenes carrying unique baked-in text / a specific face / a spoken payoff; "high" for generic b-roll / cutaways / establishing shots. Explain in "replaceability_reason".
+- "recreation_prompt": a self-contained generative brief an AI agent could paste into a text-to-image/video model to REGENERATE this exact beat — describe subject, action, setting, framing, camera move, lighting, mood, art style, and pacing in one rich paragraph. Write it in the idiom of the detected format (e.g. an ai_fruit_drama prompt reads like a stylized 3D-character render brief; a live_action prompt reads like a cinematography direction). Do NOT reference "this video" or timestamps — it must stand alone.
+- "recreation_notes": 2-5 short discrete production directives that back up the recreation_prompt (e.g. "9:16 vertical", "handheld push-in", "soft key light from left", "on-screen text: 'wait for it'", "cut on the beat at ~1.2s", "dialogue VO: <verbatim line>"). Include any exact on-screen text or spoken line that must be reproduced.
+- Match your vocabulary and framing to the detected format above.`;
+}
+async function runSceneAnnotationPass(input) {
+    if (input.frames.length === 0 || input.scenes.length === 0) {
+        return { contentFormat: DEFAULT_CONTENT_FORMAT, annotations: [], provider: null, model: null };
+    }
+    const prompt = buildSceneAnnotationPrompt({
+        durationSeconds: input.durationSeconds,
+        frameTimestamps: input.frames.map((f) => f.timestamp),
+        scenes: input.scenes,
+        viralDna: input.viralDna,
+        summary: input.summary,
+        userPrompt: input.userPrompt ?? null
+    });
+    const attempt = await callVisionWithFallback({ prompt, frames: input.frames, keys: input.keys });
+    const record = asPlainRecord(attempt.result);
+    const contentFormat = normalizeContentFormat(record.content_format);
+    const rawScenes = Array.isArray(record.scenes) ? record.scenes : [];
+    const bySlug = new Map();
+    for (const r of rawScenes) {
+        const rec = asPlainRecord(r);
+        const slug = normalizeSlug(readAnnotationString(rec.scene_slug));
+        if (slug && !bySlug.has(slug))
+            bySlug.set(slug, rec);
+    }
+    // Iterate the FINALIZED scenes (not the model output) so every scene gets
+    // exactly one annotation aligned by slug, even if the model dropped, renamed,
+    // or duplicated one. Missing fields degrade to the scene's own description.
+    const annotations = input.scenes.map((scene) => {
+        const raw = bySlug.get(scene.slug) ?? {};
+        const tags = {
+            ...sanitizeTagValues(asPlainRecord(raw.tags)),
+            transcript: transcriptForScene(input.transcript, scene) || readAnnotationString(asPlainRecord(raw.tags).transcript)
+        };
+        const literal = readAnnotationString(raw.literal) || scene.description || scene.label;
+        const scene_role = normalizeSceneRole(raw.scene_role);
+        const narrative_purpose = readAnnotationString(raw.narrative_purpose) || scene.viral_note;
+        const why_it_works = readAnnotationString(raw.why_it_works);
+        const embedding_text = buildSceneEmbeddingText({ literal, tags, scene_role, narrative_purpose, why_it_works });
+        return {
+            scene_slug: scene.slug,
+            start: scene.start,
+            duration: scene.duration,
+            content_format: contentFormat.format,
+            literal,
+            tags,
+            scene_role,
+            narrative_purpose,
+            why_it_works,
+            must_preserve: readAnnotationStringList(raw.must_preserve, 4),
+            can_vary: readAnnotationStringList(raw.can_vary, 4),
+            replaceability: normalizeReplaceability(raw.replaceability),
+            replaceability_reason: readAnnotationString(raw.replaceability_reason),
+            embedding_text,
+            match_criteria: buildSceneMatchCriteria(tags, scene, embedding_text),
+            // Fall back to the literal + purpose if the model omitted a recreation
+            // brief, so an agent always has *something* to regenerate from.
+            recreation_prompt: readAnnotationString(raw.recreation_prompt)
+                || [literal, narrative_purpose].filter((p) => p.length > 0).join(" "),
+            recreation_notes: readAnnotationStringList(raw.recreation_notes, 6)
+        };
+    });
+    return { contentFormat, annotations, provider: attempt.provider, model: attempt.model };
+}
+// Standalone entry point for the ASYNC scene-annotation pass, invoked once per
+// /scene-annotations-poll. Mirrors analyzeCastMembers: it re-downloads the
+// (caption-free) source, re-samples the same sparse frames the main decompose
+// used, and runs the annotation vision call against the ALREADY-finalized scenes
+// (passed in from the persisted smart-decompose snapshot) so annotations align
+// 1:1 by slug. Kept out of the synchronous decompose so that request stays under
+// the origin timeout.
+export async function annotateScenesFromSource(input) {
+    if (input.scenes.length === 0) {
+        return { contentFormat: DEFAULT_CONTENT_FORMAT, annotations: [], provider: null, model: null, durationSeconds: 0 };
+    }
+    const tempDir = await mkdtemp(path.join(os.tmpdir(), SMART_DECOMPOSE_TEMP_PREFIX));
+    const sourcePath = path.join(tempDir, "source.mp4");
+    const startedAt = Date.now();
+    devLog("hyperframes.scene_annotations.start", { source_url: input.sourceUrl, scene_count: input.scenes.length });
+    try {
+        await downloadVideoTo(input.sourceUrl, sourcePath);
+        const meta = await ffprobeVideo(sourcePath);
+        if (!Number.isFinite(meta.durationSeconds) || meta.durationSeconds <= 0) {
+            throw new Error("Could not read video duration");
+        }
+        const frames = await sampleFramesAsBase64(sourcePath, meta.durationSeconds, tempDir);
+        const outcome = await runSceneAnnotationPass({
+            frames,
+            keys: input.keys,
+            durationSeconds: meta.durationSeconds,
+            scenes: input.scenes,
+            viralDna: input.viralDna,
+            summary: input.summary,
+            transcript: input.transcript,
+            userPrompt: input.userPrompt ?? null
+        });
+        devLog("hyperframes.scene_annotations.completed", {
+            provider: outcome.provider,
+            model: outcome.model,
+            duration_ms: Date.now() - startedAt,
+            content_format: outcome.contentFormat.format,
+            annotation_count: outcome.annotations.length
+        });
+        return { ...outcome, durationSeconds: meta.durationSeconds };
+    }
+    catch (error) {
+        devLog("hyperframes.scene_annotations.failed", {
+            source_url: input.sourceUrl,
+            duration_ms: Date.now() - startedAt,
+            ...devErrorFields(error)
+        }, "error");
+        throw error;
+    }
+    finally {
+        await rm(tempDir, { recursive: true, force: true });
+    }
+}
 function buildSmartDecomposePrompt(input) {
     const sceneCutHint = input.sceneCuts && input.sceneCuts.length > 0
         ? `\nDETECTED SCENE CUTS (authoritative — ffmpeg measured hard visual cuts at these timestamps, in seconds):\n${input.sceneCuts.map((t) => `  cut @ ${t.toFixed(2)}s`).join("\n")}\n\nThese are ground truth for where the video visually changes. Align scene boundaries to these cuts: each detected cut should begin a new scene (a scene "start" should equal a detected cut time, within ~0.15s), and do NOT place a scene boundary where there is no detected cut unless the frames clearly show a change the detector missed. If two adjacent cuts are very close and belong to the same beat, you may merge them into one scene, but never exceed 6 scenes.\n`
@@ -1762,6 +2104,11 @@ export async function smartDecomposeVideo(input) {
         if (input.trendTaglineHint && input.trendTaglineHint.trim().length > 0) {
             viralDna.trend_tagline = input.trendTaglineHint.trim();
         }
+        // NOTE: the per-scene clip-annotation + content-format pass is deliberately
+        // NOT run here — it is a separate vision call that would risk the ~60s origin
+        // timeout on longer videos. /auto-decompose seeds scene-annotations.json
+        // "pending" and /scene-annotations-poll drives annotateScenesFromSource
+        // against these finalized scenes (persisted in smart-decompose.json).
         devLog("hyperframes.smart_decompose.completed", {
             provider: attempt.provider,
             model: attempt.model,

package/dist/src/services/scene-annotations.js

@@ -0,0 +1,32 @@
+import { DEFAULT_CONTENT_FORMAT } from "./hyperframes.js";
+// ==========================================================================
+// Scene-annotation pass state — persisted per fork under
+//   compositions/forks/{forkId}/working/scene-annotations.json
+// and copied on fork/clone/version so the annotations travel with every
+// descendant composition. Drives the async /scene-annotations-poll flow,
+// mirroring the cast.json state machine: /auto-decompose seeds it "pending",
+// and /scene-annotations-poll runs the single annotation vision call (against
+// the finalized scenes persisted in smart-decompose.json) to completion.
+//
+// It's a SINGLE unit of work (one vision call), so the state machine is simple:
+//   pending -> processing -> done | failed
+// The `processing` state exists only to keep two concurrent pollers from both
+// paying for the vision call (see the staleness guard in the poll handler).
+// ==========================================================================
+export const SCENE_ANNOTATIONS_STATE_VERSION = 1;
+export function buildPendingSceneAnnotationsState(input) {
+    return {
+        version: SCENE_ANNOTATIONS_STATE_VERSION,
+        status: "pending",
+        savedAtMs: Date.now(),
+        compositionId: input.compositionId,
+        sourceUrl: input.sourceUrl,
+        title: input.title,
+        provider: null,
+        model: null,
+        durationSeconds: 0,
+        contentFormat: DEFAULT_CONTENT_FORMAT,
+        annotations: []
+    };
+}
+//# sourceMappingURL=scene-annotations.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mevdragon/vidfarm-devcli",
-  "version": "0.7.1",
+  "version": "0.8.0",
   "description": "Local bridge for the Vidfarm Trackpad Editor. `vidfarm serve <template_id>` boots the FULL editor on localhost (disk-backed records/storage, free in-process render); edit composition.html on disk (Claude Code, Codex, etc.) and the browser live-morphs it.",
   "type": "module",
   "bin": {
@@ -9,6 +9,7 @@
   },
   "files": [
     "dist/src/**/*.js",
+    "dist/src/**/*.json",
     "demo/dist/**",
     "public/assets/**",
     "src/assets/**",
@@ -71,6 +72,7 @@
     "@sentry/node": "^10.56.0",
     "@sentry/react": "^10.56.0",
     "ai": "^6.0.191",
+    "better-sqlite3": "^12.11.1",
     "dotenv": "^16.5.0",
     "esbuild": "^0.28.1",
     "ffmpeg-static": "^5.3.0",
@@ -82,9 +84,11 @@
     "react": "^18.3.1",
     "react-dom": "^18.3.1",
     "sharp": "^0.34.5",
+    "sqlite-vec": "^0.1.9",
     "zod": "^3.25.28"
   },
   "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
     "@types/node": "^24.0.1",
     "@types/react": "^18.3.23",
     "@types/react-dom": "^18.3.7",