@mevdragon/vidfarm-devcli 0.14.0 → 0.15.0

package/dist/src/app.js

@@ -1,5 +1,5 @@
 import { createHash, randomBytes, randomUUID } from "node:crypto";
-import { createWriteStream, existsSync, mkdirSync, readFileSync, rmSync, statSync } from "node:fs";
+import { createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync } from "node:fs";
 import path from "node:path";
 import { Readable } from "node:stream";
 import { pipeline as streamPipeline } from "node:stream/promises";
@@ -35,7 +35,7 @@ import { resolveRootFileCandidates } from "./lib/package-root.js";
 import { stripTrackingParams } from "./lib/url-clean.js";
 import { addSeconds, nowIso } from "./lib/time.js";
 import { templateRegistry } from "./registry.js";
-import { buildEditorChatSystemInstruction, compactEditorChatModelMessages, defaultEditorChatModel, EDITOR_CHAT_PROVIDER_PRIORITY, extractEditorChatCompositionBrief, formatEditorChatCurrentDateTime, toEditorChatModelMessages } from "./editor-chat.js";
+import { buildAsyncJobHandoffText, buildEditorChatGuardrailNotice, buildEditorChatSystemInstruction, collectVerifiedHttpJobIds, compactEditorChatModelMessages, createLoadSkillTool, createStreamingJobIdRedactor, defaultEditorChatModel, EDITOR_CHAT_PROVIDER_PRIORITY, extractEditorChatCompositionBrief, formatEditorChatCurrentDateTime, lintReplaceCompositionAction, readAsyncJobHandoff, toEditorChatModelMessages } from "./editor-chat.js";
 import { apiCallHistoryStore } from "./services/api-call-history.js";
 import { AuthService } from "./services/auth.js";
 import { BillingService } from "./services/billing.js";
@@ -54,7 +54,7 @@ import { loadSpeechAudioLocally, MAX_LOCAL_TRANSCRIBE_AUDIO_BYTES } from "./devc
 import { DescribeExecutionCommand, SFNClient, StartExecutionCommand } from "@aws-sdk/client-sfn";
 import { clipRecords, makeUserPreset } from "./services/clip-records.js";
 import { matchScenesToClips, searchUserClips, vectorIndex } from "./services/clip-search.js";
-import { BUILTIN_PRESETS, buildEffectiveGuidance, ClipModelClient, cosineSimilarity, detectLocalAgent, effectiveHuntDurationSec, estimateScanCostFromDuration, hasFfmpeg, LocalAgentClipClient, normalizeCropFocus, normalizeHuntSpec, parseClipHuntPrompt, probeVideo, resolveDurationBand, scanVideo } from "./services/clip-curation/index.js";
+import { BUILTIN_PRESETS, buildClipEmbeddingText, buildEffectiveGuidance, ClipModelClient, cosineSimilarity, detectLocalAgent, effectiveHuntDurationSec, estimateScanCostFromDuration, hasFfmpeg, LocalAgentClipClient, normalizeCropFocus, normalizeHuntSpec, parseClipHuntPrompt, probeVideo, resolveDurationBand, scanVideo } from "./services/clip-curation/index.js";
 import { analyzeCastMembers, annotateScenesFromSource, buildSmartDecomposedHtml, HyperframesService, smartDecomposeVideo, stampCompositionIdentity } from "./services/hyperframes.js";
 import { buildPendingSceneAnnotationsState } from "./services/scene-annotations.js";
 import { createPrimitiveJobContext } from "./primitive-context.js";
@@ -3609,7 +3609,12 @@ function normalizeEditorChatPath(input) {
         // and NEVER runs GhostCut on the long-form source — "no text" is a
         // scene-selection filter; caption removal stays a per-finished-clip
         // primitive (/api/v1/primitives/videos/remove-captions).
-        CLIPS_PREFIX
+        CLIPS_PREFIX,
+        // Public skill knowledge: the vendored skill packs (load_skill fetches
+        // these too) and per-template SKILL.md manifests. Read-only markdown.
+        "/skill/",
+        "/skill-pack/",
+        "/discover/skills/"
     ];
     const parsed = trimmed.startsWith("http://") || trimmed.startsWith("https://")
         ? new URL(trimmed)
@@ -4463,95 +4468,14 @@ function createEditorActionTool() {
             explanation: z.string().describe("One-sentence reason shown in the UI.")
         }),
         execute: async (args) => {
+            const linted = lintReplaceCompositionAction(args);
+            if (linted) {
+                return linted;
+            }
             return { ...args };
         }
     });
 }
-const JOB_ID_PATTERN = /\bjob_[a-z0-9]+\b/gi;
-const HTTP_TOOL_CLAIM_PATTERN = /\b(?:http_request|REST API call|API call|GET\s+https?:\/\/|POST\s+https?:\/\/|GET\s+\/api\/|POST\s+\/api\/)/i;
-function collectJobIdsFromUnknown(value, output = new Set()) {
-    if (typeof value === "string") {
-        for (const match of value.matchAll(JOB_ID_PATTERN)) {
-            output.add(match[0]);
-        }
-        return output;
-    }
-    if (Array.isArray(value)) {
-        for (const entry of value) {
-            collectJobIdsFromUnknown(entry, output);
-        }
-        return output;
-    }
-    if (value && typeof value === "object") {
-        for (const entry of Object.values(value)) {
-            collectJobIdsFromUnknown(entry, output);
-        }
-    }
-    return output;
-}
-function collectVerifiedHttpJobIds(toolResult) {
-    if (!toolResult || typeof toolResult !== "object") {
-        return new Set();
-    }
-    const record = toolResult;
-    return collectJobIdsFromUnknown(record.response?.body ?? record.result?.body);
-}
-function readAsyncJobHandoff(output) {
-    if (!output || typeof output !== "object") {
-        return null;
-    }
-    const record = output;
-    const body = record.response?.body;
-    if (!body || typeof body !== "object") {
-        return null;
-    }
-    const job = body;
-    if (typeof job.job_id !== "string" || typeof job.status !== "string") {
-        return null;
-    }
-    if (job.status === "succeeded" || job.status === "failed" || job.status === "cancelled") {
-        return null;
-    }
-    return {
-        jobId: job.job_id,
-        tracer: typeof job.tracer === "string" ? job.tracer : null,
-        status: job.status,
-        isNewQueuedJob: record.request?.method === "POST" && record.response?.status === 202
-    };
-}
-function buildAsyncJobHandoffText(input) {
-    if (!input.isNewQueuedJob) {
-        return `Render \`${input.jobId}\` is ${input.status}.`;
-    }
-    return [
-        `Started render \`${input.jobId}\`${input.tracer ? ` with tracer \`${input.tracer}\`` : ""}.`,
-        "If it takes a while, ask for an update later and I can check it for you."
-    ].join("\n\n");
-}
-function buildEditorChatGuardrailNotice(input) {
-    const mentionedJobIds = collectJobIdsFromUnknown(input.assistantText);
-    const unverifiedJobIds = [...mentionedJobIds].filter((jobId) => !input.verifiedJobIds.has(jobId));
-    const notices = [];
-    if (input.redactedJobIds.size) {
-        notices.push("Guardrail: I withheld unverified job ID text because it did not come from a visible HTTP response card.");
-    }
-    else if (unverifiedJobIds.length) {
-        notices.push("Guardrail: I mentioned an unverified job ID. Ignore it unless it appears in a visible HTTP response card.");
-    }
-    if (!input.hasHttpRequestResult && HTTP_TOOL_CLAIM_PATTERN.test(input.assistantText)) {
-        notices.push("Guardrail: no `http_request` tool result was recorded for this response, so any claimed REST/API call should be treated as not executed.");
-    }
-    return notices.length ? `\n\n${notices.join("\n\n")}` : "";
-}
-function redactUnverifiedJobIds(input) {
-    return input.text.replace(JOB_ID_PATTERN, (jobId) => {
-        if (input.verifiedJobIds.has(jobId)) {
-            return jobId;
-        }
-        input.redactedJobIds.add(jobId);
-        return "[unverified job id]";
-    });
-}
 function normalizeHistoryAttachments(attachments) {
     return Array.isArray(attachments)
         ? attachments.map((attachment) => ({
@@ -4595,6 +4519,10 @@ async function streamEditorChatAgent(input, send) {
                     vidfarmApiKey: input.vidfarmApiKey,
                     abortSignal: input.abortSignal
                 }),
+                load_skill: createLoadSkillTool({
+                    publicBaseUrl: config.PUBLIC_BASE_URL,
+                    abortSignal: input.abortSignal
+                }),
                 frontend_action: createEditorFrontendActionTool({
                     tracerRegistry
                 }),
@@ -4621,6 +4549,7 @@ async function streamEditorChatAgent(input, send) {
     let hasHttpRequestResult = false;
     const verifiedJobIds = new Set();
     const redactedJobIds = new Set();
+    const jobIdRedactor = createStreamingJobIdRedactor({ verifiedJobIds, redactedJobIds });
     const toolExchangeOrder = [];
     const toolExchangeMap = new Map();
     const httpExchanges = [];
@@ -4635,16 +4564,14 @@ async function streamEditorChatAgent(input, send) {
             case "text-delta":
                 assistantTextForAudit += part.text;
                 {
-                    const safeText = redactUnverifiedJobIds({
-                        text: part.text,
-                        verifiedJobIds,
-                        redactedJobIds
-                    });
-                    assistantTextForUser += safeText;
-                    send({
-                        type: "text-delta",
-                        delta: safeText
-                    });
+                    const safeText = jobIdRedactor.push(part.text);
+                    if (safeText) {
+                        assistantTextForUser += safeText;
+                        send({
+                            type: "text-delta",
+                            delta: safeText
+                        });
+                    }
                 }
                 break;
             case "tool-call":
@@ -4738,6 +4665,14 @@ async function streamEditorChatAgent(input, send) {
                 break;
             case "finish":
                 {
+                    const heldText = jobIdRedactor.flush();
+                    if (heldText) {
+                        assistantTextForUser += heldText;
+                        send({
+                            type: "text-delta",
+                            delta: heldText
+                        });
+                    }
                     const guardrailNotice = buildEditorChatGuardrailNotice({
                         assistantText: assistantTextForAudit,
                         verifiedJobIds,
@@ -5677,21 +5612,193 @@ const PUBLIC_SKILL_FILES = {
     "vidfarm-director": "SKILL.director.md",
     "vidfarm-platform": "SKILL.platform.md"
 };
+// Vendored skill packs (the heygen-scrubbed hyperframes pack + vidfarm-media)
+// mirrored from .agents/skills. Vidfarm is the distribution origin: the devcli
+// (`vidfarm skills add`) and the chat copilot's load_skill tool both pull from
+// these routes, so customers never fetch skills from an upstream vendor org.
+const SKILL_PACK_NAME_PATTERN = /^[a-z0-9][a-z0-9_-]{0,64}$/;
+const SKILL_PACK_IGNORED_ENTRIES = new Set([".DS_Store", "node_modules", ".git"]);
+function resolveSkillPackDir(name) {
+    if (!SKILL_PACK_NAME_PATTERN.test(name)) {
+        return null;
+    }
+    for (const candidate of resolveRootFileCandidates(path.join(".agents", "skills", name))) {
+        try {
+            if (statSync(candidate).isDirectory()) {
+                return candidate;
+            }
+        }
+        catch {
+            continue;
+        }
+    }
+    return null;
+}
+function listSkillPackFiles(dir, prefix = "") {
+    const files = [];
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+        if (SKILL_PACK_IGNORED_ENTRIES.has(entry.name)) {
+            continue;
+        }
+        const relative = prefix ? `${prefix}/${entry.name}` : entry.name;
+        const absolute = path.join(dir, entry.name);
+        if (entry.isDirectory()) {
+            files.push(...listSkillPackFiles(absolute, relative));
+        }
+        else if (entry.isFile()) {
+            files.push(relative);
+        }
+    }
+    return files.sort();
+}
+function readSkillPackDescription(dir) {
+    try {
+        const skillMd = readFileSync(path.join(dir, "SKILL.md"), "utf8");
+        const match = skillMd.match(/^description:\s*(.+)$/m);
+        return match ? match[1].trim().slice(0, 300) : "";
+    }
+    catch {
+        return "";
+    }
+}
 app.get("/skill/:name", (c) => {
     const name = c.req.param("name").replace(/\.md$/i, "");
     const filename = PUBLIC_SKILL_FILES[name];
-    if (!filename) {
-        return c.text("Unknown skill", 404);
-    }
-    const contents = readRootSkillFile(filename);
+    const contents = filename
+        ? readRootSkillFile(filename)
+        : (() => {
+            const packDir = resolveSkillPackDir(name);
+            if (!packDir) {
+                return "";
+            }
+            try {
+                return readFileSync(path.join(packDir, "SKILL.md"), "utf8");
+            }
+            catch {
+                return "";
+            }
+        })();
     if (!contents) {
-        return c.text("Skill file unavailable", 404);
+        return c.text(filename ? "Skill file unavailable" : "Unknown skill", 404);
     }
     return c.body(contents, 200, {
         "content-type": "text/markdown; charset=utf-8",
         "cache-control": "no-cache"
     });
 });
+app.get("/skill-pack/index.json", (c) => {
+    const skills = [];
+    for (const candidate of resolveRootFileCandidates(path.join(".agents", "skills"))) {
+        let entries;
+        try {
+            entries = readdirSync(candidate);
+        }
+        catch {
+            continue;
+        }
+        for (const entry of entries.sort()) {
+            if (!SKILL_PACK_NAME_PATTERN.test(entry) || skills.some((skill) => skill.name === entry)) {
+                continue;
+            }
+            const dir = path.join(candidate, entry);
+            try {
+                if (!statSync(dir).isDirectory()) {
+                    continue;
+                }
+            }
+            catch {
+                continue;
+            }
+            skills.push({ name: entry, description: readSkillPackDescription(dir) });
+        }
+        break;
+    }
+    return c.json({ skills });
+});
+// Manifest hashing reads every file in a pack (hyperframes-animation alone is
+// ~100 files) — memoize per pack dir, invalidated by a cheap stat signature,
+// so `vidfarm skills update` sweeps don't re-hash 5MB per request.
+const skillPackManifestCache = new Map();
+app.get("/skill-pack/:name/manifest.json", (c) => {
+    const packDir = resolveSkillPackDir(c.req.param("name"));
+    if (!packDir) {
+        return c.json({ error: "Unknown skill pack." }, 404);
+    }
+    const relativePaths = listSkillPackFiles(packDir);
+    const signature = relativePaths
+        .map((relative) => {
+        try {
+            const stats = statSync(path.join(packDir, relative));
+            return `${relative}:${stats.size}:${stats.mtimeMs}`;
+        }
+        catch {
+            return `${relative}:missing`;
+        }
+    })
+        .join("|");
+    const cached = skillPackManifestCache.get(packDir);
+    if (cached && cached.signature === signature) {
+        return c.json(cached.manifest);
+    }
+    const files = relativePaths.map((relative) => {
+        const bytes = readFileSync(path.join(packDir, relative));
+        return {
+            path: relative,
+            size: bytes.length,
+            sha256: createHash("sha256").update(bytes).digest("hex")
+        };
+    });
+    const manifest = { name: path.basename(packDir), files };
+    skillPackManifestCache.set(packDir, { signature, manifest });
+    return c.json(manifest);
+});
+app.get("/skill-pack/:name/files/*", (c) => {
+    const packDir = resolveSkillPackDir(c.req.param("name"));
+    if (!packDir) {
+        return c.text("Unknown skill pack.", 404);
+    }
+    const marker = "/files/";
+    const rawPath = c.req.path.slice(c.req.path.indexOf(marker) + marker.length);
+    let relative;
+    try {
+        relative = rawPath
+            .split("/")
+            .map((segment) => decodeURIComponent(segment))
+            .join("/");
+    }
+    catch {
+        return c.text("Bad file path.", 400);
+    }
+    const resolved = path.resolve(packDir, relative);
+    if (resolved !== packDir && !resolved.startsWith(packDir + path.sep)) {
+        return c.text("Bad file path.", 400);
+    }
+    let bytes;
+    try {
+        bytes = readFileSync(resolved);
+    }
+    catch {
+        return c.text("File not found.", 404);
+    }
+    const ext = path.extname(resolved).toLowerCase();
+    const contentType = ext === ".md" || ext === ".markdown" ? "text/markdown; charset=utf-8"
+        : ext === ".json" ? "application/json; charset=utf-8"
+            : ext === ".mjs" || ext === ".cjs" || ext === ".js" ? "text/javascript; charset=utf-8"
+                : ext === ".html" ? "text/html; charset=utf-8"
+                    : ext === ".css" ? "text/css; charset=utf-8"
+                        : ext === ".txt" || ext === ".srt" || ext === ".vtt" || ext === ".sh" || ext === ".yaml" || ext === ".yml" ? "text/plain; charset=utf-8"
+                            : ext === ".png" ? "image/png"
+                                : ext === ".avif" ? "image/avif"
+                                    : ext === ".webp" ? "image/webp"
+                                        : ext === ".jpg" || ext === ".jpeg" ? "image/jpeg"
+                                            : ext === ".mp3" ? "audio/mpeg"
+                                                : ext === ".woff2" ? "font/woff2"
+                                                    : "application/octet-stream";
+    return c.body(new Uint8Array(bytes), 200, {
+        "content-type": contentType,
+        "cache-control": "no-cache"
+    });
+});
 if (config.SENTRY_DEBUG_ROUTE === "true") {
     app.get("/debug-sentry", () => {
         Sentry.logger.info("User triggered test error", {
@@ -6601,7 +6708,13 @@ app.post(`${COMPOSITIONS_PREFIX}/:forkId/captions`, async (c) => {
         const text = str("text");
         if (text) {
             const windowStart = Math.max(0, num("start") ?? 0);
-            const windowDuration = num("duration") ?? Math.max(0.4, compositionDuration - windowStart);
+            const availableWindow = compositionDuration - windowStart;
+            // When the composition declares no usable window (missing data-duration,
+            // or windowStart at/after the end), fall back to a word-count estimate
+            // (~0.32s/word, same as the /audio/captions primitive) instead of
+            // cramming the whole script into a 0.4s window of overlapping cues.
+            const estimatedWindow = Math.max(0.4, text.trim().split(/\s+/).filter(Boolean).length * 0.32);
+            const windowDuration = num("duration") ?? (availableWindow > 0.4 ? availableWindow : estimatedWindow);
             cues = cuesFromText(text, windowStart + offsetSec, windowDuration, { maxWordsPerCue: maxWords });
             sourceNote = "text";
         }
@@ -14938,6 +15051,31 @@ app.get(`${USER_PREFIX}/me/generated-media`, async (c) => {
         next_cursor: page.nextCursor
     });
 });
+// OpenAI's embedding models are named text-embedding-3-* / text-embedding-ada-*;
+// Gemini's family INCLUDES text-embedding-004 (plus gemini-embedding-*), so a
+// bare startsWith("text-embedding") misroutes a Gemini-built library to OpenAI
+// and returns cross-space (garbage) cosine scores. Match the OpenAI naming
+// precisely; everything else is gemini.
+function inferEmbeddingProviderFromModel(model) {
+    return model && /^text-embedding-(3|ada)/i.test(model) ? "openai" : "gemini";
+}
+// Embed-capable client for populating/searching a clip library. Matches the
+// library's existing embedding space when one exists (so vectors stay
+// cosine-comparable); otherwise takes the first embed-capable saved key
+// (gemini → openai; openrouter can't embed). Null → no key, keyword-only.
+async function buildClipLibraryEmbedder(customerId) {
+    const existing = await clipRecords.getLibraryEmbeddingModel(customerId);
+    const providers = existing ? [inferEmbeddingProviderFromModel(existing)] : ["gemini", "openai"];
+    for (const provider of providers) {
+        const apiKey = await resolveCustomerProviderKey(customerId, provider);
+        if (!apiKey)
+            continue;
+        const client = new ClipModelClient({ provider, apiKey });
+        if (client.canEmbed)
+            return client;
+    }
+    return null;
+}
 // Embedder for My Files notes — the same BYOK seam clip search uses. Prefer the
 // provider matching the embedding model the file library was built with (from
 // the newest annotated attachment) so query vectors stay cosine-comparable; a
@@ -14946,7 +15084,7 @@ app.get(`${USER_PREFIX}/me/generated-media`, async (c) => {
 async function buildFileNotesEmbedder(customerId) {
     const attachments = await serverlessRecords.listUserAttachments(customerId);
     const libModel = attachments.find((a) => a.notesEmbeddingModel && Array.isArray(a.notesEmbedding) && a.notesEmbedding.length > 0)?.notesEmbeddingModel ?? null;
-    const providers = libModel ? [libModel.startsWith("text-embedding") ? "openai" : "gemini"] : ["gemini", "openai"];
+    const providers = libModel ? [inferEmbeddingProviderFromModel(libModel)] : ["gemini", "openai"];
     for (const provider of providers) {
         const apiKey = await resolveCustomerProviderKey(customerId, provider);
         if (!apiKey)
@@ -16162,7 +16300,7 @@ async function pickClipProviderFromKeys(customerId) {
  */
 async function buildMatchEmbedder(customerId) {
     const libModel = await clipRecords.getLibraryEmbeddingModel(customerId);
-    const provider = libModel && libModel.startsWith("text-embedding") ? "openai" : "gemini";
+    const provider = inferEmbeddingProviderFromModel(libModel);
     const key = await resolveCustomerProviderKey(customerId, provider);
     if (!key)
         return null;
@@ -16532,29 +16670,54 @@ app.post(`${CLIPS_PREFIX}/scan`, async (c) => {
             const computeEstimate = effectiveDurationSec
                 ? estimateClipScanComputeUsd(effectiveDurationSec, huntSpec.duration_band?.target_sec)
                 : null;
-            const res = await clipScanSfn.send(new StartExecutionCommand({
-                stateMachineArn: config.VIDFARM_CLIP_SCAN_STATE_MACHINE_ARN,
-                name: scanId.replace(/[^A-Za-z0-9-_]/g, "-").slice(0, 80),
-                input: JSON.stringify({
-                    scan_id: scanId,
-                    owner_id: customer.id,
-                    source_video_id: sourceVideoId,
-                    source_filename: filename,
-                    raw_s3_uri: rawS3Uri ?? "",
-                    tier,
-                    provider,
-                    tag_api_key: tagApiKey,
-                    embedding_provider: client.embeddingProvider ?? "",
-                    embedding_api_key: embeddingKey ?? "",
-                    guidance_prompt: guidancePrompt,
-                    frames_per_scene: framesPerScene,
-                    include_audio: includeAudio,
-                    hunt_spec: huntSpec,
-                    tracer
-                })
-            }));
-            await putScanRecords({ scanStatus: "running", executionArn: res.executionArn });
-            return scanResponse({ status: "running", executionArn: res.executionArn, pipeline: "cloud", estimate, computeEstimate });
+            // Records first: a Dynamo failure here aborts submission before anything
+            // is billed, and the warm probe lambda's later status:"scanning" write is
+            // never clobbered back to "pending" by a post-start putScanRecords.
+            await putScanRecords({ scanStatus: "running" });
+            let executionArn;
+            try {
+                const res = await clipScanSfn.send(new StartExecutionCommand({
+                    stateMachineArn: config.VIDFARM_CLIP_SCAN_STATE_MACHINE_ARN,
+                    name: scanId.replace(/[^A-Za-z0-9-_]/g, "-").slice(0, 80),
+                    input: JSON.stringify({
+                        scan_id: scanId,
+                        owner_id: customer.id,
+                        source_video_id: sourceVideoId,
+                        source_filename: filename,
+                        raw_s3_uri: rawS3Uri ?? "",
+                        tier,
+                        provider,
+                        tag_api_key: tagApiKey,
+                        embedding_provider: client.embeddingProvider ?? "",
+                        embedding_api_key: embeddingKey ?? "",
+                        guidance_prompt: guidancePrompt,
+                        frames_per_scene: framesPerScene,
+                        include_audio: includeAudio,
+                        hunt_spec: huntSpec,
+                        tracer
+                    })
+                }));
+                executionArn = res.executionArn;
+            }
+            catch (error) {
+                // The execution never started — roll the scan (and its source) to failed
+                // so pollers do not spin on a "running" record that will never progress,
+                // mirroring the lazy-reconcile failure path.
+                const orphan = await clipRecords.getScan(customer.id, scanId);
+                if (orphan)
+                    await clipRecords.putScan({ ...orphan, status: "failed", error: "Failed to start the scan pipeline.", updated_at: nowIso() });
+                const orphanSource = await clipRecords.getSource(customer.id, sourceVideoId);
+                if (orphanSource && orphanSource.status !== "complete")
+                    await clipRecords.putSource({ ...orphanSource, status: "failed", updated_at: nowIso() });
+                throw error;
+            }
+            // Attach the execution ARN for lazy SFN reconciliation (best-effort — the
+            // record is already pollable; spread the current record so a runner that
+            // has already advanced the status is not regressed).
+            const startedScan = await clipRecords.getScan(customer.id, scanId);
+            if (startedScan)
+                await clipRecords.putScan({ ...startedScan, execution_arn: executionArn, updated_at: nowIso() });
+            return scanResponse({ status: "running", executionArn, pipeline: "cloud", estimate, computeEstimate });
         }
         // ── vidfarm serve (no deployed pipeline): LOCAL-FIRST, cloud as backup ──
         // Same policy as `vidfarm clips scan`: the user's local claude/codex CLI
@@ -16856,6 +17019,11 @@ async function mirrorUpstreamClipsToLocal(input) {
     if (!feed.ok || !feed.json)
         throw new Error(`Upstream clip feed unavailable (HTTP ${feed.status}).`);
     let stored = 0;
+    // Upstream /clips/feed strips the embedding vector, so re-embed each mirrored
+    // clip on the owner's BYOK key — otherwise it lands with no embedding and is
+    // permanently invisible to semantic clip search (keyword-only). Fail-soft:
+    // no embed-capable key → store without embedding (keyword-only), as before.
+    const embedder = await buildClipLibraryEmbedder(ownerId);
     for (const raw of feed.json.clips ?? []) {
         const clipId = typeof raw.clip_id === "string" ? raw.clip_id : null;
         const viewUrl = typeof raw.view_url === "string" ? raw.view_url : null;
@@ -16875,16 +17043,32 @@ async function mirrorUpstreamClipsToLocal(input) {
                 await storage.putBuffer(thumbKey, new Uint8Array(await thumbRes.arrayBuffer()), "image/jpeg");
             }
         }
-        const { view_url: _v, thumbnail_url: _t, ...rest } = raw;
+        const { view_url: _v, thumbnail_url: _t, embedding: _e, embedding_model: _em, ...rest } = raw;
         void _v;
         void _t;
-        await clipRecords.putClip({
+        void _e;
+        void _em;
+        const clip = {
             ...rest,
             owner_id: ownerId,
             source_video_id: scan.source_video_id,
             file_path: clipKey,
             thumbnail_path: thumbKey
-        });
+        };
+        if (embedder?.embeddingModelId) {
+            try {
+                const [vector] = await embedder.embedDocuments([buildClipEmbeddingText(clip.tags, clip.description)]);
+                if (vector?.length) {
+                    clip.embedding = vector;
+                    clip.embedding_model = embedder.embeddingModelId;
+                }
+            }
+            catch (error) {
+                devLog("clip_scan.upstream_mirror_embed_failed", { scan_id: scan.scan_id, clip_id: clipId, ...devErrorFields(error) }, "warn");
+            }
+        }
+        await clipRecords.putClip(clip);
+        await vectorIndex.upsert(ownerId, clip);
         stored++;
     }
     const source = await clipRecords.getSource(ownerId, scan.source_video_id);