@mevdragon/vidfarm-devcli 0.12.0 → 0.13.0

package/dist/src/cli.js

@@ -81,6 +81,9 @@ Discover & inspiration (browse the viral-video catalog, add your own source):
                             local MP4/MOV/WebM file — as a private        (files: presign + PUT via
                             template. --title (uploads; defaults to the    /discover/templates/upload)
                             template id), --tagline, --notes optional.
+                            --raw (uploads) marks it as your own raw
+                            project footage — the editor won't auto-
+                            prompt Decompose on it.
   inspiration-rm <id>       Delete a private inspiration/template you own → DELETE /discover/templates/:id
   inspiration-decompose <id>  AI-decompose an inspiration into scenes     → POST /api/v1/inspirations/:id/decompose
 
@@ -621,6 +624,11 @@ async function runServeCommand(argv) {
     // local-render by definition — cloud rendering is the EXPLICIT
     // render_target="cloud" upstream handoff (the editor's "Render in Cloud").
     process.env.VIDFARM_JOB_STATE_MACHINE_ARN = "";
+    // Same trap for clip hunts: a dev-shell .env carrying the staging clip-scan
+    // ARN would route this box's /clips/scan to the cloud Step Functions. serve
+    // hunts are LOCAL-FIRST (local agent CLI / BYOK keys); the upstream cloud is
+    // the explicit backup inside the scan route itself.
+    process.env.VIDFARM_CLIP_SCAN_STATE_MACHINE_ARN = "";
     // Same trap as the state machine ARN: a dev-shell .env can carry the staging
     // media-lambda URL, which would route probe/thumbnail/trim/etc. to a cloud
     // Lambda that cannot fetch this box's localhost storage URLs ("fetch
@@ -1265,7 +1273,7 @@ async function runVideosCommand(argv) {
     emitResult(result, ctx.json, [["Browse ", discoverFrontendUrl(ctx.host)]]);
 }
 async function runInspirationAddCommand(argv) {
-    const parsed = parseArgs({ args: argv, allowPositionals: true, options: { ...commonOptions(), title: { type: "string" }, tagline: { type: "string" }, notes: { type: "string" } } });
+    const parsed = parseArgs({ args: argv, allowPositionals: true, options: { ...commonOptions(), title: { type: "string" }, tagline: { type: "string" }, notes: { type: "string" }, raw: { type: "boolean", default: false } } });
     const source = parsed.positionals[0];
     if (!source)
         throw new Error("inspiration-add requires a video URL (TikTok/YouTube/Instagram/X) or a local video file path.");
@@ -1285,7 +1293,10 @@ async function runInspirationAddCommand(argv) {
             fileName: path.basename(localPath),
             title: parsed.values.title,
             tagline: parsed.values.tagline,
-            notes: parsed.values.notes
+            notes: parsed.values.notes,
+            // --raw: this is the user's own footage starting a new project, not an
+            // inspiration to study — the editor skips the auto Decompose prompt.
+            origin: parsed.values.raw ? "raw" : undefined
         });
         emitResult(result, ctx.json, [["Discover ", discoverFrontendUrl(ctx.host)]]);
         return;
@@ -1380,7 +1391,7 @@ async function ingestUploadedVideo(ctx, input) {
         host: ctx.host,
         path: "/discover/templates",
         auth: ctx.auth,
-        body: { upload: { storage_key: storageKey, file_name: uploadedName }, title: input.title, tagline: input.tagline, notes: input.notes }
+        body: { upload: { storage_key: storageKey, file_name: uploadedName }, title: input.title, tagline: input.tagline, notes: input.notes, origin: input.origin }
     });
     assertApiOk(result, "ingest");
     return result;
@@ -1555,7 +1566,10 @@ async function runCreateCommand(argv) {
         throw new Error(`create: could not fetch the generated video (${dl.status}).`);
     const buffer = Buffer.from(await dl.arrayBuffer());
     const title = parsed.values.title ?? prompt.slice(0, 80);
-    const add = await ingestUploadedVideo(ctx, { buffer, fileName: "generated.mp4", contentType: "video/mp4", title });
+    // origin:"raw" — a prompt-generated video is the user's own new project, not
+    // an imported inspiration; the editor must not auto-prompt Decompose (matters
+    // especially with --no-decompose).
+    const add = await ingestUploadedVideo(ctx, { buffer, fileName: "generated.mp4", contentType: "video/mp4", title, origin: "raw" });
     const inspirationId = add.json?.inspiration_id;
     if (!inspirationId)
         throw new Error("create: ingest did not return an inspiration id.");

package/dist/src/composition-runtime.js

@@ -229,16 +229,65 @@ export const COMPOSITION_RUNTIME_SCRIPT_BODY = `
   // Burns there is nothing to normalize — the scrub is simply the absolute
   // offset into the layer. Paused-by-stylesheet + scrubbed-here matches the
   // render producer's CSS adapter exactly.
+  //
+  // Stability: Chrome PERMANENTLY drops a paused fill:none CSS animation the
+  // moment its currentTime is pushed past the effect end, and never re-creates
+  // it while the computed animation-name is unchanged — so one forward scrub
+  // across a word's window would leave that word stuck in its resting state
+  // (invisible for word-pop/stack-up) on every later frame. Past the end, a
+  // parked fill:none animation looks identical to the resting styles, so park
+  // at 0 instead of overshooting. fill:forwards presets (karaoke-fill,
+  // cumulative-reveal) must overshoot to hold their final keyframe — finished
+  // + forwards animations keep applying their effect and survive. If a word's
+  // animation is missing anyway (dropped before this guard, or an innerHTML
+  // morph replaced the spans mid-scrub), restart it declaratively by cycling
+  // animation-name through none.
+  function captionWordAnimationFor(word) {
+    let anims = [];
+    try { anims = word.getAnimations(); } catch { return null; }
+    for (const anim of anims) {
+      if (typeof anim.animationName === "string" && anim.animationName.indexOf("vf-cap-") === 0) return anim;
+    }
+    return null;
+  }
+
   function syncCaptionClip(clip, timing) {
     if (typeof clip.getAnimations !== "function") return;
     const offsetMs = Math.max(0, (time - timing.start)) * 1000;
-    let anims = [];
-    try { anims = clip.getAnimations({ subtree: true }); } catch { return; }
-    for (const anim of anims) {
-      if (typeof anim.animationName !== "string" || anim.animationName.indexOf("vf-cap-") !== 0) continue;
+    let words = [];
+    try { words = Array.from(clip.querySelectorAll('[data-cap-word="true"]')); } catch { return; }
+    for (const word of words) {
+      if (typeof word.getAnimations !== "function") continue;
+      let anim = captionWordAnimationFor(word);
+      if (!anim) {
+        const inlineEndMs =
+          ((parseFloat(word.style.animationDelay) || 0) + (parseFloat(word.style.animationDuration) || 0)) * 1000;
+        let needed = offsetMs <= inlineEndMs;
+        if (!needed) {
+          try { needed = getComputedStyle(word).animationFillMode.indexOf("forward") !== -1 || getComputedStyle(word).animationFillMode.indexOf("both") !== -1; } catch {}
+        }
+        if (!needed) continue;
+        try {
+          word.style.animationName = "none";
+          void word.offsetWidth;
+          word.style.removeProperty("animation-name");
+        } catch { continue; }
+        anim = captionWordAnimationFor(word);
+        if (!anim) continue;
+      }
+      let target = offsetMs;
+      try {
+        const computed = anim.effect && typeof anim.effect.getComputedTiming === "function"
+          ? anim.effect.getComputedTiming()
+          : null;
+        if (computed && typeof computed.endTime === "number" && offsetMs >= computed.endTime
+            && computed.fill !== "forwards" && computed.fill !== "both") {
+          target = 0;
+        }
+      } catch {}
       try {
         anim.pause();
-        anim.currentTime = offsetMs;
+        anim.currentTime = target;
       } catch {}
     }
   }
@@ -369,7 +418,16 @@ export const COMPOSITION_RUNTIME_SCRIPT_BODY = `
       return playbackRate;
     },
     setMuted,
-    refreshClips
+    refreshClips,
+    // Re-scrub caption word animations at the current playhead without a full
+    // apply(). The editor calls this after live-patching a caption layer's
+    // DOM (style/text edits rebuild the word spans, whose fresh CSS-paused
+    // animations sit at currentTime 0 — resting/invisible — until scrubbed).
+    syncCaptions() {
+      for (const clip of Array.from(document.querySelectorAll("[data-caption-animation]"))) {
+        if (clip instanceof HTMLElement) syncCaptionClip(clip, readTiming(clip));
+      }
+    }
   };
 
   refreshClips();

package/dist/src/devcli/clips.js

@@ -166,6 +166,9 @@ async function runScan(argv) {
     const guidance = buildEffectiveGuidance({ contentPrompt: guidancePromptRaw, avoidText }) || undefined;
     // ── BACKUP path: run the hunt on the deployed pipeline (--cloud) ──────────
     if (values.cloud) {
+        const apiProvider = values.provider && !["agent", "local", "local-agent"].includes(values.provider)
+            ? normalizeProvider(values.provider)
+            : undefined; // unset → the cloud auto-picks from the account's saved keys
         return runScanCloud({
             videoArg: positionals[0],
             sourceUrl: values.url?.trim() || undefined,
@@ -173,6 +176,7 @@ async function runScan(argv) {
             apiKey: values["api-key"],
             tracer: values.tracer,
             prompt: guidancePromptRaw,
+            provider: apiProvider,
             windows: rawWindows,
             durationBand,
             aspect,
@@ -431,6 +435,7 @@ async function runScanCloud(input) {
         body: JSON.stringify({
             ...(input.sourceUrl ? { source_url: input.sourceUrl } : { temp_file_id: tempFileId, filename: fileName }),
             prompt: input.prompt ?? "",
+            ...(input.provider ? { provider: input.provider } : {}),
             ...(input.tracer ? { tracer: input.tracer } : {}),
             hunt_spec: {
                 ...(input.windows.length ? { windows: input.windows } : {}),

package/dist/src/editor-chat.js

@@ -80,7 +80,7 @@ export function buildTemplateEditorChatSystemPrompt(input) {
         "Use set_layer_visual for geometry (x, y, width, height in %, plus font_size and border_radius in px). Use set_layer_style for text/color styling (color, background, background_style plain|outline|highlight-solid|highlight-translucent, font_family, font_weight, italic, underline, text_align, border_radius, font_size). Use set_layer_media to swap src, volume, muted, playback_start, or object_fit on media layers. Use set_layer_timing for start, duration, track index, playback_start.",
         "KEN BURNS / ANIMATE STILL IMAGES: every still-image layer supports a built-in Ken Burns effect — a slow pan or zoom that spans the clip's full duration in both the preview and the final render. Apply it with editor_action set_layer_media and the ken_burns field (presets: zoom-in, zoom-out, pan-left, pan-right, pan-up, pan-down, zoom-in-left, zoom-in-right; 'none' removes it), optionally ken_burns_intensity (0.04-0.5, default 0.18; ~0.1 subtle, ~0.3 dramatic). When the user says 'animate the images', 'make the photos move', 'ken burns', or similar, apply a preset to EVERY still-image layer in one pass (one set_layer_media call per layer), varying presets across adjacent clips — e.g. zoom-in, then pan-left, then zoom-out — so consecutive stills don't repeat the same motion. Zoom presets suit subjects centered in frame (products, faces); pan presets suit wide scenery or tall screenshots. You can also seed ken_burns directly on add_layer (kind=image) or generate_layer (media_type=image) so freshly placed stills arrive already animated. It only applies to image layers — never set it on video, text, or audio.",
         "SCENE TRANSITIONS: every visual clip supports a first-class transition at its start — the clip animates IN over the previous clip on its track, which is automatically held on screen beneath it for the transition window, in both the preview and the final render. Apply it with editor_action set_layer_media and the transition field ON THE INCOMING CLIP — the clip AFTER the cut, never the one before it (presets: crossfade, fade-black, slide-left/right/up/down, wipe-left/right/up/down, zoom-in, zoom-out, circle-open; 'none' removes it), optionally transition_duration (0.1-2.5s, default 0.5; ~0.3 snappy, ~1.0 slow cinematic). When the user says 'add transitions', 'smooth out the cuts', 'crossfade between scenes', or similar, apply a transition to EVERY scene clip except the first in one pass (one set_layer_media call per clip). Match the preset to the template's energy: crossfade/fade-black read calm and cinematic, slide/wipe read energetic (vary directions across cuts), zoom-in/zoom-out/circle-open read punchy and meme-y; keep ONE family per video unless asked otherwise. You can also seed transition directly on add_layer or generate_layer so a newly placed scene arrives with its entrance. Transitions belong on video/image scene clips — never on audio; on text overlays prefer them only when the user asks. Timing is handled for you — never move clip starts or durations to 'make room' for a transition.",
-        "ANIMATED CAPTIONS (word-by-word, TikTok/CapCut style): the composition supports first-class animated captions — caption layers whose words animate one at a time in both the preview and the final render. Available preset looks (caption_style): spotlight (active word gets a rounded highlight pill on bold outlined text — the classic Hormozi/CapCut look), karaoke (words fill with a color as they are spoken and stay lit), word-pop (one word at a time, punchy scale-in), stack-up (words fade/rise in and accumulate), neon (active word pulses with a glow), bounce (active word bounces). When the user asks to 'add captions', 'add animated captions', 'add subtitles', or similar: FIRST call video_context to get the transcript's timestamped segments, then call editor_action action_type=set_captions with captions=[{text, start, duration, words?}] cues derived from those segments — split segment text into pages of 3-5 words, keep each cue inside its segment's time window, and pass word timings when the transcript provides them (otherwise omit words and the editor estimates). Pick caption_style to match the template's voice (composition_context), or spotlight by default; override colors with caption_active_color / caption_highlight_color / color / background / background_style, and caption_uppercase for shouty formats. set_captions REPLACES all existing animated caption layers in one call — use it for 'restyle all the captions' too (same cues, new style) when cue text/timing is already in editor_context (each caption layer's text and caption_animation are listed there). For a single cue, set_layer_style with caption_* fields restyles just that layer, set_layer_text rewrites its words (timings re-estimate), and caption_animation='none' flattens it back to static text. Never hand-build word-by-word captions out of many add_layer text layers — always use set_captions.",
+        "ANIMATED CAPTIONS (word-by-word, TikTok/CapCut style): the composition supports first-class animated captions — caption layers whose words animate one at a time in both the preview and the final render. Available preset looks (caption_style): spotlight (active word gets a rounded highlight pill on bold outlined text — the classic Hormozi/CapCut look), karaoke (words fill with a color as they are spoken and stay lit), word-pop (one word at a time, punchy scale-in), stack-up (words fade/rise in and accumulate), neon (active word pulses with a glow), bounce (active word bounces). When the user asks to 'add captions', 'add animated captions', 'add subtitles', or similar, pick the cue source by where the speech lives: (a) narration that needs TRANSCRIBING — a TTS voiceover layer, replaced audio, or when word-accurate karaoke timing matters — call the captions primitive: POST /api/v1/primitives/audio/captions with body { tracer, payload: { source_url: <the narration audio/video URL from editor_context>, style } }; the finished job's result carries captions (the cue array) and set_captions_action (fully-formed editor_action arguments) — apply them verbatim with editor_action action_type=set_captions. Real word timestamps need an OpenAI key (tried first; gemini/openrouter degrade to estimated word windows). (b) speech that is the SOURCE VIDEO's own audio on an already-decomposed fork — video_context already has the transcript's timestamped segments, so skip the job and call editor_action action_type=set_captions directly with captions=[{text, start, duration, words?}] cues derived from those segments — split segment text into pages of 3-5 words, keep each cue inside its segment's time window, and pass word timings when the transcript provides them (otherwise omit words and the editor estimates). Pick caption_style to match the template's voice (composition_context), or spotlight by default; override colors with caption_active_color / caption_highlight_color / color / background / background_style, and caption_uppercase for shouty formats. set_captions REPLACES all existing animated caption layers in one call — use it for 'restyle all the captions' too (same cues, new style) when cue text/timing is already in editor_context (each caption layer's text and caption_animation are listed there). For a single cue, set_layer_style with caption_* fields restyles just that layer, set_layer_text rewrites its words (timings re-estimate), and caption_animation='none' flattens it back to static text. Never hand-build word-by-word captions out of many add_layer text layers — always use set_captions. There is also a ONE-STEP server route, POST /api/v1/compositions/:forkId/captions { audio_url?|text?, style?, ... }, which transcribes the fork's own narration and writes the caption layers into the working composition server-side — use it ONLY for headless/automation flows, never while this editor session is open (the editor's next auto-save would overwrite the server-side change; in the editor always mutate through editor_action set_captions).",
         "Use duplicate_layer to clone an existing layer; supply layer_key for the source, and optionally start/duration/track/label on the duplicate. Use split_layer with layer_key and split_time to cut at a moment in the timeline. Use group_layers with layer_keys (>=2) and optional group_label; ungroup_layers with layer_keys of any grouped members.",
         "Prefer one editor_action call per mutation. For a multi-layer composition, sequence calls: generate or fetch each asset, then add or edit each layer with its own editor_action call. Chain edits by reusing the layer_key you assigned during add_layer.",
         "Layer/track ordering is the ONLY thing that controls visual stacking. `track` is both the timeline layer index AND the CSS z-index — the editor forces `z-index = track` on every layer at publish time. Higher track draws visually on top; lower track draws underneath. There is no separate z-index override — if the AI wants a layer above another, give it a higher track. If html/text overlays should sit ABOVE a video/image, their track must be a larger integer than the visual media below them.",

package/dist/src/frontend/homepage-view.js

@@ -378,7 +378,7 @@ function TemplateCard(input) {
             ? _jsx(LazyVideo, { src: template.previewUrl, controls: false })
             : _jsx("img", { className: "preview", src: template.previewUrl, alt: `${template.title} preview`, loading: "lazy", decoding: "async" })
         : _jsx("div", { className: "preview preview-empty", children: template.status === "processing" ? "Downloading…" : "No preview" });
-    return (_jsxs("li", { className: "discover-card row", "data-template-id": template.templateId, children: [template.previewUrl ? (_jsxs("button", { className: "discover-card-media discover-card-media-button", type: "button", "aria-label": `Open ${template.title} preview`, onClick: () => input.onOpenPreview?.(template), children: [media, _jsx("span", { className: "discover-card-media-affordance", "aria-hidden": "true", children: "View" })] })) : (_jsx("div", { className: "discover-card-media", children: media })), _jsxs("div", { className: "discover-card-copy", children: [_jsxs("div", { className: "discover-card-head", children: [_jsxs("div", { className: "discover-card-title-wrap", children: [_jsx("h2", { children: template.title }), _jsx("p", { className: "discover-card-meta", children: formatTemplateMeta(template) })] }), _jsxs("div", { className: "discover-card-actions", children: [_jsx(BookmarkButton, { templateId: template.templateId, title: template.title, isBookmarked: input.isBookmarked, onToggleBookmark: input.onToggleBookmark }), isPending ? (_jsx("span", { className: "cta-button discover-card-cta-disabled", "aria-disabled": "true", children: template.status === "failed" ? "Failed" : "Preparing…" })) : (_jsx(CreateButton, { templateId: template.templateId, accountUserId: input.accountUserId })), _jsx(CardMenu, { template: template, isBookmarked: input.isBookmarked, onToggleBookmark: input.onToggleBookmark, onDeleteTemplate: input.onDeleteTemplate, accountUserId: input.accountUserId })] })] }), _jsx("div", { className: "discover-card-body", children: _jsx("p", { children: String(template.viralDna || "").trim() || "No layer strategy provided for this composition yet." }) }), _jsxs("div", { className: "discover-card-tags", children: [template.visibility === "private" ? (_jsx("div", { className: "pill-tag private-pill", title: "Only visible to your account", children: "Only you" })) : null, template.status === "processing" ? (_jsx("div", { className: "pill-tag processing-pill", children: "Processing" })) : template.status === "failed" ? (_jsx("div", { className: "pill-tag failed-pill", children: "Download failed" })) : null, _jsx("div", { className: "pill-tag difficulty-pill", "data-difficulty": template.difficulty, children: formatDifficulty(template.difficulty) }), _jsx("div", { className: "pill-tag", children: template.sourceType || (isVideoPreview(template.previewUrl) ? "Video source" : "Image sequence") }), _jsx(CopyInlineButton, { text: template.templateId, label: template.templateId })] })] })] }));
+    return (_jsxs("li", { className: "discover-card row", "data-template-id": template.templateId, children: [template.previewUrl ? (_jsxs("button", { className: "discover-card-media discover-card-media-button", type: "button", "aria-label": `Open ${template.title} preview`, onClick: () => input.onOpenPreview?.(template), children: [media, _jsx("span", { className: "discover-card-media-affordance", "aria-hidden": "true", children: "View" })] })) : (_jsx("div", { className: "discover-card-media", children: media })), _jsxs("div", { className: "discover-card-copy", children: [_jsxs("div", { className: "discover-card-head", children: [_jsxs("div", { className: "discover-card-title-wrap", children: [_jsx("h2", { children: template.title }), _jsx("p", { className: "discover-card-meta", children: formatTemplateMeta(template) })] }), _jsxs("div", { className: "discover-card-actions", children: [_jsx(BookmarkButton, { templateId: template.templateId, title: template.title, isBookmarked: input.isBookmarked, onToggleBookmark: input.onToggleBookmark }), isPending ? (_jsx("span", { className: "cta-button discover-card-cta-disabled", "aria-disabled": "true", children: template.status === "failed" ? "Failed" : "Preparing…" })) : (_jsx(CreateButton, { templateId: template.templateId, accountUserId: input.accountUserId })), _jsx(CardMenu, { template: template, isBookmarked: input.isBookmarked, onToggleBookmark: input.onToggleBookmark, onDeleteTemplate: input.onDeleteTemplate, accountUserId: input.accountUserId })] })] }), _jsx("div", { className: "discover-card-body", children: _jsx("p", { children: String(template.viralDna || "").trim() || "No layer strategy provided for this composition yet." }) }), _jsxs("div", { className: "discover-card-tags", children: [template.origin === "raw" ? (_jsx("div", { className: "pill-tag origin-pill", title: "Your own project \u2014 created from a prompt or your raw clips, not an imported inspiration", children: "Your project" })) : null, template.visibility === "private" ? (_jsx("div", { className: "pill-tag private-pill", title: "Only visible to your account", children: "Only you" })) : null, template.status === "processing" ? (_jsx("div", { className: "pill-tag processing-pill", children: "Processing" })) : template.status === "failed" ? (_jsx("div", { className: "pill-tag failed-pill", children: "Download failed" })) : null, _jsx("div", { className: "pill-tag difficulty-pill", "data-difficulty": template.difficulty, children: formatDifficulty(template.difficulty) }), _jsx("div", { className: "pill-tag", children: template.sourceType || (isVideoPreview(template.previewUrl) ? "Video source" : "Image sequence") }), _jsx(CopyInlineButton, { text: template.templateId, label: template.templateId })] })] })] }));
 }
 function PreviewModal(input) {
     const { onClose, template } = input;

package/dist/src/frontend/template-editor-chat.js

@@ -3624,10 +3624,12 @@ export function TemplateEditorChat({ boot }) {
             }
             if (!storageKey)
                 throw new Error("upload did not return a storage key.");
+            // origin:"raw" — a prompt-generated video is the user's own new project,
+            // not an imported inspiration; the editor must not auto-prompt Decompose.
             const addRes = await fetch(new URL("/discover/templates", origin), {
                 method: "POST",
                 headers: jsonHeaders,
-                body: JSON.stringify({ upload: { storage_key: storageKey, file_name: uploadedName }, title })
+                body: JSON.stringify({ upload: { storage_key: storageKey, file_name: uploadedName }, title, origin: "raw" })
             });
             const add = await readJson(addRes);
             if (!addRes.ok)

package/dist/src/homepage.js

@@ -366,6 +366,13 @@ export function renderHomepage(input) {
         font-weight: 800;
       }
 
+      .origin-pill {
+        border-color: rgba(46, 160, 100, 0.35);
+        background: rgba(46, 160, 100, 0.1);
+        color: #1f7a4d;
+        font-weight: 800;
+      }
+
       .processing-pill {
         border-color: rgba(88, 132, 196, 0.4);
         background: rgba(88, 132, 196, 0.12);

package/dist/src/primitive-registry.js

@@ -8,6 +8,8 @@ import { definePrimitive } from "./primitive-sdk.js";
 import { PrimitiveMediaLambdaService } from "./services/primitive-media-lambda.js";
 import { estimateGhostcutCostUsd, isGhostcutConfigured, pollStatus as pollGhostcutStatus, submitSubtitleRemoval } from "./services/ghostcut.js";
 import { buildSrtFromSegments, defaultSpeechModelFor, inferSpeechProviderForVoice, speechVoiceMismatch } from "./services/speech.js";
+import { buildCaptionCues, cuesFromText } from "./services/captions.js";
+import { CAPTION_ANIMATIONS, CAPTION_STYLE_PRESETS, captionStylePresetById } from "./hyperframes/composition.js";
 import { buildHtmlImageCompositionHtml, buildMediaDedupeCompositionHtml, DEDUPE_DEFAULT_EFFECTS } from "./primitives/hyperframes-media.js";
 const imageProviderSchema = z.enum(["openai", "gemini", "openrouter"]);
 const videoProviderSchema = z.enum(["openai", "gemini", "openrouter"]);
@@ -406,9 +408,58 @@ const sttPayloadSchema = z.preprocess((raw) => {
     source_url: mediaSourceUrlSchema,
     media_type: z.enum(["auto", "video", "audio"]).default("auto"),
     diarize: z.boolean().default(true),
+    // Word-level timings on each segment (animated captions). Real timestamps
+    // come from the openai path (whisper-1); other providers keep segment-level
+    // output and downstream callers estimate word windows.
+    word_timestamps: z.boolean().default(false),
     language: z.string().min(2).max(16).optional(),
     prompt: z.string().min(1).max(2000).optional()
 }));
+// Animated captions from speech: STT with word-level timings, paged into
+// ready-to-apply cue layers. Either transcribe a source_url or page plain text.
+const captionsPayloadSchema = z.preprocess((raw) => {
+    if (!raw || typeof raw !== "object" || Array.isArray(raw))
+        return raw;
+    const payload = raw;
+    return {
+        ...payload,
+        source_url: payload.source_url ??
+            payload.source_video_url ??
+            payload.source_audio_url ??
+            payload.source_media_url ??
+            payload.video_url ??
+            payload.audio_url ??
+            payload.url
+    };
+}, z.object({
+    provider: speechProviderSchema.optional(),
+    model: z.string().min(1).optional(),
+    source_url: mediaSourceUrlSchema.optional(),
+    media_type: z.enum(["auto", "video", "audio"]).default("auto"),
+    /** Skip STT entirely: page this script across the window instead. */
+    text: z.string().min(1).max(20_000).optional(),
+    language: z.string().min(2).max(16).optional(),
+    prompt: z.string().min(1).max(2000).optional(),
+    /** Named look: spotlight | karaoke | word-pop | stack-up | neon | bounce. */
+    style: z.string().min(1).max(32).optional(),
+    animation: z.enum(CAPTION_ANIMATIONS).optional(),
+    active_color: z.string().min(1).max(64).optional(),
+    highlight_color: z.string().min(1).max(64).optional(),
+    uppercase: z.boolean().optional(),
+    color: z.string().min(1).max(64).optional(),
+    background: z.string().min(1).max(64).optional(),
+    background_style: z.enum(["plain", "outline", "highlight-solid", "highlight-translucent"]).optional(),
+    font_family: z.string().min(1).max(64).optional(),
+    font_size: z.number().int().min(8).max(200).optional(),
+    max_words_per_cue: z.number().int().min(1).max(8).default(4),
+    /** Shift every cue by this many seconds (where the narration layer starts). */
+    offset_sec: z.number().min(0).max(3600).default(0),
+    /** Text mode window: where the paged script starts and how long it spans. */
+    window_start: z.number().min(0).max(3600).default(0),
+    window_duration: z.number().positive().max(3600).optional()
+}).refine((payload) => Boolean(payload.source_url || payload.text), {
+    message: "Provide source_url (audio/video to transcribe) or text (script to page)."
+}));
 const probeVideoPayloadSchema = z.object({
     source_video_url: mediaSourceUrlSchema
 });
@@ -561,6 +612,7 @@ const PRIMITIVE_BRAINSTORM_PRODUCT_PLACEMENT_ID = "primitive:brainstorm_product_
 const PRIMITIVE_MEDIA_DEDUPE_ID = "primitive:media_dedupe";
 const PRIMITIVE_TTS_ID = "primitive:tts";
 const PRIMITIVE_STT_ID = "primitive:stt";
+const PRIMITIVE_CAPTIONS_ID = "primitive:captions";
 // Gemini inline audio caps out around 20MB; at the 64kbps mono mp3 we extract,
 // that is roughly 40 minutes of speech.
 const MAX_TRANSCRIBE_AUDIO_BYTES = 20 * 1024 * 1024;
@@ -1686,6 +1738,55 @@ const ttsPrimitive = definePrimitive({
         }
     }
 });
+// Shared source→speech-ready-audio loader for the STT-family primitives.
+// Plain audio URLs pass through; videos (and unknown containers) go through
+// the ffmpeg seam, which demuxes and normalizes into a small mono mp3 the
+// speech providers accept.
+async function loadSpeechAudioForPrimitive(ctx, payload) {
+    const sourceKind = payload.media_type === "auto" ? inferSpeechSourceKind(payload.source_url) : payload.media_type;
+    let audioBytes;
+    let audioContentType;
+    let extractionMetadata = null;
+    if (sourceKind === "audio") {
+        ctx.logger.progress(0.1, "Fetching source audio");
+        const response = await fetch(payload.source_url);
+        if (!response.ok) {
+            throw new Error(`Could not fetch source audio (${response.status}).`);
+        }
+        audioBytes = new Uint8Array(await response.arrayBuffer());
+        audioContentType = response.headers.get("content-type")?.split(";")[0]?.trim() || "audio/mpeg";
+    }
+    else {
+        ctx.logger.progress(0.1, "Extracting audio track from source", {
+            executionMode: config.VIDFARM_PRIMITIVE_MEDIA_LAMBDA_URL ? "lambda" : "local_fallback"
+        });
+        const extracted = await executePrimitiveMediaOperation(ctx, {
+            operation: "video_extract_audio",
+            payload: {
+                source_video_url: payload.source_url,
+                output_format: "mp3",
+                audio_bitrate_kbps: 64
+            },
+            outputKey: "source-audio.mp3"
+        });
+        if (!extracted.publicUrl) {
+            throw new Error("Audio extraction produced no readable audio URL.");
+        }
+        const response = await fetch(extracted.publicUrl);
+        if (!response.ok) {
+            throw new Error(`Could not read extracted audio (${response.status}).`);
+        }
+        audioBytes = new Uint8Array(await response.arrayBuffer());
+        audioContentType = extracted.contentType || "audio/mpeg";
+        extractionMetadata = extracted.metadata ?? null;
+    }
+    if (audioBytes.byteLength > MAX_TRANSCRIBE_AUDIO_BYTES) {
+        const sizeMb = Math.round(audioBytes.byteLength / (1024 * 1024));
+        const maxMb = Math.round(MAX_TRANSCRIBE_AUDIO_BYTES / (1024 * 1024));
+        throw new Error(`Audio is too large to transcribe (${sizeMb}MB > ${maxMb}MB, roughly 40 minutes of speech). Trim the source with /videos/trim or /audio/trim and transcribe in parts.`);
+    }
+    return { audioBytes, audioContentType, extractionMetadata };
+}
 const sttPrimitive = definePrimitive({
     id: PRIMITIVE_STT_ID,
     kind: "stt",
@@ -1699,53 +1800,9 @@ const sttPrimitive = definePrimitive({
     jobs: {
         async run(ctx, input) {
             const payload = sttPayloadSchema.parse(input);
-            const sourceKind = payload.media_type === "auto" ? inferSpeechSourceKind(payload.source_url) : payload.media_type;
-            let audioBytes;
-            let audioContentType;
-            let extractionMetadata = null;
-            if (sourceKind === "audio") {
-                ctx.logger.progress(0.1, "Fetching source audio");
-                const response = await fetch(payload.source_url);
-                if (!response.ok) {
-                    throw new Error(`Could not fetch source audio (${response.status}).`);
-                }
-                audioBytes = new Uint8Array(await response.arrayBuffer());
-                audioContentType = response.headers.get("content-type")?.split(";")[0]?.trim() || "audio/mpeg";
-            }
-            else {
-                // Videos (and unknown extensions) go through the ffmpeg seam: it
-                // demuxes video AND normalizes odd audio containers into a small
-                // mono mp3 the speech providers accept.
-                ctx.logger.progress(0.1, "Extracting audio track from source", {
-                    executionMode: config.VIDFARM_PRIMITIVE_MEDIA_LAMBDA_URL ? "lambda" : "local_fallback"
-                });
-                const extracted = await executePrimitiveMediaOperation(ctx, {
-                    operation: "video_extract_audio",
-                    payload: {
-                        source_video_url: payload.source_url,
-                        output_format: "mp3",
-                        audio_bitrate_kbps: 64
-                    },
-                    outputKey: "source-audio.mp3"
-                });
-                if (!extracted.publicUrl) {
-                    throw new Error("Audio extraction produced no readable audio URL.");
-                }
-                const response = await fetch(extracted.publicUrl);
-                if (!response.ok) {
-                    throw new Error(`Could not read extracted audio (${response.status}).`);
-                }
-                audioBytes = new Uint8Array(await response.arrayBuffer());
-                audioContentType = extracted.contentType || "audio/mpeg";
-                extractionMetadata = extracted.metadata ?? null;
-            }
-            if (audioBytes.byteLength > MAX_TRANSCRIBE_AUDIO_BYTES) {
-                const sizeMb = Math.round(audioBytes.byteLength / (1024 * 1024));
-                const maxMb = Math.round(MAX_TRANSCRIBE_AUDIO_BYTES / (1024 * 1024));
-                throw new Error(`Audio is too large to transcribe (${sizeMb}MB > ${maxMb}MB, roughly 40 minutes of speech). Trim the source with /videos/trim or /audio/trim and transcribe in parts.`);
-            }
+            const { audioBytes, audioContentType, extractionMetadata } = await loadSpeechAudioForPrimitive(ctx, payload);
             ctx.logger.progress(0.45, "Transcribing speech");
-            const provider = payload.provider ?? "gemini";
+            const provider = payload.provider ?? (payload.word_timestamps ? "openai" : "gemini");
             const model = payload.model ?? defaultSpeechModelFor("stt", provider) ?? "";
             const transcription = await ctx.providers.transcribeSpeech({
                 provider,
@@ -1754,7 +1811,8 @@ const sttPrimitive = definePrimitive({
                 contentType: audioContentType,
                 prompt: payload.prompt,
                 language: payload.language,
-                diarize: payload.diarize
+                diarize: payload.diarize,
+                wordTimestamps: payload.word_timestamps
             });
             ctx.logger.progress(0.85, "Storing transcript artifacts");
             const speakers = [...new Set(transcription.segments.map((segment) => segment.speaker))];
@@ -1802,6 +1860,137 @@ const sttPrimitive = definePrimitive({
         }
     }
 });
+// Speech → animated captions convenience primitive. One job: transcribe the
+// narration WITH word-level timings (openai/whisper-1 real timestamps, other
+// providers estimated), page it into CapCut-style cues, and return the cue
+// list shaped EXACTLY for the editor's `editor_action set_captions` — the
+// output even carries a ready `set_captions_action` object so an agent can
+// pipe it through without any cue math. Composition-agnostic: it never
+// mutates a fork; POST /api/v1/compositions/:forkId/captions is the one-step
+// variant that applies server-side.
+const captionsPrimitive = definePrimitive({
+    id: PRIMITIVE_CAPTIONS_ID,
+    kind: "captions",
+    operations: {
+        run: {
+            description: "Transcribe narration (or page a script) into animated word-by-word caption cues ready for the editor's set_captions action.",
+            inputSchema: captionsPayloadSchema,
+            workflow: "run"
+        }
+    },
+    jobs: {
+        async run(ctx, input) {
+            const payload = captionsPayloadSchema.parse(input);
+            const preset = payload.style
+                ? captionStylePresetById(payload.style)
+                : payload.animation
+                    ? null
+                    : CAPTION_STYLE_PRESETS[0];
+            if (payload.style && !preset) {
+                throw new Error(`Unknown caption style "${payload.style}". Use one of: ${CAPTION_STYLE_PRESETS.map((entry) => entry.id).join(", ")} — or pass animation directly.`);
+            }
+            const animation = payload.animation ?? preset?.animation ?? "highlight-pop";
+            let cues;
+            let wordTimestampSource = "estimated";
+            let transcriptText = null;
+            let transcriptArtifacts = {};
+            if (payload.text?.trim()) {
+                const tokens = payload.text.trim().split(/\s+/).filter(Boolean);
+                const windowDuration = payload.window_duration ?? Math.max(0.4, tokens.length * 0.32);
+                cues = cuesFromText(payload.text, payload.window_start + payload.offset_sec, windowDuration, {
+                    maxWordsPerCue: payload.max_words_per_cue
+                });
+            }
+            else {
+                const { audioBytes, audioContentType } = await loadSpeechAudioForPrimitive(ctx, {
+                    source_url: payload.source_url,
+                    media_type: payload.media_type
+                });
+                ctx.logger.progress(0.45, "Transcribing speech with word timings");
+                // openai first: whisper-1 is the only provider with REAL word-level
+                // timestamps; providerAttempts falls back to the caller's other keys.
+                const provider = payload.provider ?? "openai";
+                const model = payload.model ?? defaultSpeechModelFor("stt", provider) ?? "";
+                const transcription = await ctx.providers.transcribeSpeech({
+                    provider,
+                    model,
+                    audio: audioBytes,
+                    contentType: audioContentType,
+                    prompt: payload.prompt,
+                    language: payload.language,
+                    diarize: false,
+                    wordTimestamps: true
+                });
+                if (!transcription.segments.length) {
+                    throw new Error("Transcription returned no speech segments — the source may be silent.");
+                }
+                transcriptText = transcription.text || null;
+                wordTimestampSource = transcription.segments.some((segment) => segment.words?.length) ? "provider" : "estimated";
+                ctx.logger.progress(0.7, "Paging transcript into caption cues");
+                cues = buildCaptionCues(transcription.segments, {
+                    maxWordsPerCue: payload.max_words_per_cue,
+                    offsetSec: payload.offset_sec
+                });
+                const srt = buildSrtFromSegments(transcription.segments);
+                const storedSrt = srt ? await ctx.storage.putText("transcript.srt", srt, "application/x-subrip; charset=utf-8") : null;
+                if (storedSrt?.url)
+                    transcriptArtifacts.srt_url = storedSrt.url;
+            }
+            if (!cues.length) {
+                throw new Error("No caption cues produced from the source.");
+            }
+            // Style echo in the exact field names editor_action set_captions takes,
+            // so the agent can spread this straight into the action.
+            const styleFields = {
+                ...(preset ? { caption_style: preset.id } : {}),
+                caption_animation: animation,
+                ...(payload.active_color ?? preset?.activeColor ? { caption_active_color: payload.active_color ?? preset?.activeColor } : {}),
+                ...(payload.highlight_color ?? preset?.highlightColor ? { caption_highlight_color: payload.highlight_color ?? preset?.highlightColor } : {}),
+                ...((payload.uppercase ?? preset?.uppercase) !== undefined ? { caption_uppercase: payload.uppercase ?? preset?.uppercase } : {}),
+                ...(payload.color ?? preset?.color ? { color: payload.color ?? preset?.color } : {}),
+                ...(payload.background ?? preset?.background ? { background: payload.background ?? preset?.background } : {}),
+                ...(payload.background_style ?? preset?.textBackgroundStyle ? { background_style: payload.background_style ?? preset?.textBackgroundStyle } : {}),
+                ...(payload.font_family ? { font_family: payload.font_family } : {}),
+                ...(payload.font_size ? { font_size: payload.font_size } : {})
+            };
+            const setCaptionsAction = {
+                action_type: "set_captions",
+                captions: cues,
+                ...styleFields,
+                explanation: "Apply transcribed animated captions."
+            };
+            ctx.logger.progress(0.85, "Storing caption artifacts");
+            const storedCaptions = await ctx.storage.putJson("captions.json", {
+                source_url: payload.source_url ?? null,
+                word_timestamps: wordTimestampSource,
+                style: styleFields,
+                cue_count: cues.length,
+                cues,
+                set_captions_action: setCaptionsAction,
+                ...(transcriptText ? { transcript_text: transcriptText } : {})
+            });
+            const cuesInline = JSON.stringify(cues).length <= 60_000;
+            ctx.logger.progress(1, "Caption cues ready");
+            return {
+                progress: 1,
+                output: {
+                    files: compactUrls([storedCaptions.url, transcriptArtifacts.srt_url]),
+                    primary_file_url: storedCaptions.url,
+                    cue_count: cues.length,
+                    word_timestamps: wordTimestampSource,
+                    ...styleFields,
+                    // The cue list, ready to pass as editor_action set_captions `captions`.
+                    captions: cuesInline ? cues : undefined,
+                    captions_truncated: !cuesInline,
+                    // Fully-formed editor_action arguments — apply verbatim.
+                    set_captions_action: cuesInline ? setCaptionsAction : undefined,
+                    captions_json_url: storedCaptions.url,
+                    transcript: transcriptArtifacts.srt_url ? { srt_url: transcriptArtifacts.srt_url } : undefined
+                }
+            };
+        }
+    }
+});
 const videoProbePrimitive = definePrimitive({
     id: PRIMITIVE_VIDEO_PROBE_ID,
     kind: "video_probe",
@@ -2380,6 +2569,7 @@ class PrimitiveRegistry {
         [audioNormalizePrimitive.id, audioNormalizePrimitive],
         [ttsPrimitive.id, ttsPrimitive],
         [sttPrimitive.id, sttPrimitive],
+        [captionsPrimitive.id, captionsPrimitive],
         [hyperframesRenderPrimitive.id, hyperframesRenderPrimitive],
         [brainstormHooksPrimitive.id, brainstormHooksPrimitive],
         [brainstormAwarenessStagesPrimitive.id, brainstormAwarenessStagesPrimitive],

package/dist/src/services/providers.js

@@ -1555,6 +1555,7 @@ export class ProviderService {
                 prompt: input.prompt,
                 language: input.language,
                 diarize: input.diarize,
+                wordTimestamps: input.wordTimestamps,
                 apiKey: lease.secret
             })));
             await this.serverlessProviderKeys.recordProviderKeyUsage({