@mevdragon/vidfarm-devcli 0.10.0 → 0.12.0

package/dist/src/devcli/captions.js

@@ -0,0 +1,310 @@
+// devcli `vidfarm captions …` — animated word-by-word captions (TikTok/CapCut
+// style) for a composition.html on disk. LOCAL-FIRST like `place`/`stt`: the
+// transcript comes from the user's own provider key (openai gives REAL
+// word-level timings via whisper-1; gemini/openrouter fall back to estimated
+// word windows), the DOM edit is a local file write, and a running
+// `vidfarm serve` live-morphs the change into open editor tabs. No cloud job,
+// no wallet. Feed it an --srt file (e.g. from `vidfarm stt --cloud`) to skip
+// STT entirely, or --text to page a script without any transcript.
+import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import { parseArgs } from "node:util";
+import { clearCaptionLayers, computeCompositionGaps, inspectComposition, insertCaptionLayers, readCaptionCues, restyleCaptionLayers } from "./composition-edit.js";
+import { loadSpeechAudioLocally, localTranscribeSpeech, resolveLocalSpeechAuth, MAX_LOCAL_TRANSCRIBE_AUDIO_BYTES } from "./speech.js";
+import { buildCaptionCues, cuesFromText } from "../services/captions.js";
+import { CAPTION_ANIMATIONS, CAPTION_STYLE_PRESETS } from "../hyperframes/composition.js";
+export const CAPTIONS_HELP = `vidfarm captions — animated word-by-word captions (local STT + local file write)
+
+  captions generate <dir|composition.html>   Transcribe narration & lay down animated caption cues
+    --audio <file|url>        Audio/video to transcribe. Default: the composition's first
+                              audio layer src, else its first video layer src.
+    --srt <file>              Skip STT; build cues from this SRT subtitle file.
+    --text "<script>"         Skip STT; page this text across [--start, --start+--duration].
+    --style <preset>          ${CAPTION_STYLE_PRESETS.map((p) => p.id).join(" | ")}   (default: spotlight)
+    --animation <anim>        ${CAPTION_ANIMATIONS.join(" | ")}
+    --active-color <css>      Color the active word takes while spoken
+    --highlight-color <css>   Pill behind the active word (highlight-pop)
+    --color / --background / --background-style plain|outline|highlight-solid|highlight-translucent
+    --font <family> / --font-size <px> / --uppercase
+    --max-words <n>           Words per cue page (default 4)
+    --track <n> / --x / --y / --width / --height (%)
+    --provider <p>            openai | gemini | openrouter (default: openai first — it has
+                              real word timestamps; gemini/openrouter estimate word windows)
+    --language <code>         Expected audio language
+    --offset <sec>            Shift every cue (e.g. narration layer's start)
+    --start <sec> --duration <sec>   Window for --text paging
+    --json
+
+  captions style <dir|composition.html> --style <preset>   Restyle existing cues (same text/timing)
+  captions list  <dir|composition.html> [--json]           Show current animated caption cues
+  captions clear <dir|composition.html>                    Remove all animated caption layers
+
+Word timings ride inside each cue layer (per-word animation-delay/duration), so preview,
+serve live-morph, local render, and cloud render all animate identically. \`publish\` pushes
+the edited composition to the cloud as usual.
+`;
+const STYLE_FLAG_OPTIONS = {
+    style: { type: "string" },
+    animation: { type: "string" },
+    "active-color": { type: "string" },
+    "highlight-color": { type: "string" },
+    color: { type: "string" },
+    background: { type: "string" },
+    "background-style": { type: "string" },
+    font: { type: "string" },
+    "font-size": { type: "string" },
+    uppercase: { type: "boolean" },
+    track: { type: "string" },
+    x: { type: "string" },
+    y: { type: "string" },
+    width: { type: "string" },
+    height: { type: "string" }
+};
+export async function runCaptionsCommand(argv) {
+    const sub = argv[0];
+    const rest = argv.slice(1);
+    switch (sub) {
+        case "generate":
+            return runGenerate(rest);
+        case "style":
+            return runStyle(rest);
+        case "list":
+            return runListCues(rest);
+        case "clear":
+            return runClear(rest);
+        case undefined:
+        case "help":
+        case "--help":
+        case "-h":
+            console.log(CAPTIONS_HELP);
+            return;
+        default:
+            console.error(`Unknown captions subcommand: ${sub}\n`);
+            console.log(CAPTIONS_HELP);
+            process.exitCode = 1;
+    }
+}
+// Same dir-or-file resolution as `vidfarm place` (cli.ts resolveCompositionHtmlPath).
+function resolveCompositionHtml(target) {
+    if (!target)
+        throw new Error("Provide the composition dir or composition.html path.");
+    const abs = path.resolve(process.cwd(), target);
+    if (existsSync(abs) && statSync(abs).isDirectory()) {
+        const inside = path.join(abs, "composition.html");
+        if (!existsSync(inside))
+            throw new Error(`No composition.html inside ${abs}. Run \`vidfarm pull <forkId> --dir ${target}\` first.`);
+        return inside;
+    }
+    if (!existsSync(abs))
+        throw new Error(`No such composition file or dir: ${abs}.`);
+    return abs;
+}
+function styleOptionsFromFlags(values) {
+    const num = (key) => {
+        const raw = values[key];
+        if (typeof raw !== "string" || !raw.trim())
+            return undefined;
+        const parsed = Number(raw);
+        if (!Number.isFinite(parsed))
+            throw new Error(`--${key} must be a number.`);
+        return parsed;
+    };
+    return {
+        style: typeof values.style === "string" ? values.style : undefined,
+        animation: typeof values.animation === "string" ? values.animation : undefined,
+        activeColor: typeof values["active-color"] === "string" ? values["active-color"] : undefined,
+        highlightColor: typeof values["highlight-color"] === "string" ? values["highlight-color"] : undefined,
+        uppercase: values.uppercase === true ? true : undefined,
+        color: typeof values.color === "string" ? values.color : undefined,
+        background: typeof values.background === "string" ? values.background : undefined,
+        backgroundStyle: typeof values["background-style"] === "string" ? values["background-style"] : undefined,
+        fontFamily: typeof values.font === "string" ? values.font : undefined,
+        fontSize: num("font-size"),
+        track: num("track"),
+        x: num("x"),
+        y: num("y"),
+        width: num("width"),
+        height: num("height")
+    };
+}
+// Minimal SRT reader: index / "start --> end" / text lines. Produces the same
+// segment shape STT returns so cue building is one code path.
+export function parseSrtToSegments(srt) {
+    const toSeconds = (stamp) => {
+        const match = /(\d+):(\d+):(\d+)[,.](\d+)/.exec(stamp.trim());
+        if (!match)
+            return null;
+        return Number(match[1]) * 3600 + Number(match[2]) * 60 + Number(match[3]) + Number(match[4].padEnd(3, "0").slice(0, 3)) / 1000;
+    };
+    const segments = [];
+    for (const block of srt.split(/\r?\n\r?\n+/)) {
+        const lines = block.split(/\r?\n/).filter((line) => line.trim());
+        const timingIndex = lines.findIndex((line) => line.includes("-->"));
+        if (timingIndex === -1)
+            continue;
+        const [startRaw, endRaw] = lines[timingIndex].split("-->");
+        const start = toSeconds(startRaw ?? "");
+        const end = toSeconds(endRaw ?? "");
+        const text = lines.slice(timingIndex + 1).join(" ").replace(/<[^>]+>/g, "").trim();
+        if (start === null || !text)
+            continue;
+        segments.push({ speaker: "speaker_1", start_sec: start, end_sec: end, text });
+    }
+    return segments;
+}
+// Default transcription source: the composition's own narration. Prefer a
+// dedicated audio layer; fall back to the first video layer with a src.
+function defaultAudioSource(htmlPath, html) {
+    const info = inspectComposition(html);
+    const candidate = info.layers.find((layer) => layer.kind === "audio" && layer.src)
+        ?? info.layers.find((layer) => layer.kind === "video" && layer.src);
+    if (!candidate?.src)
+        return null;
+    const src = candidate.src;
+    if (/^https?:\/\//i.test(src))
+        return src;
+    // Relative srcs resolve against the composition dir (pull/serve layouts).
+    return path.resolve(path.dirname(htmlPath), src.replace(/^\.\//, ""));
+}
+async function runGenerate(argv) {
+    const { values, positionals } = parseArgs({
+        args: argv,
+        allowPositionals: true,
+        options: {
+            ...STYLE_FLAG_OPTIONS,
+            audio: { type: "string" },
+            srt: { type: "string" },
+            text: { type: "string" },
+            "max-words": { type: "string" },
+            provider: { type: "string" },
+            language: { type: "string" },
+            offset: { type: "string" },
+            start: { type: "string" },
+            duration: { type: "string" },
+            "gemini-key": { type: "string" },
+            "openai-key": { type: "string" },
+            "openrouter-key": { type: "string" },
+            json: { type: "boolean" }
+        }
+    });
+    const htmlPath = resolveCompositionHtml(positionals[0]);
+    const html = readFileSync(htmlPath, "utf8");
+    const info = inspectComposition(html);
+    const maxWords = values["max-words"] ? Math.max(1, Math.round(Number(values["max-words"]))) : 4;
+    const offsetSec = values.offset ? Number(values.offset) : 0;
+    if (!Number.isFinite(offsetSec))
+        throw new Error("--offset must be a number of seconds.");
+    let cues;
+    let sourceNote;
+    if (typeof values.text === "string" && values.text.trim()) {
+        const start = values.start ? Number(values.start) : 0;
+        const duration = values.duration ? Number(values.duration) : Math.max(0.4, info.duration_seconds - start);
+        cues = cuesFromText(values.text, start + offsetSec, duration, { maxWordsPerCue: maxWords });
+        sourceNote = "provided text (estimated word timings)";
+    }
+    else if (typeof values.srt === "string" && values.srt.trim()) {
+        const srtPath = path.resolve(process.cwd(), values.srt);
+        if (!existsSync(srtPath))
+            throw new Error(`No such SRT file: ${srtPath}`);
+        const segments = parseSrtToSegments(readFileSync(srtPath, "utf8"));
+        if (!segments.length)
+            throw new Error(`No cues found in ${srtPath}.`);
+        cues = buildCaptionCues(segments, { maxWordsPerCue: maxWords, offsetSec, fallbackDurationSec: info.duration_seconds });
+        sourceNote = `${path.basename(srtPath)} (${segments.length} SRT cues, estimated word timings)`;
+    }
+    else {
+        // openai first: whisper-1 verbose_json is the only path with REAL word
+        // timestamps, which is what makes the word-by-word animation land on beat.
+        const auth = resolveLocalSpeechAuth(values, ["openai", "gemini", "openrouter"], typeof values.provider === "string" ? values.provider : undefined);
+        if (!auth) {
+            throw new Error("No local provider key found. Export OPENAI_API_KEY (best: real word timings), GEMINI_API_KEY, or OPENROUTER_API_KEY — or pass --srt/--text instead.");
+        }
+        const audioTarget = typeof values.audio === "string" && values.audio.trim()
+            ? values.audio.trim()
+            : defaultAudioSource(htmlPath, html);
+        if (!audioTarget) {
+            throw new Error("Could not find narration to transcribe (no audio/video layer with a src). Pass --audio <file|url>, --srt, or --text.");
+        }
+        console.log(`Transcribing ${audioTarget} (${auth.provider}, local key)…`);
+        const audio = await loadSpeechAudioLocally(audioTarget);
+        if (audio.bytes.byteLength > MAX_LOCAL_TRANSCRIBE_AUDIO_BYTES) {
+            throw new Error(`Audio is ${(audio.bytes.byteLength / (1024 * 1024)).toFixed(1)}MB (limit ${(MAX_LOCAL_TRANSCRIBE_AUDIO_BYTES / (1024 * 1024)).toFixed(0)}MB). Trim it or transcribe externally and pass --srt.`);
+        }
+        const transcript = await localTranscribeSpeech({
+            auth,
+            audio: audio.bytes,
+            contentType: audio.contentType,
+            language: typeof values.language === "string" ? values.language : undefined,
+            diarize: auth.provider === "gemini",
+            wordTimestamps: true
+        });
+        if (!transcript.segments.length)
+            throw new Error("Transcription returned no speech segments.");
+        cues = buildCaptionCues(transcript.segments, { maxWordsPerCue: maxWords, offsetSec, fallbackDurationSec: info.duration_seconds });
+        const hasRealWords = transcript.segments.some((segment) => segment.words?.length);
+        sourceNote = `${transcript.provider}/${transcript.model} (${hasRealWords ? "word-level timestamps" : "estimated word timings"})`;
+    }
+    if (!cues.length)
+        throw new Error("No caption cues produced.");
+    // Keep cues inside the composition's duration.
+    cues = cues.filter((cue) => cue.start < info.duration_seconds)
+        .map((cue) => ({ ...cue, duration: Math.min(cue.duration, Math.max(0.1, info.duration_seconds - cue.start)) }));
+    const result = insertCaptionLayers(html, cues, styleOptionsFromFlags(values));
+    writeFileSync(htmlPath, result.html);
+    if (values.json) {
+        console.log(JSON.stringify({ ok: true, style: result.styleId, cues: cues.length, replaced: result.replaced, layer_keys: result.layerKeys, source: sourceNote, gaps: computeCompositionGaps(result.html) }, null, 2));
+        return;
+    }
+    console.log(`Wrote ${cues.length} animated caption cues (style: ${result.styleId}) from ${sourceNote}.`);
+    if (result.replaced > 0)
+        console.log(`Replaced ${result.replaced} previous caption cues.`);
+    console.log(`Updated ${htmlPath} — a running \`vidfarm serve\` live-morphs it; \`vidfarm publish\` pushes to cloud.`);
+}
+async function runStyle(argv) {
+    const { values, positionals } = parseArgs({
+        args: argv,
+        allowPositionals: true,
+        options: { ...STYLE_FLAG_OPTIONS, json: { type: "boolean" } }
+    });
+    const htmlPath = resolveCompositionHtml(positionals[0]);
+    const opts = styleOptionsFromFlags(values);
+    if (!opts.style && !opts.animation && !opts.activeColor && !opts.highlightColor && opts.uppercase === undefined && !opts.color && !opts.background && !opts.backgroundStyle && !opts.fontFamily && opts.fontSize === undefined) {
+        throw new Error("captions style needs at least one style flag (e.g. --style karaoke).");
+    }
+    const result = restyleCaptionLayers(readFileSync(htmlPath, "utf8"), opts);
+    writeFileSync(htmlPath, result.html);
+    if (values.json) {
+        console.log(JSON.stringify({ ok: true, style: result.styleId, cues: result.layerKeys.length }, null, 2));
+        return;
+    }
+    console.log(`Restyled ${result.layerKeys.length} caption cues to ${result.styleId}.`);
+}
+async function runListCues(argv) {
+    const { values, positionals } = parseArgs({
+        args: argv,
+        allowPositionals: true,
+        options: { json: { type: "boolean" } }
+    });
+    const htmlPath = resolveCompositionHtml(positionals[0]);
+    const cues = readCaptionCues(readFileSync(htmlPath, "utf8"));
+    if (values.json) {
+        console.log(JSON.stringify({ cues }, null, 2));
+        return;
+    }
+    if (!cues.length) {
+        console.log("No animated caption layers. Run `vidfarm captions generate <dir>` to add some.");
+        return;
+    }
+    for (const cue of cues) {
+        console.log(`${cue.start.toFixed(2).padStart(8)}s +${cue.duration.toFixed(2)}s  ${cue.text}`);
+    }
+    console.log(`${cues.length} cues on track ${cues[0].track}.`);
+}
+async function runClear(argv) {
+    const { positionals } = parseArgs({ args: argv, allowPositionals: true, options: {} });
+    const htmlPath = resolveCompositionHtml(positionals[0]);
+    const result = clearCaptionLayers(readFileSync(htmlPath, "utf8"));
+    writeFileSync(htmlPath, result.html);
+    console.log(`Removed ${result.removed} animated caption layer${result.removed === 1 ? "" : "s"}.`);
+}
+//# sourceMappingURL=captions.js.map

package/dist/src/devcli/composition-edit.js

@@ -6,9 +6,14 @@
 // pushes them identically to a browser-made edit. Pure DOM (linkedom) — no
 // ffmpeg, no network.
 import { parseHTML } from "linkedom";
+import { CAPTION_ANIMATIONS, CAPTION_STYLE_PRESETS, captionStylePresetById, estimateCaptionWords, renderHyperframeLayerHtml } from "../hyperframes/composition.js";
 const KEN_BURNS_PRESETS = new Set([
     "zoom-in", "zoom-out", "pan-left", "pan-right", "pan-up", "pan-down", "zoom-in-left", "zoom-in-right"
 ]);
+const TRANSITION_PRESET_SET = new Set([
+    "crossfade", "fade-black", "slide-left", "slide-right", "slide-up", "slide-down",
+    "wipe-left", "wipe-right", "wipe-up", "wipe-down", "zoom-in", "zoom-out", "circle-open"
+]);
 const round3 = (v) => Number(v.toFixed(3));
 const clampPercent = (v) => Math.min(100, Math.max(0, v));
 function serialize(document) {
@@ -221,6 +226,18 @@ function buildMediaClip(document, opts, id, track, geom) {
             styles.push(`--vf-kb-amount:${Math.max(0.04, Math.min(0.5, opts.kenBurnsIntensity))}`);
         }
     }
+    if (opts.transition && TRANSITION_PRESET_SET.has(opts.transition)) {
+        // Scene transition (see src/hyperframes/composition.ts): data-transition
+        // picks the keyframes; data-transition-duration + --vf-tr-dur set the span.
+        // The overlap with the previous clip is granted by the runtime/publish
+        // normalizer — the stored timeline stays butt-cut.
+        const transitionDuration = Number.isFinite(opts.transitionDuration) && opts.transitionDuration > 0
+            ? Math.max(0.1, Math.min(2.5, opts.transitionDuration))
+            : 0.5;
+        node.setAttribute("data-transition", opts.transition);
+        node.setAttribute("data-transition-duration", String(round3(transitionDuration)));
+        styles.push(`--vf-tr-dur:${round3(transitionDuration)}s`);
+    }
     node.setAttribute("style", `${styles.join(";")};`);
     return node;
 }
@@ -254,6 +271,131 @@ export function insertMediaLayer(html, opts) {
     root.append(node);
     return { html: serialize(document), layerKey: id };
 }
+const CAPTION_DEFAULT_FRAME = { x: 10, y: 70, width: 80, height: 14 };
+function resolveCaptionLook(opts) {
+    const preset = captionStylePresetById(opts.style)
+        ?? (opts.animation ? null : CAPTION_STYLE_PRESETS[0]);
+    const animation = (opts.animation ?? preset?.animation);
+    if (!animation || !CAPTION_ANIMATIONS.includes(animation)) {
+        throw new Error(`Unknown caption animation "${opts.animation ?? opts.style}". Use a preset (${CAPTION_STYLE_PRESETS.map((p) => p.id).join(", ")}) or an animation (${CAPTION_ANIMATIONS.join(", ")}).`);
+    }
+    return {
+        animation,
+        activeColor: opts.activeColor ?? preset?.activeColor,
+        highlightColor: opts.highlightColor ?? preset?.highlightColor,
+        uppercase: opts.uppercase ?? preset?.uppercase,
+        color: opts.color ?? preset?.color ?? "#ffffff",
+        background: opts.background ?? preset?.background ?? "#000000",
+        backgroundStyle: (opts.backgroundStyle ?? preset?.textBackgroundStyle ?? "plain"),
+        fontFamily: opts.fontFamily ?? "Montserrat",
+        fontWeight: opts.fontWeight ?? preset?.fontWeight ?? 800,
+        fontSize: opts.fontSize ?? 38,
+        presetId: preset?.id ?? animation
+    };
+}
+/** Read the animated-caption cues already in a composition (for restyle/list). */
+export function readCaptionCues(html) {
+    const { root } = readCompositionDoc(html);
+    return Array.from(root.querySelectorAll("[data-caption-animation]")).map((node) => {
+        const words = Array.from(node.querySelectorAll("[data-cap-word]"))
+            .map((span) => ({
+            text: String(span.textContent || "").trim(),
+            start: Number.parseFloat(span.getAttribute("data-word-start") ?? "") || 0,
+            end: Number.parseFloat(span.getAttribute("data-word-end") ?? "") || 0
+        }))
+            .filter((word) => word.text);
+        return {
+            layerKey: node.getAttribute("data-hf-id") || node.id || "",
+            track: Number.isFinite(numAttr(node, "data-track-index")) ? numAttr(node, "data-track-index") : 0,
+            text: words.length ? words.map((word) => word.text).join(" ") : String(node.textContent || "").trim(),
+            start: Number.isFinite(numAttr(node, "data-start")) ? numAttr(node, "data-start") : 0,
+            duration: Number.isFinite(numAttr(node, "data-duration")) ? numAttr(node, "data-duration") : 0,
+            words: words.length ? words : undefined
+        };
+    });
+}
+/** Remove every animated-caption layer. Returns the count removed. */
+export function clearCaptionLayers(html) {
+    const { document, root } = readCompositionDoc(html);
+    const nodes = Array.from(root.querySelectorAll("[data-caption-animation]"));
+    for (const node of nodes)
+        node.parentNode?.removeChild(node);
+    return { html: serialize(document), removed: nodes.length };
+}
+/**
+ * Insert an animated-caption cue run: one caption layer per cue on a shared
+ * collision-free track. Replaces any existing animated captions (set
+ * semantics — matching the editor's set_captions action).
+ */
+export function insertCaptionLayers(html, cues, opts = {}) {
+    const usable = cues
+        .filter((cue) => cue.text?.trim() && Number.isFinite(cue.start) && Number.isFinite(cue.duration) && cue.duration > 0)
+        .sort((a, b) => a.start - b.start);
+    if (!usable.length)
+        throw new Error("insertCaptionLayers needs at least one cue with text, start, and duration.");
+    const look = resolveCaptionLook(opts);
+    const { document, root } = readCompositionDoc(html);
+    const existing = Array.from(root.querySelectorAll("[data-caption-animation]"));
+    for (const node of existing)
+        node.parentNode?.removeChild(node);
+    const clips = collectClips(root);
+    const runStart = usable[0].start;
+    const runEnd = usable.reduce((max, cue) => Math.max(max, cue.start + cue.duration), runStart);
+    const initialTrack = opts.track !== undefined && Number.isFinite(opts.track)
+        ? Math.max(0, Math.floor(opts.track))
+        : nextAutoTrack(clips);
+    const track = findNonCollidingTrack(clips, { start: runStart, duration: Math.max(0.1, runEnd - runStart), track: initialTrack }, "");
+    const stamp = Date.now().toString(36);
+    const layerKeys = [];
+    usable.forEach((cue, index) => {
+        const id = `element_cap_${stamp}_${index}`;
+        layerKeys.push(id);
+        const layer = {
+            id,
+            kind: "caption",
+            label: cue.text.slice(0, 48),
+            start: cue.start,
+            duration: cue.duration,
+            track,
+            z: track,
+            x: opts.x ?? CAPTION_DEFAULT_FRAME.x,
+            y: opts.y ?? CAPTION_DEFAULT_FRAME.y,
+            width: opts.width ?? CAPTION_DEFAULT_FRAME.width,
+            height: opts.height ?? CAPTION_DEFAULT_FRAME.height,
+            text: cue.text,
+            color: look.color,
+            background: look.background,
+            textBackgroundStyle: look.backgroundStyle,
+            fontFamily: look.fontFamily,
+            fontWeight: look.fontWeight,
+            fontSize: look.fontSize,
+            captionAnimation: {
+                animation: look.animation,
+                words: cue.words?.length ? cue.words : estimateCaptionWords(cue.text, cue.duration),
+                activeColor: look.activeColor,
+                highlightColor: look.highlightColor,
+                uppercase: look.uppercase
+            }
+        };
+        const container = document.createElement("div");
+        container.innerHTML = renderHyperframeLayerHtml(layer);
+        const node = container.firstElementChild;
+        if (!node)
+            throw new Error(`Failed to render caption cue ${index}.`);
+        node.classList?.add?.("clip");
+        root.append(node);
+    });
+    return { html: serialize(document), layerKeys, replaced: existing.length, styleId: look.presetId };
+}
+/** Restyle existing animated captions in place: same cues/timings/track, new look. */
+export function restyleCaptionLayers(html, opts) {
+    const cues = readCaptionCues(html);
+    if (!cues.length)
+        throw new Error("No animated caption layers found. Run `captions generate` (or add captions in the editor) first.");
+    const track = opts.track ?? cues[0].track;
+    const result = insertCaptionLayers(html, cues, { ...opts, track });
+    return { html: result.html, layerKeys: result.layerKeys, styleId: result.styleId };
+}
 // Replace an existing scene/layer with a generated media clip, taking over its
 // timing + geometry (full-canvas when it was a proxy / full-canvas fill).
 export function replaceLayerWithMedia(html, targetKey, opts) {

package/dist/src/devcli/speech.js

@@ -0,0 +1,178 @@
+// Local speech engine — devcli-only. `vidfarm tts` / `vidfarm stt` run
+// LOCAL-FIRST on the user's own AI key (GEMINI_API_KEY / OPENAI_API_KEY /
+// OPENROUTER_API_KEY, or the --gemini-key/--openai-key/--openrouter-key flags)
+// by calling the shared speech provider layer (src/services/speech.ts)
+// directly: local ffmpeg for the video→audio demux, the user's key for the
+// model call — no cloud job, no wallet, no REST route. The REST primitive
+// routes (/api/v1/primitives/audio/speech|transcribe) are the explicit
+// `--cloud` BACKUP, mirroring `vidfarm clips scan`.
+//
+// Unlike clip tagging, speech can NOT ride on a claude/codex CLI agent — the
+// agent CLIs have no audio I/O (the same reason embeddings need a real key in
+// local-agent.ts) — so "local" here means a raw provider key, not the agent.
+import { spawn } from "node:child_process";
+import { existsSync, readFileSync } from "node:fs";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { buildSrtFromSegments, defaultSpeechModelFor, generateSpeechWithKey, transcribeSpeechWithKey } from "../services/speech.js";
+import { resolveFfmpeg } from "../services/clip-curation/ffmpeg.js";
+// Mirrors the env-var fallbacks used by `vidfarm clips` so one exported key
+// powers every local-first devcli surface.
+const PROVIDER_ENV = {
+    gemini: ["GEMINI_API_KEY", "GOOGLE_GENERATIVE_AI_API_KEY"],
+    openai: ["OPENAI_API_KEY"],
+    openrouter: ["OPENROUTER_API_KEY"]
+};
+const PROVIDER_FLAG = {
+    gemini: "gemini-key",
+    openai: "openai-key",
+    openrouter: "openrouter-key"
+};
+// Same inline-audio ceiling as the cloud stt primitive (~40min of 64kbps mono).
+export const MAX_LOCAL_TRANSCRIBE_AUDIO_BYTES = 20 * 1024 * 1024;
+/**
+ * Find a usable local provider key: an explicit --provider narrows the search,
+ * otherwise the first provider (in `preference` order) with a flag/env key
+ * wins. Returns null when nothing local is configured (callers point the user
+ * at env keys or --cloud).
+ */
+export function resolveLocalSpeechAuth(values, preference, requestedProvider) {
+    const providers = requestedProvider
+        ? [normalizeSpeechProvider(requestedProvider)]
+        : preference;
+    for (const provider of providers) {
+        const flagged = values[PROVIDER_FLAG[provider]];
+        if (typeof flagged === "string" && flagged.trim()) {
+            return { provider, apiKey: flagged.trim() };
+        }
+        for (const envName of PROVIDER_ENV[provider]) {
+            const fromEnv = process.env[envName]?.trim();
+            if (fromEnv) {
+                return { provider, apiKey: fromEnv };
+            }
+        }
+    }
+    return null;
+}
+export function normalizeSpeechProvider(value) {
+    const normalized = value.trim().toLowerCase();
+    if (normalized === "gemini" || normalized === "openai" || normalized === "openrouter") {
+        return normalized;
+    }
+    throw new Error(`Unsupported speech provider "${value}". Use gemini, openai, or openrouter.`);
+}
+/** Local TTS: text → audio bytes on the user's own key. */
+export async function localGenerateSpeech(input) {
+    const model = input.model?.trim() || defaultSpeechModelFor("tts", input.auth.provider);
+    if (!model) {
+        throw new Error(`No TTS model available for provider ${input.auth.provider}.`);
+    }
+    const result = await generateSpeechWithKey({
+        provider: input.auth.provider,
+        model,
+        text: input.text,
+        voice: input.voice,
+        instructions: input.instructions,
+        responseFormat: input.responseFormat,
+        apiKey: input.auth.apiKey
+    });
+    return { ...result, provider: input.auth.provider, model };
+}
+/** Local STT: audio bytes → transcript (diarized on gemini) on the user's own key.
+ * wordTimestamps requests word-level timings (real on openai/whisper-1; other
+ * providers stay segment-level and downstream callers estimate word windows). */
+export async function localTranscribeSpeech(input) {
+    const model = input.model?.trim() || defaultSpeechModelFor("stt", input.auth.provider);
+    if (!model) {
+        throw new Error(`No STT model available for provider ${input.auth.provider}.`);
+    }
+    const result = await transcribeSpeechWithKey({
+        provider: input.auth.provider,
+        model,
+        audio: input.audio,
+        contentType: input.contentType,
+        prompt: input.prompt,
+        language: input.language,
+        diarize: input.diarize,
+        wordTimestamps: input.wordTimestamps,
+        apiKey: input.auth.apiKey
+    });
+    return { ...result, provider: input.auth.provider, model, srt: buildSrtFromSegments(result.segments) };
+}
+const AUDIO_EXTENSION_CONTENT_TYPES = {
+    mp3: "audio/mpeg",
+    wav: "audio/wav",
+    m4a: "audio/mp4",
+    aac: "audio/mp4",
+    ogg: "audio/ogg",
+    oga: "audio/ogg",
+    flac: "audio/flac"
+};
+function audioExtensionOf(target) {
+    const match = /\.(mp3|wav|m4a|aac|ogg|oga|flac)(\?|#|$)/i.exec(target);
+    return match ? match[1].toLowerCase() : null;
+}
+/**
+ * Turn a local path OR http(s) URL — video or audio — into speech-provider
+ * ready audio bytes. Plain audio files pass through untouched; everything else
+ * (video, unknown containers) is demuxed to small mono mp3 by LOCAL ffmpeg
+ * (bundled ffmpeg-static, or PATH), which reads URLs directly.
+ */
+export async function loadSpeechAudioLocally(target) {
+    const isUrl = /^https?:\/\//i.test(target);
+    if (!isUrl && !existsSync(target)) {
+        throw new Error(`No such local file: ${target}`);
+    }
+    const audioExtension = audioExtensionOf(target);
+    if (audioExtension) {
+        if (isUrl) {
+            const response = await fetch(target);
+            if (!response.ok) {
+                throw new Error(`Could not fetch source audio (${response.status}).`);
+            }
+            const contentType = response.headers.get("content-type")?.split(";")[0]?.trim() || AUDIO_EXTENSION_CONTENT_TYPES[audioExtension];
+            return { bytes: new Uint8Array(await response.arrayBuffer()), contentType, demuxed: false };
+        }
+        return { bytes: readFileSync(target), contentType: AUDIO_EXTENSION_CONTENT_TYPES[audioExtension], demuxed: false };
+    }
+    const ffmpeg = await resolveFfmpeg();
+    const tempDir = await mkdtemp(path.join(os.tmpdir(), "vidfarm-stt-"));
+    const outputPath = path.join(tempDir, "audio.mp3");
+    try {
+        await runFfmpegDemux(ffmpeg, target, outputPath);
+        return { bytes: await readFile(outputPath), contentType: "audio/mpeg", demuxed: true };
+    }
+    finally {
+        await rm(tempDir, { recursive: true, force: true });
+    }
+}
+function runFfmpegDemux(bin, input, outputPath) {
+    return new Promise((resolvePromise, rejectPromise) => {
+        const child = spawn(bin, [
+            "-hide_banner",
+            "-y",
+            "-i", input,
+            "-vn",
+            "-ac", "1",
+            "-b:a", "64k",
+            outputPath
+        ], { stdio: ["ignore", "ignore", "pipe"] });
+        let stderr = "";
+        child.stderr.on("data", (chunk) => (stderr += chunk.toString()));
+        child.on("error", (error) => {
+            rejectPromise(error.code === "ENOENT"
+                ? new Error("ffmpeg not found. Install ffmpeg (or `npm i ffmpeg-static`), or run with --cloud to demux on the platform.")
+                : error);
+        });
+        child.on("close", (code) => {
+            if (code === 0) {
+                resolvePromise();
+            }
+            else {
+                rejectPromise(new Error(`ffmpeg demux exited ${code}: ${stderr.slice(-400)}`));
+            }
+        });
+    });
+}
+//# sourceMappingURL=speech.js.map