@mevdragon/vidfarm-devcli 0.10.0 → 0.11.0

package/dist/src/cli.js

@@ -10,6 +10,7 @@ import { Readable } from "node:stream";
 import { pipeline } from "node:stream/promises";
 import { computeCompositionGaps, insertMediaLayer, inspectComposition, replaceLayerWithMedia } from "./devcli/composition-edit.js";
 import { runClipsCommand } from "./devcli/clips.js";
+import { MAX_LOCAL_TRANSCRIBE_AUDIO_BYTES, loadSpeechAudioLocally, localGenerateSpeech, localTranscribeSpeech, resolveLocalSpeechAuth } from "./devcli/speech.js";
 import { initTelemetry, reportCliCrash } from "./devcli/telemetry.js";
 // vidfarm-devcli — command-line bridge for the Vidfarm video studio. The
 // `serve` command boots the FULL editor locally (single origin, disk-backed
@@ -160,6 +161,28 @@ Clip hunting (the third library — mine long-form video into a reusable clip st
   clips preset list|run <name>|save <name> --from-query '<...>'          → /clips/presets
   clips export <ids...> --to <dir>   Copy clip mp4s + metadata out
                             (run 'vidfarm clips' for the full clips help)
+
+Speech (TTS/STT) — LOCAL-FIRST on your own AI key; --cloud is the explicit backup:
+  tts "<text>"              Text → narration audio file.                 LOCAL: your OPENAI/GEMINI/
+                            The VOICE STYLE is promptable via --style.   OPENROUTER_API_KEY, no cloud job
+    --style "<direction>"     Voice-style prompt: tone/pacing/accent/emotion/persona
+                              (e.g. "calm, warm bedtime narrator", "excited sports announcer")
+    --voice <name>            Provider voice preset (openai: alloy/ash/coral…; gemini: Kore/Puck…)
+    --out <file>              Output audio path (default tts-<id>.mp3|wav)
+    --provider <p> --model <m> --format mp3|wav
+    --cloud                   BACKUP: saved-key platform job   → POST /api/v1/primitives/audio/speech (+ poll)
+  stt <file|url>            Video or audio → transcript (alias: transcribe).  LOCAL: local ffmpeg demux +
+                            Returns BOTH formats: the simple subtitle    your provider key (gemini labels
+                            version (plain text + timed SRT cues) and    speakers; openai/openrouter give
+                            the advanced multi-speaker version           a plain transcript)
+                            (speaker-labeled timed segments).
+    --out <base|.json|.srt|.txt>  Write transcript files — bare base writes all three
+    --no-diarize              Skip speaker attribution (plain transcript only)
+    --language <code>         Expected language hint (e.g. en, es)
+    --prompt "<context>"      Domain/vocabulary hint for the transcriber
+    --provider <p> --model <m>
+    --cloud                   BACKUP: saved-key platform job   → POST /api/v1/primitives/audio/transcribe (+ poll)
+                              (a local file uploads to your temp folder first — 30-day TTL)
   versions <forkId>         List immutable version snapshots             → GET  .../compositions/:forkId/versions
   snapshot <forkId>         Save the working state as a new version      → POST .../compositions/:forkId/versions
   render <forkId>           Render the fork to MP4 (--wait to poll)      → POST .../compositions/:forkId/render
@@ -252,6 +275,15 @@ Publish-mode options:
   --message <text>    Message attached to the published version snapshot
   --no-snapshot       Only update the working copy; skip the version snapshot
 `;
+// ANSI palette — declared BEFORE main() is invoked because command handlers may
+// reference these synchronously (before their first await) while the rest of
+// this module is still evaluating.
+const FRONTEND = "\x1b[1m\x1b[36m"; // bold cyan — reserved for openable URLs
+const BOLD = "\x1b[1m";
+const DIM = "\x1b[2m";
+const GREEN = "\x1b[32m";
+const RED = "\x1b[31m";
+const RESET = "\x1b[0m";
 void main().catch(async (error) => {
     console.error(error instanceof Error ? error.stack ?? error.message : String(error));
     // Report only unexpected crashes (bugs), never the deliberate user-facing
@@ -314,6 +346,13 @@ async function main() {
         case "generate":
             await runGenerateCommand(rest);
             return;
+        case "tts":
+            await runTtsCommand(rest);
+            return;
+        case "stt":
+        case "transcribe":
+            await runSttCommand(rest);
+            return;
         case "place":
             await runPlaceCommand(rest);
             return;
@@ -407,16 +446,6 @@ async function main() {
             throw new Error(`Unknown option before command: ${command}\n\n${HELP}`);
     }
 }
-// ── REST plumbing shared by every wrapper command ───────────────────────────
-// One code path for auth + host + query + JSON. This is what makes the devcli a
-// thin shell over the REST API rather than a second implementation: the named
-// commands below just assemble a method/path/body and hand it here.
-const FRONTEND = "\x1b[1m\x1b[36m"; // bold cyan — reserved for openable URLs
-const BOLD = "\x1b[1m";
-const DIM = "\x1b[2m";
-const GREEN = "\x1b[32m";
-const RED = "\x1b[31m";
-const RESET = "\x1b[0m";
 async function apiRequest(input) {
     const url = new URL(input.path, input.host);
     for (const [key, value] of Object.entries(input.query ?? {})) {
@@ -2075,6 +2104,354 @@ async function runGenerateCommand(argv) {
             console.log(`${DIM}Place it with: vidfarm place <dir> --src "${mediaUrl}" --kind ${mediaType} --at <time>|--replace <layer_key>${RESET}`);
     }
 }
+// ── Speech: tts / stt ────────────────────────────────────────────────────────
+// LOCAL-FIRST like `clips scan`: both commands default to running on the
+// user's OWN AI key (GEMINI_API_KEY / OPENAI_API_KEY / OPENROUTER_API_KEY or
+// the --*-key flags) via src/services/speech.ts — local ffmpeg demux, direct
+// provider call, no cloud job, no wallet. The REST primitive routes
+// (/api/v1/primitives/audio/speech|transcribe) are the explicit `--cloud`
+// backup. Speech cannot ride on a claude/codex CLI agent (no audio I/O), so
+// "local" means a raw provider key here, not the agent CLI.
+function speechKeyHint(kind) {
+    const preference = kind === "stt"
+        ? "GEMINI_API_KEY (best — it can label speakers), OPENAI_API_KEY, or OPENROUTER_API_KEY"
+        : "OPENAI_API_KEY, GEMINI_API_KEY, or OPENROUTER_API_KEY";
+    return `No local AI key found. Set ${preference} (or pass --gemini-key/--openai-key/--openrouter-key), or run with --cloud to use your saved Vidfarm provider keys.`;
+}
+async function downloadUrlToFile(url, dest) {
+    const res = await fetch(url);
+    if (!res.ok || !res.body)
+        throw new Error(`download failed (${res.status}) for ${url}`);
+    mkdirSync(path.dirname(dest), { recursive: true });
+    await pipeline(Readable.fromWeb(res.body), createWriteStream(dest));
+}
+async function runTtsCommand(argv) {
+    const parsed = parseArgs({
+        args: argv,
+        allowPositionals: true,
+        options: {
+            ...commonOptions(),
+            text: { type: "string" },
+            style: { type: "string" },
+            instructions: { type: "string" },
+            voice: { type: "string" },
+            provider: { type: "string" },
+            model: { type: "string" },
+            format: { type: "string" },
+            out: { type: "string" },
+            cloud: { type: "boolean", default: false },
+            "gemini-key": { type: "string" },
+            "openai-key": { type: "string" },
+            "openrouter-key": { type: "string" },
+            "no-wait": { type: "boolean", default: false },
+            tracer: { type: "string" }
+        }
+    });
+    const text = (parsed.values.text ?? parsed.positionals.join(" ")).trim();
+    if (!text)
+        throw new Error('tts requires text: vidfarm tts "Welcome back to the channel" --style "calm, warm narrator".');
+    const style = (parsed.values.instructions ?? parsed.values.style)?.trim() || undefined;
+    const rawFormat = String(parsed.values.format ?? "mp3").toLowerCase();
+    if (rawFormat !== "mp3" && rawFormat !== "wav")
+        throw new Error("tts --format must be mp3 or wav.");
+    const format = rawFormat;
+    const voice = parsed.values.voice;
+    const json = Boolean(parsed.values.json);
+    if (!parsed.values.cloud) {
+        const auth = resolveLocalSpeechAuth(parsed.values, ["openai", "gemini", "openrouter"], parsed.values.provider);
+        if (!auth)
+            throw new Error(speechKeyHint("tts"));
+        if (!json)
+            console.log(`${DIM}Generating speech locally on your ${auth.provider} key…${RESET}`);
+        const result = await localGenerateSpeech({
+            auth,
+            text,
+            voice,
+            instructions: style,
+            model: parsed.values.model,
+            responseFormat: format
+        });
+        const extension = result.contentType.includes("wav") ? "wav" : "mp3";
+        const outPath = path.resolve(process.cwd(), parsed.values.out ?? `tts-${Date.now().toString(36)}.${extension}`);
+        mkdirSync(path.dirname(outPath), { recursive: true });
+        writeFileSync(outPath, result.bytes);
+        if (json) {
+            printJson({ ok: true, mode: "local", provider: result.provider, model: result.model, voice: voice ?? null, style: style ?? null, content_type: result.contentType, bytes: result.bytes.byteLength, out: outPath });
+        }
+        else {
+            console.log(`${GREEN}Wrote ${outPath}${RESET} ${DIM}(${result.provider}/${result.model}, ${Math.round(result.bytes.byteLength / 1024)}KB)${RESET}`);
+            console.log(`${DIM}Get a shareable URL with: vidfarm upload "${outPath}" — then add it to a composition as an audio layer.${RESET}`);
+        }
+        return;
+    }
+    // BACKUP: the platform primitive route (saved provider keys, async job).
+    const ctx = commonContext(parsed.values);
+    const payload = { text, output_format: format };
+    if (voice)
+        payload.voice = voice;
+    if (style)
+        payload.instructions = style;
+    if (parsed.values.provider)
+        payload.provider = parsed.values.provider;
+    if (parsed.values.model)
+        payload.model = parsed.values.model;
+    const tracer = parsed.values.tracer ?? `devcli-tts-${Date.now().toString(36)}`;
+    const submit = await apiRequest({ method: "POST", host: ctx.host, path: "/api/v1/primitives/audio/speech", auth: ctx.auth, body: { tracer, payload } });
+    assertApiOk(submit, "tts");
+    const jobId = submit.json?.job_id;
+    if (parsed.values["no-wait"] || !jobId) {
+        if (!ctx.json && jobId)
+            console.log(`${DIM}Queued ${jobId} (tracer ${tracer}). Poll: vidfarm api GET /api/v1/user/me/jobs/${jobId} — or drop --no-wait to auto-poll.${RESET}`);
+        emitResult(submit, ctx.json);
+        return;
+    }
+    if (!ctx.json)
+        console.log(`${DIM}Generating speech (${jobId})… polling every 5s.${RESET}`);
+    const job = await pollPrimitiveJob(ctx, jobId);
+    const mediaUrl = resolveJobMediaUrl(job);
+    if (!mediaUrl) {
+        if (ctx.json) {
+            printJson({ ok: false, job_id: jobId, status: String(job?.status ?? ""), job });
+        }
+        else {
+            console.log(`${RED}TTS ${String(job?.status ?? "did not finish")} — no audio URL.${RESET}`);
+            printJson(job);
+        }
+        process.exitCode = 1;
+        return;
+    }
+    const extension = /\.wav(\?|#|$)/i.test(mediaUrl) ? "wav" : "mp3";
+    const outPath = path.resolve(process.cwd(), parsed.values.out ?? `tts-${jobId}.${extension}`);
+    await downloadUrlToFile(mediaUrl, outPath);
+    if (ctx.json) {
+        printJson({ ok: true, mode: "cloud", job_id: jobId, tracer, audio_url: mediaUrl, out: outPath });
+    }
+    else {
+        console.log(`${GREEN}Wrote ${outPath}${RESET} ${DIM}(${mediaUrl})${RESET}`);
+    }
+}
+function formatSpeechClock(totalSeconds) {
+    const minutes = Math.floor(totalSeconds / 60);
+    const seconds = Math.floor(totalSeconds % 60);
+    return `${minutes}:${String(seconds).padStart(2, "0")}`;
+}
+// Print/persist BOTH transcript formats: the simple subtitle version (plain
+// text + SRT cues) and the advanced multi-speaker version (speaker-attributed
+// segments → .json). --out picks one via extension or writes all three.
+function emitSttResult(input) {
+    const written = [];
+    if (input.out) {
+        const explicitExtension = /\.(json|srt|txt)$/i.exec(input.out)?.[1]?.toLowerCase() ?? null;
+        const base = explicitExtension ? input.out.slice(0, -(explicitExtension.length + 1)) : input.out;
+        const targets = explicitExtension ? [explicitExtension] : ["json", "srt", "txt"];
+        for (const kind of targets) {
+            const dest = path.resolve(process.cwd(), `${base}.${kind}`);
+            if (kind === "json") {
+                mkdirSync(path.dirname(dest), { recursive: true });
+                writeFileSync(dest, `${JSON.stringify({
+                    source: input.source,
+                    mode: input.mode,
+                    provider: input.provider,
+                    model: input.model,
+                    language: input.result.language,
+                    diarization: input.result.diarization,
+                    text: input.result.text,
+                    speakers: [...new Set((input.result.segments ?? []).map((segment) => segment.speaker))],
+                    segments: input.result.segments ?? []
+                }, null, 2)}\n`);
+                written.push(dest);
+            }
+            else if (kind === "srt") {
+                if (input.result.srt) {
+                    mkdirSync(path.dirname(dest), { recursive: true });
+                    writeFileSync(dest, input.result.srt);
+                    written.push(dest);
+                }
+                else if (explicitExtension === "srt") {
+                    console.log(`${RED}No timed segments — cannot write ${dest}. The plain transcript is still available (--out <base>.txt).${RESET}`);
+                }
+            }
+            else {
+                mkdirSync(path.dirname(dest), { recursive: true });
+                writeFileSync(dest, `${input.result.text ?? ""}\n`);
+                written.push(dest);
+            }
+        }
+    }
+    if (input.json) {
+        printJson({
+            ok: true,
+            mode: input.mode,
+            provider: input.provider,
+            model: input.model,
+            ...(input.jobId ? { job_id: input.jobId } : {}),
+            source: input.source,
+            language: input.result.language,
+            diarization: input.result.diarization,
+            text: input.result.text,
+            segments: input.result.segments ?? [],
+            srt: input.result.srt,
+            written
+        });
+        return;
+    }
+    if (input.result.segments?.length && input.result.diarization === "diarized") {
+        for (const segment of input.result.segments) {
+            const at = typeof segment.start_sec === "number" ? `${DIM}[${formatSpeechClock(segment.start_sec)}]${RESET} ` : "";
+            console.log(`${at}${segment.speaker}: ${segment.text}`);
+        }
+    }
+    else {
+        console.log(input.result.text || "(no speech detected)");
+    }
+    for (const dest of written) {
+        console.log(`${GREEN}Wrote ${dest}${RESET}`);
+    }
+}
+async function runSttCommand(argv) {
+    const parsed = parseArgs({
+        args: argv,
+        allowPositionals: true,
+        options: {
+            ...commonOptions(),
+            "no-diarize": { type: "boolean", default: false },
+            language: { type: "string" },
+            prompt: { type: "string" },
+            provider: { type: "string" },
+            model: { type: "string" },
+            out: { type: "string" },
+            cloud: { type: "boolean", default: false },
+            "gemini-key": { type: "string" },
+            "openai-key": { type: "string" },
+            "openrouter-key": { type: "string" },
+            "no-wait": { type: "boolean", default: false },
+            tracer: { type: "string" },
+            folder: { type: "string" }
+        }
+    });
+    const target = parsed.positionals[0];
+    if (!target)
+        throw new Error("stt requires a source: `vidfarm stt <video-or-audio file|url>`.");
+    const diarize = !parsed.values["no-diarize"];
+    const json = Boolean(parsed.values.json);
+    const language = parsed.values.language?.trim() || undefined;
+    const prompt = parsed.values.prompt?.trim() || undefined;
+    if (!parsed.values.cloud) {
+        const auth = resolveLocalSpeechAuth(parsed.values, ["gemini", "openai", "openrouter"], parsed.values.provider);
+        if (!auth)
+            throw new Error(speechKeyHint("stt"));
+        if (diarize && auth.provider !== "gemini" && !json) {
+            console.log(`${DIM}Note: speaker labels need a Gemini key; ${auth.provider} returns a plain single-speaker transcript.${RESET}`);
+        }
+        if (!json)
+            console.log(`${DIM}Transcribing locally on your ${auth.provider} key (local ffmpeg demux)…${RESET}`);
+        const audio = await loadSpeechAudioLocally(target);
+        if (audio.bytes.byteLength > MAX_LOCAL_TRANSCRIBE_AUDIO_BYTES) {
+            throw new Error(`Audio is too large to transcribe (${Math.round(audio.bytes.byteLength / (1024 * 1024))}MB > 20MB, roughly 40 minutes of speech). Trim the source first.`);
+        }
+        const result = await localTranscribeSpeech({
+            auth,
+            audio: audio.bytes,
+            contentType: audio.contentType,
+            model: parsed.values.model,
+            prompt,
+            language,
+            diarize
+        });
+        emitSttResult({
+            json,
+            out: parsed.values.out,
+            source: target,
+            mode: "local",
+            provider: result.provider,
+            model: result.model,
+            result: {
+                text: result.text,
+                language: result.language,
+                diarization: result.diarization,
+                segments: result.segments,
+                srt: result.srt
+            }
+        });
+        return;
+    }
+    // BACKUP: the platform primitive route. URLs pass through; local files
+    // upload to the ephemeral temp store first (30-day TTL).
+    const ctx = commonContext(parsed.values);
+    const sourceUrl = /^https?:\/\//i.test(target)
+        ? target
+        : await uploadLocalToTempStore(ctx, path.resolve(process.cwd(), target), parsed.values.folder ?? "temp");
+    const payload = { source_url: sourceUrl, diarize };
+    if (language)
+        payload.language = language;
+    if (prompt)
+        payload.prompt = prompt;
+    if (parsed.values.provider)
+        payload.provider = parsed.values.provider;
+    if (parsed.values.model)
+        payload.model = parsed.values.model;
+    const tracer = parsed.values.tracer ?? `devcli-stt-${Date.now().toString(36)}`;
+    const submit = await apiRequest({ method: "POST", host: ctx.host, path: "/api/v1/primitives/audio/transcribe", auth: ctx.auth, body: { tracer, payload } });
+    assertApiOk(submit, "stt");
+    const jobId = submit.json?.job_id;
+    if (parsed.values["no-wait"] || !jobId) {
+        if (!ctx.json && jobId)
+            console.log(`${DIM}Queued ${jobId} (tracer ${tracer}). Poll: vidfarm api GET /api/v1/user/me/jobs/${jobId} — or drop --no-wait to auto-poll.${RESET}`);
+        emitResult(submit, ctx.json);
+        return;
+    }
+    if (!ctx.json)
+        console.log(`${DIM}Transcribing (${jobId})… polling every 5s.${RESET}`);
+    const job = await pollPrimitiveJob(ctx, jobId);
+    const status = String(job?.status ?? "");
+    const rawResult = (job?.result && typeof job.result === "object") ? job.result : {};
+    const output = (rawResult.output && typeof rawResult.output === "object") ? rawResult.output : rawResult;
+    if (status !== "succeeded" && typeof output.text !== "string") {
+        if (ctx.json) {
+            printJson({ ok: false, job_id: jobId, status, job });
+        }
+        else {
+            console.log(`${RED}Transcription ${status || "did not finish"}.${RESET}`);
+            printJson(job);
+        }
+        process.exitCode = 1;
+        return;
+    }
+    // Long transcripts stay artifact-only; hydrate the full version from
+    // transcript.json / transcript.srt when the inline copy was truncated.
+    let text = typeof output.text === "string" ? output.text : "";
+    let segments = Array.isArray(output.segments) ? output.segments : null;
+    let srt = typeof output.subtitles?.srt === "string" ? output.subtitles.srt : null;
+    const jsonUrl = typeof output.transcript?.json_url === "string" ? output.transcript.json_url : null;
+    if ((output.text_truncated === true || !segments) && jsonUrl) {
+        const hydrated = await fetch(jsonUrl).then((res) => (res.ok ? res.json() : null)).catch(() => null);
+        if (hydrated && typeof hydrated === "object") {
+            if (typeof hydrated.text === "string")
+                text = hydrated.text;
+            if (Array.isArray(hydrated.segments))
+                segments = hydrated.segments;
+        }
+    }
+    if (!srt && typeof output.subtitles?.srt_url === "string") {
+        srt = await fetch(output.subtitles.srt_url).then((res) => (res.ok ? res.text() : null)).catch(() => null);
+    }
+    emitSttResult({
+        json: ctx.json,
+        out: parsed.values.out,
+        source: target,
+        mode: "cloud",
+        provider: typeof output.transcript?.provider === "string" ? output.transcript.provider : null,
+        model: typeof output.transcript?.model === "string" ? output.transcript.model : null,
+        jobId,
+        result: {
+            text,
+            language: typeof output.language === "string" ? output.language : null,
+            diarization: typeof output.diarization === "string" ? output.diarization : "none",
+            segments,
+            srt
+        }
+    });
+}
 async function runPlaceCommand(argv) {
     const parsed = parseArgs({
         args: argv,

package/dist/src/devcli/speech.js

@@ -0,0 +1,175 @@
+// Local speech engine — devcli-only. `vidfarm tts` / `vidfarm stt` run
+// LOCAL-FIRST on the user's own AI key (GEMINI_API_KEY / OPENAI_API_KEY /
+// OPENROUTER_API_KEY, or the --gemini-key/--openai-key/--openrouter-key flags)
+// by calling the shared speech provider layer (src/services/speech.ts)
+// directly: local ffmpeg for the video→audio demux, the user's key for the
+// model call — no cloud job, no wallet, no REST route. The REST primitive
+// routes (/api/v1/primitives/audio/speech|transcribe) are the explicit
+// `--cloud` BACKUP, mirroring `vidfarm clips scan`.
+//
+// Unlike clip tagging, speech can NOT ride on a claude/codex CLI agent — the
+// agent CLIs have no audio I/O (the same reason embeddings need a real key in
+// local-agent.ts) — so "local" here means a raw provider key, not the agent.
+import { spawn } from "node:child_process";
+import { existsSync, readFileSync } from "node:fs";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { buildSrtFromSegments, defaultSpeechModelFor, generateSpeechWithKey, transcribeSpeechWithKey } from "../services/speech.js";
+import { resolveFfmpeg } from "../services/clip-curation/ffmpeg.js";
+// Mirrors the env-var fallbacks used by `vidfarm clips` so one exported key
+// powers every local-first devcli surface.
+const PROVIDER_ENV = {
+    gemini: ["GEMINI_API_KEY", "GOOGLE_GENERATIVE_AI_API_KEY"],
+    openai: ["OPENAI_API_KEY"],
+    openrouter: ["OPENROUTER_API_KEY"]
+};
+const PROVIDER_FLAG = {
+    gemini: "gemini-key",
+    openai: "openai-key",
+    openrouter: "openrouter-key"
+};
+// Same inline-audio ceiling as the cloud stt primitive (~40min of 64kbps mono).
+export const MAX_LOCAL_TRANSCRIBE_AUDIO_BYTES = 20 * 1024 * 1024;
+/**
+ * Find a usable local provider key: an explicit --provider narrows the search,
+ * otherwise the first provider (in `preference` order) with a flag/env key
+ * wins. Returns null when nothing local is configured (callers point the user
+ * at env keys or --cloud).
+ */
+export function resolveLocalSpeechAuth(values, preference, requestedProvider) {
+    const providers = requestedProvider
+        ? [normalizeSpeechProvider(requestedProvider)]
+        : preference;
+    for (const provider of providers) {
+        const flagged = values[PROVIDER_FLAG[provider]];
+        if (typeof flagged === "string" && flagged.trim()) {
+            return { provider, apiKey: flagged.trim() };
+        }
+        for (const envName of PROVIDER_ENV[provider]) {
+            const fromEnv = process.env[envName]?.trim();
+            if (fromEnv) {
+                return { provider, apiKey: fromEnv };
+            }
+        }
+    }
+    return null;
+}
+export function normalizeSpeechProvider(value) {
+    const normalized = value.trim().toLowerCase();
+    if (normalized === "gemini" || normalized === "openai" || normalized === "openrouter") {
+        return normalized;
+    }
+    throw new Error(`Unsupported speech provider "${value}". Use gemini, openai, or openrouter.`);
+}
+/** Local TTS: text → audio bytes on the user's own key. */
+export async function localGenerateSpeech(input) {
+    const model = input.model?.trim() || defaultSpeechModelFor("tts", input.auth.provider);
+    if (!model) {
+        throw new Error(`No TTS model available for provider ${input.auth.provider}.`);
+    }
+    const result = await generateSpeechWithKey({
+        provider: input.auth.provider,
+        model,
+        text: input.text,
+        voice: input.voice,
+        instructions: input.instructions,
+        responseFormat: input.responseFormat,
+        apiKey: input.auth.apiKey
+    });
+    return { ...result, provider: input.auth.provider, model };
+}
+/** Local STT: audio bytes → transcript (diarized on gemini) on the user's own key. */
+export async function localTranscribeSpeech(input) {
+    const model = input.model?.trim() || defaultSpeechModelFor("stt", input.auth.provider);
+    if (!model) {
+        throw new Error(`No STT model available for provider ${input.auth.provider}.`);
+    }
+    const result = await transcribeSpeechWithKey({
+        provider: input.auth.provider,
+        model,
+        audio: input.audio,
+        contentType: input.contentType,
+        prompt: input.prompt,
+        language: input.language,
+        diarize: input.diarize,
+        apiKey: input.auth.apiKey
+    });
+    return { ...result, provider: input.auth.provider, model, srt: buildSrtFromSegments(result.segments) };
+}
+const AUDIO_EXTENSION_CONTENT_TYPES = {
+    mp3: "audio/mpeg",
+    wav: "audio/wav",
+    m4a: "audio/mp4",
+    aac: "audio/mp4",
+    ogg: "audio/ogg",
+    oga: "audio/ogg",
+    flac: "audio/flac"
+};
+function audioExtensionOf(target) {
+    const match = /\.(mp3|wav|m4a|aac|ogg|oga|flac)(\?|#|$)/i.exec(target);
+    return match ? match[1].toLowerCase() : null;
+}
+/**
+ * Turn a local path OR http(s) URL — video or audio — into speech-provider
+ * ready audio bytes. Plain audio files pass through untouched; everything else
+ * (video, unknown containers) is demuxed to small mono mp3 by LOCAL ffmpeg
+ * (bundled ffmpeg-static, or PATH), which reads URLs directly.
+ */
+export async function loadSpeechAudioLocally(target) {
+    const isUrl = /^https?:\/\//i.test(target);
+    if (!isUrl && !existsSync(target)) {
+        throw new Error(`No such local file: ${target}`);
+    }
+    const audioExtension = audioExtensionOf(target);
+    if (audioExtension) {
+        if (isUrl) {
+            const response = await fetch(target);
+            if (!response.ok) {
+                throw new Error(`Could not fetch source audio (${response.status}).`);
+            }
+            const contentType = response.headers.get("content-type")?.split(";")[0]?.trim() || AUDIO_EXTENSION_CONTENT_TYPES[audioExtension];
+            return { bytes: new Uint8Array(await response.arrayBuffer()), contentType, demuxed: false };
+        }
+        return { bytes: readFileSync(target), contentType: AUDIO_EXTENSION_CONTENT_TYPES[audioExtension], demuxed: false };
+    }
+    const ffmpeg = await resolveFfmpeg();
+    const tempDir = await mkdtemp(path.join(os.tmpdir(), "vidfarm-stt-"));
+    const outputPath = path.join(tempDir, "audio.mp3");
+    try {
+        await runFfmpegDemux(ffmpeg, target, outputPath);
+        return { bytes: await readFile(outputPath), contentType: "audio/mpeg", demuxed: true };
+    }
+    finally {
+        await rm(tempDir, { recursive: true, force: true });
+    }
+}
+function runFfmpegDemux(bin, input, outputPath) {
+    return new Promise((resolvePromise, rejectPromise) => {
+        const child = spawn(bin, [
+            "-hide_banner",
+            "-y",
+            "-i", input,
+            "-vn",
+            "-ac", "1",
+            "-b:a", "64k",
+            outputPath
+        ], { stdio: ["ignore", "ignore", "pipe"] });
+        let stderr = "";
+        child.stderr.on("data", (chunk) => (stderr += chunk.toString()));
+        child.on("error", (error) => {
+            rejectPromise(error.code === "ENOENT"
+                ? new Error("ffmpeg not found. Install ffmpeg (or `npm i ffmpeg-static`), or run with --cloud to demux on the platform.")
+                : error);
+        });
+        child.on("close", (code) => {
+            if (code === 0) {
+                resolvePromise();
+            }
+            else {
+                rejectPromise(new Error(`ffmpeg demux exited ${code}: ${stderr.slice(-400)}`));
+            }
+        });
+    });
+}
+//# sourceMappingURL=speech.js.map

package/dist/src/editor-chat.js

@@ -34,7 +34,7 @@ export function buildTemplateEditorChatSystemPrompt(input) {
         "Execute REST calls sequentially. Make one API call, inspect the response, then decide the next step.",
         "A queued async POST job is enough inspection for that request. Do not poll it in the same response; continue launching any remaining requested outputs, then summarize the queued jobs.",
         "Primitive routes under /api/v1/primitives are first-class allowed routes in this editor, even when the primitive is not specific to the current template.",
-        "Never say you cannot use primitive image generation, image edit, inpaint, HTML render, AI video generation, or video render routes directly from this editor chat; use the http_request tool when the requested primitive route exists.",
+        "Never say you cannot use primitive image generation, image edit, inpaint, HTML render, AI video generation, text-to-speech, speech transcription, or video render routes directly from this editor chat; use the http_request tool when the requested primitive route exists.",
         "All primitive POST routes require the same outer JSON envelope: top-level tracer, top-level payload containing the primitive-specific fields, and optional webhook_url.",
         "When the user explicitly asks to use primitives or timeline rendering, do not redirect them to old per-template operation routes unless the requested primitive route cannot accept the available inputs.",
         "If the user asks to create, generate, or make a new image similar to an attached image, call POST /api/v1/primitives/images/generate with body { tracer, payload: { prompt, prompt_attachments: [exact attachment URL] } }.",
@@ -42,6 +42,7 @@ export function buildTemplateEditorChatSystemPrompt(input) {
         "If the user asks to modify an attached image and the message includes an attachment URL, call POST /api/v1/primitives/images/edit with source_image_url inside payload set to that exact URL and instruction inside payload set to the requested edit.",
         "If the user asks to generate a new AI video from text or reference images, call POST /api/v1/primitives/videos/generate with body { tracer, payload: { prompt, input_references?, duration? } }. AI video duration is seconds; use duration: 4 for a 4-second video and never use duration_ms on /videos/generate. input_references must be direct image asset URLs, not webpage or social post URLs. Use frame_images for exact first/last-frame image-to-video.",
         "If the user asks to render HTML/design into an image, call POST /api/v1/primitives/images/render-html. If the user asks to turn existing media URLs into a slideshow/video, call POST /api/v1/primitives/videos/render-slides. If the user asks to dedupe, camouflage, or apply small zoom/tilt/rotate/filter/tint transforms to an image or video for reuse, call POST /api/v1/primitives/media/dedupe with body { tracer, payload: { source_media_url, media_type?, effects?: { zoom?, tilt?, rotate?, saturation?, speed?, horizontal_flip?, contrast?, brightness?, hue_rotate?, blur? }, tint_color?, tint_opacity?, width?, height?, duration_ms?, object_fit?, background_color?, output_format? }, webhook_url? }. Primitive media utility routes are also allowed directly here, including /videos/trim, /videos/remove-captions, /videos/extract-audio, /videos/probe, /videos/extract-frame, /videos/mute, /videos/replace-audio, /videos/concat, /audio/trim, /audio/probe, /audio/concat, and /audio/normalize. If the user asks to remove burned-in captions, subtitles, or on-screen text from a video, call POST /api/v1/primitives/videos/remove-captions with body { tracer, payload: { source_video_url } } — an async job that returns a durable caption-free MP4 and bills about $0.10 per 30 seconds of source video. Caption removal is capped at ~15 minutes of source and must NEVER be used on long-form video: when the user wants short clips out of a long video (a podcast, stream VOD, webinar, or a YouTube/TikTok/IG/X URL), call POST /clips/scan instead with body { source_url | temp_file_id | s3_key, prompt, ranges?: [\"MM:SS-MM:SS\"], target_duration_sec?, aspect?: \"9:16\"|\"16:9\"|\"4:3\"|\"1:1\", avoid_text?, tracer? } — target_duration_sec is a SOFT range (30 → 20-40s), ranges restrict the hunt to those source windows (big cost saver), aspect crops every clip, and avoid_text prefers text-free scenes via scene SELECTION (never caption removal on the source). The hunt is async (202 + scan_id): poll GET /clips/scan/:scanId, then browse GET /clips/feed?source=<source_video_id> or POST /clips/search. Hunts bill AWS compute only — the AI tagging runs on the user's own saved provider keys. To erase captions from a finished HUNTED clip, pass that short mp4 to /videos/remove-captions afterwards.",
+        "SPEECH PRIMITIVES (TTS/STT) are first-class here. VOICEOVER / NARRATION: when the user wants a script, hook, caption, or dub read aloud as AI audio, call POST /api/v1/primitives/audio/speech with body { tracer, payload: { text, voice?, instructions?, provider?, model?, output_format? } } — instructions is a free-form voice-STYLE prompt (tone, pacing, accent, emotion, persona; e.g. 'calm warm bedtime narrator' or 'excited sports announcer, fast paced'), and voice picks a provider preset (OpenAI: alloy/ash/coral/...; Gemini: Kore/Puck/Charon/...). The finished job returns a durable audio URL — place it on the timeline with editor_action add_layer (kind=audio, src=that URL), typically starting where the narration should begin. TRANSCRIPTION: when the user wants a transcript, subtitles/captions from speech, a translation source, or to know who says what, call POST /api/v1/primitives/audio/transcribe with body { tracer, payload: { source_url, diarize?, language?, prompt? } } — source_url may be a VIDEO or AUDIO URL (video audio is demuxed automatically; pick the fork's source via GET /api/v1/compositions/:forkId/remove-video-captions first when transcribing the current composition). One job returns BOTH formats: a simple subtitle version (plain transcript text + timed SRT cues, stored as transcript.txt/transcript.srt) and an advanced multi-speaker version (speaker-attributed timed segments in output.segments and transcript.json) for multi-narration sources like podcasts and interviews. diarize defaults to true and speaker attribution needs a Gemini key — openai/openrouter keys degrade to a plain single-speaker transcript. For transcript-grounded caption work on THIS fork, the read-only video_context tool may already contain the decompose-time transcript; prefer it before running a new billable-key transcription, and use /audio/transcribe for other URLs, multi-speaker attribution, or SRT output.",
         "Brainstorm primitives are also available directly here. If the user explicitly asks for hooks, angles, awareness-stage advice, a cold-start strategy questionnaire, or product-placement opportunities in a video, call the matching brainstorm primitive immediately instead of brainstorming in chat from memory. Use POST /api/v1/primitives/brainstorm/coldstart with body { tracer, payload: { user_message }, webhook_url? } when the user is clueless and needs foundational strategy questions. Use /brainstorm/awareness_stages with body { tracer, payload: { offer_description }, webhook_url? } when they are unsure what type of ads to make. Use /brainstorm/angles with body { tracer, payload: { offer_description, problem_awareness, solution_awareness }, webhook_url? } when they want persuasive ad angles; problem_awareness must be problem_unaware or problem_aware, and solution_awareness must be solution_unaware or solution_aware. Use /brainstorm/hooks with body { tracer, payload: { offer_description }, webhook_url? } when they are stuck on openings and want many hooks.",
         "Product placement is a first-class brainstorm primitive, just like angles and hooks. When the user asks to identify product-placement opportunities in a video, or to repurpose a video into a native ad for their product, prefer POST /api/v1/primitives/brainstorm/product_placement with body { tracer, payload: { source_video_url, offer_description }, webhook_url? }. source_video_url is the exact video to analyze — for the current composition, first call GET /api/v1/compositions/:forkId/remove-video-captions to pick the correct source (the ORIGINAL uploaded video vs the DECOMPOSED, caption-free video), then pass that URL. offer_description is the user's product/offer. This primitive watches the video and returns timestamped, native placement opportunities. It prefers a Gemini key for video understanding.",
         "BEFORE calling /brainstorm/product_placement for the CURRENT composition, first call the product_placement_context tool (or GET /api/v1/compositions/:forkId/product-placement.json). Decompose already scouted product-AGNOSTIC placement surfaces for this fork — real, grounded moments (timestamp, surface_type, surface_note, best_for, why_it_works). Use those surfaces to ground your answer and to seed the primitive: fold the strongest surfaces into offer_description (for example append 'Known native placement surfaces: 0:04 empty desk prop; 0:12 phone screen swap') so the product-specific pass builds on grounded moments instead of re-watching blind. If the user just wants a quick conversational take on where their product could go, the surfaces from product_placement_context are often enough to answer directly without the billable primitive.",

package/dist/src/primitive-context.js

@@ -181,6 +181,22 @@ export function createPrimitiveJobContext(input) {
                     workerId: input.workerId,
                     ...request
                 });
+            },
+            async generateSpeech(request) {
+                return input.providers.generateSpeech({
+                    customerId: input.customer.id,
+                    jobId: input.job.id,
+                    workerId: input.workerId,
+                    ...request
+                });
+            },
+            async transcribeSpeech(request) {
+                return input.providers.transcribeSpeech({
+                    customerId: input.customer.id,
+                    jobId: input.job.id,
+                    workerId: input.workerId,
+                    ...request
+                });
             }
         },
         hyperframes: {