@mevdragon/vidfarm-devcli 0.9.0 → 0.11.0

package/dist/src/cli.js

@@ -10,6 +10,7 @@ import { Readable } from "node:stream";
 import { pipeline } from "node:stream/promises";
 import { computeCompositionGaps, insertMediaLayer, inspectComposition, replaceLayerWithMedia } from "./devcli/composition-edit.js";
 import { runClipsCommand } from "./devcli/clips.js";
+import { MAX_LOCAL_TRANSCRIBE_AUDIO_BYTES, loadSpeechAudioLocally, localGenerateSpeech, localTranscribeSpeech, resolveLocalSpeechAuth } from "./devcli/speech.js";
 import { initTelemetry, reportCliCrash } from "./devcli/telemetry.js";
 // vidfarm-devcli — command-line bridge for the Vidfarm video studio. The
 // `serve` command boots the FULL editor locally (single origin, disk-backed
@@ -144,16 +145,44 @@ Generate AI media and drop it on the timeline (for local coding agents):
                             original vs decomposed (caption-free),
                             non-billing (alias: ghostcut)
 
-Clip curation (the third library — mine long-form video into a reusable clip store):
-  clips scan <video-path>   Detect scenes → tag + transcribe (Gemini)    LOCAL: ffmpeg + BYOK Gemini
-                            → embed → persist to ~/.vidfarm/clips.db.     (cloud: POST /clips/scan)
-    --tier flash|flash-lite   Gemini tier (default flash-lite, cheapest)
+Clip hunting (the third library — mine long-form video into a reusable clip store):
+  clips scan <video-path>   Hunt short clips out of a long video.        LOCAL-FIRST: local ffmpeg +
+                            Detect → tag/transcribe → embed → persist    your local claude/codex CLI
+                            to ~/.vidfarm/clips.db.                      (keys are fallback)
+    --range "MM:SS-MM:SS"     Only hunt these source windows (repeatable; big cost saver)
+    --duration <sec>          Target clip length as a SOFT band (10→5-20s, 30→20-40s)
+    --aspect 9:16|16:9|4:3|1:1  Crop clips (also: vertical/horizontal/square) [--crop-focus <f>]
+    --no-text                 Prefer scenes WITHOUT captions/on-screen text (selection only)
+    --cloud [--url <url>]     BACKUP: run on the deployed pipeline       → POST /clips/scan (+ poll)
+                              (uploads to your temp folder — 30-day TTL — bills AWS compute only)
     --dry-run                 Show the estimated cost and stop (over $1 needs --yes/confirm)
   clips list                List the local clip library                  → GET  /clips
   clips search "<text>"     Hybrid structured + semantic search          → POST /clips/search
   clips preset list|run <name>|save <name> --from-query '<...>'          → /clips/presets
   clips export <ids...> --to <dir>   Copy clip mp4s + metadata out
                             (run 'vidfarm clips' for the full clips help)
+
+Speech (TTS/STT) — LOCAL-FIRST on your own AI key; --cloud is the explicit backup:
+  tts "<text>"              Text → narration audio file.                 LOCAL: your OPENAI/GEMINI/
+                            The VOICE STYLE is promptable via --style.   OPENROUTER_API_KEY, no cloud job
+    --style "<direction>"     Voice-style prompt: tone/pacing/accent/emotion/persona
+                              (e.g. "calm, warm bedtime narrator", "excited sports announcer")
+    --voice <name>            Provider voice preset (openai: alloy/ash/coral…; gemini: Kore/Puck…)
+    --out <file>              Output audio path (default tts-<id>.mp3|wav)
+    --provider <p> --model <m> --format mp3|wav
+    --cloud                   BACKUP: saved-key platform job   → POST /api/v1/primitives/audio/speech (+ poll)
+  stt <file|url>            Video or audio → transcript (alias: transcribe).  LOCAL: local ffmpeg demux +
+                            Returns BOTH formats: the simple subtitle    your provider key (gemini labels
+                            version (plain text + timed SRT cues) and    speakers; openai/openrouter give
+                            the advanced multi-speaker version           a plain transcript)
+                            (speaker-labeled timed segments).
+    --out <base|.json|.srt|.txt>  Write transcript files — bare base writes all three
+    --no-diarize              Skip speaker attribution (plain transcript only)
+    --language <code>         Expected language hint (e.g. en, es)
+    --prompt "<context>"      Domain/vocabulary hint for the transcriber
+    --provider <p> --model <m>
+    --cloud                   BACKUP: saved-key platform job   → POST /api/v1/primitives/audio/transcribe (+ poll)
+                              (a local file uploads to your temp folder first — 30-day TTL)
   versions <forkId>         List immutable version snapshots             → GET  .../compositions/:forkId/versions
   snapshot <forkId>         Save the working state as a new version      → POST .../compositions/:forkId/versions
   render <forkId>           Render the fork to MP4 (--wait to poll)      → POST .../compositions/:forkId/render
@@ -246,6 +275,15 @@ Publish-mode options:
   --message <text>    Message attached to the published version snapshot
   --no-snapshot       Only update the working copy; skip the version snapshot
 `;
+// ANSI palette — declared BEFORE main() is invoked because command handlers may
+// reference these synchronously (before their first await) while the rest of
+// this module is still evaluating.
+const FRONTEND = "\x1b[1m\x1b[36m"; // bold cyan — reserved for openable URLs
+const BOLD = "\x1b[1m";
+const DIM = "\x1b[2m";
+const GREEN = "\x1b[32m";
+const RED = "\x1b[31m";
+const RESET = "\x1b[0m";
 void main().catch(async (error) => {
     console.error(error instanceof Error ? error.stack ?? error.message : String(error));
     // Report only unexpected crashes (bugs), never the deliberate user-facing
@@ -308,6 +346,13 @@ async function main() {
         case "generate":
             await runGenerateCommand(rest);
             return;
+        case "tts":
+            await runTtsCommand(rest);
+            return;
+        case "stt":
+        case "transcribe":
+            await runSttCommand(rest);
+            return;
         case "place":
             await runPlaceCommand(rest);
             return;
@@ -401,16 +446,6 @@ async function main() {
             throw new Error(`Unknown option before command: ${command}\n\n${HELP}`);
     }
 }
-// ── REST plumbing shared by every wrapper command ───────────────────────────
-// One code path for auth + host + query + JSON. This is what makes the devcli a
-// thin shell over the REST API rather than a second implementation: the named
-// commands below just assemble a method/path/body and hand it here.
-const FRONTEND = "\x1b[1m\x1b[36m"; // bold cyan — reserved for openable URLs
-const BOLD = "\x1b[1m";
-const DIM = "\x1b[2m";
-const GREEN = "\x1b[32m";
-const RED = "\x1b[31m";
-const RESET = "\x1b[0m";
 async function apiRequest(input) {
     const url = new URL(input.path, input.host);
     for (const [key, value] of Object.entries(input.query ?? {})) {
@@ -2069,6 +2104,354 @@ async function runGenerateCommand(argv) {
             console.log(`${DIM}Place it with: vidfarm place <dir> --src "${mediaUrl}" --kind ${mediaType} --at <time>|--replace <layer_key>${RESET}`);
     }
 }
+// ── Speech: tts / stt ────────────────────────────────────────────────────────
+// LOCAL-FIRST like `clips scan`: both commands default to running on the
+// user's OWN AI key (GEMINI_API_KEY / OPENAI_API_KEY / OPENROUTER_API_KEY or
+// the --*-key flags) via src/services/speech.ts — local ffmpeg demux, direct
+// provider call, no cloud job, no wallet. The REST primitive routes
+// (/api/v1/primitives/audio/speech|transcribe) are the explicit `--cloud`
+// backup. Speech cannot ride on a claude/codex CLI agent (no audio I/O), so
+// "local" means a raw provider key here, not the agent CLI.
+function speechKeyHint(kind) {
+    const preference = kind === "stt"
+        ? "GEMINI_API_KEY (best — it can label speakers), OPENAI_API_KEY, or OPENROUTER_API_KEY"
+        : "OPENAI_API_KEY, GEMINI_API_KEY, or OPENROUTER_API_KEY";
+    return `No local AI key found. Set ${preference} (or pass --gemini-key/--openai-key/--openrouter-key), or run with --cloud to use your saved Vidfarm provider keys.`;
+}
+async function downloadUrlToFile(url, dest) {
+    const res = await fetch(url);
+    if (!res.ok || !res.body)
+        throw new Error(`download failed (${res.status}) for ${url}`);
+    mkdirSync(path.dirname(dest), { recursive: true });
+    await pipeline(Readable.fromWeb(res.body), createWriteStream(dest));
+}
+async function runTtsCommand(argv) {
+    const parsed = parseArgs({
+        args: argv,
+        allowPositionals: true,
+        options: {
+            ...commonOptions(),
+            text: { type: "string" },
+            style: { type: "string" },
+            instructions: { type: "string" },
+            voice: { type: "string" },
+            provider: { type: "string" },
+            model: { type: "string" },
+            format: { type: "string" },
+            out: { type: "string" },
+            cloud: { type: "boolean", default: false },
+            "gemini-key": { type: "string" },
+            "openai-key": { type: "string" },
+            "openrouter-key": { type: "string" },
+            "no-wait": { type: "boolean", default: false },
+            tracer: { type: "string" }
+        }
+    });
+    const text = (parsed.values.text ?? parsed.positionals.join(" ")).trim();
+    if (!text)
+        throw new Error('tts requires text: vidfarm tts "Welcome back to the channel" --style "calm, warm narrator".');
+    const style = (parsed.values.instructions ?? parsed.values.style)?.trim() || undefined;
+    const rawFormat = String(parsed.values.format ?? "mp3").toLowerCase();
+    if (rawFormat !== "mp3" && rawFormat !== "wav")
+        throw new Error("tts --format must be mp3 or wav.");
+    const format = rawFormat;
+    const voice = parsed.values.voice;
+    const json = Boolean(parsed.values.json);
+    if (!parsed.values.cloud) {
+        const auth = resolveLocalSpeechAuth(parsed.values, ["openai", "gemini", "openrouter"], parsed.values.provider);
+        if (!auth)
+            throw new Error(speechKeyHint("tts"));
+        if (!json)
+            console.log(`${DIM}Generating speech locally on your ${auth.provider} key…${RESET}`);
+        const result = await localGenerateSpeech({
+            auth,
+            text,
+            voice,
+            instructions: style,
+            model: parsed.values.model,
+            responseFormat: format
+        });
+        const extension = result.contentType.includes("wav") ? "wav" : "mp3";
+        const outPath = path.resolve(process.cwd(), parsed.values.out ?? `tts-${Date.now().toString(36)}.${extension}`);
+        mkdirSync(path.dirname(outPath), { recursive: true });
+        writeFileSync(outPath, result.bytes);
+        if (json) {
+            printJson({ ok: true, mode: "local", provider: result.provider, model: result.model, voice: voice ?? null, style: style ?? null, content_type: result.contentType, bytes: result.bytes.byteLength, out: outPath });
+        }
+        else {
+            console.log(`${GREEN}Wrote ${outPath}${RESET} ${DIM}(${result.provider}/${result.model}, ${Math.round(result.bytes.byteLength / 1024)}KB)${RESET}`);
+            console.log(`${DIM}Get a shareable URL with: vidfarm upload "${outPath}" — then add it to a composition as an audio layer.${RESET}`);
+        }
+        return;
+    }
+    // BACKUP: the platform primitive route (saved provider keys, async job).
+    const ctx = commonContext(parsed.values);
+    const payload = { text, output_format: format };
+    if (voice)
+        payload.voice = voice;
+    if (style)
+        payload.instructions = style;
+    if (parsed.values.provider)
+        payload.provider = parsed.values.provider;
+    if (parsed.values.model)
+        payload.model = parsed.values.model;
+    const tracer = parsed.values.tracer ?? `devcli-tts-${Date.now().toString(36)}`;
+    const submit = await apiRequest({ method: "POST", host: ctx.host, path: "/api/v1/primitives/audio/speech", auth: ctx.auth, body: { tracer, payload } });
+    assertApiOk(submit, "tts");
+    const jobId = submit.json?.job_id;
+    if (parsed.values["no-wait"] || !jobId) {
+        if (!ctx.json && jobId)
+            console.log(`${DIM}Queued ${jobId} (tracer ${tracer}). Poll: vidfarm api GET /api/v1/user/me/jobs/${jobId} — or drop --no-wait to auto-poll.${RESET}`);
+        emitResult(submit, ctx.json);
+        return;
+    }
+    if (!ctx.json)
+        console.log(`${DIM}Generating speech (${jobId})… polling every 5s.${RESET}`);
+    const job = await pollPrimitiveJob(ctx, jobId);
+    const mediaUrl = resolveJobMediaUrl(job);
+    if (!mediaUrl) {
+        if (ctx.json) {
+            printJson({ ok: false, job_id: jobId, status: String(job?.status ?? ""), job });
+        }
+        else {
+            console.log(`${RED}TTS ${String(job?.status ?? "did not finish")} — no audio URL.${RESET}`);
+            printJson(job);
+        }
+        process.exitCode = 1;
+        return;
+    }
+    const extension = /\.wav(\?|#|$)/i.test(mediaUrl) ? "wav" : "mp3";
+    const outPath = path.resolve(process.cwd(), parsed.values.out ?? `tts-${jobId}.${extension}`);
+    await downloadUrlToFile(mediaUrl, outPath);
+    if (ctx.json) {
+        printJson({ ok: true, mode: "cloud", job_id: jobId, tracer, audio_url: mediaUrl, out: outPath });
+    }
+    else {
+        console.log(`${GREEN}Wrote ${outPath}${RESET} ${DIM}(${mediaUrl})${RESET}`);
+    }
+}
+function formatSpeechClock(totalSeconds) {
+    const minutes = Math.floor(totalSeconds / 60);
+    const seconds = Math.floor(totalSeconds % 60);
+    return `${minutes}:${String(seconds).padStart(2, "0")}`;
+}
+// Print/persist BOTH transcript formats: the simple subtitle version (plain
+// text + SRT cues) and the advanced multi-speaker version (speaker-attributed
+// segments → .json). --out picks one via extension or writes all three.
+function emitSttResult(input) {
+    const written = [];
+    if (input.out) {
+        const explicitExtension = /\.(json|srt|txt)$/i.exec(input.out)?.[1]?.toLowerCase() ?? null;
+        const base = explicitExtension ? input.out.slice(0, -(explicitExtension.length + 1)) : input.out;
+        const targets = explicitExtension ? [explicitExtension] : ["json", "srt", "txt"];
+        for (const kind of targets) {
+            const dest = path.resolve(process.cwd(), `${base}.${kind}`);
+            if (kind === "json") {
+                mkdirSync(path.dirname(dest), { recursive: true });
+                writeFileSync(dest, `${JSON.stringify({
+                    source: input.source,
+                    mode: input.mode,
+                    provider: input.provider,
+                    model: input.model,
+                    language: input.result.language,
+                    diarization: input.result.diarization,
+                    text: input.result.text,
+                    speakers: [...new Set((input.result.segments ?? []).map((segment) => segment.speaker))],
+                    segments: input.result.segments ?? []
+                }, null, 2)}\n`);
+                written.push(dest);
+            }
+            else if (kind === "srt") {
+                if (input.result.srt) {
+                    mkdirSync(path.dirname(dest), { recursive: true });
+                    writeFileSync(dest, input.result.srt);
+                    written.push(dest);
+                }
+                else if (explicitExtension === "srt") {
+                    console.log(`${RED}No timed segments — cannot write ${dest}. The plain transcript is still available (--out <base>.txt).${RESET}`);
+                }
+            }
+            else {
+                mkdirSync(path.dirname(dest), { recursive: true });
+                writeFileSync(dest, `${input.result.text ?? ""}\n`);
+                written.push(dest);
+            }
+        }
+    }
+    if (input.json) {
+        printJson({
+            ok: true,
+            mode: input.mode,
+            provider: input.provider,
+            model: input.model,
+            ...(input.jobId ? { job_id: input.jobId } : {}),
+            source: input.source,
+            language: input.result.language,
+            diarization: input.result.diarization,
+            text: input.result.text,
+            segments: input.result.segments ?? [],
+            srt: input.result.srt,
+            written
+        });
+        return;
+    }
+    if (input.result.segments?.length && input.result.diarization === "diarized") {
+        for (const segment of input.result.segments) {
+            const at = typeof segment.start_sec === "number" ? `${DIM}[${formatSpeechClock(segment.start_sec)}]${RESET} ` : "";
+            console.log(`${at}${segment.speaker}: ${segment.text}`);
+        }
+    }
+    else {
+        console.log(input.result.text || "(no speech detected)");
+    }
+    for (const dest of written) {
+        console.log(`${GREEN}Wrote ${dest}${RESET}`);
+    }
+}
+async function runSttCommand(argv) {
+    const parsed = parseArgs({
+        args: argv,
+        allowPositionals: true,
+        options: {
+            ...commonOptions(),
+            "no-diarize": { type: "boolean", default: false },
+            language: { type: "string" },
+            prompt: { type: "string" },
+            provider: { type: "string" },
+            model: { type: "string" },
+            out: { type: "string" },
+            cloud: { type: "boolean", default: false },
+            "gemini-key": { type: "string" },
+            "openai-key": { type: "string" },
+            "openrouter-key": { type: "string" },
+            "no-wait": { type: "boolean", default: false },
+            tracer: { type: "string" },
+            folder: { type: "string" }
+        }
+    });
+    const target = parsed.positionals[0];
+    if (!target)
+        throw new Error("stt requires a source: `vidfarm stt <video-or-audio file|url>`.");
+    const diarize = !parsed.values["no-diarize"];
+    const json = Boolean(parsed.values.json);
+    const language = parsed.values.language?.trim() || undefined;
+    const prompt = parsed.values.prompt?.trim() || undefined;
+    if (!parsed.values.cloud) {
+        const auth = resolveLocalSpeechAuth(parsed.values, ["gemini", "openai", "openrouter"], parsed.values.provider);
+        if (!auth)
+            throw new Error(speechKeyHint("stt"));
+        if (diarize && auth.provider !== "gemini" && !json) {
+            console.log(`${DIM}Note: speaker labels need a Gemini key; ${auth.provider} returns a plain single-speaker transcript.${RESET}`);
+        }
+        if (!json)
+            console.log(`${DIM}Transcribing locally on your ${auth.provider} key (local ffmpeg demux)…${RESET}`);
+        const audio = await loadSpeechAudioLocally(target);
+        if (audio.bytes.byteLength > MAX_LOCAL_TRANSCRIBE_AUDIO_BYTES) {
+            throw new Error(`Audio is too large to transcribe (${Math.round(audio.bytes.byteLength / (1024 * 1024))}MB > 20MB, roughly 40 minutes of speech). Trim the source first.`);
+        }
+        const result = await localTranscribeSpeech({
+            auth,
+            audio: audio.bytes,
+            contentType: audio.contentType,
+            model: parsed.values.model,
+            prompt,
+            language,
+            diarize
+        });
+        emitSttResult({
+            json,
+            out: parsed.values.out,
+            source: target,
+            mode: "local",
+            provider: result.provider,
+            model: result.model,
+            result: {
+                text: result.text,
+                language: result.language,
+                diarization: result.diarization,
+                segments: result.segments,
+                srt: result.srt
+            }
+        });
+        return;
+    }
+    // BACKUP: the platform primitive route. URLs pass through; local files
+    // upload to the ephemeral temp store first (30-day TTL).
+    const ctx = commonContext(parsed.values);
+    const sourceUrl = /^https?:\/\//i.test(target)
+        ? target
+        : await uploadLocalToTempStore(ctx, path.resolve(process.cwd(), target), parsed.values.folder ?? "temp");
+    const payload = { source_url: sourceUrl, diarize };
+    if (language)
+        payload.language = language;
+    if (prompt)
+        payload.prompt = prompt;
+    if (parsed.values.provider)
+        payload.provider = parsed.values.provider;
+    if (parsed.values.model)
+        payload.model = parsed.values.model;
+    const tracer = parsed.values.tracer ?? `devcli-stt-${Date.now().toString(36)}`;
+    const submit = await apiRequest({ method: "POST", host: ctx.host, path: "/api/v1/primitives/audio/transcribe", auth: ctx.auth, body: { tracer, payload } });
+    assertApiOk(submit, "stt");
+    const jobId = submit.json?.job_id;
+    if (parsed.values["no-wait"] || !jobId) {
+        if (!ctx.json && jobId)
+            console.log(`${DIM}Queued ${jobId} (tracer ${tracer}). Poll: vidfarm api GET /api/v1/user/me/jobs/${jobId} — or drop --no-wait to auto-poll.${RESET}`);
+        emitResult(submit, ctx.json);
+        return;
+    }
+    if (!ctx.json)
+        console.log(`${DIM}Transcribing (${jobId})… polling every 5s.${RESET}`);
+    const job = await pollPrimitiveJob(ctx, jobId);
+    const status = String(job?.status ?? "");
+    const rawResult = (job?.result && typeof job.result === "object") ? job.result : {};
+    const output = (rawResult.output && typeof rawResult.output === "object") ? rawResult.output : rawResult;
+    if (status !== "succeeded" && typeof output.text !== "string") {
+        if (ctx.json) {
+            printJson({ ok: false, job_id: jobId, status, job });
+        }
+        else {
+            console.log(`${RED}Transcription ${status || "did not finish"}.${RESET}`);
+            printJson(job);
+        }
+        process.exitCode = 1;
+        return;
+    }
+    // Long transcripts stay artifact-only; hydrate the full version from
+    // transcript.json / transcript.srt when the inline copy was truncated.
+    let text = typeof output.text === "string" ? output.text : "";
+    let segments = Array.isArray(output.segments) ? output.segments : null;
+    let srt = typeof output.subtitles?.srt === "string" ? output.subtitles.srt : null;
+    const jsonUrl = typeof output.transcript?.json_url === "string" ? output.transcript.json_url : null;
+    if ((output.text_truncated === true || !segments) && jsonUrl) {
+        const hydrated = await fetch(jsonUrl).then((res) => (res.ok ? res.json() : null)).catch(() => null);
+        if (hydrated && typeof hydrated === "object") {
+            if (typeof hydrated.text === "string")
+                text = hydrated.text;
+            if (Array.isArray(hydrated.segments))
+                segments = hydrated.segments;
+        }
+    }
+    if (!srt && typeof output.subtitles?.srt_url === "string") {
+        srt = await fetch(output.subtitles.srt_url).then((res) => (res.ok ? res.text() : null)).catch(() => null);
+    }
+    emitSttResult({
+        json: ctx.json,
+        out: parsed.values.out,
+        source: target,
+        mode: "cloud",
+        provider: typeof output.transcript?.provider === "string" ? output.transcript.provider : null,
+        model: typeof output.transcript?.model === "string" ? output.transcript.model : null,
+        jobId,
+        result: {
+            text,
+            language: typeof output.language === "string" ? output.language : null,
+            diarization: typeof output.diarization === "string" ? output.diarization : "none",
+            segments,
+            srt
+        }
+    });
+}
 async function runPlaceCommand(argv) {
     const parsed = parseArgs({
         args: argv,

package/dist/src/config.js

@@ -68,6 +68,11 @@ const schema = z.object({
     GHOSTCUT_KEY: z.string().optional(),
     GHOSTCUT_SECRET: z.string().optional(),
     GHOSTCUT_USD_PER_30S: z.coerce.number().min(0).default(0.10),
+    // GhostCut caption removal is for SHORT clips, never long-form sources —
+    // clip hunting must select text-free scenes instead of laundering a whole
+    // long video through GhostCut. 15 min default ceiling (per-30s pricing makes
+    // long-form runs absurd anyway).
+    GHOSTCUT_MAX_DURATION_SEC: z.coerce.number().min(0).default(900),
     // Cloud passthrough for `vidfarm serve`: the local box keeps editing/render
     // fully local but lists the upstream (prod) catalog in /discover and
     // /library, seeds forks on demand, and can hand a render off to the cloud.