@mevdragon/vidfarm-devcli 0.9.0 → 0.11.0

package/dist/src/services/speech.js

@@ -0,0 +1,467 @@
+// Speech provider layer (TTS + STT) shared by cloud and devcli. ProviderService
+// wraps these calls with BYOK key leasing/usage metering; the devcli `tts`/`stt`
+// commands call them DIRECTLY with a local env key (GEMINI_API_KEY /
+// OPENAI_API_KEY / OPENROUTER_API_KEY) so speech runs local-first without the
+// REST primitive route (which stays the explicit `--cloud` backup). Keep the
+// request/response shapes here — one implementation, zero cloud/devcli drift.
+//
+// Diarization (multi-narration): only the Gemini path can label speakers — it
+// is an audio-native LLM, so we ask for speaker-attributed JSON segments in one
+// call. OpenAI/OpenRouter transcription endpoints return plain text only and
+// degrade to a single unlabeled segment.
+import { ProviderAuthError, rateLimitOrQuotaError } from "./provider-errors.js";
+export const SUPPORTED_TTS_MODELS = {
+    openrouter: new Set(["google/gemini-3.1-flash-tts-preview"]),
+    gemini: new Set(["gemini-3.1-flash-tts-preview", "gemini-2.5-flash-preview-tts"]),
+    openai: new Set(["gpt-4o-mini-tts-2025-12-15"])
+};
+export const SUPPORTED_STT_MODELS = {
+    openrouter: new Set(["openai/gpt-4o-mini-transcribe"]),
+    gemini: new Set(["gemini-3.1-flash-lite-preview", "gemini-2.5-flash-lite"]),
+    openai: new Set(["gpt-4o-mini-transcribe-2025-12-15"])
+};
+export function defaultSpeechModelFor(kind, provider) {
+    return kind === "tts" ? SUPPORTED_TTS_MODELS[provider]?.values().next().value : SUPPORTED_STT_MODELS[provider]?.values().next().value;
+}
+/** Generate speech audio with a raw provider key. Voice style rides on `instructions`. */
+export async function generateSpeechWithKey(input) {
+    return input.provider === "gemini"
+        ? callGeminiSpeechGeneration(input)
+        : callOpenAICompatibleSpeechGeneration({
+            ...input,
+            provider: input.provider === "openrouter" ? "openrouter" : "openai"
+        });
+}
+/** Transcribe speech audio with a raw provider key; diarizes on the Gemini path. */
+export async function transcribeSpeechWithKey(input) {
+    if (input.provider === "gemini") {
+        return callGeminiSpeechTranscription(input);
+    }
+    const plain = input.provider === "openrouter"
+        ? await callOpenRouterSpeechTranscription(input)
+        : await callOpenAISpeechTranscription(input);
+    return {
+        text: plain.text,
+        language: input.language?.trim() || null,
+        segments: plain.text.trim()
+            ? [{ speaker: "speaker_1", start_sec: null, end_sec: null, text: plain.text.trim() }]
+            : [],
+        diarization: "none",
+        usage: plain.usage
+    };
+}
+async function callOpenAICompatibleSpeechGeneration(input) {
+    const requestedFormat = input.responseFormat ?? "mp3";
+    const usePcmBridge = input.provider === "openrouter" &&
+        /gemini/i.test(input.model) &&
+        requestedFormat !== "mp3";
+    const responseFormat = usePcmBridge ? "pcm" : requestedFormat;
+    const endpoint = input.provider === "openrouter"
+        ? "https://openrouter.ai/api/v1/audio/speech"
+        : "https://api.openai.com/v1/audio/speech";
+    const response = await fetch(endpoint, {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${input.apiKey}`
+        },
+        body: JSON.stringify({
+            model: input.model,
+            input: input.text,
+            voice: input.voice ?? "alloy",
+            instructions: input.instructions,
+            response_format: responseFormat
+        })
+    });
+    if (response.status === 401 || response.status === 403) {
+        throw new ProviderAuthError(`${input.provider} authentication failed`);
+    }
+    if (response.status === 429) {
+        throw await rateLimitOrQuotaError(input.provider, response);
+    }
+    if (!response.ok) {
+        const details = await response.text();
+        throw new Error(`${input.provider} speech generation returned ${response.status}${details ? `: ${details}` : ""}`);
+    }
+    const rawBytes = Buffer.from(await response.arrayBuffer());
+    const bytes = usePcmBridge ? pcm16MonoToWav(rawBytes, 24000) : rawBytes;
+    const contentType = usePcmBridge
+        ? "audio/wav"
+        : response.headers.get("content-type")?.split(";")[0]?.trim() || inferSpeechContentType(responseFormat);
+    return {
+        bytes,
+        contentType,
+        usage: {
+            inputTokens: 0,
+            outputTokens: 0,
+            costUsd: 0
+        }
+    };
+}
+async function callGeminiSpeechGeneration(input) {
+    const prompt = input.instructions?.trim()
+        ? `${input.instructions.trim()}\n\nSpeak exactly the following text:\n${input.text}`
+        : input.text;
+    const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${input.model}:generateContent?key=${input.apiKey}`, {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json"
+        },
+        body: JSON.stringify({
+            contents: [{ parts: [{ text: prompt }] }],
+            generationConfig: {
+                responseModalities: ["AUDIO"],
+                speechConfig: {
+                    voiceConfig: {
+                        prebuiltVoiceConfig: {
+                            voiceName: input.voice ?? "Kore"
+                        }
+                    }
+                }
+            }
+        })
+    });
+    if (response.status === 401 || response.status === 403) {
+        throw new ProviderAuthError("gemini authentication failed");
+    }
+    if (response.status === 429) {
+        throw await rateLimitOrQuotaError("gemini", response);
+    }
+    if (!response.ok) {
+        const details = await response.text();
+        throw new Error(`gemini speech generation returned ${response.status}${details ? `: ${details}` : ""}`);
+    }
+    const data = await response.json();
+    const part = data.candidates?.[0]?.content?.parts?.find((item) => item.inlineData?.data || item.inline_data?.data);
+    const encoded = part?.inlineData?.data ?? part?.inline_data?.data;
+    if (!encoded) {
+        throw new Error("gemini speech generation returned no audio payload");
+    }
+    const rawBytes = Buffer.from(encoded, "base64");
+    const rawContentType = String(part?.inlineData?.mimeType ?? part?.inline_data?.mime_type ?? "audio/L16;rate=24000");
+    const pcmRateMatch = /audio\/l16;?\s*rate=(\d+)/i.exec(rawContentType);
+    const looksLikeWav = isWavBuffer(rawBytes);
+    const looksLikeMp3 = isLikelyMp3Buffer(rawBytes);
+    const shouldWrapPcm = Boolean(pcmRateMatch ||
+        (!looksLikeWav && !looksLikeMp3));
+    const bytes = shouldWrapPcm
+        ? pcm16MonoToWav(rawBytes, Number(pcmRateMatch?.[1] || "24000"))
+        : rawBytes;
+    const contentType = shouldWrapPcm
+        ? "audio/wav"
+        : looksLikeWav
+            ? "audio/wav"
+            : "audio/mpeg";
+    return {
+        bytes,
+        contentType,
+        usage: {
+            inputTokens: Number(data.usageMetadata?.promptTokenCount ?? 0),
+            outputTokens: Number(data.usageMetadata?.candidatesTokenCount ?? 0),
+            costUsd: 0
+        }
+    };
+}
+async function callOpenAISpeechTranscription(input) {
+    const contentType = input.contentType ?? "audio/mpeg";
+    const form = new FormData();
+    form.set("file", new Blob([Buffer.from(input.audio)], { type: contentType }), `audio.${audioExtensionFromContentType(contentType)}`);
+    form.set("model", input.model);
+    form.set("response_format", "json");
+    if (input.prompt) {
+        form.set("prompt", input.prompt);
+    }
+    if (input.language) {
+        form.set("language", input.language);
+    }
+    const response = await fetch("https://api.openai.com/v1/audio/transcriptions", {
+        method: "POST",
+        headers: {
+            Authorization: `Bearer ${input.apiKey}`
+        },
+        body: form
+    });
+    if (response.status === 401 || response.status === 403) {
+        throw new ProviderAuthError("openai authentication failed");
+    }
+    if (response.status === 429) {
+        throw await rateLimitOrQuotaError("openai", response);
+    }
+    if (!response.ok) {
+        const details = await response.text();
+        throw new Error(`openai transcription returned ${response.status}${details ? `: ${details}` : ""}`);
+    }
+    const data = await response.json();
+    return {
+        text: String(data.text ?? ""),
+        usage: {
+            inputTokens: Number(data.usage?.input_tokens ?? 0),
+            outputTokens: Number(data.usage?.output_tokens ?? 0),
+            costUsd: Number(data.usage?.cost ?? 0)
+        }
+    };
+}
+async function callOpenRouterSpeechTranscription(input) {
+    const response = await fetch("https://openrouter.ai/api/v1/audio/transcriptions", {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${input.apiKey}`
+        },
+        body: JSON.stringify({
+            model: input.model,
+            input_audio: {
+                data: Buffer.from(input.audio).toString("base64"),
+                format: audioFormatFromContentType(input.contentType)
+            },
+            ...(input.prompt ? { prompt: input.prompt } : {}),
+            ...(input.language ? { language: input.language } : {})
+        })
+    });
+    if (response.status === 401 || response.status === 403) {
+        throw new ProviderAuthError("openrouter authentication failed");
+    }
+    if (response.status === 429) {
+        throw await rateLimitOrQuotaError("openrouter", response);
+    }
+    if (!response.ok) {
+        const details = await response.text();
+        throw new Error(`openrouter transcription returned ${response.status}${details ? `: ${details}` : ""}`);
+    }
+    const data = await response.json();
+    return {
+        text: String(data.text ?? ""),
+        usage: {
+            inputTokens: Number(data.usage?.input_tokens ?? data.usage?.prompt_tokens ?? 0),
+            outputTokens: Number(data.usage?.output_tokens ?? data.usage?.completion_tokens ?? 0),
+            costUsd: Number(data.usage?.cost ?? 0)
+        }
+    };
+}
+async function callGeminiSpeechTranscription(input) {
+    const diarize = input.diarize !== false;
+    const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${input.model}:generateContent?key=${input.apiKey}`, {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json"
+        },
+        body: JSON.stringify({
+            contents: [{
+                    parts: [
+                        {
+                            inline_data: {
+                                mime_type: input.contentType ?? "audio/mpeg",
+                                data: Buffer.from(input.audio).toString("base64")
+                            }
+                        },
+                        {
+                            text: diarize
+                                ? buildGeminiDiarizedTranscriptionPrompt(input.prompt, input.language)
+                                : buildGeminiTranscriptionPrompt(input.prompt, input.language)
+                        }
+                    ]
+                }],
+            generationConfig: {
+                temperature: 0,
+                responseMimeType: diarize ? "application/json" : "text/plain"
+            }
+        })
+    });
+    if (response.status === 401 || response.status === 403) {
+        throw new ProviderAuthError("gemini authentication failed");
+    }
+    if (response.status === 429) {
+        throw await rateLimitOrQuotaError("gemini", response);
+    }
+    if (!response.ok) {
+        const details = await response.text();
+        throw new Error(`gemini transcription returned ${response.status}${details ? `: ${details}` : ""}`);
+    }
+    const data = await response.json();
+    const usage = {
+        inputTokens: Number(data.usageMetadata?.promptTokenCount ?? 0),
+        outputTokens: Number(data.usageMetadata?.candidatesTokenCount ?? 0),
+        costUsd: 0
+    };
+    const rawText = extractGeminiText(data);
+    if (diarize) {
+        const parsed = parseDiarizedTranscription(rawText);
+        if (parsed) {
+            return { ...parsed, usage };
+        }
+        // JSON mode misbehaved — degrade to the raw text rather than failing the job.
+    }
+    return {
+        text: rawText.trim(),
+        language: input.language?.trim() || null,
+        segments: rawText.trim()
+            ? [{ speaker: "speaker_1", start_sec: null, end_sec: null, text: rawText.trim() }]
+            : [],
+        diarization: "none",
+        usage
+    };
+}
+function buildGeminiTranscriptionPrompt(prompt, language) {
+    const basePrompt = prompt?.trim() || "Transcribe this audio verbatim. Return plain text only.";
+    return language?.trim()
+        ? `${basePrompt}\n\nExpected language: ${language.trim()}.`
+        : basePrompt;
+}
+function buildGeminiDiarizedTranscriptionPrompt(prompt, language) {
+    return [
+        "Transcribe this audio verbatim with speaker diarization.",
+        "Return ONLY a JSON object with this exact shape:",
+        `{"language": "<BCP-47 code or null>", "segments": [{"speaker": "<label>", "start_sec": <number>, "end_sec": <number>, "text": "<verbatim words>"}]}`,
+        "Rules:",
+        "- Split segments on speaker turns; keep each segment's text verbatim.",
+        "- Label speakers consistently across the whole audio (speaker_1, speaker_2, …), or use a real name/role when the audio makes it unambiguous (e.g. it is spoken aloud).",
+        "- If only one voice speaks, return all segments under speaker_1.",
+        "- start_sec/end_sec are seconds from the start of the audio; use your best estimate.",
+        "- Non-speech audio (music, silence, sfx) is not a segment.",
+        ...(language?.trim() ? [`- Expected language: ${language.trim()}.`] : []),
+        ...(prompt?.trim() ? [`Context from the caller: ${prompt.trim()}`] : [])
+    ].join("\n");
+}
+function parseDiarizedTranscription(raw) {
+    const trimmed = (raw ?? "").trim();
+    if (!trimmed)
+        return null;
+    const fenced = /```(?:json)?\s*([\s\S]*?)```/.exec(trimmed);
+    for (const candidate of [fenced?.[1], trimmed]) {
+        if (!candidate)
+            continue;
+        const start = candidate.indexOf("{");
+        const end = candidate.lastIndexOf("}");
+        if (start < 0 || end <= start)
+            continue;
+        let parsed;
+        try {
+            parsed = JSON.parse(candidate.slice(start, end + 1));
+        }
+        catch {
+            continue;
+        }
+        if (!parsed || typeof parsed !== "object" || !Array.isArray(parsed.segments))
+            continue;
+        const record = parsed;
+        const segments = record.segments
+            .filter((entry) => Boolean(entry) && typeof entry === "object")
+            .map((entry, index) => ({
+            speaker: typeof entry.speaker === "string" && entry.speaker.trim() ? entry.speaker.trim() : `speaker_${index + 1}`,
+            start_sec: Number.isFinite(Number(entry.start_sec)) ? Number(entry.start_sec) : null,
+            end_sec: Number.isFinite(Number(entry.end_sec)) ? Number(entry.end_sec) : null,
+            text: String(entry.text ?? "").trim()
+        }))
+            .filter((segment) => segment.text);
+        if (!segments.length)
+            continue;
+        const speakers = new Set(segments.map((segment) => segment.speaker));
+        return {
+            text: segments.map((segment) => segment.text).join(" ").trim(),
+            language: typeof record.language === "string" && record.language.trim() ? record.language.trim() : null,
+            segments,
+            diarization: speakers.size >= 1 ? "diarized" : "none"
+        };
+    }
+    return null;
+}
+function extractGeminiText(data) {
+    return String(data.candidates?.[0]?.content?.parts?.map((part) => part.text ?? "").join("\n") ?? "");
+}
+function formatSrtTimestamp(totalSeconds) {
+    const clamped = Math.max(0, totalSeconds);
+    const hours = Math.floor(clamped / 3600);
+    const minutes = Math.floor((clamped % 3600) / 60);
+    const seconds = Math.floor(clamped % 60);
+    const millis = Math.floor((clamped * 1000) % 1000);
+    const pad = (value, length = 2) => String(value).padStart(length, "0");
+    return `${pad(hours)}:${pad(minutes)}:${pad(seconds)},${pad(millis, 3)}`;
+}
+/**
+ * Simple subtitle rendering of a transcript: one plain-text cue per timed
+ * segment, no speaker labels. Returns null when the provider gave no timings
+ * (the plain-text transcript still covers that case).
+ */
+export function buildSrtFromSegments(segments) {
+    const timed = segments.filter((segment) => typeof segment.start_sec === "number" && Number.isFinite(segment.start_sec));
+    if (!timed.length) {
+        return null;
+    }
+    const cues = timed.map((segment, index) => {
+        const start = segment.start_sec;
+        const nextStart = timed[index + 1]?.start_sec;
+        const end = typeof segment.end_sec === "number" && segment.end_sec > start
+            ? segment.end_sec
+            : typeof nextStart === "number" && nextStart > start
+                ? nextStart
+                : start + 4;
+        return `${index + 1}\n${formatSrtTimestamp(start)} --> ${formatSrtTimestamp(end)}\n${segment.text}`;
+    });
+    return `${cues.join("\n\n")}\n`;
+}
+export function inferSpeechContentType(format) {
+    return format === "wav" ? "audio/wav" : "audio/mpeg";
+}
+export function pcm16MonoToWav(pcmBuffer, sampleRate) {
+    const header = Buffer.alloc(44);
+    header.write("RIFF", 0, 4, "ascii");
+    header.writeUInt32LE(36 + pcmBuffer.byteLength, 4);
+    header.write("WAVE", 8, 4, "ascii");
+    header.write("fmt ", 12, 4, "ascii");
+    header.writeUInt32LE(16, 16);
+    header.writeUInt16LE(1, 20);
+    header.writeUInt16LE(1, 22);
+    header.writeUInt32LE(sampleRate, 24);
+    header.writeUInt32LE(sampleRate * 2, 28);
+    header.writeUInt16LE(2, 32);
+    header.writeUInt16LE(16, 34);
+    header.write("data", 36, 4, "ascii");
+    header.writeUInt32LE(pcmBuffer.byteLength, 40);
+    return Buffer.concat([header, pcmBuffer]);
+}
+function isWavBuffer(buffer) {
+    return buffer.byteLength >= 12
+        && buffer.subarray(0, 4).toString("ascii") === "RIFF"
+        && buffer.subarray(8, 12).toString("ascii") === "WAVE";
+}
+function isLikelyMp3Buffer(buffer) {
+    if (buffer.byteLength < 3) {
+        return false;
+    }
+    if (buffer.subarray(0, 3).toString("ascii") === "ID3") {
+        return true;
+    }
+    return buffer[0] === 0xff && (buffer[1] & 0xe0) === 0xe0;
+}
+export function audioFormatFromContentType(contentType) {
+    const normalized = contentType?.split(";")[0]?.trim().toLowerCase();
+    switch (normalized) {
+        case "audio/wav":
+        case "audio/x-wav":
+            return "wav";
+        case "audio/mp4":
+        case "audio/m4a":
+            return "mp4";
+        case "audio/webm":
+            return "webm";
+        case "audio/ogg":
+            return "ogg";
+        default:
+            return "mp3";
+    }
+}
+export function audioExtensionFromContentType(contentType) {
+    switch (audioFormatFromContentType(contentType)) {
+        case "wav":
+            return "wav";
+        case "mp4":
+            return "m4a";
+        case "webm":
+            return "webm";
+        case "ogg":
+            return "ogg";
+        default:
+            return "mp3";
+    }
+}
+//# sourceMappingURL=speech.js.map

package/dist/src/services/storage.js

@@ -1,6 +1,6 @@
 import { mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import path from "node:path";
-import { DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
+import { DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand, PutObjectTaggingCommand, S3Client } from "@aws-sdk/client-s3";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 import { config } from "../config.js";
 function isS3NotFoundError(error) {
@@ -107,6 +107,21 @@ export class StorageService {
         writeFileSync(filePath, value);
         return { key, url: this.getPublicUrl(key) };
     }
+    /**
+     * Tag an existing S3 object. Used to mark temp-folder uploads with
+     * `vidfarm-temp=1` AFTER the client's presigned PUT lands (server-side, so
+     * no client header contract), which the bucket's tag-scoped lifecycle rule
+     * expires after 30 days. No-op on the local filesystem driver.
+     */
+    async putObjectTags(key, tags) {
+        if (!this.s3 || !config.AWS_S3_BUCKET)
+            return;
+        await this.s3.send(new PutObjectTaggingCommand({
+            Bucket: config.AWS_S3_BUCKET,
+            Key: key,
+            Tagging: { TagSet: Object.entries(tags).map(([Key, Value]) => ({ Key, Value })) }
+        }));
+    }
     // Returns a directly-fetchable URL for a storage key when the key is
     // stored under a prefix the bucket resource policy grants public read to.
     // Callers get a real S3 URL that browsers/CDNs/GhostCut/etc. can hit

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mevdragon/vidfarm-devcli",
-  "version": "0.9.0",
+  "version": "0.11.0",
   "description": "Local bridge for the Vidfarm Trackpad Editor. `vidfarm serve <template_id>` boots the FULL editor on localhost (disk-backed records/storage, free in-process render); edit composition.html on disk (Claude Code, Codex, etc.) and the browser live-morphs it.",
   "type": "module",
   "bin": {