@mevdragon/vidfarm-devcli 0.11.0 → 0.13.0

package/dist/src/services/providers.js

@@ -5,8 +5,8 @@ import { createId } from "../lib/ids.js";
 import { addSeconds } from "../lib/time.js";
 import { ServerlessProviderKeyService } from "./serverless-provider-keys.js";
 import sharp from "sharp";
-import { ProviderAuthError, ProviderKeyUnavailableError, ProviderLeaseTimeoutError, ProviderQuotaExceededError, ProviderRateLimitError, ProviderTransientError, ProviderWaitExceededError, conciseProviderDetails, parseRetryAfterSeconds, rateLimitOrQuotaError } from "./provider-errors.js";
-import { SUPPORTED_STT_MODELS, SUPPORTED_TTS_MODELS, defaultSpeechModelFor, generateSpeechWithKey, inferSpeechContentType, transcribeSpeechWithKey } from "./speech.js";
+import { ProviderAuthError, ProviderKeyUnavailableError, ProviderLeaseTimeoutError, ProviderQuotaExceededError, ProviderRateLimitError, ProviderTransientError, ProviderWaitExceededError, SpeechVoiceMismatchError, conciseProviderDetails, parseRetryAfterSeconds, rateLimitOrQuotaError } from "./provider-errors.js";
+import { SUPPORTED_STT_MODELS, SUPPORTED_TTS_MODELS, defaultSpeechModelFor, generateSpeechWithKey, inferSpeechContentType, speechVoiceFamilyFor, speechVoiceMismatch, transcribeSpeechWithKey } from "./speech.js";
 export { ProviderAuthError, ProviderKeyUnavailableError, ProviderLeaseTimeoutError, ProviderQuotaExceededError, ProviderRateLimitError, ProviderTransientError, ProviderWaitExceededError } from "./provider-errors.js";
 const SUPPORTED_TEXT_MODELS = {
     openai: new Set(["gpt-5.4"]),
@@ -542,7 +542,9 @@ export class ProviderService {
         }
     }
     async generateSpeech(input) {
-        const attempts = await providerAttempts({
+        // Provider fallback must stay voice-aware: a Gemini preset routed to an
+        // OpenAI key (or the reverse) would 400 opaquely at the provider.
+        const attempts = (await providerAttempts({
             providerKeyService: this.serverlessProviderKeys,
             customerId: input.customerId,
             capability: "tts",
@@ -550,7 +552,11 @@ export class ProviderService {
             requestedModel: input.model,
             supportedModels: SUPPORTED_TTS_MODELS,
             defaultModelForProvider: (provider) => defaultSpeechModelFor("tts", provider)
-        });
+        })).filter((attempt) => !speechVoiceMismatch({ provider: attempt.provider, model: attempt.model, voice: input.voice }));
+        if (!attempts.length) {
+            throw new SpeechVoiceMismatchError(speechVoiceMismatch({ provider: input.provider, model: input.model, voice: input.voice }) ??
+                `Voice "${input.voice}" needs a ${speechVoiceFamilyFor(input.provider, input.model) === "gemini" ? "Gemini-family (gemini, or openrouter with a Gemini TTS model)" : "OpenAI-family (openai)"} provider key, but none is configured. Save a compatible key or pick a different voice.`);
+        }
         const retryDeadlineMs = providerLeaseRetryDeadlineMs();
         for (;;) {
             let lastError = null;
@@ -1549,6 +1555,7 @@ export class ProviderService {
                 prompt: input.prompt,
                 language: input.language,
                 diarize: input.diarize,
+                wordTimestamps: input.wordTimestamps,
                 apiKey: lease.secret
             })));
             await this.serverlessProviderKeys.recordProviderKeyUsage({

package/dist/src/services/serverless-records.js

@@ -151,6 +151,14 @@ class ServerlessRecordsServiceImpl {
         });
     }
     async getLatestApiKeyForCustomer(customerId) {
+        const keys = await this.listActiveApiKeysForCustomer(customerId, 1);
+        return keys[0] ?? null;
+    }
+    // Newest-first active API key records. Callers that hand a key to a client
+    // (boot configs, Settings) should pick one the CURRENT server can validate —
+    // after an API_KEY_SALT rotation the shared table holds keys hashed under
+    // different salts (deployed stage vs local dev boxes).
+    async listActiveApiKeysForCustomer(customerId, limit = 25) {
         const response = await dynamodb.send(new QueryCommand({
             TableName: requireMainTableName(),
             IndexName: "gsi1",
@@ -164,9 +172,9 @@ class ServerlessRecordsServiceImpl {
                 ":status": "active"
             },
             ScanIndexForward: false,
-            Limit: 1
+            Limit: limit
         }));
-        return response.Items?.[0] ?? null;
+        return response.Items ?? [];
     }
     async hasCustomerCreditGrant(customerId, grantType) {
         return Boolean(await this.getById("credit_grant", `${customerId}:${grantType}`));
@@ -390,6 +398,7 @@ class ServerlessRecordsServiceImpl {
             templateId: null,
             trendTagline: input.trendTagline ?? null,
             visibility: input.visibility ?? "public",
+            origin: input.origin ?? "inspiration",
             notes: input.notes ?? null,
             errorMessage: null,
             createdAt: timestamp,
@@ -446,6 +455,7 @@ class ServerlessRecordsServiceImpl {
             viralDna: input.viralDna ?? null,
             defaultForkId: input.defaultForkId ?? null,
             visibility: input.visibility ?? "public",
+            origin: input.origin ?? "inspiration",
             notes: input.notes ?? null,
             createdAt: timestamp,
             updatedAt: timestamp

package/dist/src/services/speech.js

@@ -9,7 +9,7 @@
 // is an audio-native LLM, so we ask for speaker-attributed JSON segments in one
 // call. OpenAI/OpenRouter transcription endpoints return plain text only and
 // degrade to a single unlabeled segment.
-import { ProviderAuthError, rateLimitOrQuotaError } from "./provider-errors.js";
+import { ProviderAuthError, SpeechVoiceMismatchError, rateLimitOrQuotaError } from "./provider-errors.js";
 export const SUPPORTED_TTS_MODELS = {
     openrouter: new Set(["google/gemini-3.1-flash-tts-preview"]),
     gemini: new Set(["gemini-3.1-flash-tts-preview", "gemini-2.5-flash-preview-tts"]),
@@ -18,25 +18,109 @@ export const SUPPORTED_TTS_MODELS = {
 export const SUPPORTED_STT_MODELS = {
     openrouter: new Set(["openai/gpt-4o-mini-transcribe"]),
     gemini: new Set(["gemini-3.1-flash-lite-preview", "gemini-2.5-flash-lite"]),
-    openai: new Set(["gpt-4o-mini-transcribe-2025-12-15"])
+    openai: new Set(["gpt-4o-mini-transcribe-2025-12-15", "whisper-1"])
 };
+// The only STT model that returns real word-level timestamps
+// (verbose_json + timestamp_granularities). Word-timestamp requests on the
+// openai path are routed here regardless of the configured chat-grade model.
+export const OPENAI_WORD_TIMESTAMP_STT_MODEL = "whisper-1";
 export function defaultSpeechModelFor(kind, provider) {
     return kind === "tts" ? SUPPORTED_TTS_MODELS[provider]?.values().next().value : SUPPORTED_STT_MODELS[provider]?.values().next().value;
 }
+// Known TTS voice presets, in the exact casing each provider accepts (OpenAI is
+// lowercase; Gemini prebuilt voices are case-sensitive TitleCase). These lists
+// exist to guard, not to gate: a voice that clearly belongs to the OTHER family
+// is rejected with an actionable message (the provider would 400 opaquely), a
+// known voice has its casing normalized, and an UNKNOWN voice passes through
+// untouched so new provider presets keep working without a code change.
+export const OPENAI_TTS_VOICES = [
+    "alloy", "ash", "ballad", "cedar", "coral", "echo", "fable", "marin", "nova", "onyx", "sage", "shimmer", "verse"
+];
+export const GEMINI_TTS_VOICES = [
+    "Achernar", "Achird", "Algenib", "Algieba", "Alnilam", "Aoede", "Autonoe", "Callirrhoe", "Charon", "Despina",
+    "Enceladus", "Erinome", "Fenrir", "Gacrux", "Iapetus", "Kore", "Laomedeia", "Leda", "Orus", "Puck",
+    "Pulcherrima", "Rasalgethi", "Sadachbia", "Sadaltager", "Schedar", "Sulafat", "Umbriel", "Vindemiatrix", "Zephyr", "Zubenelgenubi"
+];
+const OPENAI_VOICES_BY_KEY = new Map(OPENAI_TTS_VOICES.map((voice) => [voice.toLowerCase(), voice]));
+const GEMINI_VOICES_BY_KEY = new Map(GEMINI_TTS_VOICES.map((voice) => [voice.toLowerCase(), voice]));
+/** Which voice family a provider/model pair expects; openrouter follows its (typically Gemini) TTS model. */
+export function speechVoiceFamilyFor(provider, model) {
+    if (provider === "gemini") {
+        return "gemini";
+    }
+    if (provider === "openrouter") {
+        const resolvedModel = model?.trim() || defaultSpeechModelFor("tts", provider) || "";
+        return /gemini/i.test(resolvedModel) ? "gemini" : "openai";
+    }
+    return "openai";
+}
+/** Which provider a bare voice preset implies (null when the name is not a known preset). */
+export function inferSpeechProviderForVoice(voice) {
+    const key = voice?.trim().toLowerCase();
+    if (!key) {
+        return null;
+    }
+    if (OPENAI_VOICES_BY_KEY.has(key)) {
+        return "openai";
+    }
+    if (GEMINI_VOICES_BY_KEY.has(key)) {
+        return "gemini";
+    }
+    return null;
+}
+/** Actionable mismatch message when the voice belongs to the other provider family, else null (usable). */
+export function speechVoiceMismatch(input) {
+    const key = input.voice?.trim().toLowerCase();
+    if (!key) {
+        return null;
+    }
+    const family = speechVoiceFamilyFor(input.provider, input.model);
+    if ((family === "gemini" ? GEMINI_VOICES_BY_KEY : OPENAI_VOICES_BY_KEY).has(key)) {
+        return null;
+    }
+    const foreign = (family === "gemini" ? OPENAI_VOICES_BY_KEY : GEMINI_VOICES_BY_KEY).get(key);
+    if (!foreign) {
+        return null;
+    }
+    return family === "gemini"
+        ? `Voice "${foreign}" is an OpenAI preset, but provider ${input.provider} expects a Gemini voice (e.g. Kore, Puck, Charon). Pick a Gemini voice or set provider to "openai".`
+        : `Voice "${foreign}" is a Gemini preset, but provider ${input.provider} expects an OpenAI voice (e.g. alloy, ash, coral). Pick an OpenAI voice or set provider to "gemini".`;
+}
+/** Canonicalize a requested voice for the target provider/model (casing fixed, cross-family presets rejected). */
+export function resolveSpeechVoice(input) {
+    const raw = input.voice?.trim();
+    if (!raw) {
+        return undefined;
+    }
+    const mismatch = speechVoiceMismatch(input);
+    if (mismatch) {
+        throw new SpeechVoiceMismatchError(mismatch);
+    }
+    const family = speechVoiceFamilyFor(input.provider, input.model);
+    return (family === "gemini" ? GEMINI_VOICES_BY_KEY : OPENAI_VOICES_BY_KEY).get(raw.toLowerCase()) ?? raw;
+}
 /** Generate speech audio with a raw provider key. Voice style rides on `instructions`. */
 export async function generateSpeechWithKey(input) {
+    const voice = resolveSpeechVoice(input);
     return input.provider === "gemini"
-        ? callGeminiSpeechGeneration(input)
+        ? callGeminiSpeechGeneration({ ...input, voice })
         : callOpenAICompatibleSpeechGeneration({
             ...input,
+            voice,
             provider: input.provider === "openrouter" ? "openrouter" : "openai"
         });
 }
-/** Transcribe speech audio with a raw provider key; diarizes on the Gemini path. */
+/** Transcribe speech audio with a raw provider key; diarizes on the Gemini path.
+ * wordTimestamps requests word-level timings — honored only on the openai path
+ * (routed to whisper-1 verbose_json); other providers return their usual
+ * segment-level output and callers estimate word windows from text. */
 export async function transcribeSpeechWithKey(input) {
     if (input.provider === "gemini") {
         return callGeminiSpeechTranscription(input);
     }
+    if (input.provider === "openai" && input.wordTimestamps) {
+        return callOpenAIVerboseSpeechTranscription(input);
+    }
     const plain = input.provider === "openrouter"
         ? await callOpenRouterSpeechTranscription(input)
         : await callOpenAISpeechTranscription(input);
@@ -201,6 +285,84 @@ async function callOpenAISpeechTranscription(input) {
         }
     };
 }
+// whisper-1 verbose_json: the one provider path with real (non-estimated) word
+// timestamps. Returns timed segments with their word windows attached, ready
+// for animated-caption cue building.
+async function callOpenAIVerboseSpeechTranscription(input) {
+    const contentType = input.contentType ?? "audio/mpeg";
+    const model = /^whisper/i.test(input.model) ? input.model : OPENAI_WORD_TIMESTAMP_STT_MODEL;
+    const form = new FormData();
+    form.set("file", new Blob([Buffer.from(input.audio)], { type: contentType }), `audio.${audioExtensionFromContentType(contentType)}`);
+    form.set("model", model);
+    form.set("response_format", "verbose_json");
+    form.append("timestamp_granularities[]", "word");
+    form.append("timestamp_granularities[]", "segment");
+    if (input.prompt) {
+        form.set("prompt", input.prompt);
+    }
+    if (input.language) {
+        form.set("language", input.language);
+    }
+    const response = await fetch("https://api.openai.com/v1/audio/transcriptions", {
+        method: "POST",
+        headers: {
+            Authorization: `Bearer ${input.apiKey}`
+        },
+        body: form
+    });
+    if (response.status === 401 || response.status === 403) {
+        throw new ProviderAuthError("openai authentication failed");
+    }
+    if (response.status === 429) {
+        throw await rateLimitOrQuotaError("openai", response);
+    }
+    if (!response.ok) {
+        const details = await response.text();
+        throw new Error(`openai transcription returned ${response.status}${details ? `: ${details}` : ""}`);
+    }
+    const data = await response.json();
+    const words = Array.isArray(data.words)
+        ? data.words
+            .map((entry) => ({
+            text: String(entry.word ?? "").trim(),
+            start_sec: Number(entry.start),
+            end_sec: Number(entry.end)
+        }))
+            .filter((word) => word.text && Number.isFinite(word.start_sec) && Number.isFinite(word.end_sec) && word.end_sec > word.start_sec)
+        : [];
+    const rawSegments = Array.isArray(data.segments) ? data.segments : [];
+    const segments = [];
+    for (const entry of rawSegments) {
+        const start = Number(entry.start);
+        const end = Number(entry.end);
+        const text = String(entry.text ?? "").trim();
+        if (!text || !Number.isFinite(start) || !Number.isFinite(end))
+            continue;
+        const segmentWords = words.filter((word) => word.start_sec >= start - 0.05 && word.start_sec < end + 0.05);
+        segments.push({
+            speaker: "speaker_1",
+            start_sec: start,
+            end_sec: end,
+            text,
+            words: segmentWords.length ? segmentWords : undefined
+        });
+    }
+    const text = String(data.text ?? "").trim();
+    if (!segments.length && text) {
+        segments.push({ speaker: "speaker_1", start_sec: null, end_sec: null, text, words: words.length ? words : undefined });
+    }
+    return {
+        text,
+        language: typeof data.language === "string" && data.language.trim() ? data.language.trim() : input.language?.trim() || null,
+        segments,
+        diarization: "none",
+        usage: {
+            inputTokens: Number(data.usage?.input_tokens ?? 0),
+            outputTokens: Number(data.usage?.output_tokens ?? 0),
+            costUsd: Number(data.usage?.cost ?? 0)
+        }
+    };
+}
 async function callOpenRouterSpeechTranscription(input) {
     const response = await fetch("https://openrouter.ai/api/v1/audio/transcriptions", {
         method: "POST",

package/dist/src/services/storage.js

@@ -162,6 +162,18 @@ export class StorageService {
     async getReadUrl(key) {
         return this.createReadUrl(key);
     }
+    // Absolute on-disk path for a key — local driver only (null on S3), for
+    // callers that hand the file to a local process (ffmpeg) without copying.
+    getLocalPath(key) {
+        if (this.s3 && config.AWS_S3_BUCKET)
+            return null;
+        try {
+            return this.resolveLocalPath(key);
+        }
+        catch {
+            return null;
+        }
+    }
     async createWriteUrl(key, input) {
         if (!this.s3 || !config.AWS_S3_BUCKET) {
             throw new Error("Presigned uploads require S3 storage.");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mevdragon/vidfarm-devcli",
-  "version": "0.11.0",
+  "version": "0.13.0",
   "description": "Local bridge for the Vidfarm Trackpad Editor. `vidfarm serve <template_id>` boots the FULL editor on localhost (disk-backed records/storage, free in-process render); edit composition.html on disk (Claude Code, Codex, etc.) and the browser live-morphs it.",
   "type": "module",
   "bin": {