@mevdragon/vidfarm-devcli 0.19.0 → 0.20.0

package/dist/src/services/providers.js

@@ -7,6 +7,7 @@ import { ServerlessProviderKeyService } from "./serverless-provider-keys.js";
 import sharp from "sharp";
 import { ProviderAuthError, ProviderKeyUnavailableError, ProviderLeaseTimeoutError, ProviderQuotaExceededError, ProviderRateLimitError, ProviderTransientError, ProviderWaitExceededError, SpeechVoiceMismatchError, conciseProviderDetails, parseRetryAfterSeconds, rateLimitOrQuotaError } from "./provider-errors.js";
 import { SUPPORTED_STT_MODELS, SUPPORTED_TTS_MODELS, defaultSpeechModelFor, generateSpeechWithKey, inferSpeechContentType, speechVoiceFamilyFor, speechVoiceMismatch, transcribeSpeechWithKey } from "./speech.js";
+import { DEFAULT_ELEVENLABS_STT_MODEL, DEFAULT_ELEVENLABS_TTS_MODEL, elevenLabsListVoices, elevenLabsMusicCompose, elevenLabsSpeechToText, elevenLabsTextToSpeech, validateElevenLabsKey } from "./elevenlabs.js";
 export { ProviderAuthError, ProviderKeyUnavailableError, ProviderLeaseTimeoutError, ProviderQuotaExceededError, ProviderRateLimitError, ProviderTransientError, ProviderWaitExceededError } from "./provider-errors.js";
 const SUPPORTED_TEXT_MODELS = {
     openai: new Set(["gpt-5.4"]),
@@ -70,6 +71,8 @@ const OPENAI_IMAGE_SIZES = new Set([
     "auto"
 ]);
 const GEMINI_IMAGE_ASPECT_RATIOS = new Set(["1:1", "16:9", "9:16", "4:3", "3:4", "21:9", "9:21"]);
+// elevenlabs is intentionally absent — it validates against its own /v1/user
+// endpoint (see validateProviderKey), not a chat-completion probe.
 const PROVIDER_KEY_VALIDATION_MODELS = {
     openai: "gpt-4.1-mini",
     gemini: "gemini-3.1-flash-lite-preview",
@@ -334,6 +337,20 @@ export class ProviderService {
         return [...new Set(providers)];
     }
     async validateProviderKey(input) {
+        // ElevenLabs is audio-only: validate against its own account endpoint, not a
+        // chat-completion probe.
+        if (input.provider === "elevenlabs") {
+            if (config.mockProviders) {
+                return { provider: input.provider, model: "elevenlabs", mock: true };
+            }
+            try {
+                await validateElevenLabsKey(input.secret);
+                return { provider: input.provider, model: "elevenlabs", mock: false };
+            }
+            catch (error) {
+                throw new Error(toProviderValidationMessage(input.provider, error));
+            }
+        }
         const model = PROVIDER_KEY_VALIDATION_MODELS[input.provider];
         if (!model) {
             throw new Error(`Provider ${input.provider} does not support validation.`);
@@ -541,7 +558,123 @@ export class ProviderService {
             await this.releaseLease(lease);
         }
     }
+    // --- Audio (music / TTS / STT): platform-key + wallet vs BYOK routing ---
+    async resolveCustomerElevenLabsKey(customerId) {
+        const now = new Date().toISOString();
+        const secret = (await this.serverlessProviderKeys.listProviderKeysWithSecrets(customerId))
+            .filter((entry) => String(entry.provider) === "elevenlabs")
+            .filter((entry) => String(entry.status) === "active")
+            .filter((entry) => !entry.cooldown_until || String(entry.cooldown_until) <= now)
+            .map((entry) => readStoredProviderSecret(String(entry.secret ?? "")))
+            .find((value) => value.trim().length > 0);
+        return secret ?? null;
+    }
+    async customerHasByokAudioKey(customerId) {
+        const providers = await this.listAvailableProvidersAsync(customerId);
+        return providers.some((provider) => provider === "openai" || provider === "gemini" || provider === "openrouter");
+    }
+    /**
+     * Decide how an audio call is authorized. Default (`use_wallet_credits` unset
+     * or true): the platform ElevenLabs key, billed to the wallet. Explicit
+     * `false`: the customer's own ElevenLabs key, else a BYOK openai/gemini/
+     * openrouter key (TTS/STT only), else fall back to the platform key + wallet.
+     */
+    async resolveAudioAuthPlan(input) {
+        // `undefined` = legacy BYOK-only. Internal callers (captions, voice-matched
+        // regenerate-speech) never pass the flag, so they keep their exact behavior
+        // and never silently draw on the platform key or the wallet. Only surfaces
+        // that explicitly opt in (the music/tts/stt primitives) reach ElevenLabs.
+        if (input.useWalletCredits === undefined) {
+            return { mode: "byok" };
+        }
+        const platformKey = config.ELEVENLABS_API_KEY?.trim() || null;
+        if (input.useWalletCredits) {
+            if (platformKey) {
+                return { mode: "elevenlabs", apiKey: platformKey, walletBilled: true, scope: "platform" };
+            }
+            // Operator hasn't wired a platform key — honor the intent with the user's
+            // own ElevenLabs key (no charge) or a BYOK audio key before failing.
+            const userKey = await this.resolveCustomerElevenLabsKey(input.customerId);
+            if (userKey) {
+                return { mode: "elevenlabs", apiKey: userKey, walletBilled: false, scope: "user" };
+            }
+            if (input.byokFallback && (await this.customerHasByokAudioKey(input.customerId))) {
+                return { mode: "byok" };
+            }
+            throw new ProviderKeyUnavailableError("use_wallet_credits is on but no platform ElevenLabs key is configured, and you have no usable audio key. Ask the operator to set ELEVENLABS_API_KEY or save your own key.");
+        }
+        // use_wallet_credits === false: prefer the user's OWN key, then a BYOK audio
+        // key (TTS/STT), and only then fall back to the platform key + wallet.
+        const userKey = await this.resolveCustomerElevenLabsKey(input.customerId);
+        if (userKey) {
+            return { mode: "elevenlabs", apiKey: userKey, walletBilled: false, scope: "user" };
+        }
+        if (input.byokFallback && (await this.customerHasByokAudioKey(input.customerId))) {
+            return { mode: "byok" };
+        }
+        if (platformKey) {
+            return { mode: "elevenlabs", apiKey: platformKey, walletBilled: true, scope: "platform" };
+        }
+        throw new ProviderKeyUnavailableError(input.capability === "music"
+            ? "Music needs ElevenLabs: save an ElevenLabs key in Settings, or ask the operator to configure the platform ELEVENLABS_API_KEY (use_wallet_credits)."
+            : `No audio provider available for ${input.capability}. Enable use_wallet_credits (platform ElevenLabs key), save your own ElevenLabs key, or add an OpenAI/Gemini/OpenRouter key.`);
+    }
+    /** Voices available to whichever ElevenLabs account will serve this customer. */
+    async listElevenLabsVoices(input) {
+        const plan = await this.resolveAudioAuthPlan({
+            customerId: input.customerId,
+            useWalletCredits: input.useWalletCredits,
+            byokFallback: false,
+            capability: "tts"
+        });
+        if (plan.mode !== "elevenlabs") {
+            throw new ProviderKeyUnavailableError("No ElevenLabs account is available to list voices.");
+        }
+        return { scope: plan.scope, voices: await elevenLabsListVoices(plan.apiKey) };
+    }
     async generateSpeech(input) {
+        const plan = await this.resolveAudioAuthPlan({
+            customerId: input.customerId,
+            useWalletCredits: input.useWalletCredits,
+            byokFallback: true,
+            capability: "tts"
+        });
+        if (plan.mode === "elevenlabs") {
+            const model = /^eleven/i.test(input.model || "") ? input.model : DEFAULT_ELEVENLABS_TTS_MODEL;
+            const generated = await elevenLabsTextToSpeech({
+                apiKey: plan.apiKey,
+                text: input.text,
+                voice: input.voice,
+                model,
+                responseFormat: input.responseFormat
+            });
+            const billing = plan.walletBilled
+                ? {
+                    costUsd: Number(((generated.characterCount / 1000) * config.ELEVENLABS_TTS_USD_PER_1K_CHARS).toFixed(4)),
+                    costCenterSlug: "elevenlabs_tts",
+                    metadata: {
+                        operation: "tts",
+                        provider: "elevenlabs",
+                        model,
+                        character_count: generated.characterCount,
+                        wallet_billed: true,
+                        scope: plan.scope
+                    }
+                }
+                : null;
+            return {
+                bytes: generated.bytes,
+                contentType: generated.contentType,
+                usage: generated.usage,
+                effectiveProvider: "elevenlabs",
+                effectiveModel: model,
+                billing
+            };
+        }
+        const byok = await this.generateSpeechByok(input);
+        return { ...byok.result, effectiveProvider: byok.provider, effectiveModel: byok.model, billing: null };
+    }
+    async generateSpeechByok(input) {
         // Provider fallback must stay voice-aware: a Gemini preset routed to an
         // OpenAI key (or the reverse) would 400 opaquely at the provider.
         const attempts = (await providerAttempts({
@@ -562,11 +695,12 @@ export class ProviderService {
             let lastError = null;
             for (const attempt of attempts) {
                 try {
-                    return await this.generateSpeechWithProvider({
+                    const result = await this.generateSpeechWithProvider({
                         ...input,
                         provider: attempt.provider,
                         model: attempt.model
                     });
+                    return { result, provider: attempt.provider, model: attempt.model };
                 }
                 catch (error) {
                     lastError = error;
@@ -581,6 +715,42 @@ export class ProviderService {
             throw lastError instanceof Error ? lastError : new ProviderLeaseTimeoutError("No eligible AI speech provider key became available in time.");
         }
     }
+    /** ElevenLabs music generation. Platform key + wallet by default; the
+     * customer's own ElevenLabs key when use_wallet_credits is false. */
+    async generateMusic(input) {
+        const plan = await this.resolveAudioAuthPlan({
+            customerId: input.customerId,
+            useWalletCredits: input.useWalletCredits,
+            byokFallback: false,
+            capability: "music"
+        });
+        if (plan.mode !== "elevenlabs") {
+            throw new ProviderKeyUnavailableError("Music generation requires an ElevenLabs key.");
+        }
+        const generated = await elevenLabsMusicCompose({
+            apiKey: plan.apiKey,
+            prompt: input.prompt,
+            musicLengthMs: input.musicLengthMs,
+            compositionPlan: input.compositionPlan,
+            model: input.model,
+            outputFormat: input.outputFormat
+        });
+        const minutes = Math.max(0, generated.musicLengthMs) / 60000;
+        const billing = plan.walletBilled
+            ? {
+                costUsd: Number((minutes * config.ELEVENLABS_MUSIC_USD_PER_MIN).toFixed(4)),
+                costCenterSlug: "elevenlabs_music",
+                metadata: {
+                    operation: "music",
+                    provider: "elevenlabs",
+                    music_length_ms: generated.musicLengthMs,
+                    wallet_billed: true,
+                    scope: plan.scope
+                }
+            }
+            : null;
+        return { ...generated, scope: plan.scope, billing };
+    }
     async generateSpeechWithProvider(input) {
         assertSupportedSpeechModel("tts", input.provider, input.model);
         const lease = await this.leaseCustomerKey({
@@ -1475,6 +1645,45 @@ export class ProviderService {
         }
     }
     async transcribeSpeech(input) {
+        const plan = await this.resolveAudioAuthPlan({
+            customerId: input.customerId,
+            useWalletCredits: input.useWalletCredits,
+            byokFallback: true,
+            capability: "stt"
+        });
+        if (plan.mode === "elevenlabs") {
+            const model = /^scribe/i.test(input.model || "") ? input.model : DEFAULT_ELEVENLABS_STT_MODEL;
+            const transcription = await elevenLabsSpeechToText({
+                apiKey: plan.apiKey,
+                audio: input.audio,
+                contentType: input.contentType,
+                model,
+                diarize: input.diarize,
+                languageCode: input.language
+            });
+            const minutes = Math.max(0, transcription.audioDurationSec ?? 0) / 60;
+            const billing = plan.walletBilled
+                ? {
+                    costUsd: Number((minutes * config.ELEVENLABS_STT_USD_PER_MIN).toFixed(4)),
+                    costCenterSlug: "elevenlabs_stt",
+                    metadata: {
+                        operation: "stt",
+                        provider: "elevenlabs",
+                        model,
+                        audio_duration_sec: transcription.audioDurationSec,
+                        wallet_billed: true,
+                        scope: plan.scope
+                    }
+                }
+                : null;
+            const { audioDurationSec, ...rest } = transcription;
+            void audioDurationSec;
+            return { ...rest, effectiveProvider: "elevenlabs", effectiveModel: model, billing };
+        }
+        const byok = await this.transcribeSpeechByok(input);
+        return { ...byok.result, effectiveProvider: byok.provider, effectiveModel: byok.model, billing: null };
+    }
+    async transcribeSpeechByok(input) {
         const attempts = await providerAttempts({
             providerKeyService: this.serverlessProviderKeys,
             customerId: input.customerId,
@@ -1489,11 +1698,12 @@ export class ProviderService {
             let lastError = null;
             for (const attempt of attempts) {
                 try {
-                    return await this.transcribeSpeechWithProvider({
+                    const result = await this.transcribeSpeechWithProvider({
                         ...input,
                         provider: attempt.provider,
                         model: attempt.model
                     });
+                    return { result, provider: attempt.provider, model: attempt.model };
                 }
                 catch (error) {
                     lastError = error;
@@ -2041,6 +2251,10 @@ function formatProviderLabel(provider) {
             return "OpenRouter";
         case "perplexity":
             return "Perplexity";
+        case "elevenlabs":
+            return "ElevenLabs";
+        default:
+            return provider;
     }
 }
 function readStringField(value, field) {

package/dist/src/services/serverless-jobs.js

@@ -17,7 +17,7 @@ const dynamodb = config.RECORDS_DRIVER === "local"
         }
     });
 const sfn = new SFNClient({});
-const ACTIVE_ROOT_JOB_STATUSES = ["queued", "running", "waiting_for_provider", "waiting_for_child", "waiting_for_human"];
+const ACTIVE_ROOT_JOB_STATUSES = ["queued", "running", "waiting_for_provider", "waiting_for_render", "waiting_for_child", "waiting_for_human"];
 const ACTIVE_JOB_STATUSES = new Set(ACTIVE_ROOT_JOB_STATUSES);
 export class ServerlessJobsService {
     async createRootJob(input) {
@@ -282,6 +282,7 @@ export class ServerlessJobsService {
                 "#status = :queuedStatus",
                 "#status = :runningStatus",
                 "#status = :waitingForProviderStatus",
+                "#status = :waitingForRenderStatus",
                 "#status = :waitingForChildStatus",
                 "#status = :waitingForHumanStatus"
             ].join(" OR ")
@@ -293,6 +294,7 @@ export class ServerlessJobsService {
             expressionAttributeValues[":queuedStatus"] = "queued";
             expressionAttributeValues[":runningStatus"] = "running";
             expressionAttributeValues[":waitingForProviderStatus"] = "waiting_for_provider";
+            expressionAttributeValues[":waitingForRenderStatus"] = "waiting_for_render";
             expressionAttributeValues[":waitingForChildStatus"] = "waiting_for_child";
             expressionAttributeValues[":waitingForHumanStatus"] = "waiting_for_human";
         }

package/dist/src/services/swipe-customize.js

@@ -66,6 +66,7 @@ export async function reconcileSwipeRows(customerId, rows) {
     }
     return out;
 }
+// Text generation only — the audio-only elevenlabs provider never appears here.
 const TEXT_MODEL_DEFAULTS = {
     openai: "gpt-5.4-mini",
     gemini: "gemini-2.5-flash",
@@ -246,7 +247,9 @@ function parseJsonLoose(text) {
 }
 async function generateRecaption(input) {
     const available = await providers.listAvailableProvidersAsync(input.customerId);
-    const provider = TEXT_PROVIDER_PRIORITY.find((p) => available.includes(p)) ?? available[0];
+    // Text generation only — never fall back to a non-text provider (e.g. the
+    // audio-only elevenlabs key), which has no text model.
+    const provider = TEXT_PROVIDER_PRIORITY.find((p) => available.includes(p));
     if (!provider)
         throw new NoProviderKeyError();
     const generated = await providers.generateText({
@@ -254,7 +257,7 @@ async function generateRecaption(input) {
         jobId: input.jobId,
         workerId: "swipe-customize",
         provider,
-        model: TEXT_MODEL_DEFAULTS[provider],
+        model: TEXT_MODEL_DEFAULTS[provider] ?? "gpt-5.4-mini",
         prompt: input.prompt,
         temperature: 0.7,
         responseFormat: "json"

package/dist/src/template-editor-pages.js

@@ -1925,7 +1925,7 @@ export function renderCustomInpaintPage(input) {
           }
 
           function isPendingJobStatus(status) {
-            return ["queued", "running", "waiting_for_provider", "waiting_for_child", "waiting_for_human"].includes(String(status || ""));
+            return ["queued", "running", "waiting_for_provider", "waiting_for_render", "waiting_for_child", "waiting_for_human"].includes(String(status || ""));
           }
 
           function pendingHistoryItemKey(jobId) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mevdragon/vidfarm-devcli",
-  "version": "0.19.0",
+  "version": "0.20.0",
   "description": "Local bridge for the Vidfarm Trackpad Editor. `vidfarm serve <template_id>` boots the FULL editor on localhost (disk-backed records/storage, free in-process render); edit composition.html on disk (Claude Code, Codex, etc.) and the browser live-morphs it.",
   "type": "module",
   "bin": {