@mevdragon/vidfarm-devcli 0.2.7 → 0.2.8

package/dist/src/services/providers.js

@@ -15,6 +15,16 @@ export class ProviderRateLimitError extends Error {
         this.retryAfterSeconds = retryAfterSeconds;
     }
 }
+const SUPPORTED_TTS_MODELS = {
+    openrouter: new Set(["google/gemini-3.1-flash-tts-preview"]),
+    gemini: new Set(["gemini-3.1-flash-tts-preview"]),
+    openai: new Set(["gpt-4o-mini-tts-2025-12-15"])
+};
+const SUPPORTED_STT_MODELS = {
+    openrouter: new Set(["openai/gpt-4o-mini-transcribe"]),
+    gemini: new Set(["gemini-3.1-flash-lite-preview", "gemini-2.5-flash-lite"]),
+    openai: new Set(["gpt-4o-mini-transcribe-2025-12-15"])
+};
 function readStoredProviderSecret(value) {
     try {
         return decryptString(value, config.ENCRYPTION_SECRET);
@@ -129,6 +139,103 @@ export class ProviderService {
             await this.releaseLease(lease);
         }
     }
+    async generateSpeech(input) {
+        assertSupportedSpeechModel("tts", input.provider, input.model);
+        const lease = await this.leaseCustomerKey({
+            customerId: input.customerId,
+            provider: input.provider,
+            jobId: input.jobId,
+            workerId: input.workerId
+        });
+        try {
+            if (config.mockProviders) {
+                const contentType = inferSpeechContentType(input.responseFormat ?? "mp3");
+                database.recordProviderKeyUsage({
+                    id: createId("usage"),
+                    keyId: lease.keyId,
+                    jobId: input.jobId,
+                    provider: input.provider,
+                    model: input.model,
+                    eventType: "success",
+                    inputTokens: 0,
+                    outputTokens: 0,
+                    costUsd: 0.01,
+                    metadata: { mock: true, type: "speech_generation" }
+                });
+                database.touchProviderKey(lease.keyId);
+                return {
+                    bytes: Buffer.from(`mock speech:${input.text.slice(0, 120)}`, "utf8"),
+                    contentType,
+                    usage: {
+                        inputTokens: 0,
+                        outputTokens: 0,
+                        costUsd: 0.01
+                    }
+                };
+            }
+            const result = input.provider === "gemini"
+                ? await this.callGeminiSpeechGeneration({
+                    model: input.model,
+                    text: input.text,
+                    voice: input.voice,
+                    instructions: input.instructions,
+                    apiKey: lease.secret
+                })
+                : await this.callOpenAICompatibleSpeechGeneration({
+                    provider: input.provider === "openrouter" ? "openrouter" : "openai",
+                    model: input.model,
+                    text: input.text,
+                    voice: input.voice,
+                    instructions: input.instructions,
+                    responseFormat: input.responseFormat,
+                    apiKey: lease.secret
+                });
+            database.recordProviderKeyUsage({
+                id: createId("usage"),
+                keyId: lease.keyId,
+                jobId: input.jobId,
+                provider: input.provider,
+                model: input.model,
+                eventType: "success",
+                inputTokens: result.usage.inputTokens,
+                outputTokens: result.usage.outputTokens,
+                costUsd: result.usage.costUsd,
+                metadata: { type: "speech_generation" }
+            });
+            database.touchProviderKey(lease.keyId);
+            return result;
+        }
+        catch (error) {
+            if (error instanceof ProviderRateLimitError) {
+                database.recordProviderKeyUsage({
+                    id: createId("usage"),
+                    keyId: lease.keyId,
+                    jobId: input.jobId,
+                    provider: input.provider,
+                    model: input.model,
+                    eventType: "rate_limit",
+                    metadata: { retryAfterSeconds: error.retryAfterSeconds, type: "speech_generation" }
+                });
+                database.setProviderKeyCooldown(lease.keyId, addSeconds(new Date(), error.retryAfterSeconds));
+            }
+            else if (error instanceof ProviderAuthError) {
+                database.recordProviderKeyUsage({
+                    id: createId("usage"),
+                    keyId: lease.keyId,
+                    jobId: input.jobId,
+                    provider: input.provider,
+                    model: input.model,
+                    eventType: "auth_error",
+                    metadata: { type: "speech_generation" }
+                });
+                database.setProviderKeyCooldown(lease.keyId, null, "invalid", "auth_error");
+            }
+            throw error;
+        }
+        finally {
+            await this.releaseLease(lease);
+        }
+    }
     async generateImage(input) {
         const lease = await this.leaseCustomerKey({
             customerId: input.customerId,
@@ -265,6 +372,110 @@ export class ProviderService {
             await this.releaseLease(lease);
         }
     }
+    async transcribeSpeech(input) {
+        assertSupportedSpeechModel("stt", input.provider, input.model);
+        const lease = await this.leaseCustomerKey({
+            customerId: input.customerId,
+            provider: input.provider,
+            jobId: input.jobId,
+            workerId: input.workerId
+        });
+        try {
+            if (config.mockProviders) {
+                database.recordProviderKeyUsage({
+                    id: createId("usage"),
+                    keyId: lease.keyId,
+                    jobId: input.jobId,
+                    provider: input.provider,
+                    model: input.model,
+                    eventType: "success",
+                    inputTokens: 0,
+                    outputTokens: 0,
+                    costUsd: 0.01,
+                    metadata: { mock: true, type: "speech_transcription" }
+                });
+                database.touchProviderKey(lease.keyId);
+                return {
+                    text: "Mock transcription output.",
+                    usage: {
+                        inputTokens: 0,
+                        outputTokens: 0,
+                        costUsd: 0.01
+                    }
+                };
+            }
+            const result = input.provider === "gemini"
+                ? await this.callGeminiSpeechTranscription({
+                    model: input.model,
+                    audio: input.audio,
+                    contentType: input.contentType,
+                    prompt: input.prompt,
+                    language: input.language,
+                    apiKey: lease.secret
+                })
+                : input.provider === "openrouter"
+                    ? await this.callOpenRouterSpeechTranscription({
+                        model: input.model,
+                        audio: input.audio,
+                        contentType: input.contentType,
+                        prompt: input.prompt,
+                        language: input.language,
+                        apiKey: lease.secret
+                    })
+                    : await this.callOpenAISpeechTranscription({
+                        model: input.model,
+                        audio: input.audio,
+                        contentType: input.contentType,
+                        prompt: input.prompt,
+                        language: input.language,
+                        apiKey: lease.secret
+                    });
+            database.recordProviderKeyUsage({
+                id: createId("usage"),
+                keyId: lease.keyId,
+                jobId: input.jobId,
+                provider: input.provider,
+                model: input.model,
+                eventType: "success",
+                inputTokens: result.usage.inputTokens,
+                outputTokens: result.usage.outputTokens,
+                costUsd: result.usage.costUsd,
+                metadata: { type: "speech_transcription" }
+            });
+            database.touchProviderKey(lease.keyId);
+            return result;
+        }
+        catch (error) {
+            if (error instanceof ProviderRateLimitError) {
+                database.recordProviderKeyUsage({
+                    id: createId("usage"),
+                    keyId: lease.keyId,
+                    jobId: input.jobId,
+                    provider: input.provider,
+                    model: input.model,
+                    eventType: "rate_limit",
+                    metadata: { retryAfterSeconds: error.retryAfterSeconds, type: "speech_transcription" }
+                });
+                database.setProviderKeyCooldown(lease.keyId, addSeconds(new Date(), error.retryAfterSeconds));
+            }
+            else if (error instanceof ProviderAuthError) {
+                database.recordProviderKeyUsage({
+                    id: createId("usage"),
+                    keyId: lease.keyId,
+                    jobId: input.jobId,
+                    provider: input.provider,
+                    model: input.model,
+                    eventType: "auth_error",
+                    metadata: { type: "speech_transcription" }
+                });
+                database.setProviderKeyCooldown(lease.keyId, null, "invalid", "auth_error");
+            }
+            throw error;
+        }
+        finally {
+            await this.releaseLease(lease);
+        }
+    }
     async callOpenAIImageGeneration(input) {
         if (input.promptAttachments?.length) {
             throw new Error("OpenAI image generation attachments are not implemented in this provider path. Use Gemini for reference attachments.");
@@ -355,6 +566,94 @@ export class ProviderService {
             revisedPrompt: null
         };
     }
+    async callOpenAICompatibleSpeechGeneration(input) {
+        const responseFormat = input.responseFormat ?? "mp3";
+        const endpoint = input.provider === "openrouter"
+            ? "https://openrouter.ai/api/v1/audio/speech"
+            : "https://api.openai.com/v1/audio/speech";
+        const response = await fetch(endpoint, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${input.apiKey}`
+            },
+            body: JSON.stringify({
+                model: input.model,
+                input: input.text,
+                voice: input.voice ?? "alloy",
+                instructions: input.instructions,
+                response_format: responseFormat
+            })
+        });
+        if (response.status === 401 || response.status === 403) {
+            throw new ProviderAuthError(`${input.provider} authentication failed`);
+        }
+        if (response.status === 429) {
+            throw new ProviderRateLimitError(`${input.provider} rate limited`);
+        }
+        if (!response.ok) {
+            const details = await response.text();
+            throw new Error(`${input.provider} speech generation returned ${response.status}${details ? `: ${details}` : ""}`);
+        }
+        return {
+            bytes: Buffer.from(await response.arrayBuffer()),
+            contentType: response.headers.get("content-type")?.split(";")[0]?.trim() || inferSpeechContentType(responseFormat),
+            usage: {
+                inputTokens: 0,
+                outputTokens: 0,
+                costUsd: 0
+            }
+        };
+    }
+    async callGeminiSpeechGeneration(input) {
+        const prompt = input.instructions?.trim()
+            ? `${input.instructions.trim()}\n\nSpeak exactly the following text:\n${input.text}`
+            : input.text;
+        const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${input.model}:generateContent?key=${input.apiKey}`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json"
+            },
+            body: JSON.stringify({
+                contents: [{ parts: [{ text: prompt }] }],
+                generationConfig: {
+                    responseModalities: ["AUDIO"],
+                    speechConfig: {
+                        voiceConfig: {
+                            prebuiltVoiceConfig: {
+                                voiceName: input.voice ?? "Kore"
+                            }
+                        }
+                    }
+                }
+            })
+        });
+        if (response.status === 401 || response.status === 403) {
+            throw new ProviderAuthError("gemini authentication failed");
+        }
+        if (response.status === 429) {
+            throw new ProviderRateLimitError("gemini rate limited");
+        }
+        if (!response.ok) {
+            const details = await response.text();
+            throw new Error(`gemini speech generation returned ${response.status}${details ? `: ${details}` : ""}`);
+        }
+        const data = await response.json();
+        const part = data.candidates?.[0]?.content?.parts?.find((item) => item.inlineData?.data || item.inline_data?.data);
+        const encoded = part?.inlineData?.data ?? part?.inline_data?.data;
+        if (!encoded) {
+            throw new Error("gemini speech generation returned no audio payload");
+        }
+        return {
+            bytes: Buffer.from(encoded, "base64"),
+            contentType: part?.inlineData?.mimeType ?? part?.inline_data?.mime_type ?? "audio/L16;rate=24000",
+            usage: {
+                inputTokens: Number(data.usageMetadata?.promptTokenCount ?? 0),
+                outputTokens: Number(data.usageMetadata?.candidatesTokenCount ?? 0),
+                costUsd: 0
+            }
+        };
+    }
     async callOpenAICompatibleLayoutAnalysis(input) {
         const endpoint = input.provider === "openrouter"
             ? "https://openrouter.ai/api/v1/chat/completions"
@@ -458,6 +757,128 @@ export class ProviderService {
         const text = data.candidates?.[0]?.content?.parts?.map((part) => part.text ?? "").join("\n") ?? "";
         return String(text);
     }
+    async callOpenAISpeechTranscription(input) {
+        const contentType = input.contentType ?? "audio/mpeg";
+        const form = new FormData();
+        form.set("file", new Blob([Buffer.from(input.audio)], { type: contentType }), `audio.${audioExtensionFromContentType(contentType)}`);
+        form.set("model", input.model);
+        form.set("response_format", "json");
+        if (input.prompt) {
+            form.set("prompt", input.prompt);
+        }
+        if (input.language) {
+            form.set("language", input.language);
+        }
+        const response = await fetch("https://api.openai.com/v1/audio/transcriptions", {
+            method: "POST",
+            headers: {
+                Authorization: `Bearer ${input.apiKey}`
+            },
+            body: form
+        });
+        if (response.status === 401 || response.status === 403) {
+            throw new ProviderAuthError("openai authentication failed");
+        }
+        if (response.status === 429) {
+            throw new ProviderRateLimitError("openai rate limited");
+        }
+        if (!response.ok) {
+            const details = await response.text();
+            throw new Error(`openai transcription returned ${response.status}${details ? `: ${details}` : ""}`);
+        }
+        const data = await response.json();
+        return {
+            text: String(data.text ?? ""),
+            usage: {
+                inputTokens: Number(data.usage?.input_tokens ?? 0),
+                outputTokens: Number(data.usage?.output_tokens ?? 0),
+                costUsd: Number(data.usage?.cost ?? 0)
+            }
+        };
+    }
+    async callOpenRouterSpeechTranscription(input) {
+        const response = await fetch("https://openrouter.ai/api/v1/audio/transcriptions", {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${input.apiKey}`
+            },
+            body: JSON.stringify({
+                model: input.model,
+                input_audio: {
+                    data: Buffer.from(input.audio).toString("base64"),
+                    format: audioFormatFromContentType(input.contentType)
+                },
+                ...(input.prompt ? { prompt: input.prompt } : {}),
+                ...(input.language ? { language: input.language } : {})
+            })
+        });
+        if (response.status === 401 || response.status === 403) {
+            throw new ProviderAuthError("openrouter authentication failed");
+        }
+        if (response.status === 429) {
+            throw new ProviderRateLimitError("openrouter rate limited");
+        }
+        if (!response.ok) {
+            const details = await response.text();
+            throw new Error(`openrouter transcription returned ${response.status}${details ? `: ${details}` : ""}`);
+        }
+        const data = await response.json();
+        return {
+            text: String(data.text ?? ""),
+            usage: {
+                inputTokens: Number(data.usage?.input_tokens ?? data.usage?.prompt_tokens ?? 0),
+                outputTokens: Number(data.usage?.output_tokens ?? data.usage?.completion_tokens ?? 0),
+                costUsd: Number(data.usage?.cost ?? 0)
+            }
+        };
+    }
+    async callGeminiSpeechTranscription(input) {
+        const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${input.model}:generateContent?key=${input.apiKey}`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json"
+            },
+            body: JSON.stringify({
+                contents: [{
+                        parts: [
+                            {
+                                inline_data: {
+                                    mime_type: input.contentType ?? "audio/mpeg",
+                                    data: Buffer.from(input.audio).toString("base64")
+                                }
+                            },
+                            {
+                                text: buildGeminiTranscriptionPrompt(input.prompt, input.language)
+                            }
+                        ]
+                    }],
+                generationConfig: {
+                    temperature: 0,
+                    responseMimeType: "text/plain"
+                }
+            })
+        });
+        if (response.status === 401 || response.status === 403) {
+            throw new ProviderAuthError("gemini authentication failed");
+        }
+        if (response.status === 429) {
+            throw new ProviderRateLimitError("gemini rate limited");
+        }
+        if (!response.ok) {
+            const details = await response.text();
+            throw new Error(`gemini transcription returned ${response.status}${details ? `: ${details}` : ""}`);
+        }
+        const data = await response.json();
+        return {
+            text: extractGeminiText(data),
+            usage: {
+                inputTokens: Number(data.usageMetadata?.promptTokenCount ?? 0),
+                outputTokens: Number(data.usageMetadata?.candidatesTokenCount ?? 0),
+                costUsd: 0
+            }
+        };
+    }
     async callProvider(input) {
         if (input.provider === "gemini") {
             return this.callGemini({
@@ -543,6 +964,55 @@ export class ProviderService {
 function clamp(value, min, max) {
     return Math.max(min, Math.min(max, value));
 }
+function assertSupportedSpeechModel(kind, provider, model) {
+    const supported = kind === "tts" ? SUPPORTED_TTS_MODELS[provider] : SUPPORTED_STT_MODELS[provider];
+    if (!supported?.has(model)) {
+        throw new Error(`Unsupported ${kind.toUpperCase()} model for provider ${provider}: ${model}`);
+    }
+}
+function inferSpeechContentType(format) {
+    return format === "wav" ? "audio/wav" : "audio/mpeg";
+}
+function audioFormatFromContentType(contentType) {
+    const normalized = contentType?.split(";")[0]?.trim().toLowerCase();
+    switch (normalized) {
+        case "audio/wav":
+        case "audio/x-wav":
+            return "wav";
+        case "audio/mp4":
+        case "audio/m4a":
+            return "mp4";
+        case "audio/webm":
+            return "webm";
+        case "audio/ogg":
+            return "ogg";
+        default:
+            return "mp3";
+    }
+}
+function audioExtensionFromContentType(contentType) {
+    switch (audioFormatFromContentType(contentType)) {
+        case "wav":
+            return "wav";
+        case "mp4":
+            return "m4a";
+        case "webm":
+            return "webm";
+        case "ogg":
+            return "ogg";
+        default:
+            return "mp3";
+    }
+}
+function buildGeminiTranscriptionPrompt(prompt, language) {
+    const basePrompt = prompt?.trim() || "Transcribe this audio verbatim. Return plain text only.";
+    return language?.trim()
+        ? `${basePrompt}\n\nExpected language: ${language.trim()}.`
+        : basePrompt;
+}
+function extractGeminiText(data) {
+    return String(data.candidates?.[0]?.content?.parts?.map((part) => part.text ?? "").join("\n") ?? "");
+}
 function supportsGeminiImageSize(model) {
     return model === "gemini-3.1-flash-image-preview" || model === "gemini-3-pro-image-preview";
 }

package/dist/src/services/storage.js

@@ -115,6 +115,19 @@ export class StorageService {
         }
         return readFileSync(path.join(this.localRoot, key), "utf8");
     }
+    async readBuffer(key) {
+        if (this.s3 && config.AWS_S3_BUCKET) {
+            const response = await this.s3.send(new GetObjectCommand({
+                Bucket: config.AWS_S3_BUCKET,
+                Key: key
+            }));
+            if (!response.Body) {
+                return null;
+            }
+            return Buffer.from(await response.Body.transformToByteArray());
+        }
+        return readFileSync(path.join(this.localRoot, key));
+    }
     async listKeys(prefix) {
         if (this.s3 && config.AWS_S3_BUCKET) {
             const keys = [];

package/dist/src/services/template-certification.js

@@ -13,11 +13,17 @@ export class TemplateCertificationService {
             typeof template.about.visual_dna === "string" &&
             Array.isArray(template.about.preview_media) &&
             template.about.preview_media.every((entry) => typeof entry === "string") &&
-            typeof template.about.link_to_original === "string"), "Template must define id, slugId, version, and complete about metadata.", {
+            typeof template.about.link_to_original === "string" &&
+            typeof template.about.proposal_generator === "string" &&
+            Array.isArray(template.about.sample_prompts) &&
+            template.about.sample_prompts.every((entry) => typeof entry === "string") &&
+            template.about.sample_prompts.length > 0), "Template must define id, slugId, version, and complete about metadata, including proposal_generator and sample prompts.", {
             templateId: template.id,
             slugId: template.slugId,
             version: template.version,
-            previewMediaCount: template.about?.preview_media?.length ?? 0
+            previewMediaCount: template.about?.preview_media?.length ?? 0,
+            hasProposalGenerator: typeof template.about?.proposal_generator === "string" && template.about.proposal_generator.length > 0,
+            samplePromptCount: template.about?.sample_prompts?.length ?? 0
         }));
         checks.push(this.check("template.operations", Object.keys(template.operations).length > 0, "Template must define at least one operation."));
         checks.push(this.checkSkill(skillPath));
@@ -124,6 +130,10 @@ function createSmokeContext(templateId) {
                 providerCalls += 1;
                 return { text: "certification text", usage: { inputTokens: 1, outputTokens: 1, costUsd: 0 } };
             },
+            async generateSpeech() {
+                providerCalls += 1;
+                return { bytes: Uint8Array.from([0, 1, 2, 3]), contentType: "audio/mpeg", usage: { inputTokens: 1, outputTokens: 1, costUsd: 0 } };
+            },
             async generateImage() {
                 providerCalls += 1;
                 return { bytes: MOCK_PNG_BYTES, contentType: "image/png", revisedPrompt: null };
@@ -136,6 +146,10 @@ function createSmokeContext(templateId) {
                     maxWidthPercent: 80,
                     justification: "Certification mock layout."
                 };
+            },
+            async transcribeSpeech() {
+                providerCalls += 1;
+                return { text: "certification transcript", usage: { inputTokens: 1, outputTokens: 1, costUsd: 0 } };
             }
         },
         remotion: {

package/dist/src/services/template-loader.js

@@ -4,11 +4,22 @@ export async function loadTemplateFromModule(modulePath) {
     const candidates = Object.values(loaded);
     for (const candidate of candidates) {
         if (isTemplateDefinition(candidate)) {
-            return candidate;
+            return normalizeTemplateDefinition(candidate);
         }
     }
     throw new Error(`No template definition export found in ${modulePath}`);
 }
+function normalizeTemplateDefinition(template) {
+    return {
+        ...template,
+        about: {
+            ...template.about,
+            // Older runtime template releases may predate proposal metadata fields.
+            proposal_generator: template.about.proposal_generator ?? "",
+            sample_prompts: template.about.sample_prompts ?? []
+        }
+    };
+}
 function isTemplateDefinition(value) {
     if (!value || typeof value !== "object") {
         return false;
@@ -20,6 +31,9 @@ function isTemplateDefinition(value) {
     const previewMedia = about && Array.isArray(about.preview_media)
         ? about.preview_media
         : null;
+    const samplePrompts = about && Array.isArray(about.sample_prompts)
+        ? about.sample_prompts
+        : null;
     return (typeof candidate.id === "string" &&
         typeof candidate.slugId === "string" &&
         typeof candidate.version === "string" &&
@@ -31,6 +45,8 @@ function isTemplateDefinition(value) {
         Array.isArray(previewMedia) &&
         previewMedia.every((entry) => typeof entry === "string") &&
         typeof about.link_to_original === "string" &&
+        (typeof about.proposal_generator === "string" || typeof about.proposal_generator === "undefined") &&
+        (samplePrompts === null || samplePrompts.every((entry) => typeof entry === "string")) &&
         typeof candidate.configSchema === "object" &&
         typeof candidate.operations === "object" &&
         typeof candidate.jobs === "object");