@mevdragon/vidfarm-devcli 0.10.0 → 0.11.0

package/dist/src/primitive-registry.js

@@ -7,6 +7,7 @@ import { config } from "./config.js";
 import { definePrimitive } from "./primitive-sdk.js";
 import { PrimitiveMediaLambdaService } from "./services/primitive-media-lambda.js";
 import { estimateGhostcutCostUsd, isGhostcutConfigured, pollStatus as pollGhostcutStatus, submitSubtitleRemoval } from "./services/ghostcut.js";
+import { buildSrtFromSegments, defaultSpeechModelFor } from "./services/speech.js";
 import { buildHtmlImageCompositionHtml, buildMediaDedupeCompositionHtml, DEDUPE_DEFAULT_EFFECTS } from "./primitives/hyperframes-media.js";
 const imageProviderSchema = z.enum(["openai", "gemini", "openrouter"]);
 const videoProviderSchema = z.enum(["openai", "gemini", "openrouter"]);
@@ -358,6 +359,47 @@ const trimAudioPayloadSchema = z.object({
     message: "Either duration_ms or end_ms is required.",
     path: ["duration_ms"]
 });
+const speechProviderSchema = z.enum(["openai", "gemini", "openrouter"]);
+const ttsPayloadSchema = z.preprocess((raw) => {
+    if (!raw || typeof raw !== "object" || Array.isArray(raw))
+        return raw;
+    const payload = raw;
+    return {
+        ...payload,
+        text: payload.text ?? payload.input ?? payload.script,
+        instructions: payload.instructions ?? payload.style ?? payload.style_instructions ?? payload.voice_style
+    };
+}, z.object({
+    provider: speechProviderSchema.optional(),
+    model: z.string().min(1).optional(),
+    text: z.string().min(1).max(8000),
+    voice: z.string().min(1).max(120).optional(),
+    instructions: z.string().min(1).max(4000).optional(),
+    output_format: z.enum(["mp3", "wav"]).default("mp3")
+}));
+const sttPayloadSchema = z.preprocess((raw) => {
+    if (!raw || typeof raw !== "object" || Array.isArray(raw))
+        return raw;
+    const payload = raw;
+    return {
+        ...payload,
+        source_url: payload.source_url ??
+            payload.source_video_url ??
+            payload.source_audio_url ??
+            payload.source_media_url ??
+            payload.video_url ??
+            payload.audio_url ??
+            payload.url
+    };
+}, z.object({
+    provider: speechProviderSchema.optional(),
+    model: z.string().min(1).optional(),
+    source_url: mediaSourceUrlSchema,
+    media_type: z.enum(["auto", "video", "audio"]).default("auto"),
+    diarize: z.boolean().default(true),
+    language: z.string().min(2).max(16).optional(),
+    prompt: z.string().min(1).max(2000).optional()
+}));
 const probeVideoPayloadSchema = z.object({
     source_video_url: mediaSourceUrlSchema
 });
@@ -508,6 +550,14 @@ const PRIMITIVE_BRAINSTORM_ANGLES_ID = "primitive:brainstorm_angles";
 const PRIMITIVE_BRAINSTORM_COLDSTART_ID = "primitive:brainstorm_coldstart";
 const PRIMITIVE_BRAINSTORM_PRODUCT_PLACEMENT_ID = "primitive:brainstorm_product_placement";
 const PRIMITIVE_MEDIA_DEDUPE_ID = "primitive:media_dedupe";
+const PRIMITIVE_TTS_ID = "primitive:tts";
+const PRIMITIVE_STT_ID = "primitive:stt";
+// Gemini inline audio caps out around 20MB; at the 64kbps mono mp3 we extract,
+// that is roughly 40 minutes of speech.
+const MAX_TRANSCRIBE_AUDIO_BYTES = 20 * 1024 * 1024;
+// Keep job results comfortably under the DynamoDB item limit: transcripts
+// longer than this inline budget stay artifact-only (transcript.json).
+const MAX_INLINE_TRANSCRIPT_CHARS = 20_000;
 const primitiveMediaService = new PrimitiveMediaLambdaService();
 const hooksReferenceText = readPrimitiveAsset("SELLING_WITH_HOOKS.md");
 const awarenessReferenceText = readPrimitiveAsset("SELLING_AWARENESS_STAGES.md");
@@ -1566,6 +1616,181 @@ const audioTrimPrimitive = definePrimitive({
         }
     }
 });
+const ttsPrimitive = definePrimitive({
+    id: PRIMITIVE_TTS_ID,
+    kind: "tts",
+    operations: {
+        run: {
+            description: "Generate spoken narration audio from text (TTS) with a promptable voice style, persisted as a reusable platform audio URL.",
+            inputSchema: ttsPayloadSchema,
+            workflow: "run"
+        }
+    },
+    jobs: {
+        async run(ctx, input) {
+            const payload = ttsPayloadSchema.parse(input);
+            ctx.logger.progress(0.12, "Generating speech audio");
+            const provider = payload.provider ?? "openai";
+            const model = payload.model ?? defaultSpeechModelFor("tts", provider) ?? "";
+            const generated = await ctx.providers.generateSpeech({
+                provider,
+                model,
+                text: payload.text,
+                voice: payload.voice,
+                instructions: payload.instructions,
+                responseFormat: payload.output_format
+            });
+            // Gemini TTS answers in wav regardless of the requested format; name the
+            // artifact after what actually came back.
+            const extension = generated.contentType.includes("wav") ? "wav" : "mp3";
+            const stored = await ctx.storage.putBuffer(`speech.${extension}`, generated.bytes, {
+                contentType: generated.contentType,
+                kind: "audio",
+                metadata: {
+                    primitive: "tts",
+                    provider,
+                    model,
+                    voice: payload.voice ?? null,
+                    styleInstructions: payload.instructions ?? null
+                }
+            });
+            ctx.logger.progress(1, "Speech generation complete");
+            return {
+                progress: 1,
+                output: {
+                    files: compactUrls([stored.url]),
+                    primary_file_url: stored.url,
+                    audioUrl: stored.url,
+                    audio: {
+                        file_url: stored.url,
+                        content_type: generated.contentType,
+                        provider,
+                        model,
+                        voice: payload.voice ?? null,
+                        style_instructions: payload.instructions ?? null,
+                        text_length: payload.text.length
+                    }
+                }
+            };
+        }
+    }
+});
+const sttPrimitive = definePrimitive({
+    id: PRIMITIVE_STT_ID,
+    kind: "stt",
+    operations: {
+        run: {
+            description: "Transcribe speech from a video or audio URL (STT) into a simple subtitle transcript plus speaker-attributed multi-narration segments.",
+            inputSchema: sttPayloadSchema,
+            workflow: "run"
+        }
+    },
+    jobs: {
+        async run(ctx, input) {
+            const payload = sttPayloadSchema.parse(input);
+            const sourceKind = payload.media_type === "auto" ? inferSpeechSourceKind(payload.source_url) : payload.media_type;
+            let audioBytes;
+            let audioContentType;
+            let extractionMetadata = null;
+            if (sourceKind === "audio") {
+                ctx.logger.progress(0.1, "Fetching source audio");
+                const response = await fetch(payload.source_url);
+                if (!response.ok) {
+                    throw new Error(`Could not fetch source audio (${response.status}).`);
+                }
+                audioBytes = new Uint8Array(await response.arrayBuffer());
+                audioContentType = response.headers.get("content-type")?.split(";")[0]?.trim() || "audio/mpeg";
+            }
+            else {
+                // Videos (and unknown extensions) go through the ffmpeg seam: it
+                // demuxes video AND normalizes odd audio containers into a small
+                // mono mp3 the speech providers accept.
+                ctx.logger.progress(0.1, "Extracting audio track from source", {
+                    executionMode: config.VIDFARM_PRIMITIVE_MEDIA_LAMBDA_URL ? "lambda" : "local_fallback"
+                });
+                const extracted = await executePrimitiveMediaOperation(ctx, {
+                    operation: "video_extract_audio",
+                    payload: {
+                        source_video_url: payload.source_url,
+                        output_format: "mp3",
+                        audio_bitrate_kbps: 64
+                    },
+                    outputKey: "source-audio.mp3"
+                });
+                if (!extracted.publicUrl) {
+                    throw new Error("Audio extraction produced no readable audio URL.");
+                }
+                const response = await fetch(extracted.publicUrl);
+                if (!response.ok) {
+                    throw new Error(`Could not read extracted audio (${response.status}).`);
+                }
+                audioBytes = new Uint8Array(await response.arrayBuffer());
+                audioContentType = extracted.contentType || "audio/mpeg";
+                extractionMetadata = extracted.metadata ?? null;
+            }
+            if (audioBytes.byteLength > MAX_TRANSCRIBE_AUDIO_BYTES) {
+                const sizeMb = Math.round(audioBytes.byteLength / (1024 * 1024));
+                const maxMb = Math.round(MAX_TRANSCRIBE_AUDIO_BYTES / (1024 * 1024));
+                throw new Error(`Audio is too large to transcribe (${sizeMb}MB > ${maxMb}MB, roughly 40 minutes of speech). Trim the source with /videos/trim or /audio/trim and transcribe in parts.`);
+            }
+            ctx.logger.progress(0.45, "Transcribing speech");
+            const provider = payload.provider ?? "gemini";
+            const model = payload.model ?? defaultSpeechModelFor("stt", provider) ?? "";
+            const transcription = await ctx.providers.transcribeSpeech({
+                provider,
+                model,
+                audio: audioBytes,
+                contentType: audioContentType,
+                prompt: payload.prompt,
+                language: payload.language,
+                diarize: payload.diarize
+            });
+            ctx.logger.progress(0.85, "Storing transcript artifacts");
+            const speakers = [...new Set(transcription.segments.map((segment) => segment.speaker))];
+            const srt = buildSrtFromSegments(transcription.segments);
+            const storedText = await ctx.storage.putText("transcript.txt", transcription.text || "");
+            const storedSrt = srt ? await ctx.storage.putText("transcript.srt", srt, "application/x-subrip; charset=utf-8") : null;
+            const storedJson = await ctx.storage.putJson("transcript.json", {
+                source_url: payload.source_url,
+                provider,
+                model,
+                language: transcription.language,
+                diarization: transcription.diarization,
+                text: transcription.text,
+                speakers,
+                segments: transcription.segments,
+                ...(extractionMetadata ? { extraction: extractionMetadata } : {})
+            });
+            const inlineOk = (transcription.text?.length ?? 0) <= MAX_INLINE_TRANSCRIPT_CHARS;
+            ctx.logger.progress(1, "Speech transcription complete");
+            return {
+                progress: 1,
+                output: {
+                    files: compactUrls([storedJson.url, storedSrt?.url, storedText.url]),
+                    primary_file_url: storedJson.url,
+                    sourceUrl: payload.source_url,
+                    text: inlineOk ? transcription.text : transcription.text.slice(0, MAX_INLINE_TRANSCRIPT_CHARS),
+                    text_truncated: !inlineOk,
+                    language: transcription.language,
+                    diarization: transcription.diarization,
+                    // Format 1 — SIMPLE SUBTITLE VERSION: plain transcript + SRT cues (no speakers).
+                    subtitles: {
+                        srt: inlineOk ? srt : null,
+                        srt_url: storedSrt?.url ?? null,
+                        text_url: storedText.url
+                    },
+                    // Format 2 — ADVANCED MULTI-SPEAKER VERSION: speaker-attributed timed segments.
+                    speakers,
+                    segments: inlineOk ? transcription.segments : undefined,
+                    transcript: {
+                        json_url: storedJson.url,
+                        content_type: "application/json"
+                    }
+                }
+            };
+        }
+    }
+});
 const videoProbePrimitive = definePrimitive({
     id: PRIMITIVE_VIDEO_PROBE_ID,
     kind: "video_probe",
@@ -2142,6 +2367,8 @@ class PrimitiveRegistry {
         [videoConcatPrimitive.id, videoConcatPrimitive],
         [audioConcatPrimitive.id, audioConcatPrimitive],
         [audioNormalizePrimitive.id, audioNormalizePrimitive],
+        [ttsPrimitive.id, ttsPrimitive],
+        [sttPrimitive.id, sttPrimitive],
         [hyperframesRenderPrimitive.id, hyperframesRenderPrimitive],
         [brainstormHooksPrimitive.id, brainstormHooksPrimitive],
         [brainstormAwarenessStagesPrimitive.id, brainstormAwarenessStagesPrimitive],
@@ -2266,6 +2493,11 @@ function defaultTextModelForProvider(provider) {
             return "gpt-5.4-mini";
     }
 }
+// Extensions that are safe to feed a speech provider without the ffmpeg
+// normalization pass; everything else (video or unknown) gets demuxed first.
+function inferSpeechSourceKind(sourceUrl) {
+    return /\.(mp3|m4a|aac|ogg|oga|flac|wav)(\?|#|$)/i.test(sourceUrl) ? "audio" : "video";
+}
 function normalizeAiVideoDurationSeconds(explicitSeconds, milliseconds) {
     const seconds = Number(explicitSeconds);
     if (Number.isFinite(seconds) && seconds > 0) {

package/dist/src/services/provider-errors.js

@@ -0,0 +1,74 @@
+// Provider error taxonomy + HTTP-response classification, shared by every
+// surface that talks to an AI provider directly: ProviderService (cloud, leased
+// BYOK keys) and the speech module (also driven key-less from the devcli with a
+// local env key). Lives outside providers.ts so speech.ts can import it without
+// a circular dependency.
+export class ProviderKeyUnavailableError extends Error {
+}
+export class ProviderAuthError extends Error {
+}
+export class ProviderQuotaExceededError extends Error {
+}
+export class ProviderWaitExceededError extends Error {
+}
+export class ProviderLeaseTimeoutError extends Error {
+    retryAfterSeconds;
+    constructor(message, retryAfterSeconds = 5) {
+        super(message);
+        this.retryAfterSeconds = retryAfterSeconds;
+    }
+}
+export class ProviderRateLimitError extends Error {
+    retryAfterSeconds;
+    constructor(message, retryAfterSeconds = 60) {
+        super(message);
+        this.retryAfterSeconds = retryAfterSeconds;
+    }
+}
+export class ProviderTransientError extends Error {
+    retryAfterSeconds;
+    constructor(message, retryAfterSeconds = 5) {
+        super(message);
+        this.retryAfterSeconds = retryAfterSeconds;
+    }
+}
+export function parseRetryAfterSeconds(value) {
+    if (!value) {
+        return 60;
+    }
+    const numeric = Number(value);
+    if (Number.isFinite(numeric) && numeric > 0) {
+        return Math.max(1, Math.min(Math.ceil(numeric), 60 * 60));
+    }
+    const parsedDate = Date.parse(value);
+    if (Number.isFinite(parsedDate)) {
+        return Math.max(1, Math.min(Math.ceil((parsedDate - Date.now()) / 1000), 60 * 60));
+    }
+    return 60;
+}
+export function isHardQuotaError(details) {
+    const normalized = details.toLowerCase();
+    return [
+        "insufficient_quota",
+        "quota_exceeded",
+        "exceeded your current quota",
+        "billing",
+        "insufficient credits",
+        "not enough credits",
+        "credit balance",
+        "prepaid credits",
+        "resource_exhausted"
+    ].some((pattern) => normalized.includes(pattern));
+}
+export function conciseProviderDetails(details) {
+    const compact = details.replace(/\s+/g, " ").trim();
+    return compact ? `: ${compact.slice(0, 500)}` : "";
+}
+export async function rateLimitOrQuotaError(provider, response) {
+    const details = await response.text();
+    if (isHardQuotaError(details)) {
+        return new ProviderQuotaExceededError(`${provider} quota or billing limit was reached${conciseProviderDetails(details)}`);
+    }
+    return new ProviderRateLimitError(`${provider} rate limited${conciseProviderDetails(details)}`, parseRetryAfterSeconds(response.headers.get("retry-after")));
+}
+//# sourceMappingURL=provider-errors.js.map