@juspay/neurolink 9.64.0 → 9.65.0

package/dist/lib/utils/videoProcessor.js

@@ -0,0 +1,201 @@
+/**
+ * Video Generation Processing Utility
+ *
+ * Central registry + dispatch for video-generation handlers across
+ * providers (Vertex Veo, Kling, Runway, Replicate-hosted models, etc.).
+ *
+ * Mirrors the static-handler-registry pattern established by
+ * `TTSProcessor` (`utils/ttsProcessor.ts`) and `STTProcessor`
+ * (`utils/sttProcessor.ts`).
+ *
+ * @module utils/videoProcessor
+ */
+import { ErrorCategory, ErrorSeverity } from "../constants/enums.js";
+import { VIDEO_ERROR_CODES } from "../constants/videoErrors.js";
+import { SpanSerializer, SpanStatus, SpanType, getMetricsAggregator, } from "../observability/index.js";
+import { logger } from "./logger.js";
+// VideoError is canonical in vertexVideoHandler.ts (existing). Re-export
+// here so consumers of `VideoProcessor` can import the typed error from
+// the same module. Both throws and instanceof checks resolve to the same
+// class.
+import { VideoError } from "../adapters/video/vertexVideoHandler.js";
+export { VideoError, VIDEO_ERROR_CODES };
+/**
+ * Static processor managing the video handler registry.
+ *
+ * Handlers register themselves during `ProviderRegistry._doRegister()`
+ * via `VideoProcessor.registerHandler(name, instance)`. Lookups are
+ * O(1) on a normalised lower-case provider key.
+ */
+export class VideoProcessor {
+    static handlers = new Map();
+    /**
+     * Register a video handler for a specific provider.
+     */
+    static registerHandler(providerName, handler) {
+        if (!providerName) {
+            throw new Error("Provider name is required");
+        }
+        if (!handler) {
+            throw new Error("Handler is required");
+        }
+        const key = providerName.toLowerCase();
+        if (this.handlers.has(key)) {
+            logger.warn(`[VideoProcessor] Overwriting existing handler for provider: ${key}`);
+        }
+        this.handlers.set(key, handler);
+        logger.debug(`[VideoProcessor] Registered video handler: ${key}`);
+    }
+    /**
+     * Check if a provider has a registered video handler.
+     */
+    static supports(providerName) {
+        if (!providerName) {
+            return false;
+        }
+        return this.handlers.has(providerName.toLowerCase());
+    }
+    /**
+     * List the names of all registered providers.
+     */
+    static listProviders() {
+        return Array.from(this.handlers.keys());
+    }
+    static getHandler(providerName) {
+        return this.handlers.get(providerName.toLowerCase());
+    }
+    static buildSpanAttributes(provider, options) {
+        return {
+            "video.operation": "generate",
+            "video.provider": provider,
+            "video.resolution": options.resolution,
+            "video.duration": options.length,
+            "video.aspect_ratio": options.aspectRatio,
+            "video.audio": options.audio,
+        };
+    }
+    /**
+     * Generate a single video clip via the registered handler.
+     *
+     * @param provider - Registered provider name (e.g. "vertex", "kling")
+     * @param image - Source image buffer
+     * @param prompt - Text prompt describing the desired motion / content
+     * @param options - Resolution / length / aspect-ratio / audio options
+     * @param region - Optional region override (Vertex location, etc.)
+     * @throws VideoError on registry miss, handler-not-configured, or
+     *         generation failure
+     */
+    static async generate(provider, image, prompt, options, region) {
+        const span = SpanSerializer.createSpan(SpanType.MEDIA_GENERATION, "video.generate", this.buildSpanAttributes(provider, options));
+        try {
+            const handler = this.getHandler(provider);
+            if (!handler) {
+                throw new VideoError({
+                    code: VIDEO_ERROR_CODES.PROVIDER_NOT_SUPPORTED,
+                    message: `Video provider "${provider}" is not registered. Available: ${this.listProviders().join(", ")}`,
+                    category: ErrorCategory.CONFIGURATION,
+                    severity: ErrorSeverity.HIGH,
+                    retriable: false,
+                    context: { provider, available: this.listProviders() },
+                });
+            }
+            if (!handler.isConfigured()) {
+                throw new VideoError({
+                    code: VIDEO_ERROR_CODES.PROVIDER_NOT_CONFIGURED,
+                    message: `Video provider "${provider}" is not configured. Set the required credentials.`,
+                    category: ErrorCategory.CONFIGURATION,
+                    severity: ErrorSeverity.HIGH,
+                    retriable: false,
+                    context: { provider },
+                });
+            }
+            logger.debug(`[VideoProcessor] Starting video generation with provider: ${provider}`);
+            const result = await handler.generate(image, prompt, options, region);
+            const ended = SpanSerializer.endSpan(span, SpanStatus.OK);
+            getMetricsAggregator().recordSpan(ended);
+            logger.info(`[VideoProcessor] Generated ${result.data.length} bytes (${provider})`);
+            return result;
+        }
+        catch (err) {
+            const ended = SpanSerializer.endSpan(span, SpanStatus.ERROR, err instanceof Error ? err.message : String(err));
+            getMetricsAggregator().recordSpan(ended);
+            if (err instanceof VideoError) {
+                throw err;
+            }
+            const message = err instanceof Error ? err.message : String(err);
+            throw new VideoError({
+                code: VIDEO_ERROR_CODES.GENERATION_FAILED,
+                message: `Video generation failed for provider "${provider}": ${message}`,
+                category: ErrorCategory.EXECUTION,
+                severity: ErrorSeverity.HIGH,
+                retriable: true,
+                context: { provider, options, region },
+                originalError: err instanceof Error ? err : undefined,
+            });
+        }
+    }
+    /**
+     * Generate a transition clip via the registered handler (Director Mode).
+     *
+     * Providers without first-and-last-frame interpolation surface a typed
+     * `TRANSITION_NOT_SUPPORTED` error here; callers should fall back to
+     * generating a regular clip with a transition prompt.
+     */
+    static async generateTransition(provider, firstFrame, lastFrame, prompt, options, region) {
+        const handler = this.getHandler(provider);
+        if (!handler) {
+            throw new VideoError({
+                code: VIDEO_ERROR_CODES.PROVIDER_NOT_SUPPORTED,
+                message: `Video provider "${provider}" is not registered for transitions`,
+                category: ErrorCategory.CONFIGURATION,
+                severity: ErrorSeverity.HIGH,
+                retriable: false,
+                context: { provider, available: this.listProviders() },
+            });
+        }
+        if (!handler.generateTransition) {
+            throw new VideoError({
+                code: VIDEO_ERROR_CODES.TRANSITION_NOT_SUPPORTED,
+                message: `Video provider "${provider}" does not support transition clips`,
+                category: ErrorCategory.VALIDATION,
+                severity: ErrorSeverity.MEDIUM,
+                retriable: false,
+                context: { provider },
+            });
+        }
+        if (!handler.isConfigured()) {
+            throw new VideoError({
+                code: VIDEO_ERROR_CODES.PROVIDER_NOT_CONFIGURED,
+                message: `Video provider "${provider}" is not configured`,
+                category: ErrorCategory.CONFIGURATION,
+                severity: ErrorSeverity.HIGH,
+                retriable: false,
+                context: { provider },
+            });
+        }
+        try {
+            return await handler.generateTransition(firstFrame, lastFrame, prompt, options, region);
+        }
+        catch (err) {
+            if (err instanceof VideoError) {
+                throw err;
+            }
+            const message = err instanceof Error ? err.message : String(err);
+            throw new VideoError({
+                code: VIDEO_ERROR_CODES.DIRECTOR_TRANSITION_FAILED,
+                message: `Video transition generation failed for provider "${provider}": ${message}`,
+                category: ErrorCategory.EXECUTION,
+                severity: ErrorSeverity.MEDIUM,
+                retriable: true,
+                context: {
+                    provider,
+                    firstFrameSize: firstFrame.length,
+                    lastFrameSize: lastFrame.length,
+                    durationSeconds: options?.durationSeconds,
+                },
+                originalError: err instanceof Error ? err : undefined,
+            });
+        }
+    }
+}
+//# sourceMappingURL=videoProcessor.js.map

package/dist/lib/voice/providers/FishAudioTTS.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Fish Audio TTS Handler
+ *
+ * Implementation of TTS using Fish Audio API. Lower-cost alternative to
+ * ElevenLabs with strong multilingual support and 15s voice cloning.
+ *
+ * @module voice/providers/FishAudioTTS
+ * @see https://docs.fish.audio/text-to-speech/text-to-speech
+ */
+import type { TTSHandler, TTSOptions, TTSResult } from "../../types/index.js";
+/**
+ * Fish Audio Text-to-Speech Handler.
+ *
+ * Auth: `Authorization: Bearer ${FISH_AUDIO_API_KEY}`.
+ * Models: speech-1.5 (standard), speech-1.6, s1 (default; latest).
+ */
+export declare class FishAudioTTS implements TTSHandler {
+    readonly maxTextLength = 5000;
+    private readonly apiKey;
+    private readonly baseUrl;
+    constructor(apiKey?: string);
+    isConfigured(): boolean;
+    synthesize(text: string, options?: TTSOptions): Promise<TTSResult>;
+    private mapFormat;
+    private effectiveFormat;
+    private getSampleRate;
+}

package/dist/lib/voice/providers/FishAudioTTS.js

@@ -0,0 +1,183 @@
+/**
+ * Fish Audio TTS Handler
+ *
+ * Implementation of TTS using Fish Audio API. Lower-cost alternative to
+ * ElevenLabs with strong multilingual support and 15s voice cloning.
+ *
+ * @module voice/providers/FishAudioTTS
+ * @see https://docs.fish.audio/text-to-speech/text-to-speech
+ */
+import { ErrorCategory, ErrorSeverity } from "../../constants/enums.js";
+import { logger } from "../../utils/logger.js";
+import { TTS_ERROR_CODES, TTSError } from "../../utils/ttsProcessor.js";
+const DEFAULT_BASE_URL = "https://api.fish.audio";
+const REQUEST_TIMEOUT_MS = 30_000;
+/**
+ * Default reference voice — "Energetic Male" by official author `lengyue`,
+ * a long-standing public English voice on Fish Audio.
+ *
+ * @see https://fish.audio (model id 802e3bc2b27e49c2995d23ef70e6ac89)
+ *
+ * Note: the previous default `fb6c0e1ea91e427fb9a93b9bbf0a1e4d` was
+ * removed upstream and started returning 400 "Reference not found".
+ */
+const DEFAULT_REFERENCE_ID = "802e3bc2b27e49c2995d23ef70e6ac89";
+/**
+ * Fish Audio Text-to-Speech Handler.
+ *
+ * Auth: `Authorization: Bearer ${FISH_AUDIO_API_KEY}`.
+ * Models: speech-1.5 (standard), speech-1.6, s1 (default; latest).
+ */
+export class FishAudioTTS {
+    maxTextLength = 5000;
+    apiKey;
+    baseUrl;
+    constructor(apiKey) {
+        const resolved = (apiKey ?? process.env.FISH_AUDIO_API_KEY ?? "").trim();
+        this.apiKey = resolved.length > 0 ? resolved : null;
+        this.baseUrl = (process.env.FISH_AUDIO_BASE_URL ?? DEFAULT_BASE_URL).replace(/\/$/, "");
+    }
+    isConfigured() {
+        return this.apiKey !== null;
+    }
+    async synthesize(text, options = {}) {
+        if (!this.apiKey) {
+            throw new TTSError({
+                code: TTS_ERROR_CODES.PROVIDER_NOT_CONFIGURED,
+                message: "FISH_AUDIO_API_KEY not configured",
+                category: ErrorCategory.CONFIGURATION,
+                severity: ErrorSeverity.HIGH,
+                retriable: false,
+            });
+        }
+        const startTime = Date.now();
+        const referenceId = options.voice ?? DEFAULT_REFERENCE_ID;
+        const requestedFormat = options.format ?? "mp3";
+        const upstreamFormat = this.mapFormat(requestedFormat);
+        const body = {
+            text,
+            reference_id: referenceId,
+            format: upstreamFormat,
+            mp3_bitrate: 128,
+            chunk_length: 200,
+            normalize: true,
+            latency: "normal",
+        };
+        const fishOpts = options;
+        if (fishOpts.model) {
+            body.model = fishOpts.model;
+        }
+        if (fishOpts.latency) {
+            body.latency = fishOpts.latency;
+        }
+        if (fishOpts.mp3Bitrate !== undefined) {
+            body.mp3_bitrate = fishOpts.mp3Bitrate;
+        }
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
+        let response;
+        try {
+            response = await fetch(`${this.baseUrl}/v1/tts`, {
+                method: "POST",
+                headers: {
+                    Authorization: `Bearer ${this.apiKey}`,
+                    "Content-Type": "application/json",
+                },
+                body: JSON.stringify(body),
+                signal: controller.signal,
+            });
+        }
+        catch (err) {
+            if (err instanceof Error && err.name === "AbortError") {
+                throw new TTSError({
+                    code: TTS_ERROR_CODES.SYNTHESIS_FAILED,
+                    message: `Fish Audio request timed out after ${REQUEST_TIMEOUT_MS / 1000}s`,
+                    category: ErrorCategory.NETWORK,
+                    severity: ErrorSeverity.HIGH,
+                    retriable: true,
+                    originalError: err,
+                });
+            }
+            throw new TTSError({
+                code: TTS_ERROR_CODES.SYNTHESIS_FAILED,
+                message: `Fish Audio network error: ${err instanceof Error ? err.message : String(err)}`,
+                category: ErrorCategory.NETWORK,
+                severity: ErrorSeverity.HIGH,
+                retriable: true,
+                originalError: err instanceof Error ? err : undefined,
+            });
+        }
+        finally {
+            clearTimeout(timeoutId);
+        }
+        if (!response.ok) {
+            const text = await response.text();
+            const retriable = response.status === 408 ||
+                response.status === 429 ||
+                response.status >= 500;
+            throw new TTSError({
+                code: TTS_ERROR_CODES.SYNTHESIS_FAILED,
+                message: `Fish Audio synthesis failed: ${response.status} — ${text}`,
+                category: retriable ? ErrorCategory.NETWORK : ErrorCategory.EXECUTION,
+                severity: ErrorSeverity.HIGH,
+                retriable,
+                context: { status: response.status, referenceId, upstreamFormat },
+            });
+        }
+        const arrayBuffer = await response.arrayBuffer();
+        const audioBuffer = Buffer.from(arrayBuffer);
+        const latency = Date.now() - startTime;
+        const effectiveFormat = this.effectiveFormat(upstreamFormat);
+        const result = {
+            buffer: audioBuffer,
+            format: effectiveFormat,
+            size: audioBuffer.length,
+            voice: referenceId,
+            sampleRate: this.getSampleRate(effectiveFormat),
+            metadata: {
+                latency,
+                provider: "fish-audio",
+                model: fishOpts.model ?? "s1",
+                requestedFormat: options.format,
+                upstreamFormat,
+            },
+        };
+        logger.info(`[FishAudioTTS] Synthesized ${audioBuffer.length} bytes in ${latency}ms`);
+        return result;
+    }
+    mapFormat(format) {
+        const supported = {
+            mp3: "mp3",
+            wav: "wav",
+            pcm16: "pcm",
+        };
+        const mapped = supported[format];
+        if (!mapped) {
+            logger.warn(`[FishAudioTTS] Unsupported format "${format}" — falling back to "mp3"`);
+            return "mp3";
+        }
+        return mapped;
+    }
+    effectiveFormat(upstreamFormat) {
+        if (upstreamFormat === "mp3") {
+            return "mp3";
+        }
+        if (upstreamFormat === "wav") {
+            return "wav";
+        }
+        if (upstreamFormat === "pcm") {
+            return "pcm16";
+        }
+        return "mp3";
+    }
+    getSampleRate(format) {
+        if (format === "wav") {
+            return 44_100;
+        }
+        if (format === "pcm16") {
+            return 44_100;
+        }
+        return 44_100;
+    }
+}
+//# sourceMappingURL=FishAudioTTS.js.map

package/dist/lib/workflow/core/ensembleExecutor.js

@@ -120,15 +120,32 @@ async function executeModel(options, workflowDefaultSystemPrompt) {
         // 3. Workflow-level default
         // 4. undefined (provider default)
         const resolvedSystemPrompt = systemPrompt || model.systemPrompt || workflowDefaultSystemPrompt;
-        // Execute with timeout
-        const result = await executeWithTimeout(async () => {
-            return await provider.generate({
-                prompt,
-                systemPrompt: resolvedSystemPrompt,
-                temperature: model.temperature,
-                maxTokens: model.maxTokens,
-            });
-        }, timeout, `Model ${model.provider}/${model.model} timed out after ${timeout}ms`);
+        // Execute with timeout. Some upstream LLM endpoints (notably Vertex
+        // under high parallel load) occasionally respond with an empty
+        // assistant message instead of content; retry once before giving up so
+        // a transient empty response doesn't cascade into an empty ensemble
+        // result that downstream tests + judges have no way to evaluate.
+        let result;
+        let attempts = 0;
+        const MAX_ATTEMPTS = 2;
+        /* eslint-disable no-constant-condition */
+        while (true) {
+            attempts++;
+            result = await executeWithTimeout(async () => {
+                return await provider.generate({
+                    prompt,
+                    systemPrompt: resolvedSystemPrompt,
+                    temperature: model.temperature,
+                    maxTokens: model.maxTokens,
+                });
+            }, timeout, `Model ${model.provider}/${model.model} timed out after ${timeout}ms`);
+            const contentLen = result?.content?.length ?? 0;
+            if (contentLen > 0 || attempts >= MAX_ATTEMPTS) {
+                break;
+            }
+            logger.warn(`[${functionTag}] Model returned empty content — retrying once`, { provider: model.provider, model: model.model, attempt: attempts });
+        }
+        /* eslint-enable no-constant-condition */
         const responseTime = Date.now() - startTime;
         logger.debug(`[${functionTag}] Model execution successful`, {
             provider: model.provider,

package/dist/middleware/builtin/lifecycle.js

@@ -10,6 +10,31 @@
 import { logger } from "../../utils/logger.js";
 import { isRecoverableError } from "../../utils/errorHandling.js";
 import { fireOnErrorOnce } from "../../utils/lifecycleCallbacks.js";
+/**
+ * Normalize a thrown value to an `Error` while preserving structured
+ * fields (`code`, `status`, `statusCode`, `retryAfter`, `details`,
+ * `cause`, etc.) that downstream retry logic and SDK callers depend on.
+ *
+ * Previously the lifecycle middleware did
+ * `error instanceof Error ? error : new Error(String(error))`, which
+ * silently dropped every custom property on non-Error throws — so a
+ * provider's `{ code: "RATE_LIMITED", retryAfter: 30 }` rejection
+ * surfaced to consumers as a bare `Error("[object Object]")`.
+ */
+function normalizeToError(error) {
+    if (error instanceof Error) {
+        return error;
+    }
+    if (typeof error === "object" && error !== null) {
+        const obj = error;
+        const msg = typeof obj.message === "string" ? obj.message : String(error);
+        // Object.assign keeps Error's prototype chain (including the
+        // captured stack from this synthetic Error's construction) while
+        // copying enumerable fields from the original throw.
+        return Object.assign(new Error(msg), obj);
+    }
+    return new Error(String(error));
+}
 export function createLifecycleMiddleware(config = {}) {
     const metadata = {
         id: "lifecycle",
@@ -51,13 +76,18 @@ export function createLifecycleMiddleware(config = {}) {
                 return result;
             }
             catch (error) {
-                const err = error instanceof Error ? error : new Error(String(error));
-                fireOnErrorOnce(config.onError, error, {
+                const err = normalizeToError(error);
+                // fireOnErrorOnce stamps a Symbol on `err` so the SDK-level catch
+                // in neurolink.ts (and baseProvider.handleProviderError) skips
+                // its own onError fire for the same logical failure.
+                fireOnErrorOnce(config.onError, err, {
                     error: err,
                     duration: Date.now() - startTime,
                     recoverable: isRecoverableError(err),
                 });
-                throw error;
+                // Rethrow the normalized err (not the raw `error`) so the
+                // fired-mark and any preserved structured fields propagate.
+                throw err;
             }
         },
         wrapStream: async ({ doStream, }) => {
@@ -93,13 +123,13 @@ export function createLifecycleMiddleware(config = {}) {
                             controller.enqueue(chunk);
                         }
                         catch (error) {
-                            const err = error instanceof Error ? error : new Error(String(error));
-                            fireOnErrorOnce(config.onError, error, {
+                            const err = normalizeToError(error);
+                            fireOnErrorOnce(config.onError, err, {
                                 error: err,
                                 duration: Date.now() - startTime,
                                 recoverable: isRecoverableError(err),
                             });
-                            throw error;
+                            throw err;
                         }
                     },
                     flush() {
@@ -125,13 +155,13 @@ export function createLifecycleMiddleware(config = {}) {
                 };
             }
             catch (error) {
-                const err = error instanceof Error ? error : new Error(String(error));
-                fireOnErrorOnce(config.onError, error, {
+                const err = normalizeToError(error);
+                fireOnErrorOnce(config.onError, err, {
                     error: err,
                     duration: Date.now() - startTime,
                     recoverable: isRecoverableError(err),
                 });
-                throw error;
+                throw err;
             }
         },
     };

package/dist/music/index.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Music Module — Music Generation Integration for NeuroLink
+ *
+ * Provides music-generation capability across providers (Beatoven,
+ * ElevenLabs Music, Lyria, Replicate-hosted MusicGen / Riffusion).
+ *
+ * Use `MusicProcessor.generate(provider, options)` to dispatch to the
+ * registered handler for `provider`.
+ *
+ * @module music
+ */
+export { MUSIC_ERROR_CODES, MusicError, MusicProcessor, } from "../utils/musicProcessor.js";
+export { BeatovenMusic, BeatovenMusic as BeatovenMusicHandler, } from "./providers/BeatovenMusic.js";

package/dist/music/index.js

@@ -0,0 +1,13 @@
+/**
+ * Music Module — Music Generation Integration for NeuroLink
+ *
+ * Provides music-generation capability across providers (Beatoven,
+ * ElevenLabs Music, Lyria, Replicate-hosted MusicGen / Riffusion).
+ *
+ * Use `MusicProcessor.generate(provider, options)` to dispatch to the
+ * registered handler for `provider`.
+ *
+ * @module music
+ */
+export { MUSIC_ERROR_CODES, MusicError, MusicProcessor, } from "../utils/musicProcessor.js";
+export { BeatovenMusic, BeatovenMusic as BeatovenMusicHandler, } from "./providers/BeatovenMusic.js";

package/dist/music/providers/BeatovenMusic.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Beatoven.ai Music Generation Handler
+ *
+ * Async track-composition API. Submits a compose-track request, polls the
+ * task status, and downloads the resulting audio.
+ *
+ * @module music/providers/BeatovenMusic
+ * @see https://www.beatoven.ai/api-docs
+ */
+import type { MusicAudioFormat, MusicHandler, MusicOptions, MusicResult } from "../../types/index.js";
+/**
+ * Beatoven.ai Music Generation Handler.
+ *
+ * Beatoven is a royalty-free music generation API tuned for
+ * background / cinematic / brand music. Tracks are composed
+ * asynchronously: submit a prompt, poll the task, then download.
+ */
+export declare class BeatovenMusic implements MusicHandler {
+    readonly maxDurationSeconds = 300;
+    readonly supportedFormats: readonly MusicAudioFormat[];
+    readonly supportedGenres: readonly string[];
+    private readonly apiKey;
+    private readonly baseUrl;
+    constructor(apiKey?: string);
+    isConfigured(): boolean;
+    generate(options: MusicOptions): Promise<MusicResult>;
+    private submitCompose;
+    private pollUntilComposed;
+    private fetchTaskStatus;
+    private downloadTrack;
+}