@juspay/neurolink 9.64.0 → 9.65.0

package/dist/music/providers/LyriaMusic.js

@@ -0,0 +1,172 @@
+/**
+ * Google Lyria 3 Pro Music Handler
+ *
+ * Synchronous generation against the Generative Language API. Returns
+ * audio inline as base64 in the response.
+ *
+ * @module music/providers/LyriaMusic
+ * @see https://ai.google.dev/gemini-api/docs/music-generation
+ */
+import { ErrorCategory, ErrorSeverity } from "../../constants/enums.js";
+import { MUSIC_ERROR_CODES, MusicError } from "../../utils/musicProcessor.js";
+import { logger } from "../../utils/logger.js";
+import { sanitizeForLog } from "../../utils/logSanitize.js";
+const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
+const DEFAULT_MODEL = "lyria-3-pro-preview";
+const REQUEST_TIMEOUT_MS = 120_000;
+/**
+ * Google Lyria 3 Pro Music Handler.
+ *
+ * Auth: `Authorization: Bearer ${GOOGLE_API_KEY}` or query-string
+ * `?key=${GOOGLE_API_KEY}` (the latter is more compatible with the
+ * Generative Language endpoints today).
+ */
+export class LyriaMusic {
+    maxDurationSeconds = 30;
+    supportedFormats = ["wav"];
+    supportedGenres = [
+        "ambient",
+        "classical",
+        "electronic",
+        "jazz",
+        "rock",
+        "pop",
+        "lo-fi",
+        "cinematic",
+        "orchestral",
+        "world",
+    ];
+    apiKey;
+    baseUrl;
+    model;
+    constructor(apiKey) {
+        const resolved = (apiKey ??
+            process.env.GOOGLE_AI_LYRIA_API_KEY ??
+            process.env.GOOGLE_API_KEY ??
+            process.env.GOOGLE_AI_API_KEY ??
+            process.env.GEMINI_API_KEY ??
+            "").trim();
+        this.apiKey = resolved.length > 0 ? resolved : null;
+        this.baseUrl = (process.env.LYRIA_BASE_URL ?? DEFAULT_BASE_URL).replace(/\/$/, "");
+        this.model = process.env.LYRIA_MODEL ?? DEFAULT_MODEL;
+    }
+    isConfigured() {
+        return this.apiKey !== null;
+    }
+    async generate(options) {
+        if (!this.apiKey) {
+            throw new MusicError({
+                code: MUSIC_ERROR_CODES.PROVIDER_NOT_CONFIGURED,
+                message: "Lyria requires one of: GOOGLE_API_KEY, GOOGLE_AI_LYRIA_API_KEY, GOOGLE_AI_API_KEY, or GEMINI_API_KEY",
+                category: ErrorCategory.CONFIGURATION,
+                severity: ErrorSeverity.HIGH,
+                retriable: false,
+            });
+        }
+        const startTime = Date.now();
+        const duration = Math.min(options.duration ?? 16, this.maxDurationSeconds);
+        // Lyria API no longer accepts `audioGenerationOptions` — duration is
+        // controlled implicitly by the prompt/model. Only `responseModalities`
+        // is allowed under `generationConfig`. Embed duration in the prompt
+        // so the model still gets the hint.
+        const promptWithDuration = `${this.buildPrompt(options)}. Duration: ${duration} seconds`;
+        const body = {
+            contents: [
+                {
+                    parts: [
+                        {
+                            text: promptWithDuration,
+                        },
+                    ],
+                },
+            ],
+            generationConfig: {
+                responseModalities: ["AUDIO"],
+            },
+        };
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
+        let response;
+        try {
+            response = await fetch(`${this.baseUrl}/models/${this.model}:generateContent?key=${this.apiKey}`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                },
+                body: JSON.stringify(body),
+                signal: controller.signal,
+            });
+        }
+        catch (err) {
+            if (err instanceof Error && err.name === "AbortError") {
+                throw new MusicError({
+                    code: MUSIC_ERROR_CODES.GENERATION_FAILED,
+                    message: `Lyria request timed out after ${REQUEST_TIMEOUT_MS / 1000}s`,
+                    category: ErrorCategory.NETWORK,
+                    severity: ErrorSeverity.HIGH,
+                    retriable: true,
+                    originalError: err,
+                });
+            }
+            throw err;
+        }
+        finally {
+            clearTimeout(timeoutId);
+        }
+        if (!response.ok) {
+            const rawText = await response.text();
+            const retriable = response.status === 408 ||
+                response.status === 429 ||
+                response.status >= 500;
+            throw new MusicError({
+                code: MUSIC_ERROR_CODES.GENERATION_FAILED,
+                message: `Lyria generation failed: ${response.status} — ${sanitizeForLog(rawText)}`,
+                category: retriable ? ErrorCategory.NETWORK : ErrorCategory.EXECUTION,
+                severity: ErrorSeverity.HIGH,
+                retriable,
+                context: { status: response.status },
+            });
+        }
+        const data = (await response.json());
+        const audioPart = data.candidates?.[0]?.content?.parts?.find((p) => p.inlineData?.mimeType?.startsWith("audio/"));
+        if (!audioPart?.inlineData?.data) {
+            throw new MusicError({
+                code: MUSIC_ERROR_CODES.GENERATION_FAILED,
+                message: "Lyria response missing audio data",
+                category: ErrorCategory.EXECUTION,
+                severity: ErrorSeverity.HIGH,
+                retriable: false,
+                context: { response: data },
+            });
+        }
+        const buffer = Buffer.from(audioPart.inlineData.data, "base64");
+        const latency = Date.now() - startTime;
+        logger.info(`[LyriaMusic] Generated ${buffer.length} bytes in ${latency}ms — model ${this.model}`);
+        return {
+            buffer,
+            format: "wav",
+            size: buffer.length,
+            duration,
+            provider: "lyria",
+            metadata: {
+                latency,
+                provider: "lyria",
+                model: this.model,
+                sampleRate: 48_000,
+            },
+        };
+    }
+    buildPrompt(options) {
+        const parts = [options.prompt];
+        if (options.genre) {
+            parts.push(`Genre: ${options.genre}`);
+        }
+        if (options.mood) {
+            parts.push(`Mood: ${options.mood}`);
+        }
+        if (options.tempo !== undefined) {
+            parts.push(`${options.tempo} BPM`);
+        }
+        return parts.join(". ");
+    }
+}

package/dist/music/providers/ReplicateMusic.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Replicate Music Handler (MusicGen default)
+ *
+ * Routes music generation through Replicate's universal prediction
+ * lifecycle. Default model is Meta's MusicGen; alternatives include
+ * Riffusion, AudioGen, and AudioLDM via `options.model`.
+ *
+ * @module music/providers/ReplicateMusic
+ * @see https://replicate.com/meta/musicgen
+ */
+import type { MusicAudioFormat, MusicHandler, MusicOptions, MusicResult } from "../../types/index.js";
+export declare class ReplicateMusic implements MusicHandler {
+    readonly maxDurationSeconds = 30;
+    readonly supportedFormats: readonly MusicAudioFormat[];
+    readonly supportedGenres: readonly string[];
+    isConfigured(): boolean;
+    generate(options: MusicOptions): Promise<MusicResult>;
+    private buildPrompt;
+    private resolveBuffer;
+    /**
+     * Detect audio MIME subtype from magic bytes.
+     *
+     * - WAV : "RIFF" header (52 49 46 46)
+     * - MP3 : ID3 tag (49 44 33) or MPEG sync word 0xFF 0xEx
+     * - OGG : "OggS" capture pattern (4F 67 67 53)
+     * - M4A : "ftyp" box at offset 4
+     *
+     * Falls back to "mp3" when detection is inconclusive.
+     */
+    private detectAudioType;
+}

package/dist/music/providers/ReplicateMusic.js

@@ -0,0 +1,261 @@
+/**
+ * Replicate Music Handler (MusicGen default)
+ *
+ * Routes music generation through Replicate's universal prediction
+ * lifecycle. Default model is Meta's MusicGen; alternatives include
+ * Riffusion, AudioGen, and AudioLDM via `options.model`.
+ *
+ * @module music/providers/ReplicateMusic
+ * @see https://replicate.com/meta/musicgen
+ */
+import { ErrorCategory, ErrorSeverity } from "../../constants/enums.js";
+import { MUSIC_ERROR_CODES, MusicError } from "../../utils/musicProcessor.js";
+import { logger } from "../../utils/logger.js";
+import { getReplicateAuth } from "../../adapters/replicate/auth.js";
+import { downloadPredictionOutput, predict, } from "../../adapters/replicate/predictionLifecycle.js";
+import { MAX_AUDIO_BYTES, readBoundedBuffer } from "../../utils/sizeGuard.js";
+import { assertSafeUrl } from "../../utils/ssrfGuard.js";
+const DEFAULT_MODEL = "meta/musicgen:7be0f12c54a8d033a0fbd14418c9af98962da9a86f5ff7811f9b3423a1f0b7d7";
+export class ReplicateMusic {
+    maxDurationSeconds = 30;
+    supportedFormats = [
+        "mp3",
+        "wav",
+    ];
+    supportedGenres = [
+        "ambient",
+        "classical",
+        "electronic",
+        "rock",
+        "jazz",
+        "hip-hop",
+        "pop",
+        "lo-fi",
+        "cinematic",
+        "orchestral",
+    ];
+    isConfigured() {
+        return getReplicateAuth() !== null;
+    }
+    async generate(options) {
+        const auth = getReplicateAuth();
+        if (!auth) {
+            throw new MusicError({
+                code: MUSIC_ERROR_CODES.PROVIDER_NOT_CONFIGURED,
+                message: "REPLICATE_API_TOKEN not configured",
+                category: ErrorCategory.CONFIGURATION,
+                severity: ErrorSeverity.HIGH,
+                retriable: false,
+            });
+        }
+        const startTime = Date.now();
+        const model = options.model ?? DEFAULT_MODEL;
+        const requestedFormat = options.format ?? "mp3";
+        const upstreamFormat = this.supportedFormats.includes(requestedFormat)
+            ? requestedFormat
+            : "mp3";
+        // Clamp to provider max and apply default; store as `effectiveDuration`
+        // so the returned result always reflects the audio that was actually generated.
+        const effectiveDuration = Math.min(options.duration ?? 8, this.maxDurationSeconds);
+        // MusicGen accepts these inputs; other models override via cast.
+        const inputPayload = {
+            prompt: this.buildPrompt(options),
+            duration: effectiveDuration,
+            output_format: upstreamFormat,
+            model_version: "stereo-large",
+            normalization_strategy: "loudness",
+        };
+        if (options.tempo !== undefined) {
+            inputPayload.bpm = options.tempo;
+        }
+        if (options.referenceAudio) {
+            const ref = await this.resolveBuffer(options.referenceAudio);
+            inputPayload.input_audio = `data:audio/${this.detectAudioType(ref)};base64,${ref.toString("base64")}`;
+        }
+        let prediction;
+        try {
+            prediction = await predict(auth, { model, input: inputPayload });
+        }
+        catch (err) {
+            throw new MusicError({
+                code: MUSIC_ERROR_CODES.GENERATION_FAILED,
+                message: `Replicate music generation failed: ${err instanceof Error ? err.message : String(err)}`,
+                category: ErrorCategory.EXECUTION,
+                severity: ErrorSeverity.HIGH,
+                retriable: true,
+                // Sanitize context: omit raw `options` which may contain large Buffers
+                // (referenceAudio) and arbitrary user content.
+                context: {
+                    model,
+                    duration: effectiveDuration,
+                    format: upstreamFormat,
+                    hasReferenceAudio: options.referenceAudio !== undefined,
+                },
+                originalError: err instanceof Error ? err : undefined,
+            });
+        }
+        let buffer;
+        try {
+            buffer = await downloadPredictionOutput(prediction, MAX_AUDIO_BYTES);
+        }
+        catch (err) {
+            throw new MusicError({
+                code: MUSIC_ERROR_CODES.GENERATION_FAILED,
+                message: `Replicate music download failed: ${err instanceof Error ? err.message : String(err)}`,
+                category: ErrorCategory.NETWORK,
+                severity: ErrorSeverity.MEDIUM,
+                retriable: true,
+                context: { predictionId: prediction.id },
+                originalError: err instanceof Error ? err : undefined,
+            });
+        }
+        const latency = Date.now() - startTime;
+        logger.info(`[ReplicateMusic] Generated ${buffer.length} bytes in ${latency}ms — model ${model}`);
+        return {
+            buffer,
+            format: upstreamFormat,
+            size: buffer.length,
+            // Return the clamped/defaulted duration actually sent to the model,
+            // not the raw options.duration which may have been out of bounds or undefined.
+            duration: effectiveDuration,
+            provider: "replicate",
+            metadata: {
+                latency,
+                provider: "replicate",
+                model,
+                jobId: prediction.id,
+            },
+        };
+    }
+    buildPrompt(options) {
+        const parts = [options.prompt];
+        if (options.genre) {
+            parts.push(`Genre: ${options.genre}`);
+        }
+        if (options.mood) {
+            parts.push(`Mood: ${options.mood}`);
+        }
+        if (options.tempo !== undefined) {
+            parts.push(`${options.tempo} BPM`);
+        }
+        return parts.join(". ");
+    }
+    async resolveBuffer(input) {
+        if (Buffer.isBuffer(input)) {
+            return input;
+        }
+        // Reject local file paths — only Buffer or HTTPS URLs are accepted.
+        if (!/^https:\/\//.test(input)) {
+            throw new MusicError({
+                code: MUSIC_ERROR_CODES.INVALID_INPUT,
+                message: `Invalid input: expected Buffer or HTTPS URL, got string "${input}". Local file reads are not supported.`,
+                category: ErrorCategory.VALIDATION,
+                severity: ErrorSeverity.HIGH,
+                retriable: false,
+            });
+        }
+        try {
+            await assertSafeUrl(input);
+        }
+        catch (err) {
+            throw new MusicError({
+                code: MUSIC_ERROR_CODES.INVALID_INPUT,
+                message: `Unsafe URL rejected: ${err instanceof Error ? err.message : String(err)}`,
+                category: ErrorCategory.VALIDATION,
+                severity: ErrorSeverity.HIGH,
+                retriable: false,
+                context: { url: input },
+            });
+        }
+        const FETCH_TIMEOUT_MS = 60_000;
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
+        let r;
+        try {
+            r = await fetch(input, { signal: controller.signal });
+        }
+        catch (err) {
+            if (err instanceof Error && err.name === "AbortError") {
+                throw new MusicError({
+                    code: MUSIC_ERROR_CODES.GENERATION_FAILED,
+                    message: `Replicate music reference-audio fetch timed out after ${FETCH_TIMEOUT_MS / 1000}s: ${input}`,
+                    category: ErrorCategory.NETWORK,
+                    severity: ErrorSeverity.MEDIUM,
+                    retriable: true,
+                });
+            }
+            throw err;
+        }
+        finally {
+            clearTimeout(timeoutId);
+        }
+        if (!r.ok) {
+            throw new MusicError({
+                code: MUSIC_ERROR_CODES.GENERATION_FAILED,
+                message: `Failed to fetch reference audio: ${input}: ${r.status}`,
+                category: ErrorCategory.NETWORK,
+                severity: ErrorSeverity.MEDIUM,
+                retriable: r.status >= 500,
+            });
+        }
+        try {
+            return await readBoundedBuffer(r, MAX_AUDIO_BYTES, "Replicate reference audio");
+        }
+        catch (err) {
+            throw new MusicError({
+                code: MUSIC_ERROR_CODES.INVALID_INPUT,
+                message: `Replicate reference audio too large: ${err instanceof Error ? err.message : String(err)}`,
+                category: ErrorCategory.VALIDATION,
+                severity: ErrorSeverity.HIGH,
+                retriable: false,
+                context: { url: input },
+            });
+        }
+    }
+    /**
+     * Detect audio MIME subtype from magic bytes.
+     *
+     * - WAV : "RIFF" header (52 49 46 46)
+     * - MP3 : ID3 tag (49 44 33) or MPEG sync word 0xFF 0xEx
+     * - OGG : "OggS" capture pattern (4F 67 67 53)
+     * - M4A : "ftyp" box at offset 4
+     *
+     * Falls back to "mp3" when detection is inconclusive.
+     */
+    detectAudioType(buffer) {
+        if (buffer.length < 4) {
+            return "mp3";
+        }
+        // WAV: starts with RIFF
+        if (buffer[0] === 0x52 &&
+            buffer[1] === 0x49 &&
+            buffer[2] === 0x46 &&
+            buffer[3] === 0x46) {
+            return "wav";
+        }
+        // OGG: starts with OggS
+        if (buffer[0] === 0x4f &&
+            buffer[1] === 0x67 &&
+            buffer[2] === 0x67 &&
+            buffer[3] === 0x53) {
+            return "ogg";
+        }
+        // MP3: ID3 header
+        if (buffer[0] === 0x49 && buffer[1] === 0x44 && buffer[2] === 0x33) {
+            return "mp3";
+        }
+        // MP3: MPEG sync word (0xFF 0xE0–0xFF)
+        if (buffer[0] === 0xff && (buffer[1] & 0xe0) === 0xe0) {
+            return "mpeg";
+        }
+        // M4A / AAC: "ftyp" box at offset 4
+        if (buffer.length >= 8 &&
+            buffer[4] === 0x66 &&
+            buffer[5] === 0x74 &&
+            buffer[6] === 0x79 &&
+            buffer[7] === 0x70) {
+            return "mp4";
+        }
+        return "mp3";
+    }
+}

package/dist/neurolink.d.ts

@@ -40,7 +40,26 @@ export declare const STREAM_DEDUP_CONTEXT_KEY: "_streamDedupContext";
 export declare function markStreamProviderEmittedGenerationEnd(options: {
     _streamDedupContext?: StreamGenerationEndContext;
 } | undefined): void;
+/**
+ * Symbol-based brand for cross-module identification without circular imports.
+ *
+ * Provider constructors receive `sdk?: unknown` (the factory layer's
+ * contract). Rather than duck-typing via `"getInMemoryServers" in sdk`,
+ * use `isNeuroLink(value)` from this module to do a brand check —
+ * survives minification AND doesn't rely on method-name stability.
+ */
+export declare const NEUROLINK_BRAND: unique symbol;
+/**
+ * Type-guard for opaque values that should be a {@link NeuroLink} instance.
+ *
+ * Designed for the provider-factory boundary where TS can't carry the type
+ * through `UnknownRecord` without forcing every caller into a circular
+ * dependency. Cheap to call and unaffected by minification.
+ */
+export declare function isNeuroLink(value: unknown): value is NeuroLink;
 export declare class NeuroLink {
+    /** @internal Brand for cross-module identification — see {@link isNeuroLink}. */
+    readonly [NEUROLINK_BRAND]: true;
     private mcpInitialized;
     private mcpSkipped;
     private mcpInitPromise;
@@ -570,6 +589,7 @@ export declare class NeuroLink {
      * @since 1.0.0
      */
     generate(optionsOrPrompt: GenerateOptions | DynamicOptions | string): Promise<GenerateResult>;
+    private fireConsumerOnErrorIfNotFired;
     /**
      * Curator P2-3: wraps a generate/stream call with the fallback
      * orchestration (`providerFallback` callback + `modelChain` walker).
@@ -603,6 +623,16 @@ export declare class NeuroLink {
      * Handle PPT generation mode
      */
     private generateWithPPT;
+    /**
+     * Dispatch a music-generation request to the registered music handler
+     * for the provider named in `options.output.music.provider`.
+     */
+    private generateWithMusic;
+    /**
+     * Dispatch an avatar (lip-sync) request to the registered avatar handler
+     * for the provider named in `options.output.avatar.provider`.
+     */
+    private generateWithAvatar;
     /**
      * Generate with workflow engine integration
      * Returns both original and processed responses for AB testing