@juspay/neurolink 9.64.0 → 9.65.0

package/dist/types/generate.d.ts

@@ -11,6 +11,8 @@ import type { DirectorModeOptions, DirectorSegment, VideoGenerationResult, Video
 import type { PPTGenerationResult, PPTOutputOptions } from "./ppt.js";
 import type { TTSOptions, TTSResult } from "./tts.js";
 import type { STTOptions, STTResult } from "./stt.js";
+import type { AvatarOptions, AvatarResult } from "./avatar.js";
+import type { MusicOptions, MusicResult } from "./music.js";
 import type { StandardRecord, ValidationSchema, ZodUnknownSchema } from "./aliases.js";
 import type { NeurolinkCredentials } from "./providers.js";
 import type { FileWithMetadata } from "./file.js";
@@ -20,8 +22,14 @@ import type { WorkflowConfig } from "./workflow.js";
  * Supports multimodal content while maintaining backward compatibility
  */
 export type GenerateOptions = {
-    input: {
-        text: string;
+    /**
+     * Input content for generation. Optional for media-only modes (avatar, music,
+     * video) where all configuration lives in `output`; the SDK synthesises an
+     * empty `input` automatically when this field is omitted.
+     */
+    input?: {
+        /** Prompt text. Optional for media-only modes (avatar, music) that are driven by uploaded files rather than a prompt. */
+        text?: string;
         /**
          * Images to include in the request.
          * Supports simple image data (Buffer, string) or objects with alt text for accessibility.
@@ -81,8 +89,10 @@ export type GenerateOptions = {
          * - "text": Standard text generation (default)
          * - "video": Video generation using models like Veo 3.1
          * - "ppt": PowerPoint presentation generation
+         * - "avatar": Talking-head / lip-sync video (D-ID, HeyGen, Replicate-MuseTalk)
+         * - "music": Music / sound generation (Beatoven, ElevenLabs Music, Lyria, Replicate)
          */
-        mode?: "text" | "video" | "ppt";
+        mode?: "text" | "video" | "ppt" | "avatar" | "music";
         /**
          * Video generation configuration (used when mode is "video")
          * Requires an input image and text prompt
@@ -98,6 +108,17 @@ export type GenerateOptions = {
          * Controls transition prompts, durations, and concurrency.
          */
         director?: DirectorModeOptions;
+        /**
+         * Avatar generation configuration (used when mode is "avatar")
+         * Combines a portrait image with audio (or text via TTS) to produce
+         * a lip-synced talking-head video.
+         */
+        avatar?: AvatarOptions;
+        /**
+         * Music generation configuration (used when mode is "music")
+         * Generates music / sound from a text prompt.
+         */
+        music?: MusicOptions;
     };
     csvOptions?: {
         maxRows?: number;
@@ -595,6 +616,14 @@ export type GenerateResult = {
      * ```
      */
     video?: VideoGenerationResult;
+    /**
+     * Avatar (talking-head) generation result (present when output.mode is "avatar")
+     */
+    avatar?: AvatarResult;
+    /**
+     * Music generation result (present when output.mode is "music")
+     */
+    music?: MusicResult;
     /**
      * PowerPoint generation result (present when output.mode is "ppt")
      *
@@ -743,7 +772,8 @@ export type TextGenerationOptions = {
      * Supports text, images, and other multimodal inputs.
      */
     input?: {
-        text: string;
+        /** Prompt text. Optional for media-only modes (avatar, music) that are driven by uploaded files rather than a prompt. */
+        text?: string;
         /**
          * Images to include in the request.
          * For video generation, the first image is used as the source frame.
@@ -785,8 +815,10 @@ export type TextGenerationOptions = {
          * - "text": Standard text generation (default)
          * - "video": Video generation using models like Veo 3.1
          * - "ppt": PowerPoint presentation generation
+         * - "avatar": Talking-head / lip-sync video (D-ID, HeyGen, Replicate-MuseTalk)
+         * - "music": Music / sound generation (Beatoven, ElevenLabs Music, Lyria, Replicate)
          */
-        mode?: "text" | "video" | "ppt";
+        mode?: "text" | "video" | "ppt" | "avatar" | "music";
         /**
          * Video generation configuration (used when mode is "video")
          */
@@ -799,6 +831,14 @@ export type TextGenerationOptions = {
          * Director Mode configuration (only used when input.segments is provided)
          */
         director?: DirectorModeOptions;
+        /**
+         * Avatar generation configuration (used when mode is "avatar")
+         */
+        avatar?: AvatarOptions;
+        /**
+         * Music generation configuration (used when mode is "music")
+         */
+        music?: MusicOptions;
     };
     tools?: Record<string, Tool>;
     /**
@@ -1096,6 +1136,10 @@ export type TextGenerationResult = {
     transcription?: STTResult;
     /** Video generation result */
     video?: VideoGenerationResult;
+    /** Avatar (talking-head) generation result */
+    avatar?: AvatarResult;
+    /** Music generation result */
+    music?: MusicResult;
     /** PowerPoint generation result */
     ppt?: PPTGenerationResult;
     /** Image generation output */
@@ -1104,6 +1148,10 @@ export type TextGenerationResult = {
     } | null;
     /** Gemini 3 thought signature for reasoning continuity across turns */
     thoughtSignature?: string;
+    /** Thinking/reasoning text from provider (Anthropic thinking blocks, Gemini thought parts, DeepSeek/NIM reasoning_content) */
+    reasoning?: string;
+    /** Token count for reasoning content */
+    reasoningTokens?: number;
     retries?: {
         count: number;
         errors: Array<{
@@ -1130,3 +1178,12 @@ export type ModelAliasConfig = {
         reason?: string;
     }>;
 };
+/**
+ * Internal alias used by messageBuilder helpers after the entry-point
+ * (`buildMultimodalMessagesArray`) has guaranteed that `input` is non-null.
+ * All private helper functions that receive post-normalised options should
+ * accept this type to avoid repetitive null checks on every `input.*` access.
+ */
+export type GenerateOptionsNormalized = GenerateOptions & {
+    input: NonNullable<GenerateOptions["input"]>;
+};

package/dist/types/index.d.ts

@@ -60,3 +60,8 @@ export * from "./elicitation.js";
 export * from "./dynamic.js";
 export * from "./streamDedup.js";
 export * from "./noOutputSentinel.js";
+export * from "./video.js";
+export * from "./avatar.js";
+export * from "./music.js";
+export * from "./replicate.js";
+export * from "./safeFetch.js";

package/dist/types/index.js

@@ -65,3 +65,10 @@ export * from "./dynamic.js";
 export * from "./streamDedup.js";
 // Curator P3-6: NoOutputGeneratedError sentinel chunk shape
 export * from "./noOutputSentinel.js";
+// New modality categories (M9.1+)
+export * from "./video.js";
+export * from "./avatar.js";
+export * from "./music.js";
+export * from "./replicate.js";
+// Safe-fetch helper types (SSRF-hardened download)
+export * from "./safeFetch.js";

package/dist/types/middleware.d.ts

@@ -285,6 +285,33 @@ export type LifecycleMiddlewareConfig = {
     onFinish?: OnFinishCallback;
     onError?: OnErrorCallback;
     onChunk?: OnChunkCallback;
+    /**
+     * Per-callback deadline in milliseconds applied to every
+     * `onChunk` / `onFinish` / `onError` invocation. When a callback
+     * exceeds this it is logged and abandoned — generate()/stream()
+     * still resolves or rejects on schedule.
+     *
+     * Defaults to the `NEUROLINK_LIFECYCLE_TIMEOUT_MS` env var (also
+     * read by the CLI) and ultimately falls back to 5_000. Set `0`
+     * to make consumer callbacks effectively fire-and-forget.
+     */
+    timeoutMs?: number;
+};
+/**
+ * Structural view of the nested lifecycle config buried inside a request's
+ * middleware blob. Extracted so call sites that need to read it (e.g.
+ * `BaseProvider.wrapStreamWithLifecycleCallbacks`,
+ * `BaseProvider.fireLifecycleErrorCallback`) don't each inline the same
+ * three-level cast.
+ */
+export type OptionsWithLifecycleMiddleware = {
+    middleware?: {
+        middlewareConfig?: {
+            lifecycle?: {
+                config?: LifecycleMiddlewareConfig;
+            };
+        };
+    };
 };
 /** Options for the abort-signal middleware. */
 export type AbortSignalMiddlewareOptions = {

package/dist/types/multimodal.d.ts

@@ -141,14 +141,47 @@ export type AudioContent = {
  * ```
  */
 export type VideoOutputOptions = {
+    /**
+     * Per-call cancellation signal forwarded to provider requests and polling
+     * loops. When aborted, long-running video generation is interrupted and
+     * the handler throws a non-retriable abort error.
+     */
+    abortSignal?: AbortSignal;
+    /**
+     * Override the video-gen provider. Defaults to "vertex" or to the LLM
+     * provider name if it is also a registered video handler.
+     *
+     * Registered providers are managed via `VideoProcessor.registerHandler`
+     * (see src/lib/utils/videoProcessor.ts). Examples: "vertex", "kling",
+     * "runway", "replicate".
+     */
+    provider?: string;
+    /**
+     * Specific model to use within the provider. Provider-specific shape
+     * (e.g. "veo-3.1-generate-001" for vertex; "atonamy/wan-alpha:..." for
+     * replicate).
+     */
+    model?: string;
     /** Output resolution - "720p" (1280x720) or "1080p" (1920x1080) */
     resolution?: "720p" | "1080p";
     /** Video duration in seconds (4, 6, or 8 seconds supported) */
     length?: 4 | 6 | 8;
-    /** Aspect ratio - "9:16" for portrait or "16:9" for landscape */
-    aspectRatio?: "9:16" | "16:9";
+    /** Aspect ratio - "9:16" for portrait, "16:9" for landscape, "1:1" for square */
+    aspectRatio?: "9:16" | "16:9" | "1:1";
     /** Enable audio generation (default: true) */
     audio?: boolean;
+    /**
+     * Publicly accessible URL of the input image.
+     * Required by providers that do not accept inline base64 data (e.g. PiAPI Kling).
+     * When provided and the provider requires a URL, this takes precedence over the
+     * `image` Buffer argument passed to `generate()`.
+     */
+    imageUrl?: string;
+    /**
+     * Per-call provider credentials. Takes precedence over instance-level
+     * credentials set at construction time, which in turn override env vars.
+     */
+    credentials?: import("./providers.js").NeurolinkCredentials;
 };
 /**
  * A single segment in Director Mode, representing one video clip.

package/dist/types/music.d.ts

@@ -0,0 +1,165 @@
+/**
+ * Music Generation Type Definitions
+ *
+ * Types for music / sound-effect generation across providers (Beatoven,
+ * ElevenLabs Music, Lyria, Replicate-hosted MusicGen / Riffusion / AudioGen).
+ *
+ * Music is a separate modality from TTS — TTS produces voiced speech with
+ * prosody; Music produces melodic / harmonic / textural audio.
+ *
+ * @module types/music
+ */
+/**
+ * Output audio formats supported across music providers.
+ */
+export type MusicAudioFormat = "mp3" | "wav" | "flac" | "ogg";
+/**
+ * Music genre — provider-specific, free-text accepted.
+ *
+ * Common genres: "ambient", "cinematic", "rock", "pop", "jazz", "classical",
+ * "electronic", "lo-fi", "hip-hop", "orchestral", "synthwave", "folk".
+ */
+export type MusicGenre = string;
+/**
+ * Music mood — provider-specific, free-text accepted.
+ *
+ * Common moods: "uplifting", "melancholic", "tense", "calm", "energetic",
+ * "mysterious", "romantic", "epic".
+ */
+export type MusicMood = string;
+/**
+ * Options for music generation requests.
+ */
+export type MusicOptions = {
+    /** Text prompt describing the music to generate (required). */
+    prompt: string;
+    /** Target duration in seconds. Provider-clamped to its supported range. */
+    duration?: number;
+    /** Output format (default: "mp3"). */
+    format?: MusicAudioFormat;
+    /** Genre hint (e.g. "ambient", "cinematic"). */
+    genre?: MusicGenre;
+    /** Mood / emotion hint (e.g. "uplifting", "tense"). */
+    mood?: MusicMood;
+    /** Tempo in BPM (provider-specific support). */
+    tempo?: number;
+    /** Override the music provider (e.g. "beatoven", "elevenlabs-music", "lyria", "replicate"). */
+    provider?: string;
+    /** Reference audio for melody / style guidance (Buffer or path). */
+    referenceAudio?: Buffer | string;
+    /** Output file path (optional — buffer is always returned in result). */
+    output?: string;
+    /** Per-call timeout in ms (default: 5 minutes). */
+    timeout?: number;
+    /** Provider-specific additional options. */
+    [key: string]: unknown;
+};
+/**
+ * Result of a music generation request.
+ */
+export type MusicResult = {
+    /** Generated audio buffer. */
+    buffer: Buffer;
+    /** Output format. */
+    format: MusicAudioFormat;
+    /** File size in bytes. */
+    size: number;
+    /** Duration in seconds (when reported by the provider). */
+    duration?: number;
+    /** Provider used for generation. */
+    provider?: string;
+    /** Performance / request metadata. */
+    metadata?: {
+        /** Request latency in milliseconds. */
+        latency: number;
+        /** Provider name. */
+        provider?: string;
+        /** Model variant used (when applicable). */
+        model?: string;
+        /** Sample rate (when known). */
+        sampleRate?: number;
+        /** Bit rate (when known). */
+        bitRate?: number;
+        /** Track / job identifier from the upstream. */
+        jobId?: string;
+        /** Any additional provider-specific metadata. */
+        [key: string]: unknown;
+    };
+};
+/**
+ * Handler contract for music-generation providers.
+ *
+ * Implementations enforce their own timeouts. Recommended:
+ *   - Per-request fetch timeout: 30 seconds
+ *   - Total job-completion timeout: 5 minutes
+ */
+export type MusicHandler = {
+    /**
+     * Generate a music track from prompt + options.
+     *
+     * @param options - prompt, duration, format, genre, mood, etc.
+     */
+    generate(options: MusicOptions): Promise<MusicResult>;
+    /**
+     * Validate the provider is configured (auth, base URL, etc.).
+     */
+    isConfigured(): boolean;
+    /** Maximum supported track duration in seconds (provider-specific). */
+    readonly maxDurationSeconds?: number;
+    /** Output formats supported by this handler. */
+    readonly supportedFormats?: readonly MusicAudioFormat[];
+    /** Genres / styles the upstream advertises (informational). */
+    readonly supportedGenres?: readonly string[];
+};
+/**
+ * Valid music audio formats — runtime validation array.
+ */
+export declare const VALID_MUSIC_FORMATS: readonly MusicAudioFormat[];
+/**
+ * Beatoven.ai task status response.
+ *
+ * Used by `BeatovenMusic` handler to type-check polling responses.
+ */
+export type BeatovenTaskStatus = {
+    status: "composing" | "running" | "composed" | "failed";
+    meta?: {
+        track_url?: string;
+        project_id?: string;
+        track_id?: string;
+        duration?: number;
+    };
+    message?: string;
+};
+/**
+ * Beatoven.ai compose-track submit response.
+ */
+export type BeatovenComposeResponse = {
+    task_id: string;
+    status?: string;
+};
+/**
+ * Google Lyria 3 Pro :generateContent response shape.
+ *
+ * The audio comes back as a part with `inlineData: { mimeType, data }`
+ * where `data` is base64-encoded WAV.
+ */
+export type LyriaResponse = {
+    candidates?: {
+        content?: {
+            parts?: {
+                inlineData?: {
+                    mimeType?: string;
+                    data?: string;
+                };
+                text?: string;
+            }[];
+        };
+        finishReason?: string;
+        index?: number;
+    }[];
+    usageMetadata?: {
+        promptTokenCount?: number;
+        candidatesTokenCount?: number;
+        totalTokenCount?: number;
+    };
+};

package/dist/types/music.js

@@ -0,0 +1,20 @@
+/**
+ * Music Generation Type Definitions
+ *
+ * Types for music / sound-effect generation across providers (Beatoven,
+ * ElevenLabs Music, Lyria, Replicate-hosted MusicGen / Riffusion / AudioGen).
+ *
+ * Music is a separate modality from TTS — TTS produces voiced speech with
+ * prosody; Music produces melodic / harmonic / textural audio.
+ *
+ * @module types/music
+ */
+/**
+ * Valid music audio formats — runtime validation array.
+ */
+export const VALID_MUSIC_FORMATS = [
+    "mp3",
+    "wav",
+    "flac",
+    "ogg",
+];

package/dist/types/providers.d.ts

@@ -163,6 +163,141 @@ export type NeurolinkCredentials = {
         apiKey?: string;
         baseURL?: string;
     };
+    xai?: {
+        apiKey?: string;
+        baseURL?: string;
+    };
+    groq?: {
+        apiKey?: string;
+        baseURL?: string;
+    };
+    cohere?: {
+        apiKey?: string;
+        baseURL?: string;
+    };
+    together?: {
+        apiKey?: string;
+        baseURL?: string;
+    };
+    fireworks?: {
+        apiKey?: string;
+        baseURL?: string;
+    };
+    perplexity?: {
+        apiKey?: string;
+        baseURL?: string;
+    };
+    cloudflare?: {
+        apiKey?: string;
+        accountId?: string;
+        baseURL?: string;
+    };
+    replicate?: {
+        apiToken?: string;
+        baseUrl?: string;
+    };
+    voyage?: {
+        apiKey?: string;
+        baseURL?: string;
+    };
+    jina?: {
+        apiKey?: string;
+        baseURL?: string;
+    };
+    stability?: {
+        apiKey?: string;
+        baseURL?: string;
+    };
+    ideogram?: {
+        apiKey?: string;
+        baseURL?: string;
+    };
+    recraft?: {
+        apiKey?: string;
+        baseURL?: string;
+    };
+};
+/**
+ * Voyage AI /embeddings response shape.
+ */
+export type VoyageEmbeddingsResponse = {
+    object: "list";
+    data: {
+        object: "embedding";
+        embedding: number[];
+        index: number;
+    }[];
+    model: string;
+    usage?: {
+        total_tokens?: number;
+    };
+};
+/**
+ * Jina AI /embeddings response shape (compatible with OpenAI's shape).
+ */
+export type JinaEmbeddingsResponse = {
+    object?: string;
+    data: {
+        object?: string;
+        embedding: number[];
+        index: number;
+    }[];
+    model?: string;
+    usage?: {
+        total_tokens?: number;
+        prompt_tokens?: number;
+    };
+};
+/**
+ * Jina AI /rerank response shape.
+ */
+export type JinaRerankResponse = {
+    model?: string;
+    results: {
+        index: number;
+        relevance_score: number;
+        document?: {
+            text?: string;
+        };
+    }[];
+    usage?: {
+        total_tokens?: number;
+    };
+};
+/**
+ * Stability AI /v2beta/stable-image/generate/{model} response shape
+ * (returns either binary directly, or JSON with base64 when Accept is set
+ * to application/json). We always request JSON for uniformity.
+ */
+export type StabilityImageResponse = {
+    image?: string;
+    finish_reason?: "SUCCESS" | "ERROR" | "CONTENT_FILTERED";
+    seed?: number;
+};
+/**
+ * Ideogram /api/v1/ideogram-v3/generate response shape.
+ */
+export type IdeogramImageResponse = {
+    created?: string;
+    data: {
+        prompt?: string;
+        resolution?: string;
+        is_image_safe?: boolean;
+        seed?: number;
+        url?: string;
+        style_type?: string;
+    }[];
+};
+/**
+ * Recraft /v1/images/generations response shape.
+ */
+export type RecraftImageResponse = {
+    created?: number;
+    data: {
+        url?: string;
+        b64_json?: string;
+        image_id?: string;
+    }[];
 };
 /**
  * NVIDIA NIM extra request body parameters passed via `providerOptions.openai.body`.
@@ -1473,10 +1608,18 @@ export type NativeFunctionDeclaration = {
 export type NativeToolsConfig = Array<{
     functionDeclarations: NativeFunctionDeclaration[];
 }>;
-/** Return value of buildNativeToolDeclarations. */
+/**
+ * Return value of buildNativeToolDeclarations.
+ *
+ * `originalNameMap` lets callers translate a Google-safe (sanitized,
+ * suffix-disambiguated) tool name back to the original identifier the
+ * SDK consumer registered. Sanitized names are transport-only — they
+ * MUST be hidden from tool-call metadata exposed to consumers.
+ */
 export type NativeToolDeclarationsResult = {
     toolsConfig: NativeToolsConfig;
     executeMap: Map<string, Tool["execute"]>;
+    originalNameMap: Map<string, string>;
 };
 /** A single function call returned by the Gemini model. */
 export type NativeFunctionCall = {

package/dist/types/replicate.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Replicate Type Definitions
+ *
+ * Shared types for the Replicate prediction lifecycle (LLM, video,
+ * avatar, music handlers all consume these via the types barrel).
+ *
+ * @module types/replicate
+ */
+/**
+ * Replicate auth payload — bearer token + optional base URL.
+ */
+export type ReplicateAuth = {
+    apiToken: string;
+    baseUrl?: string;
+};
+/**
+ * Replicate prediction status (terminal: succeeded / failed / canceled).
+ */
+export type ReplicatePredictionStatus = "starting" | "processing" | "succeeded" | "failed" | "canceled";
+/**
+ * Replicate prediction object as returned by /v1/predictions POST and
+ * /v1/predictions/:id GET. Output shape varies by model — caller narrows.
+ */
+export type ReplicatePrediction = {
+    id: string;
+    model?: string;
+    version?: string;
+    status: ReplicatePredictionStatus;
+    /** URL string, array of URL strings, base64, or model-specific JSON. */
+    output?: unknown;
+    error?: string | null;
+    metrics?: {
+        predict_time?: number;
+    };
+    urls?: {
+        get: string;
+        cancel: string;
+    };
+    logs?: string;
+    created_at?: string;
+    started_at?: string;
+    completed_at?: string;
+};
+/**
+ * Input shape for Replicate's createPrediction helper.
+ */
+export type ReplicateCreatePredictionInput = {
+    /** Model in "owner/name" or "owner/name:version" form. */
+    model: string;
+    /** Provider/model-specific input shape. */
+    input: Record<string, unknown>;
+    /** Optional webhook URL for completion notifications. */
+    webhook?: string;
+    /** Optional webhook events filter. */
+    webhookEventsFilter?: ("start" | "output" | "logs" | "completed")[];
+};
+/**
+ * Options for the Replicate poll loop.
+ */
+export type ReplicatePollOptions = {
+    /** Total time to wait before throwing a timeout error (default 5 min). */
+    timeoutMs?: number;
+    /** Poll interval in milliseconds (default 2 s). */
+    pollIntervalMs?: number;
+    /** Caller-supplied AbortSignal to cancel polling early. */
+    abortSignal?: AbortSignal;
+};

package/dist/types/replicate.js

@@ -0,0 +1,9 @@
+/**
+ * Replicate Type Definitions
+ *
+ * Shared types for the Replicate prediction lifecycle (LLM, video,
+ * avatar, music handlers all consume these via the types barrel).
+ *
+ * @module types/replicate
+ */
+export {};

package/dist/types/safeFetch.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Types for the safe-fetch utility (SSRF-hardened binary downloads).
+ *
+ * Runtime helper lives in `src/lib/utils/safeFetch.ts`.
+ */
+export type SafeDownloadOptions = {
+    /** Hard cap on response size in bytes. Pass MAX_VIDEO_BYTES/MAX_AUDIO_BYTES/MAX_IMAGE_BYTES from sizeGuard. */
+    maxBytes: number;
+    /** Human-readable identifier used in error messages (e.g. "HeyGen video"). */
+    label: string;
+    /** Optional abort signal for caller-driven cancellation. */
+    signal?: AbortSignal;
+    /** Optional per-call request timeout (ms). Default: 60_000. */
+    timeoutMs?: number;
+};