@juspay/neurolink 9.63.1 → 9.65.0

package/dist/lib/types/generate.d.ts

@@ -11,6 +11,8 @@ import type { DirectorModeOptions, DirectorSegment, VideoGenerationResult, Video
 import type { PPTGenerationResult, PPTOutputOptions } from "./ppt.js";
 import type { TTSOptions, TTSResult } from "./tts.js";
 import type { STTOptions, STTResult } from "./stt.js";
+import type { AvatarOptions, AvatarResult } from "./avatar.js";
+import type { MusicOptions, MusicResult } from "./music.js";
 import type { StandardRecord, ValidationSchema, ZodUnknownSchema } from "./aliases.js";
 import type { NeurolinkCredentials } from "./providers.js";
 import type { FileWithMetadata } from "./file.js";
@@ -20,8 +22,14 @@ import type { WorkflowConfig } from "./workflow.js";
  * Supports multimodal content while maintaining backward compatibility
  */
 export type GenerateOptions = {
-    input: {
-        text: string;
+    /**
+     * Input content for generation. Optional for media-only modes (avatar, music,
+     * video) where all configuration lives in `output`; the SDK synthesises an
+     * empty `input` automatically when this field is omitted.
+     */
+    input?: {
+        /** Prompt text. Optional for media-only modes (avatar, music) that are driven by uploaded files rather than a prompt. */
+        text?: string;
         /**
          * Images to include in the request.
          * Supports simple image data (Buffer, string) or objects with alt text for accessibility.
@@ -81,8 +89,10 @@ export type GenerateOptions = {
          * - "text": Standard text generation (default)
          * - "video": Video generation using models like Veo 3.1
          * - "ppt": PowerPoint presentation generation
+         * - "avatar": Talking-head / lip-sync video (D-ID, HeyGen, Replicate-MuseTalk)
+         * - "music": Music / sound generation (Beatoven, ElevenLabs Music, Lyria, Replicate)
          */
-        mode?: "text" | "video" | "ppt";
+        mode?: "text" | "video" | "ppt" | "avatar" | "music";
         /**
          * Video generation configuration (used when mode is "video")
          * Requires an input image and text prompt
@@ -98,6 +108,17 @@ export type GenerateOptions = {
          * Controls transition prompts, durations, and concurrency.
          */
         director?: DirectorModeOptions;
+        /**
+         * Avatar generation configuration (used when mode is "avatar")
+         * Combines a portrait image with audio (or text via TTS) to produce
+         * a lip-synced talking-head video.
+         */
+        avatar?: AvatarOptions;
+        /**
+         * Music generation configuration (used when mode is "music")
+         * Generates music / sound from a text prompt.
+         */
+        music?: MusicOptions;
     };
     csvOptions?: {
         maxRows?: number;
@@ -217,6 +238,12 @@ export type GenerateOptions = {
     region?: string;
     temperature?: number;
     maxTokens?: number;
+    /** Top-p (nucleus) sampling parameter. Controls diversity of generated tokens. */
+    topP?: number;
+    /** Top-k sampling parameter. Limits the number of tokens considered. (Google/Gemini models only) */
+    topK?: number;
+    /** Stop sequences that will halt generation when encountered. */
+    stopSequences?: string[];
     systemPrompt?: string;
     /**
      * Zod schema for structured output validation
@@ -589,6 +616,14 @@ export type GenerateResult = {
      * ```
      */
     video?: VideoGenerationResult;
+    /**
+     * Avatar (talking-head) generation result (present when output.mode is "avatar")
+     */
+    avatar?: AvatarResult;
+    /**
+     * Music generation result (present when output.mode is "music")
+     */
+    music?: MusicResult;
     /**
      * PowerPoint generation result (present when output.mode is "ppt")
      *
@@ -737,7 +772,8 @@ export type TextGenerationOptions = {
      * Supports text, images, and other multimodal inputs.
      */
     input?: {
-        text: string;
+        /** Prompt text. Optional for media-only modes (avatar, music) that are driven by uploaded files rather than a prompt. */
+        text?: string;
         /**
          * Images to include in the request.
          * For video generation, the first image is used as the source frame.
@@ -753,6 +789,12 @@ export type TextGenerationOptions = {
     region?: string;
     temperature?: number;
     maxTokens?: number;
+    /** Top-p (nucleus) sampling parameter. Controls diversity of generated tokens. */
+    topP?: number;
+    /** Top-k sampling parameter. Limits the number of tokens considered. (Google/Gemini models only) */
+    topK?: number;
+    /** Stop sequences that will halt generation when encountered. */
+    stopSequences?: string[];
     systemPrompt?: string;
     schema?: ZodUnknownSchema | Schema<unknown>;
     /**
@@ -773,8 +815,10 @@ export type TextGenerationOptions = {
          * - "text": Standard text generation (default)
          * - "video": Video generation using models like Veo 3.1
          * - "ppt": PowerPoint presentation generation
+         * - "avatar": Talking-head / lip-sync video (D-ID, HeyGen, Replicate-MuseTalk)
+         * - "music": Music / sound generation (Beatoven, ElevenLabs Music, Lyria, Replicate)
          */
-        mode?: "text" | "video" | "ppt";
+        mode?: "text" | "video" | "ppt" | "avatar" | "music";
         /**
          * Video generation configuration (used when mode is "video")
          */
@@ -787,6 +831,14 @@ export type TextGenerationOptions = {
          * Director Mode configuration (only used when input.segments is provided)
          */
         director?: DirectorModeOptions;
+        /**
+         * Avatar generation configuration (used when mode is "avatar")
+         */
+        avatar?: AvatarOptions;
+        /**
+         * Music generation configuration (used when mode is "music")
+         */
+        music?: MusicOptions;
     };
     tools?: Record<string, Tool>;
     /**
@@ -926,6 +978,8 @@ export type TextGenerationOptions = {
     conversationMemoryConfig?: Partial<ConversationMemoryConfig>;
     originalPrompt?: string;
     middleware?: MiddlewareFactoryOptions;
+    onFinish?: OnFinishCallback;
+    onError?: OnErrorCallback;
     expectedOutcome?: string;
     evaluationCriteria?: string[];
     csvOptions?: {
@@ -1082,6 +1136,10 @@ export type TextGenerationResult = {
     transcription?: STTResult;
     /** Video generation result */
     video?: VideoGenerationResult;
+    /** Avatar (talking-head) generation result */
+    avatar?: AvatarResult;
+    /** Music generation result */
+    music?: MusicResult;
     /** PowerPoint generation result */
     ppt?: PPTGenerationResult;
     /** Image generation output */
@@ -1090,6 +1148,10 @@ export type TextGenerationResult = {
     } | null;
     /** Gemini 3 thought signature for reasoning continuity across turns */
     thoughtSignature?: string;
+    /** Thinking/reasoning text from provider (Anthropic thinking blocks, Gemini thought parts, DeepSeek/NIM reasoning_content) */
+    reasoning?: string;
+    /** Token count for reasoning content */
+    reasoningTokens?: number;
     retries?: {
         count: number;
         errors: Array<{
@@ -1116,3 +1178,12 @@ export type ModelAliasConfig = {
         reason?: string;
     }>;
 };
+/**
+ * Internal alias used by messageBuilder helpers after the entry-point
+ * (`buildMultimodalMessagesArray`) has guaranteed that `input` is non-null.
+ * All private helper functions that receive post-normalised options should
+ * accept this type to avoid repetitive null checks on every `input.*` access.
+ */
+export type GenerateOptionsNormalized = GenerateOptions & {
+    input: NonNullable<GenerateOptions["input"]>;
+};

package/dist/lib/types/index.d.ts

@@ -30,6 +30,7 @@ export * from "./guardrails.js";
 export * from "./hitl.js";
 export * from "./mcp.js";
 export * from "./mcpOutput.js";
+export * from "./memory.js";
 export * from "./middleware.js";
 export * from "./model.js";
 export * from "./multimodal.js";
@@ -59,3 +60,8 @@ export * from "./elicitation.js";
 export * from "./dynamic.js";
 export * from "./streamDedup.js";
 export * from "./noOutputSentinel.js";
+export * from "./video.js";
+export * from "./avatar.js";
+export * from "./music.js";
+export * from "./replicate.js";
+export * from "./safeFetch.js";

package/dist/lib/types/index.js

@@ -31,6 +31,7 @@ export * from "./guardrails.js";
 export * from "./hitl.js";
 export * from "./mcp.js";
 export * from "./mcpOutput.js";
+export * from "./memory.js";
 export * from "./middleware.js";
 export * from "./model.js";
 export * from "./multimodal.js";
@@ -64,4 +65,11 @@ export * from "./dynamic.js";
 export * from "./streamDedup.js";
 // Curator P3-6: NoOutputGeneratedError sentinel chunk shape
 export * from "./noOutputSentinel.js";
+// New modality categories (M9.1+)
+export * from "./video.js";
+export * from "./avatar.js";
+export * from "./music.js";
+export * from "./replicate.js";
+// Safe-fetch helper types (SSRF-hardened download)
+export * from "./safeFetch.js";
 //# sourceMappingURL=index.js.map

package/dist/lib/types/memory.d.ts

@@ -0,0 +1,96 @@
+/**
+ * Local structural types for the optional @juspay/hippocampus integration.
+ *
+ * These mirror the public shapes that ship with @juspay/hippocampus's
+ * `dist/types.d.ts` so NeuroLink's public type surface stays compatible
+ * for consumers that already configure memory, while the runtime package
+ * itself becomes an optional peer dependency. The previous setup (a hard
+ * value import of @juspay/hippocampus) made pnpm pull a registry copy of
+ * @juspay/neurolink to satisfy Hippocampus's peer, which transitively
+ * dragged @ai-sdk/google + @ai-sdk/google-vertex into the production
+ * dependency graph.
+ *
+ * Naming:
+ *  - Hippocampus's own `StorageType` and `RedisStorageConfig` collide with
+ *    NeuroLink's in-house Redis manager types in `common.ts` /
+ *    `conversation.ts`. To satisfy the `unique-type-names` ESLint rule,
+ *    the storage variants get a `Memory*` prefix here.
+ *  - `HippocampusMemory` (consumer-facing) and `StorageConfig` (legacy
+ *    re-export) keep their original public names — only their definitions
+ *    move from `import("@juspay/hippocampus").Foo` to local structural form.
+ */
+export type MemorySqliteStorageConfig = {
+    type: "sqlite";
+    /** Path to SQLite file. Default: ./data/hippocampus.sqlite */
+    path?: string;
+};
+export type MemoryRedisStorageConfig = {
+    type: "redis";
+    host?: string;
+    port?: number;
+    password?: string;
+    db?: number;
+    keyPrefix?: string;
+    ttl?: number;
+};
+export type MemoryS3StorageConfig = {
+    type: "s3";
+    bucket: string;
+    prefix?: string;
+};
+export type MemoryCustomStorageConfig = {
+    type: "custom";
+    onGet: (ownerId: string) => Promise<string | null>;
+    onSet: (ownerId: string, memory: string) => Promise<void>;
+    onDelete: (ownerId: string) => Promise<void>;
+    onClose?: () => Promise<void>;
+};
+/**
+ * Storage configuration accepted by the optional Hippocampus client.
+ * Re-exported with the legacy `StorageConfig` name from `conversation.ts`
+ * to preserve the existing public type surface.
+ */
+export type HippocampusStorageConfig = MemorySqliteStorageConfig | MemoryRedisStorageConfig | MemoryS3StorageConfig | MemoryCustomStorageConfig;
+/** Per-call options accepted by `Hippocampus.add`. */
+export type HippocampusAddOptions = {
+    prompt?: string;
+    maxWords?: number;
+};
+/** Constructor config accepted by the Hippocampus class. */
+export type HippocampusConfig = {
+    storage?: HippocampusStorageConfig;
+    prompt?: string;
+    neurolink?: {
+        provider?: string;
+        model?: string;
+        temperature?: number;
+    };
+    maxWords?: number;
+};
+/**
+ * Subset of the @juspay/hippocampus client surface that NeuroLink core
+ * actually calls. Defining this locally lets the initializer / SDK code
+ * avoid a value or even a type import from the optional package.
+ */
+export type HippocampusLike = {
+    add: (ownerId: string, content: string, options?: HippocampusAddOptions) => Promise<string>;
+    get: (ownerId: string) => Promise<string | null>;
+    delete: (ownerId: string) => Promise<void>;
+    close: () => Promise<void>;
+};
+/**
+ * Consumer-facing memory config. Same shape as before — the `enabled`
+ * flag toggles activation while the rest is passed straight through to
+ * the Hippocampus constructor when the optional package is installed.
+ */
+export type HippocampusMemory = HippocampusConfig & {
+    enabled?: boolean;
+};
+/**
+ * Shape of the dynamically-required `@juspay/hippocampus` module surface
+ * that NeuroLink's lazy initializer reaches for. Only the constructor is
+ * surfaced here; the rest of the module is irrelevant to core.
+ */
+export type HippocampusModule = {
+    Hippocampus: new (config?: HippocampusConfig) => HippocampusLike;
+};

package/dist/lib/types/memory.js

@@ -0,0 +1,23 @@
+/**
+ * Local structural types for the optional @juspay/hippocampus integration.
+ *
+ * These mirror the public shapes that ship with @juspay/hippocampus's
+ * `dist/types.d.ts` so NeuroLink's public type surface stays compatible
+ * for consumers that already configure memory, while the runtime package
+ * itself becomes an optional peer dependency. The previous setup (a hard
+ * value import of @juspay/hippocampus) made pnpm pull a registry copy of
+ * @juspay/neurolink to satisfy Hippocampus's peer, which transitively
+ * dragged @ai-sdk/google + @ai-sdk/google-vertex into the production
+ * dependency graph.
+ *
+ * Naming:
+ *  - Hippocampus's own `StorageType` and `RedisStorageConfig` collide with
+ *    NeuroLink's in-house Redis manager types in `common.ts` /
+ *    `conversation.ts`. To satisfy the `unique-type-names` ESLint rule,
+ *    the storage variants get a `Memory*` prefix here.
+ *  - `HippocampusMemory` (consumer-facing) and `StorageConfig` (legacy
+ *    re-export) keep their original public names — only their definitions
+ *    move from `import("@juspay/hippocampus").Foo` to local structural form.
+ */
+export {};
+//# sourceMappingURL=memory.js.map

package/dist/lib/types/middleware.d.ts

@@ -285,6 +285,33 @@ export type LifecycleMiddlewareConfig = {
     onFinish?: OnFinishCallback;
     onError?: OnErrorCallback;
     onChunk?: OnChunkCallback;
+    /**
+     * Per-callback deadline in milliseconds applied to every
+     * `onChunk` / `onFinish` / `onError` invocation. When a callback
+     * exceeds this it is logged and abandoned — generate()/stream()
+     * still resolves or rejects on schedule.
+     *
+     * Defaults to the `NEUROLINK_LIFECYCLE_TIMEOUT_MS` env var (also
+     * read by the CLI) and ultimately falls back to 5_000. Set `0`
+     * to make consumer callbacks effectively fire-and-forget.
+     */
+    timeoutMs?: number;
+};
+/**
+ * Structural view of the nested lifecycle config buried inside a request's
+ * middleware blob. Extracted so call sites that need to read it (e.g.
+ * `BaseProvider.wrapStreamWithLifecycleCallbacks`,
+ * `BaseProvider.fireLifecycleErrorCallback`) don't each inline the same
+ * three-level cast.
+ */
+export type OptionsWithLifecycleMiddleware = {
+    middleware?: {
+        middlewareConfig?: {
+            lifecycle?: {
+                config?: LifecycleMiddlewareConfig;
+            };
+        };
+    };
 };
 /** Options for the abort-signal middleware. */
 export type AbortSignalMiddlewareOptions = {

package/dist/lib/types/multimodal.d.ts

@@ -141,14 +141,47 @@ export type AudioContent = {
  * ```
  */
 export type VideoOutputOptions = {
+    /**
+     * Per-call cancellation signal forwarded to provider requests and polling
+     * loops. When aborted, long-running video generation is interrupted and
+     * the handler throws a non-retriable abort error.
+     */
+    abortSignal?: AbortSignal;
+    /**
+     * Override the video-gen provider. Defaults to "vertex" or to the LLM
+     * provider name if it is also a registered video handler.
+     *
+     * Registered providers are managed via `VideoProcessor.registerHandler`
+     * (see src/lib/utils/videoProcessor.ts). Examples: "vertex", "kling",
+     * "runway", "replicate".
+     */
+    provider?: string;
+    /**
+     * Specific model to use within the provider. Provider-specific shape
+     * (e.g. "veo-3.1-generate-001" for vertex; "atonamy/wan-alpha:..." for
+     * replicate).
+     */
+    model?: string;
     /** Output resolution - "720p" (1280x720) or "1080p" (1920x1080) */
     resolution?: "720p" | "1080p";
     /** Video duration in seconds (4, 6, or 8 seconds supported) */
     length?: 4 | 6 | 8;
-    /** Aspect ratio - "9:16" for portrait or "16:9" for landscape */
-    aspectRatio?: "9:16" | "16:9";
+    /** Aspect ratio - "9:16" for portrait, "16:9" for landscape, "1:1" for square */
+    aspectRatio?: "9:16" | "16:9" | "1:1";
     /** Enable audio generation (default: true) */
     audio?: boolean;
+    /**
+     * Publicly accessible URL of the input image.
+     * Required by providers that do not accept inline base64 data (e.g. PiAPI Kling).
+     * When provided and the provider requires a URL, this takes precedence over the
+     * `image` Buffer argument passed to `generate()`.
+     */
+    imageUrl?: string;
+    /**
+     * Per-call provider credentials. Takes precedence over instance-level
+     * credentials set at construction time, which in turn override env vars.
+     */
+    credentials?: import("./providers.js").NeurolinkCredentials;
 };
 /**
  * A single segment in Director Mode, representing one video clip.

package/dist/lib/types/music.d.ts

@@ -0,0 +1,165 @@
+/**
+ * Music Generation Type Definitions
+ *
+ * Types for music / sound-effect generation across providers (Beatoven,
+ * ElevenLabs Music, Lyria, Replicate-hosted MusicGen / Riffusion / AudioGen).
+ *
+ * Music is a separate modality from TTS — TTS produces voiced speech with
+ * prosody; Music produces melodic / harmonic / textural audio.
+ *
+ * @module types/music
+ */
+/**
+ * Output audio formats supported across music providers.
+ */
+export type MusicAudioFormat = "mp3" | "wav" | "flac" | "ogg";
+/**
+ * Music genre — provider-specific, free-text accepted.
+ *
+ * Common genres: "ambient", "cinematic", "rock", "pop", "jazz", "classical",
+ * "electronic", "lo-fi", "hip-hop", "orchestral", "synthwave", "folk".
+ */
+export type MusicGenre = string;
+/**
+ * Music mood — provider-specific, free-text accepted.
+ *
+ * Common moods: "uplifting", "melancholic", "tense", "calm", "energetic",
+ * "mysterious", "romantic", "epic".
+ */
+export type MusicMood = string;
+/**
+ * Options for music generation requests.
+ */
+export type MusicOptions = {
+    /** Text prompt describing the music to generate (required). */
+    prompt: string;
+    /** Target duration in seconds. Provider-clamped to its supported range. */
+    duration?: number;
+    /** Output format (default: "mp3"). */
+    format?: MusicAudioFormat;
+    /** Genre hint (e.g. "ambient", "cinematic"). */
+    genre?: MusicGenre;
+    /** Mood / emotion hint (e.g. "uplifting", "tense"). */
+    mood?: MusicMood;
+    /** Tempo in BPM (provider-specific support). */
+    tempo?: number;
+    /** Override the music provider (e.g. "beatoven", "elevenlabs-music", "lyria", "replicate"). */
+    provider?: string;
+    /** Reference audio for melody / style guidance (Buffer or path). */
+    referenceAudio?: Buffer | string;
+    /** Output file path (optional — buffer is always returned in result). */
+    output?: string;
+    /** Per-call timeout in ms (default: 5 minutes). */
+    timeout?: number;
+    /** Provider-specific additional options. */
+    [key: string]: unknown;
+};
+/**
+ * Result of a music generation request.
+ */
+export type MusicResult = {
+    /** Generated audio buffer. */
+    buffer: Buffer;
+    /** Output format. */
+    format: MusicAudioFormat;
+    /** File size in bytes. */
+    size: number;
+    /** Duration in seconds (when reported by the provider). */
+    duration?: number;
+    /** Provider used for generation. */
+    provider?: string;
+    /** Performance / request metadata. */
+    metadata?: {
+        /** Request latency in milliseconds. */
+        latency: number;
+        /** Provider name. */
+        provider?: string;
+        /** Model variant used (when applicable). */
+        model?: string;
+        /** Sample rate (when known). */
+        sampleRate?: number;
+        /** Bit rate (when known). */
+        bitRate?: number;
+        /** Track / job identifier from the upstream. */
+        jobId?: string;
+        /** Any additional provider-specific metadata. */
+        [key: string]: unknown;
+    };
+};
+/**
+ * Handler contract for music-generation providers.
+ *
+ * Implementations enforce their own timeouts. Recommended:
+ *   - Per-request fetch timeout: 30 seconds
+ *   - Total job-completion timeout: 5 minutes
+ */
+export type MusicHandler = {
+    /**
+     * Generate a music track from prompt + options.
+     *
+     * @param options - prompt, duration, format, genre, mood, etc.
+     */
+    generate(options: MusicOptions): Promise<MusicResult>;
+    /**
+     * Validate the provider is configured (auth, base URL, etc.).
+     */
+    isConfigured(): boolean;
+    /** Maximum supported track duration in seconds (provider-specific). */
+    readonly maxDurationSeconds?: number;
+    /** Output formats supported by this handler. */
+    readonly supportedFormats?: readonly MusicAudioFormat[];
+    /** Genres / styles the upstream advertises (informational). */
+    readonly supportedGenres?: readonly string[];
+};
+/**
+ * Valid music audio formats — runtime validation array.
+ */
+export declare const VALID_MUSIC_FORMATS: readonly MusicAudioFormat[];
+/**
+ * Beatoven.ai task status response.
+ *
+ * Used by `BeatovenMusic` handler to type-check polling responses.
+ */
+export type BeatovenTaskStatus = {
+    status: "composing" | "running" | "composed" | "failed";
+    meta?: {
+        track_url?: string;
+        project_id?: string;
+        track_id?: string;
+        duration?: number;
+    };
+    message?: string;
+};
+/**
+ * Beatoven.ai compose-track submit response.
+ */
+export type BeatovenComposeResponse = {
+    task_id: string;
+    status?: string;
+};
+/**
+ * Google Lyria 3 Pro :generateContent response shape.
+ *
+ * The audio comes back as a part with `inlineData: { mimeType, data }`
+ * where `data` is base64-encoded WAV.
+ */
+export type LyriaResponse = {
+    candidates?: {
+        content?: {
+            parts?: {
+                inlineData?: {
+                    mimeType?: string;
+                    data?: string;
+                };
+                text?: string;
+            }[];
+        };
+        finishReason?: string;
+        index?: number;
+    }[];
+    usageMetadata?: {
+        promptTokenCount?: number;
+        candidatesTokenCount?: number;
+        totalTokenCount?: number;
+    };
+};

package/dist/lib/types/music.js

@@ -0,0 +1,21 @@
+/**
+ * Music Generation Type Definitions
+ *
+ * Types for music / sound-effect generation across providers (Beatoven,
+ * ElevenLabs Music, Lyria, Replicate-hosted MusicGen / Riffusion / AudioGen).
+ *
+ * Music is a separate modality from TTS — TTS produces voiced speech with
+ * prosody; Music produces melodic / harmonic / textural audio.
+ *
+ * @module types/music
+ */
+/**
+ * Valid music audio formats — runtime validation array.
+ */
+export const VALID_MUSIC_FORMATS = [
+    "mp3",
+    "wav",
+    "flac",
+    "ogg",
+];
+//# sourceMappingURL=music.js.map