@juspay/neurolink 11.28.0 → 11.29.0

package/dist/core/baseProvider.js

@@ -1,9 +1,9 @@
 import { context, SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
 import { directAgentTools } from "../agent/directTools.js";
 import { defaultProviderFor } from "../factories/mediaHandlerCatalog.js";
-import { isImageGenerationModel } from "./constants.js";
 import { MiddlewareFactory } from "../middleware/factory.js";
 import { modelSupports } from "../models/modelRegistry.js";
+import { resolveRequestKind } from "./resolveRequestKind.js";
 import { ATTR, tracers } from "../telemetry/index.js";
 import { ERROR_CODES, isAbortError, NeuroLinkError, } from "../utils/errorHandling.js";
 import { ProviderError } from "../types/index.js";
@@ -239,14 +239,12 @@ export class BaseProvider {
         }
         // CRITICAL: Image generation models don't support real streaming
         // Force fake streaming for image models to ensure image output is yielded.
-        // Skip this path when the caller explicitly requests non-image output (e.g.
-        // JSON analysis) so dual-mode models like gemini-3.1-flash-image-preview
-        // can still perform text/structured generation.
-        const isImageModel = isImageGenerationModel(this.modelName);
-        const requestsNonImageOutput = options.output?.format === "json" ||
-            options.output?.format === "structured" ||
-            options.output?.format === "text";
-        if (isImageModel && !requestsNonImageOutput) {
+        // resolveRequestKind() skips this path when the caller explicitly requests
+        // non-image output (e.g. JSON analysis) so dual-mode models like
+        // gemini-3.1-flash-image-preview can still perform text/structured
+        // generation — see its doc comment for the full precedence table.
+        const requestKind = resolveRequestKind(options, this.modelName);
+        if (requestKind === "image") {
             logger.info(`Image model detected, forcing fake streaming`, {
                 provider: this.providerName,
                 model: this.modelName,
@@ -1067,14 +1065,13 @@ export class BaseProvider {
     }
     async runGenerateInActiveContext(options, startTime, otelSpan, otelSpanState) {
         try {
-            if (options.output?.mode === "video") {
+            // Single source of truth for "what kind of request is this" — see
+            // resolveRequestKind's doc comment for the full precedence table.
+            const requestKind = resolveRequestKind(options, this.modelName);
+            if (requestKind === "video") {
                 return await this.handleVideoGeneration(options, startTime);
             }
-            const isImageModel = isImageGenerationModel(this.modelName);
-            const requestsNonImageOutput = options.output?.format === "json" ||
-                options.output?.format === "structured" ||
-                options.output?.format === "text";
-            if (isImageModel && !requestsNonImageOutput) {
+            if (requestKind === "image") {
                 logger.info(`Image generation model detected, routing to executeImageGeneration`, {
                     provider: this.providerName,
                     model: this.modelName,
@@ -1082,7 +1079,7 @@ export class BaseProvider {
                 const imageResult = await this.executeImageGeneration(options);
                 return await this.enhanceResult(imageResult, options, startTime);
             }
-            if (options.tts?.enabled && !options.tts?.useAiResponse) {
+            if (requestKind === "tts-direct") {
                 return this.handleDirectTTSSynthesis(options, startTime);
             }
             const { tools, model } = await this.prepareGenerationContext(options);
@@ -2179,7 +2176,12 @@ export class BaseProvider {
         // shared timeout helper so standard video gen honors the caller's
         // timeout the same way director mode does (see above ~Line 2062).
         const videoTimeout = options.timeout ?? 600_000; // 10 min default
-        const videoResult = await this.executeWithTimeout(() => VideoProcessor.generate(requestedProvider, imageBuffer, prompt, options.output?.video ?? {}, options.region), { timeout: videoTimeout, operationType: "generate" });
+        const videoResult = await this.executeWithTimeout(() => VideoProcessor.generate(requestedProvider, {
+            ...(options.output?.video ?? {}),
+            image: imageBuffer,
+            prompt,
+            region: options.region,
+        }), { timeout: videoTimeout, operationType: "generate" });
         // Prefer the handler's own model id (more accurate — it knows the exact
         // checkpoint that ran). Fall back to the request-time value, and finally
         // to the Vertex default only when we're on the Vertex route.

package/dist/core/resolveRequestKind.d.ts

@@ -0,0 +1,27 @@
+import type { RequestKind, RequestKindInput } from "../types/index.js";
+/**
+ * The dispatch decision for "what kind of request is this" — text, image,
+ * video, music, avatar, direct TTS synthesis, or PPT generation — at the
+ * CORE call sites: neurolink.ts's maybeHandleEarlyGenerateResult
+ * (music/avatar/ppt/workflow routing) and baseProvider.ts's
+ * stream()/runGenerateInActiveContext (image/video/tts-direct routing) call
+ * this instead of independently re-deriving the decision.
+ *
+ * NOT yet the only copy: replicate.ts's generate() override and
+ * googleVertex/client.ts's native dispatch (~6672-6718) still carry their
+ * own provider-internal versions — the Vertex one with a cruder
+ * startsWith() image match. Migrating those two is queued follow-up work;
+ * until it lands, an edit to this precedence table does not reach them.
+ *
+ * Precedence, checked in order:
+ *   1. output.mode (music/avatar/video/ppt) — an explicit mode always wins.
+ *   2. an image-generation model, unless the caller explicitly asked for a
+ *      non-image output.format (json/structured/text) — this lets dual-mode
+ *      models like gemini-3.1-flash-image-preview still perform text or
+ *      structured generation when requested.
+ *   3. tts.enabled without tts.useAiResponse — direct synthesis, bypassing
+ *      the LLM turn entirely (useAiResponse means the LLM's own text
+ *      response gets synthesized afterward, which is NOT this branch).
+ *   4. otherwise, "text".
+ */
+export declare function resolveRequestKind(options: RequestKindInput, modelName?: string): RequestKind;

package/dist/core/resolveRequestKind.js

@@ -0,0 +1,50 @@
+import { isImageGenerationModel } from "./constants.js";
+/**
+ * The dispatch decision for "what kind of request is this" — text, image,
+ * video, music, avatar, direct TTS synthesis, or PPT generation — at the
+ * CORE call sites: neurolink.ts's maybeHandleEarlyGenerateResult
+ * (music/avatar/ppt/workflow routing) and baseProvider.ts's
+ * stream()/runGenerateInActiveContext (image/video/tts-direct routing) call
+ * this instead of independently re-deriving the decision.
+ *
+ * NOT yet the only copy: replicate.ts's generate() override and
+ * googleVertex/client.ts's native dispatch (~6672-6718) still carry their
+ * own provider-internal versions — the Vertex one with a cruder
+ * startsWith() image match. Migrating those two is queued follow-up work;
+ * until it lands, an edit to this precedence table does not reach them.
+ *
+ * Precedence, checked in order:
+ *   1. output.mode (music/avatar/video/ppt) — an explicit mode always wins.
+ *   2. an image-generation model, unless the caller explicitly asked for a
+ *      non-image output.format (json/structured/text) — this lets dual-mode
+ *      models like gemini-3.1-flash-image-preview still perform text or
+ *      structured generation when requested.
+ *   3. tts.enabled without tts.useAiResponse — direct synthesis, bypassing
+ *      the LLM turn entirely (useAiResponse means the LLM's own text
+ *      response gets synthesized afterward, which is NOT this branch).
+ *   4. otherwise, "text".
+ */
+export function resolveRequestKind(options, modelName) {
+    if (options.output?.mode === "music") {
+        return "music";
+    }
+    if (options.output?.mode === "avatar") {
+        return "avatar";
+    }
+    if (options.output?.mode === "video") {
+        return "video";
+    }
+    if (options.output?.mode === "ppt") {
+        return "ppt";
+    }
+    const requestsNonImageOutput = options.output?.format === "json" ||
+        options.output?.format === "structured" ||
+        options.output?.format === "text";
+    if (isImageGenerationModel(modelName) && !requestsNonImageOutput) {
+        return "image";
+    }
+    if (options.tts?.enabled && !options.tts?.useAiResponse) {
+        return "tts-direct";
+    }
+    return "text";
+}

package/dist/neurolink.js

@@ -34,6 +34,7 @@ import { buildToolRoutingCatalog, buildRoutingQueryFromHistory, resolveToolRouti
 import { ToolRoutingCache } from "./core/toolRoutingCache.js";
 import { DEFAULT_RECENT_TURNS, KnowledgeGroundingEngine, } from "./knowledge/index.js";
 import { AIProviderFactory } from "./core/factory.js";
+import { resolveRequestKind } from "./core/resolveRequestKind.js";
 import { createToolEventPayload } from "./core/toolEvents.js";
 import { ProviderFactory } from "./factories/providerFactory.js";
 import { ProviderRegistry } from "./factories/providerRegistry.js";
@@ -3576,13 +3577,16 @@ Current user's request: ${currentInput}`;
             }
             return this.generateWithWorkflow(options);
         }
-        if (options.output?.mode === "music") {
+        // Single source of truth for "what kind of request is this" — see
+        // resolveRequestKind's doc comment for the full precedence table.
+        const requestKind = resolveRequestKind(options, options.model);
+        if (requestKind === "music") {
             return this.generateWithMusic(options, generateSpan);
         }
-        if (options.output?.mode === "avatar") {
+        if (requestKind === "avatar") {
             return this.generateWithAvatar(options, generateSpan);
         }
-        if (options.output?.mode !== "ppt") {
+        if (requestKind !== "ppt") {
             return null;
         }
         if (options.stt?.enabled && options.stt?.audio) {

package/dist/types/dispatch.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Types backing resolveRequestKind() (src/lib/core/resolveRequestKind.ts) —
+ * the single function that decides which of NeuroLink's output modes a
+ * generate/stream request is asking for.
+ */
+export type RequestKind = "text" | "image" | "video" | "music" | "avatar" | "tts-direct" | "ppt";
+/**
+ * Narrow structural subset of TextGenerationOptions/GenerateOptions that
+ * resolveRequestKind() actually reads. Kept intentionally minimal (rather
+ * than importing the full options type) so this module has no dependency
+ * on the wider options type graph.
+ */
+export type RequestKindInput = {
+    output?: {
+        mode?: string;
+        format?: string;
+    };
+    tts?: {
+        enabled?: boolean;
+        useAiResponse?: boolean;
+    };
+};

package/dist/types/dispatch.js

@@ -0,0 +1,6 @@
+/**
+ * Types backing resolveRequestKind() (src/lib/core/resolveRequestKind.ts) —
+ * the single function that decides which of NeuroLink's output modes a
+ * generate/stream request is asking for.
+ */
+export {};

package/dist/types/generate.d.ts

@@ -802,23 +802,12 @@ export type ToolExecutionCaptureOptions = {
  */
 export type GenerateStopReason = "completed" | "step-cap" | "context-cap" | "time-limit" | "stalled" | "aborted" | "provider-error";
 /**
- * Generate function result type - Primary output format
- * Future-ready for multi-modal outputs while maintaining text focus
+ * Media generation/processing outputs shared by GenerateResult and
+ * TextGenerationResult. Extracted so both result types intersect (&) this
+ * single definition instead of each declaring its own drifting copy of the
+ * same audio/video/avatar/music/ppt/image/transcription fields.
  */
-export type GenerateResult = {
-    content: string;
-    /** Knowledge-grounding diagnostics for this turn (present only when grounding ran). */
-    knowledge?: KnowledgeGroundingMetadata;
-    /**
-     * Parsed structured object when a `schema` was requested. Populated from
-     * AI-SDK experimental_output, or from text-mode coercion (balanced-scan +
-     * jsonrepair). Prefer this over JSON.parse(content) — it never requires the
-     * caller to re-parse hand-escaped model text.
-     */
-    structuredData?: unknown;
-    outputs?: {
-        text: string;
-    };
+export type MediaGenerationOutputs = {
     /**
      * Text-to-Speech audio result
      *
@@ -911,9 +900,31 @@ export type GenerateResult = {
      * ```
      */
     ppt?: PPTGenerationResult;
+    /** Standard format for image generation */
     imageOutput?: {
         base64: string;
     } | null;
+    /** STT transcription result (present when stt.enabled is true and audio input was provided) */
+    transcription?: STTResult;
+};
+/**
+ * Generate function result type - Primary output format
+ * Future-ready for multi-modal outputs while maintaining text focus
+ */
+export type GenerateResult = {
+    content: string;
+    /** Knowledge-grounding diagnostics for this turn (present only when grounding ran). */
+    knowledge?: KnowledgeGroundingMetadata;
+    /**
+     * Parsed structured object when a `schema` was requested. Populated from
+     * AI-SDK experimental_output, or from text-mode coercion (balanced-scan +
+     * jsonrepair). Prefer this over JSON.parse(content) — it never requires the
+     * caller to re-parse hand-escaped model text.
+     */
+    structuredData?: unknown;
+    outputs?: {
+        text: string;
+    };
     provider?: string;
     model?: string;
     finishReason?: string;
@@ -1013,8 +1024,6 @@ export type GenerateResult = {
     reasoning?: string;
     /** Token count for reasoning content */
     reasoningTokens?: number;
-    /** STT transcription result (present when stt.enabled is true and audio input was provided) */
-    transcription?: STTResult;
     retries?: {
         count: number;
         errors: Array<{
@@ -1033,7 +1042,7 @@ export type GenerateResult = {
      * absolute `requestsRemaining` / `tokensRemaining`.
      */
     limits?: ClaudeLimitSnapshot;
-};
+} & MediaGenerationOutputs;
 /**
  * Unified options for both generation and streaming
  * Supports factory patterns and domain configuration
@@ -1467,23 +1476,6 @@ export type TextGenerationResult = {
     }>;
     analytics?: AnalyticsData;
     evaluation?: EvaluationData;
-    audio?: TTSResult;
-    /** Outcome of TTS synthesis, including the failure reason. */
-    ttsMetadata?: TTSMetadata;
-    /** STT transcription result (present when stt input was processed) */
-    transcription?: STTResult;
-    /** Video generation result */
-    video?: VideoGenerationResult;
-    /** Avatar (talking-head) generation result */
-    avatar?: AvatarResult;
-    /** Music generation result */
-    music?: MusicResult;
-    /** PowerPoint generation result */
-    ppt?: PPTGenerationResult;
-    /** Image generation output */
-    imageOutput?: {
-        base64: string;
-    } | null;
     /** Gemini 3 thought signature for reasoning continuity across turns */
     thoughtSignature?: string;
     /** Thinking/reasoning text from provider (Anthropic thinking blocks, Gemini thought parts, DeepSeek/NIM reasoning_content) */
@@ -1497,7 +1489,7 @@ export type TextGenerationResult = {
             message: string;
         }>;
     };
-};
+} & MediaGenerationOutputs;
 /**
  * Enhanced result type with optional analytics/evaluation
  */

package/dist/types/index.d.ts

@@ -86,3 +86,4 @@ export * from "./requestRouter.js";
 export * from "./classifierRouter.js";
 export * from "./agentNetwork.js";
 export * from "./localUsage.js";
+export * from "./dispatch.js";

package/dist/types/index.js

@@ -97,3 +97,5 @@ export * from "./classifierRouter.js";
 // Multi-Agent orchestration types
 export * from "./agentNetwork.js";
 export * from "./localUsage.js";
+// resolveRequestKind() dispatch-decision types
+export * from "./dispatch.js";

package/dist/types/video.d.ts

@@ -13,6 +13,19 @@
  */
 import type { VideoGenerationResult, VideoOutputOptions } from "./multimodal.js";
 export type { VideoGenerationResult, VideoOutputOptions, } from "./multimodal.js";
+/**
+ * Bag-form input to `VideoProcessor.generate()` — the primary data (image,
+ * prompt, region) alongside the video-specific output options, collapsed
+ * into a single object matching Music/Avatar's existing `generate(provider,
+ * options)` shape. `VideoHandler.generate()`'s own 4-positional-argument
+ * signature is unchanged; `VideoProcessor.generate()` translates between the
+ * two internally.
+ */
+export type VideoGenerateOptions = VideoOutputOptions & {
+    image: Buffer;
+    prompt: string;
+    region?: string;
+};
 /**
  * Director-mode transition options.
  *

package/dist/utils/videoProcessor.d.ts

@@ -11,7 +11,7 @@
  * @module utils/videoProcessor
  */
 import { VIDEO_ERROR_CODES } from "../constants/videoErrors.js";
-import type { VideoGenerationResult, VideoHandler, VideoOutputOptions, VideoTransitionOptions } from "../types/index.js";
+import type { VideoGenerateOptions, VideoGenerationResult, VideoHandler, VideoOutputOptions, VideoTransitionOptions } from "../types/index.js";
 import { VideoError } from "../adapters/video/vertexVideoHandler.js";
 export { VideoError, VIDEO_ERROR_CODES };
 /**
@@ -45,13 +45,19 @@ export declare class VideoProcessor {
      * Generate a single video clip via the registered handler.
      *
      * @param provider - Registered provider name (e.g. "vertex", "kling")
-     * @param image - Source image buffer
-     * @param prompt - Text prompt describing the desired motion / content
-     * @param options - Resolution / length / aspect-ratio / audio options
-     * @param region - Optional region override (Vertex location, etc.)
+     * @param options - Bag of the source image, prompt, optional region
+     *   override, and resolution / length / aspect-ratio / audio options.
+     *   Translated internally into the handler-level 4-positional-argument
+     *   call — `VideoHandler.generate()`'s own signature is unchanged.
      * @throws VideoError on registry miss, handler-not-configured, or
      *         generation failure
      */
+    static generate(provider: string, options: VideoGenerateOptions): Promise<VideoGenerationResult>;
+    /**
+     * @deprecated Positional form kept for backward compatibility with
+     * pre-bag callers (VideoProcessor is a public export). Use the
+     * options-bag overload.
+     */
     static generate(provider: string, image: Buffer, prompt: string, options: VideoOutputOptions, region?: string): Promise<VideoGenerationResult>;
     /**
      * Generate a transition clip via the registered handler (Director Mode).

package/dist/utils/videoProcessor.js

@@ -14,6 +14,11 @@ import { ErrorCategory, ErrorSeverity } from "../constants/enums.js";
 import { VIDEO_ERROR_CODES } from "../constants/videoErrors.js";
 import { SpanSerializer, SpanStatus, SpanType, getMetricsAggregator, } from "../observability/index.js";
 import { logger } from "./logger.js";
+import { withTimeout } from "./async/withTimeout.js";
+// Video generation is legitimately minutes-long (Kling/Runway render queues),
+// so the bound is generous — its job is to convert a wedged handler into an
+// error rather than an eternal hang, not to police normal latency.
+const VIDEO_GENERATION_TIMEOUT_MS = 600_000;
 // VideoError is canonical in vertexVideoHandler.ts (existing). Re-export
 // here so consumers of `VideoProcessor` can import the typed error from
 // the same module. Both throws and instanceof checks resolve to the same
@@ -69,19 +74,17 @@ export class VideoProcessor {
             "video.audio": options.audio,
         };
     }
-    /**
-     * Generate a single video clip via the registered handler.
-     *
-     * @param provider - Registered provider name (e.g. "vertex", "kling")
-     * @param image - Source image buffer
-     * @param prompt - Text prompt describing the desired motion / content
-     * @param options - Resolution / length / aspect-ratio / audio options
-     * @param region - Optional region override (Vertex location, etc.)
-     * @throws VideoError on registry miss, handler-not-configured, or
-     *         generation failure
-     */
-    static async generate(provider, image, prompt, options, region) {
-        const span = SpanSerializer.createSpan(SpanType.MEDIA_GENERATION, "video.generate", this.buildSpanAttributes(provider, options));
+    static async generate(provider, optionsOrImage, legacyPrompt, legacyOptions, legacyRegion) {
+        const bag = Buffer.isBuffer(optionsOrImage)
+            ? {
+                image: optionsOrImage,
+                prompt: legacyPrompt ?? "",
+                ...(legacyRegion !== undefined ? { region: legacyRegion } : {}),
+                ...(legacyOptions ?? {}),
+            }
+            : optionsOrImage;
+        const { image, prompt, region, ...videoOptions } = bag;
+        const span = SpanSerializer.createSpan(SpanType.MEDIA_GENERATION, "video.generate", this.buildSpanAttributes(provider, videoOptions));
         try {
             const handler = this.getHandler(provider);
             if (!handler) {
@@ -105,7 +108,10 @@ export class VideoProcessor {
                 });
             }
             logger.debug(`[VideoProcessor] Starting video generation with provider: ${provider}`);
-            const result = await handler.generate(image, prompt, options, region);
+            // Bounded per repo guideline (async provider calls wrap withTimeout):
+            // video generation is legitimately slow, so the deadline is generous —
+            // but a wedged handler must error, never hang the caller forever.
+            const result = await withTimeout(handler.generate(image, prompt, videoOptions, region), VIDEO_GENERATION_TIMEOUT_MS, `Video generation via "${provider}" timed out after ${VIDEO_GENERATION_TIMEOUT_MS}ms`);
             const ended = SpanSerializer.endSpan(span, SpanStatus.OK);
             getMetricsAggregator().recordSpan(ended);
             logger.info(`[VideoProcessor] Generated ${result.data.length} bytes (${provider})`);
@@ -124,7 +130,7 @@ export class VideoProcessor {
                 category: ErrorCategory.EXECUTION,
                 severity: ErrorSeverity.HIGH,
                 retriable: true,
-                context: { provider, options, region },
+                context: { provider, options: videoOptions, region },
                 originalError: err instanceof Error ? err : undefined,
             });
         }
@@ -169,7 +175,8 @@ export class VideoProcessor {
             });
         }
         try {
-            return await handler.generateTransition(firstFrame, lastFrame, prompt, options, region);
+            // Same bound as generate(): a wedged transition must error, not hang.
+            return await withTimeout(handler.generateTransition(firstFrame, lastFrame, prompt, options, region), VIDEO_GENERATION_TIMEOUT_MS, `Video transition via "${provider}" timed out after ${VIDEO_GENERATION_TIMEOUT_MS}ms`);
         }
         catch (err) {
             if (err instanceof VideoError) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "11.28.0",
+  "version": "11.29.0",
   "packageManager": "pnpm@10.15.1",
   "description": "TypeScript AI SDK with 24+ LLM providers behind one consistent API. MCP-native (connect any MCP server), voice TTS/STT/realtime, RAG, agents, memory, context compaction. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · DeepSeek · NVIDIA NIM and more.",
   "author": {
@@ -73,6 +73,7 @@
     "test:context": "pnpm exec tsx test/continuous-test-suite-context.ts",
     "test:evaluation": "pnpm exec tsx test/continuous-test-suite-evaluation.ts",
     "test:handler-registry": "pnpm exec tsx test/continuous-test-suite-handler-registry.ts",
+    "test:resolve-request-kind": "pnpm exec tsx test/continuous-test-suite-resolve-request-kind.ts",
     "test:mcp": "pnpm exec tsx test/continuous-test-suite-mcp-infra.ts",
     "test:tool-resolution": "pnpm exec tsx test/continuous-test-suite-tool-resolution.ts",
     "test:mcp:http": "pnpm exec tsx test/continuous-test-suite-mcp-http.ts",