@juspay/neurolink 8.5.1 → 8.6.0

package/CHANGELOG.md

@@ -1,3 +1,14 @@
+## [8.6.0](https://github.com/juspay/neurolink/compare/v8.5.1...v8.6.0) (2025-12-06)
+
+### Features
+
+- **(multimodal):** add altText support to ImageContent for accessibility ([27118c8](https://github.com/juspay/neurolink/commit/27118c87c73bc1eb6389bbc49dd2e59f1cc4c523)), closes [#565](https://github.com/juspay/neurolink/issues/565)
+
+### Bug Fixes
+
+- **(guardrails):** added fallback for guardrail errors on azure's jailbreak errors ([ae42552](https://github.com/juspay/neurolink/commit/ae4255255657c00ea164730dbd61fbad9f65f339))
+- **(observability):** add support to let applications customize traces ([608d991](https://github.com/juspay/neurolink/commit/608d991114c5df2335be73f44a24a187f424373a))
+
 ## [8.5.1](https://github.com/juspay/neurolink/compare/v8.5.0...v8.5.1) (2025-12-04)
 
 ### Bug Fixes

package/dist/adapters/providerImageAdapter.d.ts

@@ -2,7 +2,7 @@
  * Provider Image Adapter - Smart routing for multimodal content
  * Handles provider-specific image formatting and vision capability validation
  */
-import type { Content } from "../types/multimodal.js";
+import type { Content, ImageWithAltText } from "../types/multimodal.js";
 /**
  * Simplified logger for essential error reporting only
  */
@@ -39,8 +39,10 @@ export declare class ProviderImageAdapter {
     private static validateVisionSupport;
     /**
      * Convert simple images array to advanced content format
+     * @param text - Text content to include
+     * @param images - Array of images (Buffer, string, or ImageWithAltText)
      */
-    static convertToContent(text: string, images?: Array<Buffer | string>): Content[];
+    static convertToContent(text: string, images?: Array<Buffer | string | ImageWithAltText>): Content[];
     /**
      * Check if provider supports multimodal content
      */

package/dist/adapters/providerImageAdapter.js

@@ -400,15 +400,29 @@ export class ProviderImageAdapter {
     }
     /**
      * Convert simple images array to advanced content format
+     * @param text - Text content to include
+     * @param images - Array of images (Buffer, string, or ImageWithAltText)
      */
     static convertToContent(text, images) {
         const content = [{ type: "text", text }];
         if (images && images.length > 0) {
             images.forEach((image) => {
+                // Handle both simple images and images with alt text
+                const imageData = typeof image === "object" &&
+                    "data" in image &&
+                    !Buffer.isBuffer(image)
+                    ? image.data
+                    : image;
+                const altText = typeof image === "object" &&
+                    "data" in image &&
+                    !Buffer.isBuffer(image)
+                    ? image.altText
+                    : undefined;
                 content.push({
                     type: "image",
-                    data: image,
-                    mediaType: ImageProcessor.detectImageType(image),
+                    data: imageData,
+                    altText,
+                    mediaType: ImageProcessor.detectImageType(imageData),
                 });
             });
         }

package/dist/core/baseProvider.js

@@ -4,7 +4,6 @@ import { MiddlewareFactory } from "../middleware/factory.js";
 import { logger } from "../utils/logger.js";
 import { directAgentTools } from "../agent/directTools.js";
 import { createTimeoutController, TimeoutError } from "../utils/timeout.js";
-import { nanoid } from "nanoid";
 import { shouldDisableBuiltinTools } from "../utils/toolUtils.js";
 import { getKeysAsString, getKeyCount } from "../utils/transformationUtils.js";
 // Import modules for composition
@@ -658,12 +657,17 @@ export class BaseProvider {
         if (!this.neurolink?.isTelemetryEnabled()) {
             return undefined;
         }
-        const functionId = `${this.providerName}-${operationType}-${nanoid()}`;
+        const context = options.context;
+        const traceName = context?.traceName;
+        const userId = context?.userId;
+        const functionId = traceName ? traceName : userId ? userId : "guest";
         const metadata = {
             provider: this.providerName,
             model: this.modelName,
             toolsEnabled: !options.disableTools,
             neurolink: true,
+            operationType,
+            originalProvider: this.providerName,
         };
         // Add sessionId if available
         if ("sessionId" in options && options.sessionId) {

package/dist/core/modules/TelemetryHandler.js

@@ -120,12 +120,17 @@ export class TelemetryHandler {
         if (!this.neurolink?.isTelemetryEnabled()) {
             return undefined;
         }
-        const functionId = `${this.providerName}-${operationType}-${nanoid()}`;
+        const context = options.context;
+        const traceName = context?.traceName;
+        const userId = context?.userId;
+        const functionId = traceName ? traceName : userId ? userId : "guest";
         const metadata = {
             provider: this.providerName,
             model: this.modelName,
             toolsEnabled: !options.disableTools,
             neurolink: true,
+            operationType,
+            originalProvider: this.providerName,
         };
         // Add sessionId if available
         if ("sessionId" in options && options.sessionId) {

package/dist/lib/adapters/providerImageAdapter.d.ts

@@ -2,7 +2,7 @@
  * Provider Image Adapter - Smart routing for multimodal content
  * Handles provider-specific image formatting and vision capability validation
  */
-import type { Content } from "../types/multimodal.js";
+import type { Content, ImageWithAltText } from "../types/multimodal.js";
 /**
  * Simplified logger for essential error reporting only
  */
@@ -39,8 +39,10 @@ export declare class ProviderImageAdapter {
     private static validateVisionSupport;
     /**
      * Convert simple images array to advanced content format
+     * @param text - Text content to include
+     * @param images - Array of images (Buffer, string, or ImageWithAltText)
      */
-    static convertToContent(text: string, images?: Array<Buffer | string>): Content[];
+    static convertToContent(text: string, images?: Array<Buffer | string | ImageWithAltText>): Content[];
     /**
      * Check if provider supports multimodal content
      */

package/dist/lib/adapters/providerImageAdapter.js

@@ -400,15 +400,29 @@ export class ProviderImageAdapter {
     }
     /**
      * Convert simple images array to advanced content format
+     * @param text - Text content to include
+     * @param images - Array of images (Buffer, string, or ImageWithAltText)
      */
     static convertToContent(text, images) {
         const content = [{ type: "text", text }];
         if (images && images.length > 0) {
             images.forEach((image) => {
+                // Handle both simple images and images with alt text
+                const imageData = typeof image === "object" &&
+                    "data" in image &&
+                    !Buffer.isBuffer(image)
+                    ? image.data
+                    : image;
+                const altText = typeof image === "object" &&
+                    "data" in image &&
+                    !Buffer.isBuffer(image)
+                    ? image.altText
+                    : undefined;
                 content.push({
                     type: "image",
-                    data: image,
-                    mediaType: ImageProcessor.detectImageType(image),
+                    data: imageData,
+                    altText,
+                    mediaType: ImageProcessor.detectImageType(imageData),
                 });
             });
         }

package/dist/lib/core/baseProvider.js

@@ -4,7 +4,6 @@ import { MiddlewareFactory } from "../middleware/factory.js";
 import { logger } from "../utils/logger.js";
 import { directAgentTools } from "../agent/directTools.js";
 import { createTimeoutController, TimeoutError } from "../utils/timeout.js";
-import { nanoid } from "nanoid";
 import { shouldDisableBuiltinTools } from "../utils/toolUtils.js";
 import { getKeysAsString, getKeyCount } from "../utils/transformationUtils.js";
 // Import modules for composition
@@ -658,12 +657,17 @@ export class BaseProvider {
         if (!this.neurolink?.isTelemetryEnabled()) {
             return undefined;
         }
-        const functionId = `${this.providerName}-${operationType}-${nanoid()}`;
+        const context = options.context;
+        const traceName = context?.traceName;
+        const userId = context?.userId;
+        const functionId = traceName ? traceName : userId ? userId : "guest";
         const metadata = {
             provider: this.providerName,
             model: this.modelName,
             toolsEnabled: !options.disableTools,
             neurolink: true,
+            operationType,
+            originalProvider: this.providerName,
         };
         // Add sessionId if available
         if ("sessionId" in options && options.sessionId) {

package/dist/lib/core/modules/TelemetryHandler.js

@@ -120,12 +120,17 @@ export class TelemetryHandler {
         if (!this.neurolink?.isTelemetryEnabled()) {
             return undefined;
         }
-        const functionId = `${this.providerName}-${operationType}-${nanoid()}`;
+        const context = options.context;
+        const traceName = context?.traceName;
+        const userId = context?.userId;
+        const functionId = traceName ? traceName : userId ? userId : "guest";
         const metadata = {
             provider: this.providerName,
             model: this.modelName,
             toolsEnabled: !options.disableTools,
             neurolink: true,
+            operationType,
+            originalProvider: this.providerName,
         };
         // Add sessionId if available
         if ("sessionId" in options && options.sessionId) {

package/dist/lib/middleware/builtin/guardrails.js

@@ -69,8 +69,10 @@ export function createGuardrailsMiddleware(config = {}) {
                 };
             }
             const { stream, ...rest } = await doStream();
+            let hasYieldedChunks = false;
             const transformStream = new TransformStream({
                 transform(chunk, controller) {
+                    hasYieldedChunks = true;
                     let filteredChunk = chunk;
                     if (typeof filteredChunk === "object" &&
                         "textDelta" in filteredChunk) {
@@ -84,6 +86,11 @@ export function createGuardrailsMiddleware(config = {}) {
                     }
                     controller.enqueue(filteredChunk);
                 },
+                flush() {
+                    if (!hasYieldedChunks) {
+                        logger.warn(`[GuardrailsMiddleware] Stream ended without yielding any chunks`);
+                    }
+                },
             });
             return {
                 stream: stream.pipeThrough(transformStream),

package/dist/lib/neurolink.js

@@ -1998,19 +1998,85 @@ Current user's request: ${currentInput}`;
                     }
                 }
                 const { stream: mcpStream, provider: providerName } = await this.createMCPStream(enhancedOptions);
-                // Create a wrapper around the stream that accumulates content
                 let accumulatedContent = "";
+                let chunkCount = 0;
+                const metadata = {
+                    fallbackAttempted: false,
+                    guardrailsBlocked: false,
+                    error: undefined,
+                };
                 const processedStream = (async function* (self) {
                     try {
                         for await (const chunk of mcpStream) {
+                            chunkCount++;
                             if (chunk &&
                                 "content" in chunk &&
                                 typeof chunk.content === "string") {
                                 accumulatedContent += chunk.content;
-                                // Emit chunk event for compatibility
                                 self.emitter.emit("response:chunk", chunk.content);
                             }
-                            yield chunk; // Preserve original streaming behavior
+                            yield chunk;
+                        }
+                        if (chunkCount === 0 && !metadata.fallbackAttempted) {
+                            metadata.fallbackAttempted = true;
+                            const errorMsg = "Stream completed with 0 chunks (possible guardrails block)";
+                            metadata.error = errorMsg;
+                            const fallbackRoute = ModelRouter.getFallbackRoute(originalPrompt || enhancedOptions.input.text || "", {
+                                provider: providerName,
+                                model: enhancedOptions.model || "gpt-4o",
+                                reasoning: "primary failed",
+                                confidence: 0.5,
+                            }, { fallbackStrategy: "auto" });
+                            logger.warn("Retrying with fallback provider", {
+                                originalProvider: providerName,
+                                fallbackProvider: fallbackRoute.provider,
+                                reason: errorMsg,
+                            });
+                            try {
+                                const fallbackProvider = await AIProviderFactory.createProvider(fallbackRoute.provider, fallbackRoute.model);
+                                // Ensure fallback provider can execute tools
+                                fallbackProvider.setupToolExecutor({
+                                    customTools: self.getCustomTools(),
+                                    executeTool: self.executeTool.bind(self),
+                                }, "NeuroLink.fallbackStream");
+                                // Get conversation messages for context (same as primary stream)
+                                const conversationMessages = await getConversationMessages(self.conversationMemory, {
+                                    prompt: enhancedOptions.input.text,
+                                    context: enhancedOptions.context,
+                                });
+                                const fallbackResult = await fallbackProvider.stream({
+                                    ...enhancedOptions,
+                                    model: fallbackRoute.model,
+                                    conversationMessages,
+                                });
+                                let fallbackChunkCount = 0;
+                                for await (const fallbackChunk of fallbackResult.stream) {
+                                    fallbackChunkCount++;
+                                    if (fallbackChunk &&
+                                        "content" in fallbackChunk &&
+                                        typeof fallbackChunk.content === "string") {
+                                        accumulatedContent += fallbackChunk.content;
+                                        self.emitter.emit("response:chunk", fallbackChunk.content);
+                                    }
+                                    yield fallbackChunk;
+                                }
+                                if (fallbackChunkCount === 0) {
+                                    throw new Error(`Fallback provider ${fallbackRoute.provider} also returned 0 chunks`);
+                                }
+                                // Fallback succeeded - likely guardrails blocked primary
+                                metadata.guardrailsBlocked = true;
+                            }
+                            catch (fallbackError) {
+                                const fallbackErrorMsg = fallbackError instanceof Error
+                                    ? fallbackError.message
+                                    : String(fallbackError);
+                                metadata.error = `${errorMsg}; Fallback failed: ${fallbackErrorMsg}`;
+                                logger.error("Fallback provider failed", {
+                                    fallbackProvider: fallbackRoute.provider,
+                                    error: fallbackErrorMsg,
+                                });
+                                throw fallbackError;
+                            }
                         }
                     }
                     finally {
@@ -2053,7 +2119,7 @@ Current user's request: ${currentInput}`;
                         }
                     }
                 })(this);
-                const streamResult = await this.processStreamResult(mcpStream, enhancedOptions, factoryResult);
+                const streamResult = await this.processStreamResult(processedStream, enhancedOptions, factoryResult);
                 const responseTime = Date.now() - startTime;
                 this.emitStreamEndEvents(streamResult);
                 return this.createStreamResponse(streamResult, processedStream, {
@@ -2062,7 +2128,9 @@ Current user's request: ${currentInput}`;
                     startTime,
                     responseTime,
                     streamId,
-                    fallback: false,
+                    fallback: metadata.fallbackAttempted,
+                    guardrailsBlocked: metadata.guardrailsBlocked,
+                    error: metadata.error,
                 });
             }
             catch (error) {
@@ -2181,6 +2249,8 @@ Current user's request: ${currentInput}`;
                 startTime: config.startTime,
                 responseTime: config.responseTime,
                 fallback: config.fallback || false,
+                guardrailsBlocked: config.guardrailsBlocked,
+                error: config.error,
             },
         };
     }

package/dist/lib/telemetry/telemetryService.d.ts

@@ -31,7 +31,7 @@ export declare class TelemetryService {
     private initializeTelemetry;
     private initializeMetrics;
     initialize(): Promise<void>;
-    traceAIRequest<T>(provider: string, operation: () => Promise<T>): Promise<T>;
+    traceAIRequest<T>(provider: string, operation: () => Promise<T>, operationType?: string): Promise<T>;
     recordAIRequest(provider: string, model: string, tokens: number, duration: number): void;
     recordAIError(provider: string, error: Error): void;
     recordMCPToolCall(toolName: string, duration: number, success: boolean): void;

package/dist/lib/telemetry/telemetryService.js

@@ -108,14 +108,14 @@ export class TelemetryService {
         }
     }
     // AI Operation Tracing (NO-OP when disabled)
-    async traceAIRequest(provider, operation) {
+    async traceAIRequest(provider, operation, operationType = "generate_text") {
         if (!this.enabled || !this.tracer) {
-            return await operation(); // Direct execution when disabled
+            return await operation();
         }
-        const span = this.tracer.startSpan(`ai.${provider}.generate_text`, {
+        const span = this.tracer.startSpan(`ai.${provider}.${operationType}`, {
             attributes: {
                 "ai.provider": provider,
-                "ai.operation": "generate_text",
+                "ai.operation": operationType,
             },
         });
         try {

package/dist/lib/types/common.d.ts

@@ -129,3 +129,8 @@ export type TypedEventEmitter<TEvents extends Record<string, unknown>> = {
     listenerCount<K extends keyof TEvents>(event: K): number;
     listeners<K extends keyof TEvents>(event: K): Array<(...args: unknown[]) => void>;
 };
+export type Context = {
+    traceName?: string;
+    userId?: string;
+    sessionId?: string;
+};

package/dist/lib/types/content.d.ts

@@ -14,5 +14,5 @@
  * import type { MultimodalInput } from './types/multimodal.js';
  * ```
  */
-export type { TextContent, ImageContent, CSVContent, PDFContent, AudioContent, VideoContent, Content, MultimodalInput, MultimodalMessage, VisionCapability, ProviderImageFormat, ProcessedImage, ProviderMultimodalPayload, } from "./multimodal.js";
+export type { TextContent, ImageContent, CSVContent, PDFContent, AudioContent, VideoContent, Content, ImageWithAltText, MultimodalInput, MultimodalMessage, VisionCapability, ProviderImageFormat, ProcessedImage, ProviderMultimodalPayload, } from "./multimodal.js";
 export { isTextContent, isImageContent, isCSVContent, isPDFContent, isAudioContent, isVideoContent, isMultimodalInput, } from "./multimodal.js";

package/dist/lib/types/generateTypes.d.ts

@@ -6,7 +6,7 @@ import type { EvaluationData } from "./evaluation.js";
 import type { ChatMessage, ConversationMemoryConfig } from "./conversation.js";
 import type { MiddlewareFactoryOptions } from "./middlewareTypes.js";
 import type { JsonValue } from "./common.js";
-import type { Content } from "./content.js";
+import type { Content, ImageWithAltText } from "./content.js";
 /**
  * Generate function options type - Primary method for content generation
  * Supports multimodal content while maintaining backward compatibility
@@ -14,7 +14,24 @@ import type { Content } from "./content.js";
 export type GenerateOptions = {
     input: {
         text: string;
-        images?: Array<Buffer | string>;
+        /**
+         * Images to include in the request.
+         * Supports simple image data (Buffer, string) or objects with alt text for accessibility.
+         *
+         * @example Simple usage
+         * ```typescript
+         * images: [imageBuffer, "https://example.com/image.jpg"]
+         * ```
+         *
+         * @example With alt text for accessibility
+         * ```typescript
+         * images: [
+         *   { data: imageBuffer, altText: "Product screenshot showing main dashboard" },
+         *   { data: "https://example.com/chart.png", altText: "Sales chart for Q3 2024" }
+         * ]
+         * ```
+         */
+        images?: Array<Buffer | string | ImageWithAltText>;
         csvFiles?: Array<Buffer | string>;
         pdfFiles?: Array<Buffer | string>;
         files?: Array<Buffer | string>;

package/dist/lib/types/multimodal.d.ts

@@ -52,6 +52,8 @@ export type TextContent = {
 export type ImageContent = {
     type: "image";
     data: Buffer | string;
+    /** Alternative text for accessibility (screen readers, SEO) */
+    altText?: string;
     mediaType?: "image/jpeg" | "image/png" | "image/gif" | "image/webp" | "image/bmp" | "image/tiff";
     metadata?: {
         description?: string;
@@ -164,13 +166,48 @@ export type VideoContent = {
  * Covers text, images, documents, and multimedia
  */
 export type Content = TextContent | ImageContent | CSVContent | PDFContent | AudioContent | VideoContent;
+/**
+ * Image data with optional alt text for accessibility
+ * Use this when you need to provide alt text for screen readers and SEO
+ *
+ * @example
+ * ```typescript
+ * const imageWithAlt: ImageWithAltText = {
+ *   data: imageBuffer,
+ *   altText: "A dashboard showing quarterly sales trends"
+ * };
+ * ```
+ */
+export type ImageWithAltText = {
+    /** Image data as Buffer, base64 string, URL, or data URI */
+    data: Buffer | string;
+    /** Alternative text for accessibility (screen readers, SEO) */
+    altText?: string;
+};
 /**
  * Multimodal input type for options that may contain images or content arrays
  * This is the primary interface for users to provide multimodal content
  */
 export type MultimodalInput = {
     text: string;
-    images?: Array<Buffer | string>;
+    /**
+     * Images to include in the request.
+     * Can be simple image data (Buffer, string) or objects with alt text for accessibility.
+     *
+     * @example Simple usage
+     * ```typescript
+     * images: [imageBuffer, "https://example.com/image.jpg"]
+     * ```
+     *
+     * @example With alt text for accessibility
+     * ```typescript
+     * images: [
+     *   { data: imageBuffer, altText: "Product screenshot showing main dashboard" },
+     *   { data: "https://example.com/chart.png", altText: "Sales chart for Q3 2024" }
+     * ]
+     * ```
+     */
+    images?: Array<Buffer | string | ImageWithAltText>;
     content?: Content[];
     csvFiles?: Array<Buffer | string>;
     pdfFiles?: Array<Buffer | string>;

package/dist/lib/types/streamTypes.d.ts

@@ -1,7 +1,7 @@
 import type { Tool } from "ai";
 import type { ValidationSchema, StandardRecord } from "./typeAliases.js";
 import type { AIModelProviderConfig } from "./providers.js";
-import type { Content } from "./content.js";
+import type { Content, ImageWithAltText } from "./content.js";
 import type { AnalyticsData, ToolExecutionEvent, ToolExecutionSummary } from "../types/index.js";
 import { AIProviderName } from "../constants/enums.js";
 import type { TokenUsage } from "./analytics.js";
@@ -125,7 +125,24 @@ export type StreamOptions = {
     input: {
         text: string;
         audio?: AudioInputSpec;
-        images?: Array<Buffer | string>;
+        /**
+         * Images to include in the request.
+         * Supports simple image data (Buffer, string) or objects with alt text for accessibility.
+         *
+         * @example Simple usage
+         * ```typescript
+         * images: [imageBuffer, "https://example.com/image.jpg"]
+         * ```
+         *
+         * @example With alt text for accessibility
+         * ```typescript
+         * images: [
+         *   { data: imageBuffer, altText: "Product screenshot showing main dashboard" },
+         *   { data: "https://example.com/chart.png", altText: "Sales chart for Q3 2024" }
+         * ]
+         * ```
+         */
+        images?: Array<Buffer | string | ImageWithAltText>;
         csvFiles?: Array<Buffer | string>;
         pdfFiles?: Array<Buffer | string>;
         files?: Array<Buffer | string>;
@@ -211,6 +228,8 @@ export type StreamResult = {
         totalToolExecutions?: number;
         toolExecutionTime?: number;
         hasToolErrors?: boolean;
+        guardrailsBlocked?: boolean;
+        error?: string;
     };
     analytics?: AnalyticsData | Promise<AnalyticsData>;
     evaluation?: EvaluationData | Promise<EvaluationData>;

package/dist/lib/utils/messageBuilder.js

@@ -10,6 +10,30 @@ import { FileDetector } from "./fileDetector.js";
 import { PDFProcessor } from "./pdfProcessor.js";
 import { request, getGlobalDispatcher, interceptors } from "undici";
 import { readFileSync, existsSync } from "fs";
+/**
+ * Type guard to check if an image input has alt text
+ */
+function isImageWithAltText(image) {
+    return (typeof image === "object" && !Buffer.isBuffer(image) && "data" in image);
+}
+/**
+ * Extract image data from an image input (handles both simple and alt text formats)
+ */
+function extractImageData(image) {
+    if (isImageWithAltText(image)) {
+        return image.data;
+    }
+    return image;
+}
+/**
+ * Extract alt text from an image input if available
+ */
+function extractAltText(image) {
+    if (isImageWithAltText(image)) {
+        return image.altText;
+    }
+    return undefined;
+}
 /**
  * Type guard for validating message roles
  */
@@ -639,28 +663,47 @@ async function downloadImageFromUrl(url) {
  * - URLs: Downloaded and converted to base64 for Vercel AI SDK compatibility
  * - Local files: Converted to base64 for Vercel AI SDK compatibility
  * - Buffers/Data URIs: Processed normally
+ * - Supports alt text for accessibility (included as context in text parts)
  */
 async function convertSimpleImagesToProviderFormat(text, images, provider, _model) {
     // For Vercel AI SDK, we need to return the content in the standard format
     // The Vercel AI SDK will handle provider-specific formatting internally
+    // IMPORTANT: Generate alt text descriptions BEFORE URL downloading to maintain correct image numbering
+    // This ensures image numbers match the original order provided by users, even if some URLs fail to download
+    const altTextDescriptions = images
+        .map((image, idx) => {
+        const altText = extractAltText(image);
+        return altText ? `[Image ${idx + 1}: ${altText}]` : null;
+    })
+        .filter(Boolean);
+    // Build enhanced text with alt text context for accessibility
+    // NOTE: Alt text is appended to the user's prompt as contextual information because most AI providers
+    // don't have native alt text fields in their APIs. This approach ensures accessibility metadata
+    // is preserved and helps AI models better understand image content.
+    const enhancedText = altTextDescriptions.length > 0
+        ? `${text}\n\nImage descriptions for context: ${altTextDescriptions.join(" ")}`
+        : text;
     // Smart auto-detection: separate URLs from actual image data
+    // Also track alt text for each image
     const urlImages = [];
     const actualImages = [];
     images.forEach((image, _index) => {
-        if (typeof image === "string" && isInternetUrl(image)) {
+        const imageData = extractImageData(image);
+        const altText = extractAltText(image);
+        if (typeof imageData === "string" && isInternetUrl(imageData)) {
             // Internet URL - will be downloaded and converted to base64
-            urlImages.push(image);
+            urlImages.push({ url: imageData, altText });
         }
         else {
             // Actual image data (file path, Buffer, data URI) - process for Vercel AI SDK
-            actualImages.push(image);
+            actualImages.push({ data: imageData, altText });
         }
     });
     // Download URL images and add to actual images
-    for (const url of urlImages) {
+    for (const { url, altText } of urlImages) {
         try {
             const downloadedDataUri = await downloadImageFromUrl(url);
-            actualImages.push(downloadedDataUri);
+            actualImages.push({ data: downloadedDataUri, altText });
         }
         catch (error) {
             MultimodalLogger.logError("URL_DOWNLOAD_FAILED_SKIPPING", error, { url });
@@ -668,9 +711,11 @@ async function convertSimpleImagesToProviderFormat(text, images, provider, _mode
             logger.warn(`Failed to download image from ${url}, skipping: ${error instanceof Error ? error.message : String(error)}`);
         }
     }
-    const content = [{ type: "text", text }];
+    const content = [
+        { type: "text", text: enhancedText },
+    ];
     // Process all images (including downloaded URLs) for Vercel AI SDK
-    actualImages.forEach((image, index) => {
+    actualImages.forEach(({ data: image }, index) => {
         try {
             // Vercel AI SDK expects { type: 'image', image: Buffer | string, mimeType?: string }
             // For Vertex AI, we need to include mimeType

package/dist/lib/utils/multimodalOptionsBuilder.d.ts

@@ -44,7 +44,7 @@ import type { StreamOptions } from "../types/streamTypes.js";
 export declare function buildMultimodalOptions(options: StreamOptions, providerName: string, modelName: string): {
     input: {
         text: string;
-        images: (string | Buffer<ArrayBufferLike>)[] | undefined;
+        images: (string | Buffer<ArrayBufferLike> | import("../types/multimodal.js").ImageWithAltText)[] | undefined;
         content: import("../types/multimodal.js").Content[] | undefined;
         files: (string | Buffer<ArrayBufferLike>)[] | undefined;
         csvFiles: (string | Buffer<ArrayBufferLike>)[] | undefined;

package/dist/middleware/builtin/guardrails.js

@@ -69,8 +69,10 @@ export function createGuardrailsMiddleware(config = {}) {
                 };
             }
             const { stream, ...rest } = await doStream();
+            let hasYieldedChunks = false;
             const transformStream = new TransformStream({
                 transform(chunk, controller) {
+                    hasYieldedChunks = true;
                     let filteredChunk = chunk;
                     if (typeof filteredChunk === "object" &&
                         "textDelta" in filteredChunk) {
@@ -84,6 +86,11 @@ export function createGuardrailsMiddleware(config = {}) {
                     }
                     controller.enqueue(filteredChunk);
                 },
+                flush() {
+                    if (!hasYieldedChunks) {
+                        logger.warn(`[GuardrailsMiddleware] Stream ended without yielding any chunks`);
+                    }
+                },
             });
             return {
                 stream: stream.pipeThrough(transformStream),

package/dist/neurolink.js

@@ -1998,19 +1998,85 @@ Current user's request: ${currentInput}`;
                     }
                 }
                 const { stream: mcpStream, provider: providerName } = await this.createMCPStream(enhancedOptions);
-                // Create a wrapper around the stream that accumulates content
                 let accumulatedContent = "";
+                let chunkCount = 0;
+                const metadata = {
+                    fallbackAttempted: false,
+                    guardrailsBlocked: false,
+                    error: undefined,
+                };
                 const processedStream = (async function* (self) {
                     try {
                         for await (const chunk of mcpStream) {
+                            chunkCount++;
                             if (chunk &&
                                 "content" in chunk &&
                                 typeof chunk.content === "string") {
                                 accumulatedContent += chunk.content;
-                                // Emit chunk event for compatibility
                                 self.emitter.emit("response:chunk", chunk.content);
                             }
-                            yield chunk; // Preserve original streaming behavior
+                            yield chunk;
+                        }
+                        if (chunkCount === 0 && !metadata.fallbackAttempted) {
+                            metadata.fallbackAttempted = true;
+                            const errorMsg = "Stream completed with 0 chunks (possible guardrails block)";
+                            metadata.error = errorMsg;
+                            const fallbackRoute = ModelRouter.getFallbackRoute(originalPrompt || enhancedOptions.input.text || "", {
+                                provider: providerName,
+                                model: enhancedOptions.model || "gpt-4o",
+                                reasoning: "primary failed",
+                                confidence: 0.5,
+                            }, { fallbackStrategy: "auto" });
+                            logger.warn("Retrying with fallback provider", {
+                                originalProvider: providerName,
+                                fallbackProvider: fallbackRoute.provider,
+                                reason: errorMsg,
+                            });
+                            try {
+                                const fallbackProvider = await AIProviderFactory.createProvider(fallbackRoute.provider, fallbackRoute.model);
+                                // Ensure fallback provider can execute tools
+                                fallbackProvider.setupToolExecutor({
+                                    customTools: self.getCustomTools(),
+                                    executeTool: self.executeTool.bind(self),
+                                }, "NeuroLink.fallbackStream");
+                                // Get conversation messages for context (same as primary stream)
+                                const conversationMessages = await getConversationMessages(self.conversationMemory, {
+                                    prompt: enhancedOptions.input.text,
+                                    context: enhancedOptions.context,
+                                });
+                                const fallbackResult = await fallbackProvider.stream({
+                                    ...enhancedOptions,
+                                    model: fallbackRoute.model,
+                                    conversationMessages,
+                                });
+                                let fallbackChunkCount = 0;
+                                for await (const fallbackChunk of fallbackResult.stream) {
+                                    fallbackChunkCount++;
+                                    if (fallbackChunk &&
+                                        "content" in fallbackChunk &&
+                                        typeof fallbackChunk.content === "string") {
+                                        accumulatedContent += fallbackChunk.content;
+                                        self.emitter.emit("response:chunk", fallbackChunk.content);
+                                    }
+                                    yield fallbackChunk;
+                                }
+                                if (fallbackChunkCount === 0) {
+                                    throw new Error(`Fallback provider ${fallbackRoute.provider} also returned 0 chunks`);
+                                }
+                                // Fallback succeeded - likely guardrails blocked primary
+                                metadata.guardrailsBlocked = true;
+                            }
+                            catch (fallbackError) {
+                                const fallbackErrorMsg = fallbackError instanceof Error
+                                    ? fallbackError.message
+                                    : String(fallbackError);
+                                metadata.error = `${errorMsg}; Fallback failed: ${fallbackErrorMsg}`;
+                                logger.error("Fallback provider failed", {
+                                    fallbackProvider: fallbackRoute.provider,
+                                    error: fallbackErrorMsg,
+                                });
+                                throw fallbackError;
+                            }
                         }
                     }
                     finally {
@@ -2053,7 +2119,7 @@ Current user's request: ${currentInput}`;
                         }
                     }
                 })(this);
-                const streamResult = await this.processStreamResult(mcpStream, enhancedOptions, factoryResult);
+                const streamResult = await this.processStreamResult(processedStream, enhancedOptions, factoryResult);
                 const responseTime = Date.now() - startTime;
                 this.emitStreamEndEvents(streamResult);
                 return this.createStreamResponse(streamResult, processedStream, {
@@ -2062,7 +2128,9 @@ Current user's request: ${currentInput}`;
                     startTime,
                     responseTime,
                     streamId,
-                    fallback: false,
+                    fallback: metadata.fallbackAttempted,
+                    guardrailsBlocked: metadata.guardrailsBlocked,
+                    error: metadata.error,
                 });
             }
             catch (error) {
@@ -2181,6 +2249,8 @@ Current user's request: ${currentInput}`;
                 startTime: config.startTime,
                 responseTime: config.responseTime,
                 fallback: config.fallback || false,
+                guardrailsBlocked: config.guardrailsBlocked,
+                error: config.error,
             },
         };
     }

package/dist/telemetry/telemetryService.d.ts

@@ -31,7 +31,7 @@ export declare class TelemetryService {
     private initializeTelemetry;
     private initializeMetrics;
     initialize(): Promise<void>;
-    traceAIRequest<T>(provider: string, operation: () => Promise<T>): Promise<T>;
+    traceAIRequest<T>(provider: string, operation: () => Promise<T>, operationType?: string): Promise<T>;
     recordAIRequest(provider: string, model: string, tokens: number, duration: number): void;
     recordAIError(provider: string, error: Error): void;
     recordMCPToolCall(toolName: string, duration: number, success: boolean): void;

package/dist/telemetry/telemetryService.js

@@ -108,14 +108,14 @@ export class TelemetryService {
         }
     }
     // AI Operation Tracing (NO-OP when disabled)
-    async traceAIRequest(provider, operation) {
+    async traceAIRequest(provider, operation, operationType = "generate_text") {
         if (!this.enabled || !this.tracer) {
-            return await operation(); // Direct execution when disabled
+            return await operation();
         }
-        const span = this.tracer.startSpan(`ai.${provider}.generate_text`, {
+        const span = this.tracer.startSpan(`ai.${provider}.${operationType}`, {
             attributes: {
                 "ai.provider": provider,
-                "ai.operation": "generate_text",
+                "ai.operation": operationType,
             },
         });
         try {

package/dist/types/common.d.ts

@@ -129,3 +129,8 @@ export type TypedEventEmitter<TEvents extends Record<string, unknown>> = {
     listenerCount<K extends keyof TEvents>(event: K): number;
     listeners<K extends keyof TEvents>(event: K): Array<(...args: unknown[]) => void>;
 };
+export type Context = {
+    traceName?: string;
+    userId?: string;
+    sessionId?: string;
+};

package/dist/types/content.d.ts

@@ -14,5 +14,5 @@
  * import type { MultimodalInput } from './types/multimodal.js';
  * ```
  */
-export type { TextContent, ImageContent, CSVContent, PDFContent, AudioContent, VideoContent, Content, MultimodalInput, MultimodalMessage, VisionCapability, ProviderImageFormat, ProcessedImage, ProviderMultimodalPayload, } from "./multimodal.js";
+export type { TextContent, ImageContent, CSVContent, PDFContent, AudioContent, VideoContent, Content, ImageWithAltText, MultimodalInput, MultimodalMessage, VisionCapability, ProviderImageFormat, ProcessedImage, ProviderMultimodalPayload, } from "./multimodal.js";
 export { isTextContent, isImageContent, isCSVContent, isPDFContent, isAudioContent, isVideoContent, isMultimodalInput, } from "./multimodal.js";

package/dist/types/generateTypes.d.ts

@@ -6,7 +6,7 @@ import type { EvaluationData } from "./evaluation.js";
 import type { ChatMessage, ConversationMemoryConfig } from "./conversation.js";
 import type { MiddlewareFactoryOptions } from "./middlewareTypes.js";
 import type { JsonValue } from "./common.js";
-import type { Content } from "./content.js";
+import type { Content, ImageWithAltText } from "./content.js";
 /**
  * Generate function options type - Primary method for content generation
  * Supports multimodal content while maintaining backward compatibility
@@ -14,7 +14,24 @@ import type { Content } from "./content.js";
 export type GenerateOptions = {
     input: {
         text: string;
-        images?: Array<Buffer | string>;
+        /**
+         * Images to include in the request.
+         * Supports simple image data (Buffer, string) or objects with alt text for accessibility.
+         *
+         * @example Simple usage
+         * ```typescript
+         * images: [imageBuffer, "https://example.com/image.jpg"]
+         * ```
+         *
+         * @example With alt text for accessibility
+         * ```typescript
+         * images: [
+         *   { data: imageBuffer, altText: "Product screenshot showing main dashboard" },
+         *   { data: "https://example.com/chart.png", altText: "Sales chart for Q3 2024" }
+         * ]
+         * ```
+         */
+        images?: Array<Buffer | string | ImageWithAltText>;
         csvFiles?: Array<Buffer | string>;
         pdfFiles?: Array<Buffer | string>;
         files?: Array<Buffer | string>;

package/dist/types/multimodal.d.ts

@@ -52,6 +52,8 @@ export type TextContent = {
 export type ImageContent = {
     type: "image";
     data: Buffer | string;
+    /** Alternative text for accessibility (screen readers, SEO) */
+    altText?: string;
     mediaType?: "image/jpeg" | "image/png" | "image/gif" | "image/webp" | "image/bmp" | "image/tiff";
     metadata?: {
         description?: string;
@@ -164,13 +166,48 @@ export type VideoContent = {
  * Covers text, images, documents, and multimedia
  */
 export type Content = TextContent | ImageContent | CSVContent | PDFContent | AudioContent | VideoContent;
+/**
+ * Image data with optional alt text for accessibility
+ * Use this when you need to provide alt text for screen readers and SEO
+ *
+ * @example
+ * ```typescript
+ * const imageWithAlt: ImageWithAltText = {
+ *   data: imageBuffer,
+ *   altText: "A dashboard showing quarterly sales trends"
+ * };
+ * ```
+ */
+export type ImageWithAltText = {
+    /** Image data as Buffer, base64 string, URL, or data URI */
+    data: Buffer | string;
+    /** Alternative text for accessibility (screen readers, SEO) */
+    altText?: string;
+};
 /**
  * Multimodal input type for options that may contain images or content arrays
  * This is the primary interface for users to provide multimodal content
  */
 export type MultimodalInput = {
     text: string;
-    images?: Array<Buffer | string>;
+    /**
+     * Images to include in the request.
+     * Can be simple image data (Buffer, string) or objects with alt text for accessibility.
+     *
+     * @example Simple usage
+     * ```typescript
+     * images: [imageBuffer, "https://example.com/image.jpg"]
+     * ```
+     *
+     * @example With alt text for accessibility
+     * ```typescript
+     * images: [
+     *   { data: imageBuffer, altText: "Product screenshot showing main dashboard" },
+     *   { data: "https://example.com/chart.png", altText: "Sales chart for Q3 2024" }
+     * ]
+     * ```
+     */
+    images?: Array<Buffer | string | ImageWithAltText>;
     content?: Content[];
     csvFiles?: Array<Buffer | string>;
     pdfFiles?: Array<Buffer | string>;

package/dist/types/streamTypes.d.ts

@@ -1,7 +1,7 @@
 import type { Tool } from "ai";
 import type { ValidationSchema, StandardRecord } from "./typeAliases.js";
 import type { AIModelProviderConfig } from "./providers.js";
-import type { Content } from "./content.js";
+import type { Content, ImageWithAltText } from "./content.js";
 import type { AnalyticsData, ToolExecutionEvent, ToolExecutionSummary } from "../types/index.js";
 import { AIProviderName } from "../constants/enums.js";
 import type { TokenUsage } from "./analytics.js";
@@ -125,7 +125,24 @@ export type StreamOptions = {
     input: {
         text: string;
         audio?: AudioInputSpec;
-        images?: Array<Buffer | string>;
+        /**
+         * Images to include in the request.
+         * Supports simple image data (Buffer, string) or objects with alt text for accessibility.
+         *
+         * @example Simple usage
+         * ```typescript
+         * images: [imageBuffer, "https://example.com/image.jpg"]
+         * ```
+         *
+         * @example With alt text for accessibility
+         * ```typescript
+         * images: [
+         *   { data: imageBuffer, altText: "Product screenshot showing main dashboard" },
+         *   { data: "https://example.com/chart.png", altText: "Sales chart for Q3 2024" }
+         * ]
+         * ```
+         */
+        images?: Array<Buffer | string | ImageWithAltText>;
         csvFiles?: Array<Buffer | string>;
         pdfFiles?: Array<Buffer | string>;
         files?: Array<Buffer | string>;
@@ -211,6 +228,8 @@ export type StreamResult = {
         totalToolExecutions?: number;
         toolExecutionTime?: number;
         hasToolErrors?: boolean;
+        guardrailsBlocked?: boolean;
+        error?: string;
     };
     analytics?: AnalyticsData | Promise<AnalyticsData>;
     evaluation?: EvaluationData | Promise<EvaluationData>;

package/dist/utils/messageBuilder.js

@@ -10,6 +10,30 @@ import { FileDetector } from "./fileDetector.js";
 import { PDFProcessor } from "./pdfProcessor.js";
 import { request, getGlobalDispatcher, interceptors } from "undici";
 import { readFileSync, existsSync } from "fs";
+/**
+ * Type guard to check if an image input has alt text
+ */
+function isImageWithAltText(image) {
+    return (typeof image === "object" && !Buffer.isBuffer(image) && "data" in image);
+}
+/**
+ * Extract image data from an image input (handles both simple and alt text formats)
+ */
+function extractImageData(image) {
+    if (isImageWithAltText(image)) {
+        return image.data;
+    }
+    return image;
+}
+/**
+ * Extract alt text from an image input if available
+ */
+function extractAltText(image) {
+    if (isImageWithAltText(image)) {
+        return image.altText;
+    }
+    return undefined;
+}
 /**
  * Type guard for validating message roles
  */
@@ -639,28 +663,47 @@ async function downloadImageFromUrl(url) {
  * - URLs: Downloaded and converted to base64 for Vercel AI SDK compatibility
  * - Local files: Converted to base64 for Vercel AI SDK compatibility
  * - Buffers/Data URIs: Processed normally
+ * - Supports alt text for accessibility (included as context in text parts)
  */
 async function convertSimpleImagesToProviderFormat(text, images, provider, _model) {
     // For Vercel AI SDK, we need to return the content in the standard format
     // The Vercel AI SDK will handle provider-specific formatting internally
+    // IMPORTANT: Generate alt text descriptions BEFORE URL downloading to maintain correct image numbering
+    // This ensures image numbers match the original order provided by users, even if some URLs fail to download
+    const altTextDescriptions = images
+        .map((image, idx) => {
+        const altText = extractAltText(image);
+        return altText ? `[Image ${idx + 1}: ${altText}]` : null;
+    })
+        .filter(Boolean);
+    // Build enhanced text with alt text context for accessibility
+    // NOTE: Alt text is appended to the user's prompt as contextual information because most AI providers
+    // don't have native alt text fields in their APIs. This approach ensures accessibility metadata
+    // is preserved and helps AI models better understand image content.
+    const enhancedText = altTextDescriptions.length > 0
+        ? `${text}\n\nImage descriptions for context: ${altTextDescriptions.join(" ")}`
+        : text;
     // Smart auto-detection: separate URLs from actual image data
+    // Also track alt text for each image
     const urlImages = [];
     const actualImages = [];
     images.forEach((image, _index) => {
-        if (typeof image === "string" && isInternetUrl(image)) {
+        const imageData = extractImageData(image);
+        const altText = extractAltText(image);
+        if (typeof imageData === "string" && isInternetUrl(imageData)) {
             // Internet URL - will be downloaded and converted to base64
-            urlImages.push(image);
+            urlImages.push({ url: imageData, altText });
         }
         else {
             // Actual image data (file path, Buffer, data URI) - process for Vercel AI SDK
-            actualImages.push(image);
+            actualImages.push({ data: imageData, altText });
         }
     });
     // Download URL images and add to actual images
-    for (const url of urlImages) {
+    for (const { url, altText } of urlImages) {
         try {
             const downloadedDataUri = await downloadImageFromUrl(url);
-            actualImages.push(downloadedDataUri);
+            actualImages.push({ data: downloadedDataUri, altText });
         }
         catch (error) {
             MultimodalLogger.logError("URL_DOWNLOAD_FAILED_SKIPPING", error, { url });
@@ -668,9 +711,11 @@ async function convertSimpleImagesToProviderFormat(text, images, provider, _mode
             logger.warn(`Failed to download image from ${url}, skipping: ${error instanceof Error ? error.message : String(error)}`);
         }
     }
-    const content = [{ type: "text", text }];
+    const content = [
+        { type: "text", text: enhancedText },
+    ];
     // Process all images (including downloaded URLs) for Vercel AI SDK
-    actualImages.forEach((image, index) => {
+    actualImages.forEach(({ data: image }, index) => {
         try {
             // Vercel AI SDK expects { type: 'image', image: Buffer | string, mimeType?: string }
             // For Vertex AI, we need to include mimeType

package/dist/utils/multimodalOptionsBuilder.d.ts

@@ -44,7 +44,7 @@ import type { StreamOptions } from "../types/streamTypes.js";
 export declare function buildMultimodalOptions(options: StreamOptions, providerName: string, modelName: string): {
     input: {
         text: string;
-        images: (string | Buffer<ArrayBufferLike>)[] | undefined;
+        images: (string | Buffer<ArrayBufferLike> | import("../types/multimodal.js").ImageWithAltText)[] | undefined;
         content: import("../types/multimodal.js").Content[] | undefined;
         files: (string | Buffer<ArrayBufferLike>)[] | undefined;
         csvFiles: (string | Buffer<ArrayBufferLike>)[] | undefined;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "8.5.1",
+  "version": "8.6.0",
   "description": "Universal AI Development Platform with working MCP integration, multi-provider support, and professional CLI. Built-in tools operational, 58+ external MCP servers discoverable. Connect to filesystem, GitHub, database operations, and more. Build, test, and deploy AI applications with 9 major providers: OpenAI, Anthropic, Google AI, AWS Bedrock, Azure, Hugging Face, Ollama, and Mistral AI.",
   "author": {
     "name": "Juspay Technologies",