@juspay/neurolink 8.5.0 → 8.6.0

package/dist/types/generateTypes.d.ts

@@ -6,7 +6,7 @@ import type { EvaluationData } from "./evaluation.js";
 import type { ChatMessage, ConversationMemoryConfig } from "./conversation.js";
 import type { MiddlewareFactoryOptions } from "./middlewareTypes.js";
 import type { JsonValue } from "./common.js";
-import type { Content } from "./content.js";
+import type { Content, ImageWithAltText } from "./content.js";
 /**
  * Generate function options type - Primary method for content generation
  * Supports multimodal content while maintaining backward compatibility
@@ -14,7 +14,24 @@ import type { Content } from "./content.js";
 export type GenerateOptions = {
     input: {
         text: string;
-        images?: Array<Buffer | string>;
+        /**
+         * Images to include in the request.
+         * Supports simple image data (Buffer, string) or objects with alt text for accessibility.
+         *
+         * @example Simple usage
+         * ```typescript
+         * images: [imageBuffer, "https://example.com/image.jpg"]
+         * ```
+         *
+         * @example With alt text for accessibility
+         * ```typescript
+         * images: [
+         *   { data: imageBuffer, altText: "Product screenshot showing main dashboard" },
+         *   { data: "https://example.com/chart.png", altText: "Sales chart for Q3 2024" }
+         * ]
+         * ```
+         */
+        images?: Array<Buffer | string | ImageWithAltText>;
         csvFiles?: Array<Buffer | string>;
         pdfFiles?: Array<Buffer | string>;
         files?: Array<Buffer | string>;
@@ -34,9 +51,58 @@ export type GenerateOptions = {
     temperature?: number;
     maxTokens?: number;
     systemPrompt?: string;
+    /**
+     * Zod schema for structured output validation
+     *
+     * @important Google Gemini Limitation
+     * Google Vertex AI and Google AI Studio cannot combine function calling with
+     * structured output. You MUST use `disableTools: true` when using schemas with
+     * Google providers.
+     *
+     * Error without disableTools: "Function calling with a response mime type:
+     * 'application/json' is unsupported"
+     *
+     * This is a documented Google API limitation, not a NeuroLink bug.
+     * All frameworks (LangChain, Vercel AI SDK, Agno, Instructor) use this approach.
+     *
+     * @example
+     * ```typescript
+     * // ✅ Correct for Google providers
+     * const result = await neurolink.generate({
+     *   schema: MySchema,
+     *   provider: "vertex",
+     *   disableTools: true  // Required for Google
+     * });
+     *
+     * // ✅ No restriction for other providers
+     * const result = await neurolink.generate({
+     *   schema: MySchema,
+     *   provider: "openai"  // Works without disableTools
+     * });
+     * ```
+     *
+     * @see https://ai.google.dev/gemini-api/docs/function-calling
+     */
     schema?: ValidationSchema;
     tools?: Record<string, Tool>;
     timeout?: number | string;
+    /**
+     * Disable tool execution (including built-in tools)
+     *
+     * @required For Google Gemini providers when using schemas
+     * Google Vertex AI and Google AI Studio require this flag when using
+     * structured output (schemas) due to Google API limitations.
+     *
+     * @example
+     * ```typescript
+     * // Required for Google providers with schemas
+     * await neurolink.generate({
+     *   schema: MySchema,
+     *   provider: "vertex",
+     *   disableTools: true
+     * });
+     * ```
+     */
     disableTools?: boolean;
     enableEvaluation?: boolean;
     enableAnalytics?: boolean;

package/dist/types/multimodal.d.ts

@@ -52,6 +52,8 @@ export type TextContent = {
 export type ImageContent = {
     type: "image";
     data: Buffer | string;
+    /** Alternative text for accessibility (screen readers, SEO) */
+    altText?: string;
     mediaType?: "image/jpeg" | "image/png" | "image/gif" | "image/webp" | "image/bmp" | "image/tiff";
     metadata?: {
         description?: string;
@@ -164,13 +166,48 @@ export type VideoContent = {
  * Covers text, images, documents, and multimedia
  */
 export type Content = TextContent | ImageContent | CSVContent | PDFContent | AudioContent | VideoContent;
+/**
+ * Image data with optional alt text for accessibility
+ * Use this when you need to provide alt text for screen readers and SEO
+ *
+ * @example
+ * ```typescript
+ * const imageWithAlt: ImageWithAltText = {
+ *   data: imageBuffer,
+ *   altText: "A dashboard showing quarterly sales trends"
+ * };
+ * ```
+ */
+export type ImageWithAltText = {
+    /** Image data as Buffer, base64 string, URL, or data URI */
+    data: Buffer | string;
+    /** Alternative text for accessibility (screen readers, SEO) */
+    altText?: string;
+};
 /**
  * Multimodal input type for options that may contain images or content arrays
  * This is the primary interface for users to provide multimodal content
  */
 export type MultimodalInput = {
     text: string;
-    images?: Array<Buffer | string>;
+    /**
+     * Images to include in the request.
+     * Can be simple image data (Buffer, string) or objects with alt text for accessibility.
+     *
+     * @example Simple usage
+     * ```typescript
+     * images: [imageBuffer, "https://example.com/image.jpg"]
+     * ```
+     *
+     * @example With alt text for accessibility
+     * ```typescript
+     * images: [
+     *   { data: imageBuffer, altText: "Product screenshot showing main dashboard" },
+     *   { data: "https://example.com/chart.png", altText: "Sales chart for Q3 2024" }
+     * ]
+     * ```
+     */
+    images?: Array<Buffer | string | ImageWithAltText>;
     content?: Content[];
     csvFiles?: Array<Buffer | string>;
     pdfFiles?: Array<Buffer | string>;

package/dist/types/streamTypes.d.ts

@@ -1,7 +1,7 @@
 import type { Tool } from "ai";
 import type { ValidationSchema, StandardRecord } from "./typeAliases.js";
 import type { AIModelProviderConfig } from "./providers.js";
-import type { Content } from "./content.js";
+import type { Content, ImageWithAltText } from "./content.js";
 import type { AnalyticsData, ToolExecutionEvent, ToolExecutionSummary } from "../types/index.js";
 import { AIProviderName } from "../constants/enums.js";
 import type { TokenUsage } from "./analytics.js";
@@ -125,7 +125,24 @@ export type StreamOptions = {
     input: {
         text: string;
         audio?: AudioInputSpec;
-        images?: Array<Buffer | string>;
+        /**
+         * Images to include in the request.
+         * Supports simple image data (Buffer, string) or objects with alt text for accessibility.
+         *
+         * @example Simple usage
+         * ```typescript
+         * images: [imageBuffer, "https://example.com/image.jpg"]
+         * ```
+         *
+         * @example With alt text for accessibility
+         * ```typescript
+         * images: [
+         *   { data: imageBuffer, altText: "Product screenshot showing main dashboard" },
+         *   { data: "https://example.com/chart.png", altText: "Sales chart for Q3 2024" }
+         * ]
+         * ```
+         */
+        images?: Array<Buffer | string | ImageWithAltText>;
         csvFiles?: Array<Buffer | string>;
         pdfFiles?: Array<Buffer | string>;
         files?: Array<Buffer | string>;
@@ -211,6 +228,8 @@ export type StreamResult = {
         totalToolExecutions?: number;
         toolExecutionTime?: number;
         hasToolErrors?: boolean;
+        guardrailsBlocked?: boolean;
+        error?: string;
     };
     analytics?: AnalyticsData | Promise<AnalyticsData>;
     evaluation?: EvaluationData | Promise<EvaluationData>;

package/dist/utils/messageBuilder.js

@@ -3,13 +3,37 @@
  * Centralized logic for building message arrays from TextGenerationOptions
  * Enhanced with multimodal support for images
  */
-import { CONVERSATION_INSTRUCTIONS } from "../config/conversationMemory.js";
+import { CONVERSATION_INSTRUCTIONS, STRUCTURED_OUTPUT_INSTRUCTIONS, } from "../config/conversationMemory.js";
 import { ProviderImageAdapter, MultimodalLogger, } from "../adapters/providerImageAdapter.js";
 import { logger } from "./logger.js";
 import { FileDetector } from "./fileDetector.js";
 import { PDFProcessor } from "./pdfProcessor.js";
 import { request, getGlobalDispatcher, interceptors } from "undici";
 import { readFileSync, existsSync } from "fs";
+/**
+ * Type guard to check if an image input has alt text
+ */
+function isImageWithAltText(image) {
+    return (typeof image === "object" && !Buffer.isBuffer(image) && "data" in image);
+}
+/**
+ * Extract image data from an image input (handles both simple and alt text formats)
+ */
+function extractImageData(image) {
+    if (isImageWithAltText(image)) {
+        return image.data;
+    }
+    return image;
+}
+/**
+ * Extract alt text from an image input if available
+ */
+function extractAltText(image) {
+    if (isImageWithAltText(image)) {
+        return image.altText;
+    }
+    return undefined;
+}
 /**
  * Type guard for validating message roles
  */
@@ -199,6 +223,15 @@ function formatCSVMetadata(metadata) {
     }
     return parts.length > 0 ? `**Metadata**: ${parts.join(" | ")}` : "";
 }
+/**
+ * Check if structured output mode should be enabled
+ * Structured output is used when a schema is provided with json/structured format
+ */
+function shouldUseStructuredOutput(options) {
+    return (!!options.schema &&
+        (options.output?.format === "json" ||
+            options.output?.format === "structured"));
+}
 /**
  * Build a properly formatted message array for AI providers
  * Combines system prompt, conversation history, and current user prompt
@@ -215,6 +248,10 @@ export async function buildMessagesArray(options) {
     if (hasConversationHistory) {
         systemPrompt = `${systemPrompt.trim()}${CONVERSATION_INSTRUCTIONS}`;
     }
+    // Add structured output instructions when schema is provided with json/structured format
+    if (shouldUseStructuredOutput(options)) {
+        systemPrompt = `${systemPrompt.trim()}${STRUCTURED_OUTPUT_INSTRUCTIONS}`;
+    }
     // Add system message if we have one
     if (systemPrompt.trim()) {
         messages.push({
@@ -473,6 +510,10 @@ export async function buildMultimodalMessagesArray(options, provider, model) {
     if (hasConversationHistory) {
         systemPrompt = `${systemPrompt.trim()}${CONVERSATION_INSTRUCTIONS}`;
     }
+    // Add structured output instructions when schema is provided with json/structured format
+    if (shouldUseStructuredOutput(options)) {
+        systemPrompt = `${systemPrompt.trim()}${STRUCTURED_OUTPUT_INSTRUCTIONS}`;
+    }
     // Add file handling guidance when multimodal files are present
     const hasCSVFiles = (options.input.csvFiles && options.input.csvFiles.length > 0) ||
         (options.input.files &&
@@ -622,28 +663,47 @@ async function downloadImageFromUrl(url) {
  * - URLs: Downloaded and converted to base64 for Vercel AI SDK compatibility
  * - Local files: Converted to base64 for Vercel AI SDK compatibility
  * - Buffers/Data URIs: Processed normally
+ * - Supports alt text for accessibility (included as context in text parts)
  */
 async function convertSimpleImagesToProviderFormat(text, images, provider, _model) {
     // For Vercel AI SDK, we need to return the content in the standard format
     // The Vercel AI SDK will handle provider-specific formatting internally
+    // IMPORTANT: Generate alt text descriptions BEFORE URL downloading to maintain correct image numbering
+    // This ensures image numbers match the original order provided by users, even if some URLs fail to download
+    const altTextDescriptions = images
+        .map((image, idx) => {
+        const altText = extractAltText(image);
+        return altText ? `[Image ${idx + 1}: ${altText}]` : null;
+    })
+        .filter(Boolean);
+    // Build enhanced text with alt text context for accessibility
+    // NOTE: Alt text is appended to the user's prompt as contextual information because most AI providers
+    // don't have native alt text fields in their APIs. This approach ensures accessibility metadata
+    // is preserved and helps AI models better understand image content.
+    const enhancedText = altTextDescriptions.length > 0
+        ? `${text}\n\nImage descriptions for context: ${altTextDescriptions.join(" ")}`
+        : text;
     // Smart auto-detection: separate URLs from actual image data
+    // Also track alt text for each image
     const urlImages = [];
     const actualImages = [];
     images.forEach((image, _index) => {
-        if (typeof image === "string" && isInternetUrl(image)) {
+        const imageData = extractImageData(image);
+        const altText = extractAltText(image);
+        if (typeof imageData === "string" && isInternetUrl(imageData)) {
             // Internet URL - will be downloaded and converted to base64
-            urlImages.push(image);
+            urlImages.push({ url: imageData, altText });
         }
         else {
             // Actual image data (file path, Buffer, data URI) - process for Vercel AI SDK
-            actualImages.push(image);
+            actualImages.push({ data: imageData, altText });
         }
     });
     // Download URL images and add to actual images
-    for (const url of urlImages) {
+    for (const { url, altText } of urlImages) {
         try {
             const downloadedDataUri = await downloadImageFromUrl(url);
-            actualImages.push(downloadedDataUri);
+            actualImages.push({ data: downloadedDataUri, altText });
         }
         catch (error) {
             MultimodalLogger.logError("URL_DOWNLOAD_FAILED_SKIPPING", error, { url });
@@ -651,9 +711,11 @@ async function convertSimpleImagesToProviderFormat(text, images, provider, _mode
             logger.warn(`Failed to download image from ${url}, skipping: ${error instanceof Error ? error.message : String(error)}`);
         }
     }
-    const content = [{ type: "text", text }];
+    const content = [
+        { type: "text", text: enhancedText },
+    ];
     // Process all images (including downloaded URLs) for Vercel AI SDK
-    actualImages.forEach((image, index) => {
+    actualImages.forEach(({ data: image }, index) => {
         try {
             // Vercel AI SDK expects { type: 'image', image: Buffer | string, mimeType?: string }
             // For Vertex AI, we need to include mimeType

package/dist/utils/multimodalOptionsBuilder.d.ts

@@ -44,7 +44,7 @@ import type { StreamOptions } from "../types/streamTypes.js";
 export declare function buildMultimodalOptions(options: StreamOptions, providerName: string, modelName: string): {
     input: {
         text: string;
-        images: (string | Buffer<ArrayBufferLike>)[] | undefined;
+        images: (string | Buffer<ArrayBufferLike> | import("../types/multimodal.js").ImageWithAltText)[] | undefined;
         content: import("../types/multimodal.js").Content[] | undefined;
         files: (string | Buffer<ArrayBufferLike>)[] | undefined;
         csvFiles: (string | Buffer<ArrayBufferLike>)[] | undefined;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "8.5.0",
+  "version": "8.6.0",
   "description": "Universal AI Development Platform with working MCP integration, multi-provider support, and professional CLI. Built-in tools operational, 58+ external MCP servers discoverable. Connect to filesystem, GitHub, database operations, and more. Build, test, and deploy AI applications with 9 major providers: OpenAI, Anthropic, Google AI, AWS Bedrock, Azure, Hugging Face, Ollama, and Mistral AI.",
   "author": {
     "name": "Juspay Technologies",