@juspay/neurolink 8.5.1 → 8.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/CHANGELOG.md +30 -0
  2. package/dist/adapters/providerImageAdapter.d.ts +4 -2
  3. package/dist/adapters/providerImageAdapter.js +16 -2
  4. package/dist/cli/factories/commandFactory.d.ts +5 -0
  5. package/dist/cli/factories/commandFactory.js +96 -0
  6. package/dist/cli/utils/audioFileUtils.d.ts +70 -0
  7. package/dist/cli/utils/audioFileUtils.js +174 -0
  8. package/dist/core/baseProvider.js +6 -2
  9. package/dist/core/modules/TelemetryHandler.js +6 -1
  10. package/dist/lib/adapters/providerImageAdapter.d.ts +4 -2
  11. package/dist/lib/adapters/providerImageAdapter.js +16 -2
  12. package/dist/lib/core/baseProvider.js +6 -2
  13. package/dist/lib/core/modules/TelemetryHandler.js +6 -1
  14. package/dist/lib/middleware/builtin/guardrails.js +7 -0
  15. package/dist/lib/neurolink.js +75 -5
  16. package/dist/lib/telemetry/telemetryService.d.ts +1 -1
  17. package/dist/lib/telemetry/telemetryService.js +4 -4
  18. package/dist/lib/types/cli.d.ts +2 -0
  19. package/dist/lib/types/common.d.ts +5 -0
  20. package/dist/lib/types/content.d.ts +1 -1
  21. package/dist/lib/types/fileTypes.d.ts +13 -12
  22. package/dist/lib/types/generateTypes.d.ts +19 -2
  23. package/dist/lib/types/index.d.ts +1 -0
  24. package/dist/lib/types/index.js +2 -0
  25. package/dist/lib/types/multimodal.d.ts +38 -1
  26. package/dist/lib/types/streamTypes.d.ts +21 -2
  27. package/dist/lib/types/ttsTypes.d.ts +91 -0
  28. package/dist/lib/types/ttsTypes.js +58 -0
  29. package/dist/lib/utils/imageProcessor.d.ts +38 -5
  30. package/dist/lib/utils/imageProcessor.js +131 -7
  31. package/dist/lib/utils/messageBuilder.js +52 -7
  32. package/dist/lib/utils/multimodalOptionsBuilder.d.ts +1 -1
  33. package/dist/lib/utils/pdfProcessor.js +24 -2
  34. package/dist/middleware/builtin/guardrails.js +7 -0
  35. package/dist/neurolink.js +75 -5
  36. package/dist/telemetry/telemetryService.d.ts +1 -1
  37. package/dist/telemetry/telemetryService.js +4 -4
  38. package/dist/types/cli.d.ts +2 -0
  39. package/dist/types/common.d.ts +5 -0
  40. package/dist/types/content.d.ts +1 -1
  41. package/dist/types/fileTypes.d.ts +13 -12
  42. package/dist/types/generateTypes.d.ts +19 -2
  43. package/dist/types/index.d.ts +1 -0
  44. package/dist/types/index.js +2 -0
  45. package/dist/types/multimodal.d.ts +38 -1
  46. package/dist/types/streamTypes.d.ts +21 -2
  47. package/dist/types/ttsTypes.d.ts +91 -0
  48. package/dist/types/ttsTypes.js +57 -0
  49. package/dist/utils/imageProcessor.d.ts +38 -5
  50. package/dist/utils/imageProcessor.js +131 -7
  51. package/dist/utils/messageBuilder.js +52 -7
  52. package/dist/utils/multimodalOptionsBuilder.d.ts +1 -1
  53. package/dist/utils/pdfProcessor.js +24 -2
  54. package/package.json +7 -4
@@ -52,6 +52,8 @@ export type TextContent = {
52
52
  export type ImageContent = {
53
53
  type: "image";
54
54
  data: Buffer | string;
55
+ /** Alternative text for accessibility (screen readers, SEO) */
56
+ altText?: string;
55
57
  mediaType?: "image/jpeg" | "image/png" | "image/gif" | "image/webp" | "image/bmp" | "image/tiff";
56
58
  metadata?: {
57
59
  description?: string;
@@ -164,13 +166,48 @@ export type VideoContent = {
164
166
  * Covers text, images, documents, and multimedia
165
167
  */
166
168
  export type Content = TextContent | ImageContent | CSVContent | PDFContent | AudioContent | VideoContent;
169
+ /**
170
+ * Image data with optional alt text for accessibility
171
+ * Use this when you need to provide alt text for screen readers and SEO
172
+ *
173
+ * @example
174
+ * ```typescript
175
+ * const imageWithAlt: ImageWithAltText = {
176
+ * data: imageBuffer,
177
+ * altText: "A dashboard showing quarterly sales trends"
178
+ * };
179
+ * ```
180
+ */
181
+ export type ImageWithAltText = {
182
+ /** Image data as Buffer, base64 string, URL, or data URI */
183
+ data: Buffer | string;
184
+ /** Alternative text for accessibility (screen readers, SEO) */
185
+ altText?: string;
186
+ };
167
187
  /**
168
188
  * Multimodal input type for options that may contain images or content arrays
169
189
  * This is the primary interface for users to provide multimodal content
170
190
  */
171
191
  export type MultimodalInput = {
172
192
  text: string;
173
- images?: Array<Buffer | string>;
193
+ /**
194
+ * Images to include in the request.
195
+ * Can be simple image data (Buffer, string) or objects with alt text for accessibility.
196
+ *
197
+ * @example Simple usage
198
+ * ```typescript
199
+ * images: [imageBuffer, "https://example.com/image.jpg"]
200
+ * ```
201
+ *
202
+ * @example With alt text for accessibility
203
+ * ```typescript
204
+ * images: [
205
+ * { data: imageBuffer, altText: "Product screenshot showing main dashboard" },
206
+ * { data: "https://example.com/chart.png", altText: "Sales chart for Q3 2024" }
207
+ * ]
208
+ * ```
209
+ */
210
+ images?: Array<Buffer | string | ImageWithAltText>;
174
211
  content?: Content[];
175
212
  csvFiles?: Array<Buffer | string>;
176
213
  pdfFiles?: Array<Buffer | string>;
@@ -1,7 +1,7 @@
1
1
  import type { Tool } from "ai";
2
2
  import type { ValidationSchema, StandardRecord } from "./typeAliases.js";
3
3
  import type { AIModelProviderConfig } from "./providers.js";
4
- import type { Content } from "./content.js";
4
+ import type { Content, ImageWithAltText } from "./content.js";
5
5
  import type { AnalyticsData, ToolExecutionEvent, ToolExecutionSummary } from "../types/index.js";
6
6
  import { AIProviderName } from "../constants/enums.js";
7
7
  import type { TokenUsage } from "./analytics.js";
@@ -125,7 +125,24 @@ export type StreamOptions = {
125
125
  input: {
126
126
  text: string;
127
127
  audio?: AudioInputSpec;
128
- images?: Array<Buffer | string>;
128
+ /**
129
+ * Images to include in the request.
130
+ * Supports simple image data (Buffer, string) or objects with alt text for accessibility.
131
+ *
132
+ * @example Simple usage
133
+ * ```typescript
134
+ * images: [imageBuffer, "https://example.com/image.jpg"]
135
+ * ```
136
+ *
137
+ * @example With alt text for accessibility
138
+ * ```typescript
139
+ * images: [
140
+ * { data: imageBuffer, altText: "Product screenshot showing main dashboard" },
141
+ * { data: "https://example.com/chart.png", altText: "Sales chart for Q3 2024" }
142
+ * ]
143
+ * ```
144
+ */
145
+ images?: Array<Buffer | string | ImageWithAltText>;
129
146
  csvFiles?: Array<Buffer | string>;
130
147
  pdfFiles?: Array<Buffer | string>;
131
148
  files?: Array<Buffer | string>;
@@ -211,6 +228,8 @@ export type StreamResult = {
211
228
  totalToolExecutions?: number;
212
229
  toolExecutionTime?: number;
213
230
  hasToolErrors?: boolean;
231
+ guardrailsBlocked?: boolean;
232
+ error?: string;
214
233
  };
215
234
  analytics?: AnalyticsData | Promise<AnalyticsData>;
216
235
  evaluation?: EvaluationData | Promise<EvaluationData>;
@@ -0,0 +1,91 @@
1
+ /**
2
+ * Text-to-Speech (TTS) Type Definitions for NeuroLink
3
+ *
4
+ * This module defines types for TTS audio generation and output.
5
+ *
6
+ * @module types/ttsTypes
7
+ */
8
+ /**
9
+ * Supported audio formats for TTS output
10
+ */
11
+ export type AudioFormat = "mp3" | "wav" | "ogg" | "opus";
12
+ /**
13
+ * TTS quality settings
14
+ */
15
+ export type TTSQuality = "standard" | "hd";
16
+ /**
17
+ * TTS configuration options
18
+ */
19
+ export type TTSOptions = {
20
+ /** Enable TTS output */
21
+ enabled?: boolean;
22
+ /** Voice identifier (e.g., "en-US-Neural2-C") */
23
+ voice?: string;
24
+ /** Audio format (default: mp3) */
25
+ format?: AudioFormat;
26
+ /** Speaking rate 0.25-4.0 (default: 1.0) */
27
+ speed?: number;
28
+ /** Audio quality (default: standard) */
29
+ quality?: TTSQuality;
30
+ /** Output file path (optional) */
31
+ output?: string;
32
+ /** Auto-play audio after generation (default: false) */
33
+ play?: boolean;
34
+ };
35
+ /**
36
+ * TTS audio result returned from generation
37
+ */
38
+ export type TTSResult = {
39
+ /** Audio data as Buffer */
40
+ buffer: Buffer;
41
+ /** Audio format */
42
+ format: AudioFormat;
43
+ /** Audio file size in bytes */
44
+ size: number;
45
+ /** Duration in seconds (if available) */
46
+ duration?: number;
47
+ /** Voice used for generation */
48
+ voice?: string;
49
+ /** Sample rate in Hz */
50
+ sampleRate?: number;
51
+ };
52
+ /**
53
+ * Result of saving audio to file
54
+ */
55
+ export type AudioSaveResult = {
56
+ /** Whether the save was successful */
57
+ success: boolean;
58
+ /** Full path to the saved file */
59
+ path: string;
60
+ /** File size in bytes */
61
+ size: number;
62
+ /** Error message if failed */
63
+ error?: string;
64
+ };
65
+ /**
66
+ * TTS voice information
67
+ */
68
+ export type TTSVoice = {
69
+ /** Voice identifier */
70
+ id: string;
71
+ /** Display name */
72
+ name: string;
73
+ /** Language code (e.g., "en-US") */
74
+ languageCode: string;
75
+ /** Gender */
76
+ gender: "male" | "female" | "neutral";
77
+ /** Voice type */
78
+ type: "neural" | "wavenet" | "standard";
79
+ };
80
+ /** Valid audio formats as an array for runtime validation */
81
+ export declare const VALID_AUDIO_FORMATS: readonly AudioFormat[];
82
+ /** Valid TTS quality levels as an array for runtime validation */
83
+ export declare const VALID_TTS_QUALITIES: readonly TTSQuality[];
84
+ /**
85
+ * Type guard to check if an object is a TTSResult
86
+ */
87
+ export declare function isTTSResult(value: unknown): value is TTSResult;
88
+ /**
89
+ * Type guard to check if TTSOptions are valid
90
+ */
91
+ export declare function isValidTTSOptions(options: unknown): options is TTSOptions;
@@ -0,0 +1,57 @@
1
+ /**
2
+ * Text-to-Speech (TTS) Type Definitions for NeuroLink
3
+ *
4
+ * This module defines types for TTS audio generation and output.
5
+ *
6
+ * @module types/ttsTypes
7
+ */
8
+ /** Valid audio formats as an array for runtime validation */
9
+ export const VALID_AUDIO_FORMATS = [
10
+ "mp3",
11
+ "wav",
12
+ "ogg",
13
+ "opus",
14
+ ];
15
+ /** Valid TTS quality levels as an array for runtime validation */
16
+ export const VALID_TTS_QUALITIES = ["standard", "hd"];
17
+ /**
18
+ * Type guard to check if an object is a TTSResult
19
+ */
20
+ export function isTTSResult(value) {
21
+ if (!value || typeof value !== "object") {
22
+ return false;
23
+ }
24
+ const obj = value;
25
+ return (Buffer.isBuffer(obj.buffer) &&
26
+ typeof obj.format === "string" &&
27
+ VALID_AUDIO_FORMATS.includes(obj.format) &&
28
+ typeof obj.size === "number" &&
29
+ obj.size >= 0);
30
+ }
31
+ /**
32
+ * Type guard to check if TTSOptions are valid
33
+ */
34
+ export function isValidTTSOptions(options) {
35
+ if (!options || typeof options !== "object") {
36
+ return false;
37
+ }
38
+ const opts = options;
39
+ if (opts.speed !== undefined) {
40
+ if (typeof opts.speed !== "number" ||
41
+ opts.speed < 0.25 ||
42
+ opts.speed > 4.0) {
43
+ return false;
44
+ }
45
+ }
46
+ if (opts.format !== undefined) {
47
+ if (!VALID_AUDIO_FORMATS.includes(opts.format)) {
48
+ return false;
49
+ }
50
+ }
51
+ if (opts.quality !== undefined) {
52
+ if (!VALID_TTS_QUALITIES.includes(opts.quality)) {
53
+ return false;
54
+ }
55
+ }
56
+ return true;
57
+ }
@@ -17,6 +17,18 @@ export declare class ImageProcessor {
17
17
  * @returns Processed image as data URI
18
18
  */
19
19
  static process(content: Buffer, _options?: unknown): Promise<FileProcessingResult>;
20
+ /**
21
+ * Validate processed output meets required format
22
+ * Checks:
23
+ * - Base64 content is non-empty
24
+ * - Data URI format is valid (data:{mimeType};base64,{content})
25
+ * - MIME type is in the allowed list
26
+ * @param dataUri - The complete data URI string
27
+ * @param base64 - The base64-encoded content
28
+ * @param mediaType - The MIME type of the image
29
+ * @throws Error if any validation fails
30
+ */
31
+ private static validateProcessOutput;
20
32
  /**
21
33
  * Process image for OpenAI (requires data URI format)
22
34
  */
@@ -104,11 +116,32 @@ export declare const imageUtils: {
104
116
  */
105
117
  fileToBase64DataUri: (filePath: string, maxBytes?: number) => Promise<string>;
106
118
  /**
107
- * Convert URL to base64 data URI by downloading the image
108
- */
109
- urlToBase64DataUri: (url: string, { timeoutMs, maxBytes }?: {
110
- timeoutMs?: number | undefined;
111
- maxBytes?: number | undefined;
119
+ * Convert URL to base64 data URI by downloading the image.
120
+ * Implements retry logic with exponential backoff for network errors.
121
+ *
122
+ * Retries are performed for:
123
+ * - Network errors (ECONNRESET, ENOTFOUND, ECONNREFUSED, ETIMEDOUT, ERR_NETWORK, AbortError)
124
+ * - Server errors (5xx status codes)
125
+ * - Rate limiting (429 Too Many Requests)
126
+ * - Request timeouts (408 Request Timeout)
127
+ *
128
+ * Retries are NOT performed for:
129
+ * - Client errors (4xx status codes except 408, 429)
130
+ * - Invalid content type
131
+ * - Content size limit exceeded
132
+ * - Unsupported protocol
133
+ *
134
+ * @param url - The URL of the image to download
135
+ * @param options - Configuration options
136
+ * @param options.timeoutMs - Timeout for each download attempt (default: 15000ms)
137
+ * @param options.maxBytes - Maximum allowed file size (default: 10MB)
138
+ * @param options.maxAttempts - Maximum number of total attempts including initial attempt (default: 3)
139
+ * @returns Promise<string> - Base64 data URI of the downloaded image
140
+ */
141
+ urlToBase64DataUri: (url: string, { timeoutMs, maxBytes, maxAttempts, }?: {
142
+ timeoutMs?: number;
143
+ maxBytes?: number;
144
+ maxAttempts?: number;
112
145
  }) => Promise<string>;
113
146
  /**
114
147
  * Extract base64 data from data URI
@@ -3,6 +3,57 @@
3
3
  * Handles format conversion for different AI providers
4
4
  */
5
5
  import { logger } from "./logger.js";
6
+ import { withRetry } from "./retryHandler.js";
7
+ import { SYSTEM_LIMITS } from "../core/constants.js";
8
+ /**
9
+ * Network error codes that should trigger a retry
10
+ */
11
+ const RETRYABLE_ERROR_CODES = new Set([
12
+ "ECONNRESET",
13
+ "ENOTFOUND",
14
+ "ECONNREFUSED",
15
+ "ETIMEDOUT",
16
+ "ERR_NETWORK",
17
+ ]);
18
+ /**
19
+ * Determines if an HTTP error is retryable based on status code
20
+ * Only network errors and certain HTTP status codes should be retried
21
+ * 4xx client errors like 404 (Not Found) and 403 (Forbidden) should NOT be retried
22
+ *
23
+ * @param error - The error to check
24
+ * @returns true if the error is retryable, false otherwise
25
+ */
26
+ function isRetryableDownloadError(error) {
27
+ // Network-related errors should be retried
28
+ if (error && typeof error === "object") {
29
+ const errorCode = error.code;
30
+ const errorName = error.name;
31
+ if (RETRYABLE_ERROR_CODES.has(errorCode || "") ||
32
+ errorName === "AbortError") {
33
+ return true;
34
+ }
35
+ }
36
+ // Check for HTTP status code in error message for retryable errors
37
+ // Only retry on 5xx server errors, 429 (Too Many Requests), and 408 (Request Timeout)
38
+ // Do NOT retry on 4xx client errors like 404 (Not Found) or 403 (Forbidden)
39
+ if (error instanceof Error) {
40
+ const message = error.message;
41
+ // Extract HTTP status from error message like "HTTP 503: Service Unavailable"
42
+ const statusMatch = message.match(/HTTP (\d{3}):/);
43
+ if (statusMatch) {
44
+ const status = parseInt(statusMatch[1], 10);
45
+ // Retry on 5xx server errors, 429 (rate limit), 408 (timeout)
46
+ return status >= 500 || status === 429 || status === 408;
47
+ }
48
+ // Check for timeout/network-related error messages
49
+ // Use more precise matching to avoid false positives like "No timeout specified"
50
+ if (/\b(request timed out|operation timed out|connection timed out|timed out)\b/i.test(message) ||
51
+ /\bnetwork (error|failure|unreachable|down)\b/i.test(message)) {
52
+ return true;
53
+ }
54
+ }
55
+ return false;
56
+ }
6
57
  /**
7
58
  * Image processor class for handling provider-specific image formatting
8
59
  */
@@ -16,9 +67,16 @@ export class ImageProcessor {
16
67
  * @returns Processed image as data URI
17
68
  */
18
69
  static async process(content, _options) {
70
+ // Validate content is non-empty before processing
71
+ if (content.length === 0) {
72
+ logger.error("Empty buffer provided");
73
+ throw new Error("Invalid image processing: buffer is empty");
74
+ }
19
75
  const mediaType = this.detectImageType(content);
20
76
  const base64 = content.toString("base64");
21
77
  const dataUri = `data:${mediaType};base64,${base64}`;
78
+ // Validate output before returning
79
+ this.validateProcessOutput(dataUri, base64, mediaType);
22
80
  return {
23
81
  type: "image",
24
82
  content: dataUri,
@@ -29,6 +87,37 @@ export class ImageProcessor {
29
87
  },
30
88
  };
31
89
  }
90
+ /**
91
+ * Validate processed output meets required format
92
+ * Checks:
93
+ * - Base64 content is non-empty
94
+ * - Data URI format is valid (data:{mimeType};base64,{content})
95
+ * - MIME type is in the allowed list
96
+ * @param dataUri - The complete data URI string
97
+ * @param base64 - The base64-encoded content
98
+ * @param mediaType - The MIME type of the image
99
+ * @throws Error if any validation fails
100
+ */
101
+ static validateProcessOutput(dataUri, base64, mediaType) {
102
+ // Validate base64 is non-empty (check first for better error message)
103
+ if (base64.length === 0) {
104
+ logger.error("Empty base64 content generated");
105
+ throw new Error("Invalid image processing: base64 content is empty");
106
+ }
107
+ // Validate data URI format with proper base64 character validation
108
+ // Base64 can only have 0, 1, or 2 padding characters at the end
109
+ const dataUriRegex = /^data:[^;]+;base64,[A-Za-z0-9+/]*={0,2}$/;
110
+ if (!dataUriRegex.test(dataUri)) {
111
+ logger.error("Invalid data URI format generated", { dataUri });
112
+ throw new Error("Invalid data URI format: must be data:{mimeType};base64,{content}");
113
+ }
114
+ // Defensive check: ensure detectImageType() returns valid MIME type
115
+ // This validation protects against future changes to detectImageType()
116
+ if (!this.validateImageFormat(mediaType)) {
117
+ logger.error("Invalid MIME type generated", { mediaType });
118
+ throw new Error(`Invalid MIME type: ${mediaType} is not in allowed list`);
119
+ }
120
+ }
32
121
  /**
33
122
  * Process image for OpenAI (requires data URI format)
34
123
  */
@@ -434,14 +523,35 @@ export const imageUtils = {
434
523
  }
435
524
  },
436
525
  /**
437
- * Convert URL to base64 data URI by downloading the image
526
+ * Convert URL to base64 data URI by downloading the image.
527
+ * Implements retry logic with exponential backoff for network errors.
528
+ *
529
+ * Retries are performed for:
530
+ * - Network errors (ECONNRESET, ENOTFOUND, ECONNREFUSED, ETIMEDOUT, ERR_NETWORK, AbortError)
531
+ * - Server errors (5xx status codes)
532
+ * - Rate limiting (429 Too Many Requests)
533
+ * - Request timeouts (408 Request Timeout)
534
+ *
535
+ * Retries are NOT performed for:
536
+ * - Client errors (4xx status codes except 408, 429)
537
+ * - Invalid content type
538
+ * - Content size limit exceeded
539
+ * - Unsupported protocol
540
+ *
541
+ * @param url - The URL of the image to download
542
+ * @param options - Configuration options
543
+ * @param options.timeoutMs - Timeout for each download attempt (default: 15000ms)
544
+ * @param options.maxBytes - Maximum allowed file size (default: 10MB)
545
+ * @param options.maxAttempts - Maximum number of total attempts including initial attempt (default: 3)
546
+ * @returns Promise<string> - Base64 data URI of the downloaded image
438
547
  */
439
- urlToBase64DataUri: async (url, { timeoutMs = 15000, maxBytes = 10 * 1024 * 1024 } = {}) => {
440
- try {
441
- // Basic protocol whitelist
442
- if (!/^https?:\/\//i.test(url)) {
443
- throw new Error("Unsupported protocol");
444
- }
548
+ urlToBase64DataUri: async (url, { timeoutMs = 15000, maxBytes = 10 * 1024 * 1024, maxAttempts = 3, } = {}) => {
549
+ // Basic protocol whitelist - fail fast, no retry needed
550
+ if (!/^https?:\/\//i.test(url)) {
551
+ throw new Error("Unsupported protocol");
552
+ }
553
+ // Perform the actual download with retry logic
554
+ const performDownload = async () => {
445
555
  const controller = new AbortController();
446
556
  const t = setTimeout(() => controller.abort(), timeoutMs);
447
557
  try {
@@ -467,6 +577,20 @@ export const imageUtils = {
467
577
  finally {
468
578
  clearTimeout(t);
469
579
  }
580
+ };
581
+ try {
582
+ return await withRetry(performDownload, {
583
+ maxAttempts,
584
+ initialDelay: SYSTEM_LIMITS.DEFAULT_INITIAL_DELAY,
585
+ backoffMultiplier: SYSTEM_LIMITS.DEFAULT_BACKOFF_MULTIPLIER,
586
+ maxDelay: SYSTEM_LIMITS.DEFAULT_MAX_DELAY,
587
+ retryCondition: isRetryableDownloadError,
588
+ onRetry: (attempt, error) => {
589
+ const message = error instanceof Error ? error.message : String(error);
590
+ const attemptsLeft = maxAttempts - attempt;
591
+ logger.warn(`⚠️ Image download attempt ${attempt} failed for ${url}: ${message}. ${attemptsLeft} ${attemptsLeft === 1 ? "attempt" : "attempts"} remaining...`);
592
+ },
593
+ });
470
594
  }
471
595
  catch (error) {
472
596
  throw new Error(`Failed to download and convert URL to base64: ${error instanceof Error ? error.message : "Unknown error"}`);
@@ -10,6 +10,30 @@ import { FileDetector } from "./fileDetector.js";
10
10
  import { PDFProcessor } from "./pdfProcessor.js";
11
11
  import { request, getGlobalDispatcher, interceptors } from "undici";
12
12
  import { readFileSync, existsSync } from "fs";
13
+ /**
14
+ * Type guard to check if an image input has alt text
15
+ */
16
+ function isImageWithAltText(image) {
17
+ return (typeof image === "object" && !Buffer.isBuffer(image) && "data" in image);
18
+ }
19
+ /**
20
+ * Extract image data from an image input (handles both simple and alt text formats)
21
+ */
22
+ function extractImageData(image) {
23
+ if (isImageWithAltText(image)) {
24
+ return image.data;
25
+ }
26
+ return image;
27
+ }
28
+ /**
29
+ * Extract alt text from an image input if available
30
+ */
31
+ function extractAltText(image) {
32
+ if (isImageWithAltText(image)) {
33
+ return image.altText;
34
+ }
35
+ return undefined;
36
+ }
13
37
  /**
14
38
  * Type guard for validating message roles
15
39
  */
@@ -639,28 +663,47 @@ async function downloadImageFromUrl(url) {
639
663
  * - URLs: Downloaded and converted to base64 for Vercel AI SDK compatibility
640
664
  * - Local files: Converted to base64 for Vercel AI SDK compatibility
641
665
  * - Buffers/Data URIs: Processed normally
666
+ * - Supports alt text for accessibility (included as context in text parts)
642
667
  */
643
668
  async function convertSimpleImagesToProviderFormat(text, images, provider, _model) {
644
669
  // For Vercel AI SDK, we need to return the content in the standard format
645
670
  // The Vercel AI SDK will handle provider-specific formatting internally
671
+ // IMPORTANT: Generate alt text descriptions BEFORE URL downloading to maintain correct image numbering
672
+ // This ensures image numbers match the original order provided by users, even if some URLs fail to download
673
+ const altTextDescriptions = images
674
+ .map((image, idx) => {
675
+ const altText = extractAltText(image);
676
+ return altText ? `[Image ${idx + 1}: ${altText}]` : null;
677
+ })
678
+ .filter(Boolean);
679
+ // Build enhanced text with alt text context for accessibility
680
+ // NOTE: Alt text is appended to the user's prompt as contextual information because most AI providers
681
+ // don't have native alt text fields in their APIs. This approach ensures accessibility metadata
682
+ // is preserved and helps AI models better understand image content.
683
+ const enhancedText = altTextDescriptions.length > 0
684
+ ? `${text}\n\nImage descriptions for context: ${altTextDescriptions.join(" ")}`
685
+ : text;
646
686
  // Smart auto-detection: separate URLs from actual image data
687
+ // Also track alt text for each image
647
688
  const urlImages = [];
648
689
  const actualImages = [];
649
690
  images.forEach((image, _index) => {
650
- if (typeof image === "string" && isInternetUrl(image)) {
691
+ const imageData = extractImageData(image);
692
+ const altText = extractAltText(image);
693
+ if (typeof imageData === "string" && isInternetUrl(imageData)) {
651
694
  // Internet URL - will be downloaded and converted to base64
652
- urlImages.push(image);
695
+ urlImages.push({ url: imageData, altText });
653
696
  }
654
697
  else {
655
698
  // Actual image data (file path, Buffer, data URI) - process for Vercel AI SDK
656
- actualImages.push(image);
699
+ actualImages.push({ data: imageData, altText });
657
700
  }
658
701
  });
659
702
  // Download URL images and add to actual images
660
- for (const url of urlImages) {
703
+ for (const { url, altText } of urlImages) {
661
704
  try {
662
705
  const downloadedDataUri = await downloadImageFromUrl(url);
663
- actualImages.push(downloadedDataUri);
706
+ actualImages.push({ data: downloadedDataUri, altText });
664
707
  }
665
708
  catch (error) {
666
709
  MultimodalLogger.logError("URL_DOWNLOAD_FAILED_SKIPPING", error, { url });
@@ -668,9 +711,11 @@ async function convertSimpleImagesToProviderFormat(text, images, provider, _mode
668
711
  logger.warn(`Failed to download image from ${url}, skipping: ${error instanceof Error ? error.message : String(error)}`);
669
712
  }
670
713
  }
671
- const content = [{ type: "text", text }];
714
+ const content = [
715
+ { type: "text", text: enhancedText },
716
+ ];
672
717
  // Process all images (including downloaded URLs) for Vercel AI SDK
673
- actualImages.forEach((image, index) => {
718
+ actualImages.forEach(({ data: image }, index) => {
674
719
  try {
675
720
  // Vercel AI SDK expects { type: 'image', image: Buffer | string, mimeType?: string }
676
721
  // For Vertex AI, we need to include mimeType
@@ -44,7 +44,7 @@ import type { StreamOptions } from "../types/streamTypes.js";
44
44
  export declare function buildMultimodalOptions(options: StreamOptions, providerName: string, modelName: string): {
45
45
  input: {
46
46
  text: string;
47
- images: (string | Buffer<ArrayBufferLike>)[] | undefined;
47
+ images: (string | Buffer<ArrayBufferLike> | import("../types/multimodal.js").ImageWithAltText)[] | undefined;
48
48
  content: import("../types/multimodal.js").Content[] | undefined;
49
49
  files: (string | Buffer<ArrayBufferLike>)[] | undefined;
50
50
  csvFiles: (string | Buffer<ArrayBufferLike>)[] | undefined;
@@ -1,6 +1,6 @@
1
1
  import { logger } from "./logger.js";
2
- import * as pdfjs from "pdfjs-dist/legacy/build/pdf.mjs";
3
- import { createCanvas } from "canvas";
2
+ // Lazy-load pdfjs-dist to avoid DOMMatrix errors in Node.js server environment
3
+ // import * as pdfjs from "pdfjs-dist/legacy/build/pdf.mjs";
4
4
  const PDF_PROVIDER_CONFIGS = {
5
5
  anthropic: {
6
6
  maxSizeMB: 5,
@@ -196,6 +196,28 @@ export class PDFProcessor {
196
196
  }
197
197
  }
198
198
  static async convertPDFToImages(pdfBuffer, options) {
199
+ // Dynamic import canvas - only load when actually needed
200
+ let createCanvas;
201
+ try {
202
+ const canvasModule = await import("canvas");
203
+ createCanvas = canvasModule.createCanvas;
204
+ }
205
+ catch {
206
+ throw new Error("Canvas dependency not available. " +
207
+ "PDF-to-image conversion requires the 'canvas' package with native bindings. " +
208
+ "Install with: pnpm install canvas\n" +
209
+ "Note: This requires native build tools (Python, C++ compiler).");
210
+ }
211
+ // Dynamic import pdfjs - only load when actually needed to avoid DOMMatrix errors
212
+ let pdfjs;
213
+ try {
214
+ pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs");
215
+ }
216
+ catch {
217
+ throw new Error("pdfjs-dist dependency not available. " +
218
+ "PDF processing requires the 'pdfjs-dist' package. " +
219
+ "Install with: pnpm install pdfjs-dist");
220
+ }
199
221
  const maxPages = options?.maxPages || 10;
200
222
  const scale = options?.scale || 2.0;
201
223
  const format = options?.format || "png";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@juspay/neurolink",
3
- "version": "8.5.1",
3
+ "version": "8.7.0",
4
4
  "description": "Universal AI Development Platform with working MCP integration, multi-provider support, and professional CLI. Built-in tools operational, 58+ external MCP servers discoverable. Connect to filesystem, GitHub, database operations, and more. Build, test, and deploy AI applications with 9 major providers: OpenAI, Anthropic, Google AI, AWS Bedrock, Azure, Hugging Face, Ollama, and Mistral AI.",
5
5
  "author": {
6
6
  "name": "Juspay Technologies",
@@ -179,7 +179,6 @@
179
179
  "@opentelemetry/sdk-trace-node": "^2.1.0",
180
180
  "@opentelemetry/semantic-conventions": "^1.30.1",
181
181
  "ai": "4.3.16",
182
- "canvas": "^3.2.0",
183
182
  "chalk": "^5.6.2",
184
183
  "csv-parser": "^3.2.0",
185
184
  "dotenv": "^16.6.1",
@@ -201,6 +200,9 @@
201
200
  "zod": "^3.22.0",
202
201
  "zod-to-json-schema": "^3.24.6"
203
202
  },
203
+ "optionalDependencies": {
204
+ "canvas": "^3.2.0"
205
+ },
204
206
  "devDependencies": {
205
207
  "@biomejs/biome": "^2.2.4",
206
208
  "@changesets/changelog-github": "^0.5.1",
@@ -210,7 +212,7 @@
210
212
  "@semantic-release/commit-analyzer": "^13.0.1",
211
213
  "@semantic-release/git": "^10.0.1",
212
214
  "@semantic-release/github": "^11.0.6",
213
- "@semantic-release/npm": "^12.0.2",
215
+ "@semantic-release/npm": "^13.1.2",
214
216
  "@semantic-release/release-notes-generator": "^14.1.0",
215
217
  "@smithy/types": "^4.5.0",
216
218
  "@sveltejs/adapter-auto": "^6.1.0",
@@ -297,7 +299,8 @@
297
299
  "@eslint/plugin-kit@<0.3.4": ">=0.3.4",
298
300
  "tmp@<=0.2.3": ">=0.2.4",
299
301
  "axios@<1.8.2": ">=1.8.2",
300
- "glob@>=10.3.7 <=11.0.3": ">=11.1.0"
302
+ "glob@>=10.3.7 <=11.0.3": ">=11.1.0",
303
+ "@semantic-release/npm": "^13.1.2"
301
304
  }
302
305
  },
303
306
  "os": [