@juspay/neurolink 9.10.0 → 9.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/dist/adapters/video/videoAnalyzer.d.ts +3 -3
  3. package/dist/adapters/video/videoAnalyzer.js +39 -25
  4. package/dist/agent/directTools.d.ts +3 -3
  5. package/dist/cli/commands/config.d.ts +9 -9
  6. package/dist/cli/loop/optionsSchema.d.ts +1 -1
  7. package/dist/constants/contextWindows.d.ts +6 -3
  8. package/dist/constants/contextWindows.js +30 -3
  9. package/dist/constants/index.d.ts +3 -3
  10. package/dist/constants/retry.d.ts +4 -4
  11. package/dist/constants/retry.js +1 -1
  12. package/dist/context/contextCompactor.d.ts +1 -1
  13. package/dist/context/contextCompactor.js +59 -1
  14. package/dist/context/summarizationEngine.d.ts +2 -2
  15. package/dist/context/summarizationEngine.js +44 -18
  16. package/dist/context/toolOutputLimits.d.ts +22 -13
  17. package/dist/context/toolOutputLimits.js +58 -64
  18. package/dist/core/baseProvider.d.ts +11 -2
  19. package/dist/core/baseProvider.js +30 -1
  20. package/dist/core/conversationMemoryManager.d.ts +13 -1
  21. package/dist/core/conversationMemoryManager.js +36 -5
  22. package/dist/core/modules/GenerationHandler.d.ts +6 -0
  23. package/dist/core/modules/GenerationHandler.js +192 -7
  24. package/dist/core/modules/MessageBuilder.js +42 -4
  25. package/dist/core/modules/TelemetryHandler.js +4 -1
  26. package/dist/core/redisConversationMemoryManager.d.ts +19 -3
  27. package/dist/core/redisConversationMemoryManager.js +253 -58
  28. package/dist/index.d.ts +2 -0
  29. package/dist/index.js +3 -0
  30. package/dist/lib/adapters/video/videoAnalyzer.d.ts +3 -3
  31. package/dist/lib/adapters/video/videoAnalyzer.js +39 -25
  32. package/dist/lib/agent/directTools.d.ts +7 -7
  33. package/dist/lib/constants/contextWindows.d.ts +6 -3
  34. package/dist/lib/constants/contextWindows.js +30 -3
  35. package/dist/lib/constants/index.d.ts +3 -3
  36. package/dist/lib/constants/retry.d.ts +4 -4
  37. package/dist/lib/constants/retry.js +1 -1
  38. package/dist/lib/context/contextCompactor.d.ts +1 -1
  39. package/dist/lib/context/contextCompactor.js +59 -1
  40. package/dist/lib/context/summarizationEngine.d.ts +2 -2
  41. package/dist/lib/context/summarizationEngine.js +44 -18
  42. package/dist/lib/context/toolOutputLimits.d.ts +22 -13
  43. package/dist/lib/context/toolOutputLimits.js +58 -64
  44. package/dist/lib/core/baseProvider.d.ts +11 -2
  45. package/dist/lib/core/baseProvider.js +30 -1
  46. package/dist/lib/core/conversationMemoryManager.d.ts +13 -1
  47. package/dist/lib/core/conversationMemoryManager.js +36 -5
  48. package/dist/lib/core/modules/GenerationHandler.d.ts +6 -0
  49. package/dist/lib/core/modules/GenerationHandler.js +192 -7
  50. package/dist/lib/core/modules/MessageBuilder.js +42 -4
  51. package/dist/lib/core/modules/TelemetryHandler.js +4 -1
  52. package/dist/lib/core/redisConversationMemoryManager.d.ts +19 -3
  53. package/dist/lib/core/redisConversationMemoryManager.js +253 -58
  54. package/dist/lib/files/fileTools.d.ts +3 -3
  55. package/dist/lib/index.d.ts +2 -0
  56. package/dist/lib/index.js +3 -0
  57. package/dist/lib/mcp/externalServerManager.js +36 -1
  58. package/dist/lib/memory/memoryRetrievalTools.d.ts +166 -0
  59. package/dist/lib/memory/memoryRetrievalTools.js +145 -0
  60. package/dist/lib/neurolink.d.ts +35 -1
  61. package/dist/lib/neurolink.js +471 -16
  62. package/dist/lib/providers/amazonBedrock.d.ts +1 -1
  63. package/dist/lib/providers/amazonBedrock.js +78 -45
  64. package/dist/lib/providers/amazonSagemaker.d.ts +1 -1
  65. package/dist/lib/providers/amazonSagemaker.js +1 -1
  66. package/dist/lib/providers/anthropic.d.ts +1 -1
  67. package/dist/lib/providers/anthropic.js +7 -7
  68. package/dist/lib/providers/anthropicBaseProvider.d.ts +1 -1
  69. package/dist/lib/providers/anthropicBaseProvider.js +7 -6
  70. package/dist/lib/providers/azureOpenai.d.ts +1 -1
  71. package/dist/lib/providers/azureOpenai.js +1 -1
  72. package/dist/lib/providers/googleAiStudio.d.ts +1 -1
  73. package/dist/lib/providers/googleAiStudio.js +5 -5
  74. package/dist/lib/providers/googleVertex.d.ts +1 -1
  75. package/dist/lib/providers/googleVertex.js +74 -17
  76. package/dist/lib/providers/huggingFace.d.ts +1 -1
  77. package/dist/lib/providers/huggingFace.js +1 -1
  78. package/dist/lib/providers/litellm.d.ts +1 -1
  79. package/dist/lib/providers/litellm.js +18 -16
  80. package/dist/lib/providers/mistral.d.ts +1 -1
  81. package/dist/lib/providers/mistral.js +1 -1
  82. package/dist/lib/providers/ollama.d.ts +1 -1
  83. package/dist/lib/providers/ollama.js +8 -7
  84. package/dist/lib/providers/openAI.d.ts +1 -1
  85. package/dist/lib/providers/openAI.js +6 -6
  86. package/dist/lib/providers/openRouter.d.ts +1 -1
  87. package/dist/lib/providers/openRouter.js +6 -2
  88. package/dist/lib/providers/openaiCompatible.d.ts +1 -1
  89. package/dist/lib/providers/openaiCompatible.js +1 -1
  90. package/dist/lib/proxy/proxyFetch.js +291 -65
  91. package/dist/lib/server/utils/validation.d.ts +4 -4
  92. package/dist/lib/services/server/ai/observability/instrumentation.js +12 -3
  93. package/dist/lib/telemetry/telemetryService.d.ts +2 -1
  94. package/dist/lib/telemetry/telemetryService.js +8 -1
  95. package/dist/lib/types/contextTypes.d.ts +26 -2
  96. package/dist/lib/types/conversation.d.ts +72 -40
  97. package/dist/lib/types/conversationMemoryInterface.d.ts +5 -1
  98. package/dist/lib/types/generateTypes.d.ts +26 -0
  99. package/dist/lib/types/modelTypes.d.ts +2 -2
  100. package/dist/lib/types/multimodal.d.ts +2 -0
  101. package/dist/lib/types/observability.d.ts +10 -0
  102. package/dist/lib/types/sdkTypes.d.ts +1 -1
  103. package/dist/lib/utils/conversationMemory.d.ts +4 -3
  104. package/dist/lib/utils/conversationMemory.js +44 -6
  105. package/dist/lib/utils/errorHandling.d.ts +5 -0
  106. package/dist/lib/utils/errorHandling.js +7 -2
  107. package/dist/lib/utils/logger.d.ts +8 -0
  108. package/dist/lib/utils/logger.js +56 -1
  109. package/dist/lib/utils/messageBuilder.js +74 -4
  110. package/dist/lib/utils/redis.js +6 -1
  111. package/dist/lib/utils/tokenEstimation.d.ts +2 -2
  112. package/dist/lib/utils/tokenEstimation.js +16 -1
  113. package/dist/lib/utils/videoAnalysisProcessor.d.ts +2 -1
  114. package/dist/lib/utils/videoAnalysisProcessor.js +7 -2
  115. package/dist/lib/workflow/config.d.ts +110 -110
  116. package/dist/mcp/externalServerManager.js +36 -1
  117. package/dist/memory/memoryRetrievalTools.d.ts +166 -0
  118. package/dist/memory/memoryRetrievalTools.js +144 -0
  119. package/dist/neurolink.d.ts +35 -1
  120. package/dist/neurolink.js +471 -16
  121. package/dist/providers/amazonBedrock.d.ts +1 -1
  122. package/dist/providers/amazonBedrock.js +78 -45
  123. package/dist/providers/amazonSagemaker.d.ts +1 -1
  124. package/dist/providers/amazonSagemaker.js +1 -1
  125. package/dist/providers/anthropic.d.ts +1 -1
  126. package/dist/providers/anthropic.js +7 -7
  127. package/dist/providers/anthropicBaseProvider.d.ts +1 -1
  128. package/dist/providers/anthropicBaseProvider.js +7 -6
  129. package/dist/providers/azureOpenai.d.ts +1 -1
  130. package/dist/providers/azureOpenai.js +1 -1
  131. package/dist/providers/googleAiStudio.d.ts +1 -1
  132. package/dist/providers/googleAiStudio.js +5 -5
  133. package/dist/providers/googleVertex.d.ts +1 -1
  134. package/dist/providers/googleVertex.js +74 -17
  135. package/dist/providers/huggingFace.d.ts +1 -1
  136. package/dist/providers/huggingFace.js +1 -1
  137. package/dist/providers/litellm.d.ts +1 -1
  138. package/dist/providers/litellm.js +18 -16
  139. package/dist/providers/mistral.d.ts +1 -1
  140. package/dist/providers/mistral.js +1 -1
  141. package/dist/providers/ollama.d.ts +1 -1
  142. package/dist/providers/ollama.js +8 -7
  143. package/dist/providers/openAI.d.ts +1 -1
  144. package/dist/providers/openAI.js +6 -6
  145. package/dist/providers/openRouter.d.ts +1 -1
  146. package/dist/providers/openRouter.js +6 -2
  147. package/dist/providers/openaiCompatible.d.ts +1 -1
  148. package/dist/providers/openaiCompatible.js +1 -1
  149. package/dist/proxy/proxyFetch.js +291 -65
  150. package/dist/services/server/ai/observability/instrumentation.js +12 -3
  151. package/dist/telemetry/telemetryService.d.ts +2 -1
  152. package/dist/telemetry/telemetryService.js +8 -1
  153. package/dist/types/contextTypes.d.ts +26 -2
  154. package/dist/types/conversation.d.ts +72 -40
  155. package/dist/types/conversationMemoryInterface.d.ts +5 -1
  156. package/dist/types/generateTypes.d.ts +26 -0
  157. package/dist/types/modelTypes.d.ts +10 -10
  158. package/dist/types/multimodal.d.ts +2 -0
  159. package/dist/types/observability.d.ts +10 -0
  160. package/dist/types/sdkTypes.d.ts +1 -1
  161. package/dist/utils/conversationMemory.d.ts +4 -3
  162. package/dist/utils/conversationMemory.js +44 -6
  163. package/dist/utils/errorHandling.d.ts +5 -0
  164. package/dist/utils/errorHandling.js +7 -2
  165. package/dist/utils/logger.d.ts +8 -0
  166. package/dist/utils/logger.js +56 -1
  167. package/dist/utils/messageBuilder.js +74 -4
  168. package/dist/utils/redis.js +6 -1
  169. package/dist/utils/tokenEstimation.d.ts +2 -2
  170. package/dist/utils/tokenEstimation.js +16 -1
  171. package/dist/utils/videoAnalysisProcessor.d.ts +2 -1
  172. package/dist/utils/videoAnalysisProcessor.js +7 -2
  173. package/dist/workflow/config.d.ts +12 -12
  174. package/package.json +1 -1
@@ -4,6 +4,7 @@
4
4
  */
5
5
  import { randomUUID } from "crypto";
6
6
  import { MESSAGES_PER_TURN } from "../config/conversationMemory.js";
7
+ import { generateToolOutputPreview } from "../context/toolOutputLimits.js";
7
8
  import { SummarizationEngine } from "../context/summarizationEngine.js";
8
9
  import { NeuroLink } from "../neurolink.js";
9
10
  import { ConversationMemoryError } from "../types/conversation.js";
@@ -102,7 +103,7 @@ export class RedisConversationMemoryManager {
102
103
  /**
103
104
  * Get session by ID, reconstructing a SessionMemory from Redis storage.
104
105
  */
105
- async getSession(sessionId, userId) {
106
+ async getSession(sessionId, userId, requestId) {
106
107
  await this.ensureInitialized();
107
108
  if (!this.redisClient) {
108
109
  return undefined;
@@ -114,6 +115,34 @@ export class RedisConversationMemoryManager {
114
115
  if (!conversation) {
115
116
  return undefined;
116
117
  }
118
+ // Log session load metadata for observability
119
+ const blobSizeBytes = conversationData
120
+ ? Buffer.byteLength(conversationData, "utf8")
121
+ : 0;
122
+ const messageCount = conversation.messages.length;
123
+ const hasSummary = !!conversation.summarizedUpToMessageId;
124
+ const pointerIndex = hasSummary
125
+ ? conversation.messages.findIndex((msg) => msg.id === conversation.summarizedUpToMessageId)
126
+ : -1;
127
+ const recentMessageCount = hasSummary && pointerIndex !== -1
128
+ ? messageCount - pointerIndex - 1
129
+ : messageCount;
130
+ logger.info("[ConversationMemory] Session loaded", {
131
+ requestId,
132
+ sessionId,
133
+ blobSizeBytes,
134
+ messageCount,
135
+ hasSummary,
136
+ recentMessageCount,
137
+ });
138
+ if (blobSizeBytes > 512 * 1024) {
139
+ logger.warn("[ConversationMemory] Large session blob", {
140
+ requestId,
141
+ sessionId,
142
+ blobSizeBytes,
143
+ messageCount,
144
+ });
145
+ }
117
146
  return {
118
147
  sessionId: conversation.sessionId,
119
148
  userId: conversation.userId,
@@ -137,6 +166,30 @@ export class RedisConversationMemoryManager {
137
166
  return undefined;
138
167
  }
139
168
  }
169
+ /**
170
+ * Get raw session data without any filtering or transformation.
171
+ * Used by the memory retrieval tool and internal APIs that need
172
+ * access to full message data including unmodified tool outputs.
173
+ */
174
+ async getSessionRaw(sessionId, userId) {
175
+ try {
176
+ await this.ensureInitialized();
177
+ if (!this.redisClient) {
178
+ return null;
179
+ }
180
+ const redisKey = getSessionKey(this.redisConfig, sessionId, userId);
181
+ const conversationData = await this.redisClient.get(redisKey);
182
+ return deserializeConversation(conversationData || null);
183
+ }
184
+ catch (error) {
185
+ logger.error("[RedisConversationMemoryManager] Failed to get raw session", {
186
+ sessionId,
187
+ userId,
188
+ error: error instanceof Error ? error.message : String(error),
189
+ });
190
+ return null;
191
+ }
192
+ }
140
193
  /**
141
194
  * Get all sessions for a specific user
142
195
  */
@@ -377,7 +430,7 @@ export class RedisConversationMemoryManager {
377
430
  if (!this.summarizationInProgress.has(summarizationKey)) {
378
431
  setImmediate(async () => {
379
432
  try {
380
- await this.checkAndSummarize(conversation, tokenThreshold, options.sessionId, options.userId);
433
+ await this.checkAndSummarize(conversation, tokenThreshold, options.sessionId, options.userId, options.requestId);
381
434
  }
382
435
  catch (error) {
383
436
  logger.error("Background summarization failed", {
@@ -396,6 +449,24 @@ export class RedisConversationMemoryManager {
396
449
  }
397
450
  const serializedData = serializeConversation(conversation);
398
451
  await this.redisClient.set(redisKey, serializedData);
452
+ // Log turn storage metadata for observability
453
+ const blobSizeBytes = Buffer.byteLength(serializedData, "utf8");
454
+ logger.info("[ConversationMemory] Turn stored", {
455
+ requestId: options.requestId,
456
+ sessionId: options.sessionId,
457
+ blobSizeBytes,
458
+ totalMessages: conversation.messages.length,
459
+ userMsgChars: options.userMessage.length,
460
+ assistantMsgChars: options.aiResponse.length,
461
+ });
462
+ if (blobSizeBytes > 512 * 1024) {
463
+ logger.warn("[ConversationMemory] Large session blob", {
464
+ requestId: options.requestId,
465
+ sessionId: options.sessionId,
466
+ blobSizeBytes,
467
+ messageCount: conversation.messages.length,
468
+ });
469
+ }
399
470
  if (this.redisConfig.ttl > 0) {
400
471
  await this.redisClient.expire(redisKey, this.redisConfig.ttl);
401
472
  }
@@ -418,7 +489,7 @@ export class RedisConversationMemoryManager {
418
489
  /**
419
490
  * Check if summarization is needed based on token count
420
491
  */
421
- async checkAndSummarize(conversation, threshold, sessionId, userId) {
492
+ async checkAndSummarize(conversation, threshold, sessionId, userId, requestId) {
422
493
  const normalizedUserId = userId || "randomUser";
423
494
  const summarizationKey = `${sessionId}:${normalizedUserId}`;
424
495
  if (this.summarizationInProgress.has(summarizationKey)) {
@@ -442,7 +513,7 @@ export class RedisConversationMemoryManager {
442
513
  createdAt: new Date(conversation.createdAt).getTime(),
443
514
  lastActivity: new Date(conversation.updatedAt).getTime(),
444
515
  };
445
- const summarized = await this.summarizationEngine.checkAndSummarize(session, threshold, this.config, "[RedisConversationMemoryManager]");
516
+ const summarized = await this.summarizationEngine.checkAndSummarize(session, threshold, this.config, "[RedisConversationMemoryManager]", requestId);
446
517
  conversation.lastTokenCount = session.lastTokenCount;
447
518
  conversation.lastCountedAt = session.lastCountedAt;
448
519
  if (summarized) {
@@ -471,9 +542,9 @@ export class RedisConversationMemoryManager {
471
542
  /**
472
543
  * Build context messages for AI prompt injection (TOKEN-BASED)
473
544
  * Returns messages from pointer onwards (or all if no pointer)
474
- * Filters out tool_call and tool_result messages when summarization is enabled
545
+ * Applies sendToolPreview toggle and hydrates result.result for backward compat
475
546
  */
476
- async buildContextMessages(sessionId, userId, enableSummarization) {
547
+ async buildContextMessages(sessionId, userId, enableSummarization, requestId) {
477
548
  logger.info("[RedisConversationMemoryManager] Building context messages", {
478
549
  sessionId,
479
550
  userId,
@@ -497,14 +568,34 @@ export class RedisConversationMemoryManager {
497
568
  createdAt: new Date(conversation.createdAt).getTime(),
498
569
  lastActivity: new Date(conversation.updatedAt).getTime(),
499
570
  };
500
- const contextMessages = buildContextFromPointer(session);
501
- const isSummarizationEnabled = enableSummarization !== undefined
502
- ? enableSummarization
503
- : this.config.enableSummarization === true;
504
- let finalMessages = contextMessages;
505
- if (isSummarizationEnabled) {
506
- finalMessages = contextMessages.filter((msg) => msg.role !== "tool_call" && msg.role !== "tool_result");
507
- }
571
+ const contextMessages = buildContextFromPointer(session, requestId);
572
+ const sendToolPreview = this.config?.contextCompaction?.sendToolPreview === true;
573
+ // Map tool_result messages: apply preview toggle + hydrate result.result
574
+ const finalMessages = contextMessages.map((msg) => {
575
+ if (msg.role !== "tool_result") {
576
+ return msg;
577
+ }
578
+ // Toggle: swap content to preview if enabled AND a preview exists
579
+ const content = sendToolPreview && msg.metadata?.toolOutputPreview
580
+ ? msg.metadata.toolOutputPreview
581
+ : msg.content;
582
+ // Hydrate result.result from content for backward compatibility
583
+ // (result.result is no longer stored — inferred from content at read time)
584
+ let hydratedResult = msg.result;
585
+ if (msg.result && msg.result.result === undefined) {
586
+ let parsedResult = content;
587
+ try {
588
+ parsedResult = JSON.parse(content);
589
+ }
590
+ catch {
591
+ /* plain text — use as-is */
592
+ }
593
+ hydratedResult = { ...msg.result, result: parsedResult };
594
+ }
595
+ return { ...msg, content, result: hydratedResult };
596
+ });
597
+ // Tool messages now have real content and participate in context properly.
598
+ // The tool output pruner (Stage 1) handles bounding old tool outputs.
508
599
  logger.info("[RedisConversationMemoryManager] Retrieved context messages", {
509
600
  sessionId,
510
601
  userId,
@@ -756,6 +847,75 @@ User message: "${userMessage}"`;
756
847
  },
757
848
  };
758
849
  }
850
+ /**
851
+ * Get the raw messages array for a session.
852
+ * Returns the full messages list without context filtering or summarization.
853
+ */
854
+ async getSessionMessages(sessionId, userId) {
855
+ await this.ensureInitialized();
856
+ if (!this.redisClient) {
857
+ return [];
858
+ }
859
+ try {
860
+ const redisKey = getSessionKey(this.redisConfig, sessionId, userId);
861
+ const conversationData = await this.redisClient.get(redisKey);
862
+ const conversation = deserializeConversation(conversationData || null);
863
+ return conversation?.messages ?? [];
864
+ }
865
+ catch (error) {
866
+ logger.error("[RedisConversationMemoryManager] Failed to get session messages", {
867
+ sessionId,
868
+ userId,
869
+ error: error instanceof Error ? error.message : String(error),
870
+ });
871
+ return [];
872
+ }
873
+ }
874
+ /**
875
+ * Replace the entire messages array for a session.
876
+ * The session must already exist in Redis.
877
+ */
878
+ async setSessionMessages(sessionId, messages, userId) {
879
+ await this.ensureInitialized();
880
+ if (!this.redisClient) {
881
+ throw new ConversationMemoryError("Redis client not initialized", "STORAGE_ERROR", { sessionId });
882
+ }
883
+ try {
884
+ const redisKey = getSessionKey(this.redisConfig, sessionId, userId);
885
+ const conversationData = await this.redisClient.get(redisKey);
886
+ const conversation = deserializeConversation(conversationData || null);
887
+ if (!conversation) {
888
+ throw new ConversationMemoryError(`Session ${sessionId} not found`, "STORAGE_ERROR", { sessionId });
889
+ }
890
+ conversation.messages = messages;
891
+ conversation.updatedAt = new Date().toISOString();
892
+ // Reset summarization pointers — the old summary no longer applies
893
+ // to the replaced messages array
894
+ conversation.summarizedUpToMessageId = undefined;
895
+ conversation.summarizedMessage = undefined;
896
+ conversation.lastTokenCount = undefined;
897
+ conversation.lastCountedAt = undefined;
898
+ const serializedData = serializeConversation(conversation);
899
+ await this.redisClient.set(redisKey, serializedData);
900
+ if (this.redisConfig.ttl > 0) {
901
+ await this.redisClient.expire(redisKey, this.redisConfig.ttl);
902
+ }
903
+ logger.debug("[RedisConversationMemoryManager] Session messages replaced", {
904
+ sessionId,
905
+ userId,
906
+ messageCount: messages.length,
907
+ });
908
+ }
909
+ catch (error) {
910
+ if (error instanceof ConversationMemoryError) {
911
+ throw error;
912
+ }
913
+ throw new ConversationMemoryError(`Failed to set session messages for session ${sessionId}`, "STORAGE_ERROR", {
914
+ sessionId,
915
+ error: error instanceof Error ? error.message : String(error),
916
+ });
917
+ }
918
+ }
759
919
  /**
760
920
  * Close Redis connection
761
921
  */
@@ -966,52 +1126,87 @@ User message: "${userMessage}"`;
966
1126
  toolCallsCount: pendingData.toolCalls.length,
967
1127
  toolResultsCount: pendingData.toolResults.length,
968
1128
  });
969
- // Create a mapping from toolCallId to toolName for matching tool results
970
- const toolCallMap = new Map();
971
- // Create separate messages for tool calls and build the mapping
972
- for (const toolCall of pendingData.toolCalls) {
973
- const toolCallId = String(toolCall.toolCallId);
974
- const toolName = String(toolCall.toolName);
975
- // Store in mapping for tool results
976
- toolCallMap.set(toolCallId, toolName);
977
- const toolCallMessage = {
978
- id: randomUUID(),
979
- timestamp: toolCall.timestamp?.toISOString() || this.generateTimestamp(),
980
- role: "tool_call",
981
- content: "", // Can be empty for tool calls
982
- tool: toolName,
983
- args: (toolCall.args ||
984
- toolCall.arguments ||
985
- toolCall.parameters ||
986
- {}),
987
- };
988
- conversation.messages.push(toolCallMessage);
989
- }
990
- // Create separate messages for tool results using the mapping
991
- for (const toolResult of pendingData.toolResults) {
992
- const toolCallId = String(toolResult.toolCallId || toolResult.id || "unknown");
993
- const toolName = toolCallMap.get(toolCallId) || "unknown";
994
- const toolResultMessage = {
995
- id: randomUUID(),
996
- timestamp: toolResult.timestamp?.toISOString() || this.generateTimestamp(),
997
- role: "tool_result",
998
- content: "", // Can be empty for tool results
999
- tool: toolName, // Now correctly extracted from tool call mapping
1000
- result: {
1129
+ try {
1130
+ // Create a mapping from toolCallId to toolName for matching tool results
1131
+ const toolCallMap = new Map();
1132
+ // Create separate messages for tool calls and build the mapping
1133
+ for (const toolCall of pendingData.toolCalls) {
1134
+ const toolCallId = toolCall.toolCallId ?? "";
1135
+ const toolName = toolCall.toolName ?? "";
1136
+ // Store in mapping for tool results
1137
+ toolCallMap.set(toolCallId, toolName);
1138
+ const toolCallMessage = {
1139
+ id: randomUUID(),
1140
+ timestamp: toolCall.timestamp?.toISOString() || this.generateTimestamp(),
1141
+ role: "tool_call",
1142
+ content: "", // Can be empty for tool calls
1143
+ tool: toolName,
1144
+ args: (toolCall.args ||
1145
+ toolCall.arguments ||
1146
+ toolCall.parameters ||
1147
+ {}),
1148
+ };
1149
+ conversation.messages.push(toolCallMessage);
1150
+ }
1151
+ // Create separate messages for tool results using the mapping
1152
+ for (const toolResult of pendingData.toolResults) {
1153
+ const toolCallId = String(toolResult.toolCallId || toolResult.id || "unknown");
1154
+ const toolName = toolCallMap.get(toolCallId) || "unknown";
1155
+ // Serialize the tool result to string for content field
1156
+ let serializedResult;
1157
+ if (typeof toolResult.result === "string") {
1158
+ serializedResult = toolResult.result;
1159
+ }
1160
+ else if (toolResult.result === undefined ||
1161
+ toolResult.result === null) {
1162
+ serializedResult = String(toolResult.result ?? "null");
1163
+ }
1164
+ else {
1165
+ try {
1166
+ serializedResult = JSON.stringify(toolResult.result, null, 2);
1167
+ }
1168
+ catch (serializeError) {
1169
+ serializedResult = `[Serialization failed: ${serializeError instanceof Error ? serializeError.message : String(serializeError)}]`;
1170
+ }
1171
+ }
1172
+ // Generate preview (uses existing config fields that were previously unused)
1173
+ const { preview, truncated, originalSize } = generateToolOutputPreview(serializedResult, {
1174
+ maxBytes: this.config?.contextCompaction?.maxToolOutputBytes,
1175
+ maxLines: this.config?.contextCompaction?.maxToolOutputLines,
1176
+ });
1177
+ // Build metadata — only store preview when truncation occurred (no duplication)
1178
+ const metadata = {
1179
+ truncated,
1180
+ ...(truncated && { toolOutputPreview: preview }),
1181
+ ...(truncated && { originalSize }),
1182
+ };
1183
+ // Build result — success/error metadata only, NOT the output data
1184
+ const result = {
1001
1185
  success: !toolResult.error,
1002
- result: toolResult.result,
1186
+ // result.result intentionally NOT stored — inferred from content at read time
1003
1187
  error: toolResult.error ? String(toolResult.error) : undefined,
1004
- },
1005
- };
1006
- conversation.messages.push(toolResultMessage);
1188
+ };
1189
+ const toolResultMessage = {
1190
+ id: randomUUID(),
1191
+ timestamp: toolResult.timestamp?.toISOString() || this.generateTimestamp(),
1192
+ role: "tool_result",
1193
+ content: serializedResult, // Full output (was "")
1194
+ tool: toolName,
1195
+ result,
1196
+ metadata,
1197
+ };
1198
+ conversation.messages.push(toolResultMessage);
1199
+ }
1200
+ logger.debug("[RedisConversationMemoryManager] Successfully flushed pending tool data", {
1201
+ sessionId,
1202
+ userId,
1203
+ toolMessagesAdded: pendingData.toolCalls.length + pendingData.toolResults.length,
1204
+ totalMessages: conversation.messages.length,
1205
+ });
1206
+ }
1207
+ finally {
1208
+ // Always clean up pending data, even on failure, to prevent infinite retry loops
1209
+ this.pendingToolExecutions.delete(pendingKey);
1007
1210
  }
1008
- // Remove the pending data now that it's been flushed
1009
- this.pendingToolExecutions.delete(pendingKey);
1010
- logger.debug("[RedisConversationMemoryManager] Successfully flushed pending tool data", {
1011
- sessionId,
1012
- userId,
1013
- toolMessagesAdded: pendingData.toolCalls.length + pendingData.toolResults.length,
1014
- totalMessages: conversation.messages.length,
1015
- });
1016
1211
  }
1017
1212
  }
package/dist/index.d.ts CHANGED
@@ -42,6 +42,8 @@ export type { DynamicModelConfig, ModelRegistry } from "./types/modelTypes.js";
42
42
  export { getAvailableProviders, getBestProvider, isValidProvider, } from "./utils/providerUtils.js";
43
43
  export { calculateCost, hasPricing } from "./utils/pricing.js";
44
44
  export { isAbortError } from "./utils/errorHandling.js";
45
+ export { TTSProcessor } from "./utils/ttsProcessor.js";
46
+ export { GoogleTTSHandler } from "./adapters/tts/googleTTSHandler.js";
45
47
  import { NeuroLink } from "./neurolink.js";
46
48
  export { NeuroLink };
47
49
  export type { MCPServerInfo } from "./types/mcpTypes.js";
package/dist/index.js CHANGED
@@ -49,6 +49,9 @@ export { getAvailableProviders, getBestProvider, isValidProvider, } from "./util
49
49
  export { calculateCost, hasPricing } from "./utils/pricing.js";
50
50
  // Error utilities
51
51
  export { isAbortError } from "./utils/errorHandling.js";
52
+ // TTS utilities
53
+ export { TTSProcessor } from "./utils/ttsProcessor.js";
54
+ export { GoogleTTSHandler } from "./adapters/tts/googleTTSHandler.js";
52
55
  // Main NeuroLink wrapper class and diagnostic types
53
56
  import { NeuroLink } from "./neurolink.js";
54
57
  export { NeuroLink };
@@ -8,16 +8,16 @@
8
8
  */
9
9
  import { AIProviderName } from "../../constants/enums.js";
10
10
  import type { CoreMessage } from "ai";
11
- export declare function analyzeVideoWithVertexAI(frames: CoreMessage, options?: {
11
+ export declare function analyzeVideoWithVertexAI(messages: CoreMessage[], options?: {
12
12
  project?: string;
13
13
  location?: string;
14
14
  model?: string;
15
15
  }): Promise<string>;
16
- export declare function analyzeVideoWithGeminiAPI(frames: CoreMessage, options?: {
16
+ export declare function analyzeVideoWithGeminiAPI(messages: CoreMessage[], options?: {
17
17
  apiKey?: string;
18
18
  model?: string;
19
19
  }): Promise<string>;
20
- export declare function analyzeVideo(frames: CoreMessage, options?: {
20
+ export declare function analyzeVideo(messages: CoreMessage[], options?: {
21
21
  provider?: AIProviderName;
22
22
  project?: string;
23
23
  location?: string;
@@ -9,23 +9,35 @@
9
9
  import { AIProviderName, ErrorSeverity, ErrorCategory, } from "../../constants/enums.js";
10
10
  import { logger } from "../../utils/logger.js";
11
11
  import { readFile } from "node:fs/promises";
12
- import { NeuroLinkError } from "../../utils/errorHandling.js";
12
+ import { NeuroLinkError, ErrorFactory } from "../../utils/errorHandling.js";
13
13
  // ---------------------------------------------------------------------------
14
14
  // Shared config
15
15
  // ---------------------------------------------------------------------------
16
16
  const DEFAULT_MODEL = "gemini-2.0-flash";
17
17
  const DEFAULT_LOCATION = "us-central1";
18
+ /**
19
+ * Extract content items from user messages
20
+ *
21
+ * @param messages - Array of CoreMessage objects
22
+ * @returns Flattened array of content items from user messages
23
+ */
24
+ function extractUserContent(messages) {
25
+ const userMessages = messages.filter((msg) => msg.role === "user");
26
+ return userMessages.flatMap((msg) => Array.isArray(msg.content) ? msg.content : []);
27
+ }
18
28
  /**
19
29
  * Convert CoreMessage content array to Gemini parts format
20
30
  *
21
- * @param contentArray - Array of content items from CoreMessage
31
+ * @param messages - Array of CoreMessage objects
22
32
  * @returns Array of parts in Gemini API format
23
33
  */
24
- function buildContentParts(frames) {
25
- const contentArray = Array.isArray(frames.content) ? frames.content : [];
26
- return contentArray.map((item) => {
27
- if (item.type === "text" && item.text) {
28
- return { text: item.text };
34
+ function buildContentParts(messages) {
35
+ const allContent = extractUserContent(messages);
36
+ return allContent
37
+ .map((item) => {
38
+ if (item.type === "text") {
39
+ // Accept text parts regardless of whether text is empty
40
+ return { text: item.text || "" };
29
41
  }
30
42
  else if (item.type === "image" && item.image) {
31
43
  let base64Data;
@@ -38,7 +50,7 @@ function buildContentParts(frames) {
38
50
  base64Data = item.image.replace(/^data:image\/[a-z]+;base64,/, "");
39
51
  }
40
52
  else {
41
- throw new Error(`Invalid image data type: expected string, Buffer, or Uint8Array, got ${typeof item.image}`);
53
+ throw ErrorFactory.invalidConfiguration("image data type", `expected string, Buffer, or Uint8Array, got ${typeof item.image}`, { itemType: item.type, dataType: typeof item.image });
42
54
  }
43
55
  return {
44
56
  inlineData: {
@@ -47,8 +59,14 @@ function buildContentParts(frames) {
47
59
  },
48
60
  };
49
61
  }
50
- throw new Error(`Invalid content type: ${item.type}`);
51
- });
62
+ else if (item.type === "file") {
63
+ // Skip file parts - not supported in Gemini parts format
64
+ return null;
65
+ }
66
+ // Return null for unsupported types
67
+ return null;
68
+ })
69
+ .filter((part) => part !== null);
52
70
  }
53
71
  /**
54
72
  * Configuration for video frame analysis.
@@ -88,7 +106,7 @@ Ensure the final response is fully self-sufficient and does not reference extern
88
106
  // ---------------------------------------------------------------------------
89
107
  // Vertex AI
90
108
  // ---------------------------------------------------------------------------
91
- export async function analyzeVideoWithVertexAI(frames, options = {}) {
109
+ export async function analyzeVideoWithVertexAI(messages, options = {}) {
92
110
  const startTime = Date.now();
93
111
  const { GoogleGenAI } = await import("@google/genai");
94
112
  // Get default config and merge with provided options
@@ -96,9 +114,9 @@ export async function analyzeVideoWithVertexAI(frames, options = {}) {
96
114
  const project = options.project ?? config.project;
97
115
  const location = options.location ?? config.location;
98
116
  const model = options.model || DEFAULT_MODEL;
99
- // Extract content array from CoreMessage
100
- const contentArray = Array.isArray(frames.content) ? frames.content : [];
101
- const frameCount = contentArray.filter((item) => item.type === "image").length;
117
+ // Convert frames content to parts array for Gemini
118
+ const parts = buildContentParts(messages);
119
+ const frameCount = parts.filter((part) => "inlineData" in part && part.inlineData).length;
102
120
  logger.debug("[GeminiVideoAnalyzer] Analyzing video with Vertex AI", {
103
121
  project,
104
122
  location,
@@ -106,8 +124,6 @@ export async function analyzeVideoWithVertexAI(frames, options = {}) {
106
124
  frameCount,
107
125
  });
108
126
  const ai = new GoogleGenAI({ vertexai: true, project, location });
109
- // Convert frames content to parts array for Gemini
110
- const parts = buildContentParts(frames);
111
127
  const response = await ai.models.generateContent({
112
128
  model,
113
129
  config: buildConfig(),
@@ -129,7 +145,7 @@ export async function analyzeVideoWithVertexAI(frames, options = {}) {
129
145
  // ---------------------------------------------------------------------------
130
146
  // Gemini API (Google AI)
131
147
  // ---------------------------------------------------------------------------
132
- export async function analyzeVideoWithGeminiAPI(frames, options = {}) {
148
+ export async function analyzeVideoWithGeminiAPI(messages, options = {}) {
133
149
  const startTime = Date.now();
134
150
  const { GoogleGenAI } = await import("@google/genai");
135
151
  const apiKey = options.apiKey || process.env.GOOGLE_AI_API_KEY;
@@ -137,16 +153,14 @@ export async function analyzeVideoWithGeminiAPI(frames, options = {}) {
137
153
  if (!apiKey) {
138
154
  throw new Error("GOOGLE_AI_API_KEY environment variable is required for Gemini API video analysis");
139
155
  }
140
- // Extract content array from CoreMessage
141
- const contentArray = Array.isArray(frames.content) ? frames.content : [];
142
- const frameCount = contentArray.filter((item) => item.type === "image").length;
156
+ // Convert frames content to parts array for Gemini
157
+ const parts = buildContentParts(messages);
158
+ const frameCount = parts.filter((part) => "inlineData" in part && part.inlineData).length;
143
159
  logger.debug("[GeminiVideoAnalyzer] Analyzing video with Gemini API", {
144
160
  model,
145
161
  frameCount,
146
162
  });
147
163
  const ai = new GoogleGenAI({ apiKey });
148
- // Convert frames content to parts array for Gemini
149
- const parts = buildContentParts(frames);
150
164
  logger.debug("[GeminiVideoAnalyzer] Generating analysis with frames");
151
165
  const response = await ai.models.generateContent({
152
166
  model,
@@ -207,15 +221,15 @@ async function getVertexConfig() {
207
221
  }
208
222
  return { project, location };
209
223
  }
210
- export async function analyzeVideo(frames, options = {}) {
224
+ export async function analyzeVideo(messages, options = {}) {
211
225
  const provider = options.provider || AIProviderName.AUTO;
212
226
  // Vertex — only when GOOGLE_VERTEX_PROJECT is explicitly set
213
227
  if (provider === AIProviderName.VERTEX || provider === AIProviderName.AUTO) {
214
- return analyzeVideoWithVertexAI(frames, options);
228
+ return analyzeVideoWithVertexAI(messages, options);
215
229
  }
216
230
  // Gemini API — when GOOGLE_AI_API_KEY is set
217
231
  if (provider === AIProviderName.GOOGLE_AI && process.env.GOOGLE_AI_API_KEY) {
218
- return analyzeVideoWithGeminiAPI(frames, options);
232
+ return analyzeVideoWithGeminiAPI(messages, options);
219
233
  }
220
234
  throw new Error("No valid provider configuration found. " +
221
235
  "Set GOOGLE_VERTEX_PROJECT for Vertex AI or GOOGLE_AI_API_KEY for Gemini API.");
@@ -221,11 +221,11 @@ export declare const directAgentTools: {
221
221
  }, "strip", z.ZodTypeAny, {
222
222
  content: string;
223
223
  path: string;
224
- mode: "create" | "append" | "overwrite";
224
+ mode: "create" | "overwrite" | "append";
225
225
  }, {
226
226
  content: string;
227
227
  path: string;
228
- mode?: "create" | "append" | "overwrite" | undefined;
228
+ mode?: "create" | "overwrite" | "append" | undefined;
229
229
  }>, {
230
230
  success: boolean;
231
231
  error: string;
@@ -236,7 +236,7 @@ export declare const directAgentTools: {
236
236
  } | {
237
237
  success: boolean;
238
238
  path: string;
239
- mode: "create" | "append" | "overwrite";
239
+ mode: "create" | "overwrite" | "append";
240
240
  size: number;
241
241
  written: number;
242
242
  error?: undefined;
@@ -251,7 +251,7 @@ export declare const directAgentTools: {
251
251
  execute: (args: {
252
252
  content: string;
253
253
  path: string;
254
- mode: "create" | "append" | "overwrite";
254
+ mode: "create" | "overwrite" | "append";
255
255
  }, options: import("ai").ToolExecutionOptions) => PromiseLike<{
256
256
  success: boolean;
257
257
  error: string;
@@ -262,7 +262,7 @@ export declare const directAgentTools: {
262
262
  } | {
263
263
  success: boolean;
264
264
  path: string;
265
- mode: "create" | "append" | "overwrite";
265
+ mode: "create" | "overwrite" | "append";
266
266
  size: number;
267
267
  written: number;
268
268
  error?: undefined;
@@ -282,8 +282,8 @@ export declare const directAgentTools: {
282
282
  maxRows: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
283
283
  }, "strip", z.ZodTypeAny, {
284
284
  operation: "count_by_column" | "sum_by_column" | "average_by_column" | "min_max_by_column" | "describe";
285
- filePath: string;
286
285
  maxRows: number;
286
+ filePath: string;
287
287
  column: string;
288
288
  }, {
289
289
  operation: "count_by_column" | "sum_by_column" | "average_by_column" | "min_max_by_column" | "describe";
@@ -309,8 +309,8 @@ export declare const directAgentTools: {
309
309
  }> & {
310
310
  execute: (args: {
311
311
  operation: "count_by_column" | "sum_by_column" | "average_by_column" | "min_max_by_column" | "describe";
312
- filePath: string;
313
312
  maxRows: number;
313
+ filePath: string;
314
314
  column: string;
315
315
  }, options: import("ai").ToolExecutionOptions) => PromiseLike<{
316
316
  success: boolean;
@@ -25,9 +25,12 @@ export declare const MODEL_CONTEXT_WINDOWS: Record<string, Record<string, number
25
25
  * Resolve context window size for a provider/model combination.
26
26
  *
27
27
  * Priority:
28
- * 1. Exact model match under provider
29
- * 2. Provider's _default
30
- * 3. Global DEFAULT_CONTEXT_WINDOW
28
+ * 0. Dynamic model registry (DynamicModelProvider) — resolves cross-provider
29
+ * models (e.g. Claude on Vertex) that the static table cannot handle
30
+ * 1. Exact model match under provider in static registry
31
+ * 2. Prefix match under provider in static registry
32
+ * 3. Provider's _default in static registry
33
+ * 4. Global DEFAULT_CONTEXT_WINDOW
31
34
  */
32
35
  export declare function getContextWindowSize(provider: string, model?: string): number;
33
36
  /**