@juspay/neurolink 9.15.0 → 9.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/dist/adapters/video/videoAnalyzer.d.ts +1 -1
  3. package/dist/adapters/video/videoAnalyzer.js +10 -8
  4. package/dist/cli/commands/setup-anthropic.js +1 -14
  5. package/dist/cli/commands/setup-azure.js +1 -12
  6. package/dist/cli/commands/setup-bedrock.js +1 -9
  7. package/dist/cli/commands/setup-google-ai.js +1 -12
  8. package/dist/cli/commands/setup-openai.js +1 -14
  9. package/dist/cli/commands/workflow.d.ts +27 -0
  10. package/dist/cli/commands/workflow.js +216 -0
  11. package/dist/cli/factories/commandFactory.js +79 -20
  12. package/dist/cli/index.js +0 -1
  13. package/dist/cli/parser.js +4 -1
  14. package/dist/cli/utils/maskCredential.d.ts +11 -0
  15. package/dist/cli/utils/maskCredential.js +23 -0
  16. package/dist/constants/contextWindows.js +107 -16
  17. package/dist/constants/enums.d.ts +99 -15
  18. package/dist/constants/enums.js +152 -22
  19. package/dist/context/budgetChecker.js +1 -1
  20. package/dist/context/contextCompactor.js +31 -4
  21. package/dist/context/emergencyTruncation.d.ts +21 -0
  22. package/dist/context/emergencyTruncation.js +88 -0
  23. package/dist/context/errorDetection.d.ts +16 -0
  24. package/dist/context/errorDetection.js +48 -1
  25. package/dist/context/errors.d.ts +19 -0
  26. package/dist/context/errors.js +21 -0
  27. package/dist/context/stages/slidingWindowTruncator.d.ts +6 -0
  28. package/dist/context/stages/slidingWindowTruncator.js +159 -24
  29. package/dist/core/baseProvider.js +306 -200
  30. package/dist/core/conversationMemoryManager.js +104 -61
  31. package/dist/core/evaluationProviders.js +16 -33
  32. package/dist/core/factory.js +237 -164
  33. package/dist/core/modules/GenerationHandler.js +175 -116
  34. package/dist/core/modules/MessageBuilder.js +222 -170
  35. package/dist/core/modules/StreamHandler.d.ts +1 -0
  36. package/dist/core/modules/StreamHandler.js +95 -27
  37. package/dist/core/modules/TelemetryHandler.d.ts +10 -1
  38. package/dist/core/modules/TelemetryHandler.js +25 -7
  39. package/dist/core/modules/ToolsManager.js +115 -191
  40. package/dist/core/redisConversationMemoryManager.js +418 -282
  41. package/dist/factories/providerRegistry.d.ts +5 -0
  42. package/dist/factories/providerRegistry.js +20 -2
  43. package/dist/index.d.ts +2 -2
  44. package/dist/index.js +4 -2
  45. package/dist/lib/adapters/video/videoAnalyzer.d.ts +1 -1
  46. package/dist/lib/adapters/video/videoAnalyzer.js +10 -8
  47. package/dist/lib/constants/contextWindows.js +107 -16
  48. package/dist/lib/constants/enums.d.ts +99 -15
  49. package/dist/lib/constants/enums.js +152 -22
  50. package/dist/lib/context/budgetChecker.js +1 -1
  51. package/dist/lib/context/contextCompactor.js +31 -4
  52. package/dist/lib/context/emergencyTruncation.d.ts +21 -0
  53. package/dist/lib/context/emergencyTruncation.js +89 -0
  54. package/dist/lib/context/errorDetection.d.ts +16 -0
  55. package/dist/lib/context/errorDetection.js +48 -1
  56. package/dist/lib/context/errors.d.ts +19 -0
  57. package/dist/lib/context/errors.js +22 -0
  58. package/dist/lib/context/stages/slidingWindowTruncator.d.ts +6 -0
  59. package/dist/lib/context/stages/slidingWindowTruncator.js +159 -24
  60. package/dist/lib/core/baseProvider.js +306 -200
  61. package/dist/lib/core/conversationMemoryManager.js +104 -61
  62. package/dist/lib/core/evaluationProviders.js +16 -33
  63. package/dist/lib/core/factory.js +237 -164
  64. package/dist/lib/core/modules/GenerationHandler.js +175 -116
  65. package/dist/lib/core/modules/MessageBuilder.js +222 -170
  66. package/dist/lib/core/modules/StreamHandler.d.ts +1 -0
  67. package/dist/lib/core/modules/StreamHandler.js +95 -27
  68. package/dist/lib/core/modules/TelemetryHandler.d.ts +10 -1
  69. package/dist/lib/core/modules/TelemetryHandler.js +25 -7
  70. package/dist/lib/core/modules/ToolsManager.js +115 -191
  71. package/dist/lib/core/redisConversationMemoryManager.js +418 -282
  72. package/dist/lib/factories/providerRegistry.d.ts +5 -0
  73. package/dist/lib/factories/providerRegistry.js +20 -2
  74. package/dist/lib/index.d.ts +2 -2
  75. package/dist/lib/index.js +4 -2
  76. package/dist/lib/mcp/externalServerManager.js +66 -0
  77. package/dist/lib/mcp/mcpCircuitBreaker.js +24 -0
  78. package/dist/lib/mcp/mcpClientFactory.js +16 -0
  79. package/dist/lib/mcp/toolDiscoveryService.js +32 -6
  80. package/dist/lib/mcp/toolRegistry.js +193 -123
  81. package/dist/lib/neurolink.d.ts +6 -0
  82. package/dist/lib/neurolink.js +1162 -646
  83. package/dist/lib/providers/amazonBedrock.d.ts +1 -1
  84. package/dist/lib/providers/amazonBedrock.js +521 -319
  85. package/dist/lib/providers/anthropic.js +73 -17
  86. package/dist/lib/providers/anthropicBaseProvider.js +77 -17
  87. package/dist/lib/providers/googleAiStudio.d.ts +1 -1
  88. package/dist/lib/providers/googleAiStudio.js +292 -227
  89. package/dist/lib/providers/googleVertex.d.ts +36 -1
  90. package/dist/lib/providers/googleVertex.js +553 -260
  91. package/dist/lib/providers/ollama.js +329 -278
  92. package/dist/lib/providers/openAI.js +77 -19
  93. package/dist/lib/providers/sagemaker/parsers.js +3 -3
  94. package/dist/lib/providers/sagemaker/streaming.js +3 -3
  95. package/dist/lib/proxy/proxyFetch.js +81 -48
  96. package/dist/lib/rag/ChunkerFactory.js +1 -1
  97. package/dist/lib/rag/chunkers/MarkdownChunker.d.ts +22 -0
  98. package/dist/lib/rag/chunkers/MarkdownChunker.js +213 -9
  99. package/dist/lib/rag/chunking/markdownChunker.d.ts +16 -0
  100. package/dist/lib/rag/chunking/markdownChunker.js +174 -2
  101. package/dist/lib/rag/pipeline/contextAssembly.js +2 -1
  102. package/dist/lib/rag/ragIntegration.d.ts +18 -1
  103. package/dist/lib/rag/ragIntegration.js +94 -14
  104. package/dist/lib/rag/retrieval/vectorQueryTool.js +21 -4
  105. package/dist/lib/server/abstract/baseServerAdapter.js +4 -1
  106. package/dist/lib/server/adapters/fastifyAdapter.js +35 -30
  107. package/dist/lib/services/server/ai/observability/instrumentation.d.ts +32 -0
  108. package/dist/lib/services/server/ai/observability/instrumentation.js +39 -0
  109. package/dist/lib/telemetry/attributes.d.ts +52 -0
  110. package/dist/lib/telemetry/attributes.js +61 -0
  111. package/dist/lib/telemetry/index.d.ts +3 -0
  112. package/dist/lib/telemetry/index.js +3 -0
  113. package/dist/lib/telemetry/telemetryService.d.ts +6 -0
  114. package/dist/lib/telemetry/telemetryService.js +6 -0
  115. package/dist/lib/telemetry/tracers.d.ts +15 -0
  116. package/dist/lib/telemetry/tracers.js +17 -0
  117. package/dist/lib/telemetry/withSpan.d.ts +9 -0
  118. package/dist/lib/telemetry/withSpan.js +35 -0
  119. package/dist/lib/types/contextTypes.d.ts +10 -0
  120. package/dist/lib/types/streamTypes.d.ts +14 -0
  121. package/dist/lib/utils/conversationMemory.js +121 -82
  122. package/dist/lib/utils/logger.d.ts +5 -0
  123. package/dist/lib/utils/logger.js +50 -2
  124. package/dist/lib/utils/messageBuilder.js +22 -42
  125. package/dist/lib/utils/modelDetection.js +3 -3
  126. package/dist/lib/utils/providerRetry.d.ts +41 -0
  127. package/dist/lib/utils/providerRetry.js +114 -0
  128. package/dist/lib/utils/retryability.d.ts +14 -0
  129. package/dist/lib/utils/retryability.js +23 -0
  130. package/dist/lib/utils/sanitizers/svg.js +4 -5
  131. package/dist/lib/utils/tokenEstimation.d.ts +11 -1
  132. package/dist/lib/utils/tokenEstimation.js +19 -4
  133. package/dist/lib/utils/videoAnalysisProcessor.js +7 -3
  134. package/dist/mcp/externalServerManager.js +66 -0
  135. package/dist/mcp/mcpCircuitBreaker.js +24 -0
  136. package/dist/mcp/mcpClientFactory.js +16 -0
  137. package/dist/mcp/toolDiscoveryService.js +32 -6
  138. package/dist/mcp/toolRegistry.js +193 -123
  139. package/dist/neurolink.d.ts +6 -0
  140. package/dist/neurolink.js +1162 -646
  141. package/dist/providers/amazonBedrock.d.ts +1 -1
  142. package/dist/providers/amazonBedrock.js +521 -319
  143. package/dist/providers/anthropic.js +73 -17
  144. package/dist/providers/anthropicBaseProvider.js +77 -17
  145. package/dist/providers/googleAiStudio.d.ts +1 -1
  146. package/dist/providers/googleAiStudio.js +292 -227
  147. package/dist/providers/googleVertex.d.ts +36 -1
  148. package/dist/providers/googleVertex.js +553 -260
  149. package/dist/providers/ollama.js +329 -278
  150. package/dist/providers/openAI.js +77 -19
  151. package/dist/providers/sagemaker/parsers.js +3 -3
  152. package/dist/providers/sagemaker/streaming.js +3 -3
  153. package/dist/proxy/proxyFetch.js +81 -48
  154. package/dist/rag/ChunkerFactory.js +1 -1
  155. package/dist/rag/chunkers/MarkdownChunker.d.ts +22 -0
  156. package/dist/rag/chunkers/MarkdownChunker.js +213 -9
  157. package/dist/rag/chunking/markdownChunker.d.ts +16 -0
  158. package/dist/rag/chunking/markdownChunker.js +174 -2
  159. package/dist/rag/pipeline/contextAssembly.js +2 -1
  160. package/dist/rag/ragIntegration.d.ts +18 -1
  161. package/dist/rag/ragIntegration.js +94 -14
  162. package/dist/rag/retrieval/vectorQueryTool.js +21 -4
  163. package/dist/server/abstract/baseServerAdapter.js +4 -1
  164. package/dist/server/adapters/fastifyAdapter.js +35 -30
  165. package/dist/services/server/ai/observability/instrumentation.d.ts +32 -0
  166. package/dist/services/server/ai/observability/instrumentation.js +39 -0
  167. package/dist/telemetry/attributes.d.ts +52 -0
  168. package/dist/telemetry/attributes.js +60 -0
  169. package/dist/telemetry/index.d.ts +3 -0
  170. package/dist/telemetry/index.js +3 -0
  171. package/dist/telemetry/telemetryService.d.ts +6 -0
  172. package/dist/telemetry/telemetryService.js +6 -0
  173. package/dist/telemetry/tracers.d.ts +15 -0
  174. package/dist/telemetry/tracers.js +16 -0
  175. package/dist/telemetry/withSpan.d.ts +9 -0
  176. package/dist/telemetry/withSpan.js +34 -0
  177. package/dist/types/contextTypes.d.ts +10 -0
  178. package/dist/types/streamTypes.d.ts +14 -0
  179. package/dist/utils/conversationMemory.js +121 -82
  180. package/dist/utils/logger.d.ts +5 -0
  181. package/dist/utils/logger.js +50 -2
  182. package/dist/utils/messageBuilder.js +22 -42
  183. package/dist/utils/modelDetection.js +3 -3
  184. package/dist/utils/providerRetry.d.ts +41 -0
  185. package/dist/utils/providerRetry.js +113 -0
  186. package/dist/utils/retryability.d.ts +14 -0
  187. package/dist/utils/retryability.js +22 -0
  188. package/dist/utils/sanitizers/svg.js +4 -5
  189. package/dist/utils/tokenEstimation.d.ts +11 -1
  190. package/dist/utils/tokenEstimation.js +19 -4
  191. package/dist/utils/videoAnalysisProcessor.js +7 -3
  192. package/dist/workflow/config.d.ts +26 -26
  193. package/package.json +1 -1
@@ -14,6 +14,22 @@ export declare class MarkdownChunker implements Chunker {
14
14
  chunk(text: string, config?: MarkdownChunkerConfig): Promise<Chunk[]>;
15
15
  private splitByHeaders;
16
16
  private splitContent;
17
+ /**
18
+ * Detect contiguous table blocks in lines.
19
+ * Returns array of { start, end } line index ranges (inclusive).
20
+ */
21
+ private detectTableRanges;
22
+ /** Check if a line is a markdown table separator (e.g. |---|---|). */
23
+ private isTableSeparator;
24
+ /**
25
+ * Split content while preserving markdown tables.
26
+ */
27
+ private splitContentTableAware;
28
+ /**
29
+ * Split a table on row boundaries, repeating header + separator in each chunk.
30
+ */
31
+ private splitTableByRows;
32
+ private splitPlainContent;
17
33
  private stripMarkdown;
18
34
  validateConfig(config: BaseChunkerConfig): ChunkerValidationResult;
19
35
  }
@@ -129,10 +129,182 @@ export class MarkdownChunker {
129
129
  if (content.length <= effectiveMaxSize) {
130
130
  return [content];
131
131
  }
132
+ // Use table-aware splitting
133
+ const lines = content.split("\n");
134
+ const tableRanges = this.detectTableRanges(lines);
135
+ if (tableRanges.length > 0) {
136
+ return this.splitContentTableAware(content, lines, tableRanges, effectiveMaxSize, effectiveOverlap);
137
+ }
138
+ return this.splitPlainContent(content, effectiveMaxSize, effectiveOverlap);
139
+ }
140
+ /**
141
+ * Detect contiguous table blocks in lines.
142
+ * Returns array of { start, end } line index ranges (inclusive).
143
+ */
144
+ detectTableRanges(lines) {
145
+ // Simple pipe-prefixed line check (single character class — no backtracking)
146
+ const TABLE_ROW_RE = /^\|[^\r\n]{1,10000}/;
147
+ // Per-cell separator regex applied AFTER splitting on "|" — safe because
148
+ // each cell is short and bounded by pipe delimiters (CodeQL: js/polynomial-redos)
149
+ const SEPARATOR_CELL_RE = /^[\t ]*:?-+:?[\t ]*$/;
150
+ const ranges = [];
151
+ let i = 0;
152
+ while (i < lines.length) {
153
+ if (i + 1 < lines.length &&
154
+ TABLE_ROW_RE.test(lines[i]) &&
155
+ this.isTableSeparator(lines[i + 1], SEPARATOR_CELL_RE)) {
156
+ const start = i;
157
+ i += 2;
158
+ while (i < lines.length && TABLE_ROW_RE.test(lines[i])) {
159
+ i++;
160
+ }
161
+ ranges.push({ start, end: i - 1 });
162
+ }
163
+ else {
164
+ i++;
165
+ }
166
+ }
167
+ return ranges;
168
+ }
169
+ /** Check if a line is a markdown table separator (e.g. |---|---|). */
170
+ isTableSeparator(line, cellRe) {
171
+ const trimmed = line.trimEnd();
172
+ if (!trimmed.startsWith("|")) {
173
+ return false;
174
+ }
175
+ // Split by "|" → ["", "---", "---", ""] for "|---|---|"
176
+ const cells = trimmed.split("|");
177
+ cells.shift(); // remove leading empty element
178
+ if (cells.length > 0 && cells[cells.length - 1].trim() === "") {
179
+ cells.pop(); // remove trailing empty element
180
+ }
181
+ if (cells.length === 0) {
182
+ return false;
183
+ }
184
+ return cells.every((cell) => cellRe.test(cell));
185
+ }
186
+ /**
187
+ * Split content while preserving markdown tables.
188
+ */
189
+ splitContentTableAware(content, lines, tableRanges, maxSize, overlap) {
190
+ // Build segments: alternating non-table and table blocks
191
+ const segments = [];
192
+ let lineIdx = 0;
193
+ for (const range of tableRanges) {
194
+ if (lineIdx < range.start) {
195
+ const text = lines.slice(lineIdx, range.start).join("\n").trim();
196
+ if (text) {
197
+ segments.push({ text, isTable: false });
198
+ }
199
+ }
200
+ const tableText = lines.slice(range.start, range.end + 1).join("\n");
201
+ segments.push({ text: tableText, isTable: true });
202
+ lineIdx = range.end + 1;
203
+ }
204
+ if (lineIdx < lines.length) {
205
+ const text = lines.slice(lineIdx).join("\n").trim();
206
+ if (text) {
207
+ segments.push({ text, isTable: false });
208
+ }
209
+ }
210
+ const result = [];
211
+ let current = "";
212
+ for (const seg of segments) {
213
+ if (!seg.isTable) {
214
+ const pieces = this.splitPlainContent(seg.text, maxSize, overlap);
215
+ for (const piece of pieces) {
216
+ if (current.length === 0) {
217
+ current = piece;
218
+ }
219
+ else if (current.length + 1 + piece.length <= maxSize) {
220
+ current += "\n" + piece;
221
+ }
222
+ else {
223
+ result.push(current);
224
+ current = piece;
225
+ }
226
+ }
227
+ }
228
+ else {
229
+ if (seg.text.length <= maxSize) {
230
+ if (current.length === 0) {
231
+ current = seg.text;
232
+ }
233
+ else if (current.length + 2 + seg.text.length <= maxSize) {
234
+ current += "\n\n" + seg.text;
235
+ }
236
+ else {
237
+ result.push(current);
238
+ current = seg.text;
239
+ }
240
+ }
241
+ else {
242
+ if (current) {
243
+ result.push(current);
244
+ current = "";
245
+ }
246
+ const tableChunks = this.splitTableByRows(seg.text, maxSize);
247
+ result.push(...tableChunks);
248
+ }
249
+ }
250
+ }
251
+ if (current) {
252
+ result.push(current);
253
+ }
254
+ return result.length > 0 ? result : [content];
255
+ }
256
+ /**
257
+ * Split a table on row boundaries, repeating header + separator in each chunk.
258
+ */
259
+ splitTableByRows(tableText, maxSize) {
260
+ const rows = tableText.split("\n");
261
+ if (rows.length < 3) {
262
+ return [tableText];
263
+ }
264
+ const headerRow = rows[0];
265
+ const separatorRow = rows[1];
266
+ const headerBlock = headerRow + "\n" + separatorRow;
267
+ const dataRows = rows.slice(2);
268
+ if (headerBlock.length > maxSize) {
269
+ return this.splitPlainContent(tableText, maxSize, 0);
270
+ }
271
+ const chunks = [];
272
+ let currentChunk = headerBlock;
273
+ for (const row of dataRows) {
274
+ // Guard: single row exceeds budget — flush and emit as standalone chunk
275
+ const singleRowChunk = `${headerBlock}\n${row}`;
276
+ if (singleRowChunk.length > maxSize) {
277
+ if (currentChunk.length > headerBlock.length) {
278
+ chunks.push(currentChunk);
279
+ }
280
+ chunks.push(singleRowChunk);
281
+ currentChunk = headerBlock;
282
+ continue;
283
+ }
284
+ const candidate = currentChunk + "\n" + row;
285
+ if (candidate.length <= maxSize) {
286
+ currentChunk = candidate;
287
+ }
288
+ else {
289
+ if (currentChunk.length > headerBlock.length) {
290
+ chunks.push(currentChunk);
291
+ }
292
+ currentChunk = headerBlock + "\n" + row;
293
+ }
294
+ }
295
+ if (currentChunk.length > headerBlock.length) {
296
+ chunks.push(currentChunk);
297
+ }
298
+ return chunks.length > 0 ? chunks : [tableText];
299
+ }
300
+ splitPlainContent(content, maxSize, overlap) {
301
+ if (content.length <= maxSize) {
302
+ return [content];
303
+ }
132
304
  const chunks = [];
133
305
  let start = 0;
134
306
  while (start < content.length) {
135
- let end = Math.min(start + effectiveMaxSize, content.length);
307
+ let end = Math.min(start + maxSize, content.length);
136
308
  // Try to break at a paragraph or sentence boundary
137
309
  if (end < content.length) {
138
310
  const searchStart = Math.max(start, end - 200);
@@ -151,7 +323,7 @@ export class MarkdownChunker {
151
323
  }
152
324
  }
153
325
  chunks.push(content.slice(start, end));
154
- start = Math.max(start + 1, end - effectiveOverlap);
326
+ start = Math.max(start + 1, end - overlap);
155
327
  }
156
328
  return chunks;
157
329
  }
@@ -12,6 +12,7 @@
12
12
  * - Context summarization
13
13
  */
14
14
  import { logger } from "../../utils/logger.js";
15
+ import { estimateTokens } from "../../utils/tokenEstimation.js";
15
16
  /**
16
17
  * Assemble context from retrieved results
17
18
  *
@@ -184,7 +185,7 @@ export function createContextWindow(results, options) {
184
185
  text,
185
186
  chunkCount,
186
187
  charCount: text.length,
187
- tokenCount: Math.ceil(text.length / 4),
188
+ tokenCount: estimateTokens(text),
188
189
  truncatedChunks,
189
190
  citations,
190
191
  };
@@ -7,7 +7,20 @@
7
7
  * so developers only need to pass `rag: { files: [...] }`.
8
8
  */
9
9
  import type { Tool } from "ai";
10
- import type { RAGConfig } from "./types.js";
10
+ import type { RAGConfig, VectorQueryResult } from "./types.js";
11
+ /**
12
+ * Generate deterministic embeddings for chunks.
13
+ * Combines character-frequency (40%) with word-level hash features (60%)
14
+ * for better semantic discrimination than pure character frequency.
15
+ * When a real embedding provider is configured, it will be used instead.
16
+ */
17
+ declare function generateSimpleEmbedding(text: string, dimension: number): number[];
18
+ /**
19
+ * Diversify retrieval results via round-robin across source files.
20
+ * Ensures at least one chunk per source file appears in the top-K results,
21
+ * preventing any single file from dominating retrieval.
22
+ */
23
+ declare function diversifyResults(results: VectorQueryResult[], topK: number): VectorQueryResult[];
11
24
  /**
12
25
  * Result of preparing RAG for a generate/stream call
13
26
  */
@@ -36,3 +49,7 @@ export type RAGPreparedTool = {
36
49
  * @returns Prepared RAG tool to inject into the tools record
37
50
  */
38
51
  export declare function prepareRAGTool(ragConfig: RAGConfig, fallbackProvider?: string): Promise<RAGPreparedTool>;
52
+ /** @internal Exported for testing only */
53
+ export { generateSimpleEmbedding as _generateSimpleEmbedding };
54
+ /** @internal Exported for testing only */
55
+ export { diversifyResults as _diversifyResults };
@@ -10,7 +10,7 @@ import { existsSync, readFileSync } from "fs";
10
10
  import { extname, resolve } from "path";
11
11
  import { z } from "zod";
12
12
  import { logger } from "../utils/logger.js";
13
- import { ChunkerRegistry } from "./chunking/index.js";
13
+ import { createChunker } from "./ChunkerFactory.js";
14
14
  import { createVectorQueryTool, InMemoryVectorStore, } from "./retrieval/vectorQueryTool.js";
15
15
  /**
16
16
  * Maps file extensions to recommended chunking strategies
@@ -48,27 +48,91 @@ function detectStrategy(filePath) {
48
48
  const ext = extname(filePath).toLowerCase();
49
49
  return EXTENSION_TO_STRATEGY[ext] || "recursive";
50
50
  }
51
+ /**
52
+ * Simple hash function for strings (FNV-1a variant).
53
+ * Maps a word to a bucket index deterministically.
54
+ */
55
+ function hashWord(word, buckets) {
56
+ let hash = 2166136261;
57
+ for (let i = 0; i < word.length; i++) {
58
+ hash ^= word.charCodeAt(i);
59
+ hash = (hash * 16777619) >>> 0;
60
+ }
61
+ return hash % buckets;
62
+ }
51
63
  /**
52
64
  * Generate deterministic embeddings for chunks.
53
- * Uses a simple hash-based approach for the in-memory vector store.
65
+ * Combines character-frequency (40%) with word-level hash features (60%)
66
+ * for better semantic discrimination than pure character frequency.
54
67
  * When a real embedding provider is configured, it will be used instead.
55
68
  */
56
69
  function generateSimpleEmbedding(text, dimension) {
57
- const embedding = new Array(dimension).fill(0);
58
- // Simple character-frequency based embedding
70
+ const charEmbedding = new Array(dimension).fill(0);
71
+ const wordEmbedding = new Array(dimension).fill(0);
72
+ // Character-frequency features
59
73
  for (let i = 0; i < text.length; i++) {
60
74
  const charCode = text.charCodeAt(i);
61
75
  const idx = charCode % dimension;
62
- embedding[idx] += 1;
76
+ charEmbedding[idx] += 1;
77
+ }
78
+ // Word-level hash features (TF-IDF-like)
79
+ const words = text
80
+ .toLowerCase()
81
+ .replace(/[^a-z0-9\s]/g, "")
82
+ .split(/\s+/)
83
+ .filter((w) => w.length > 1);
84
+ for (const word of words) {
85
+ const idx = hashWord(word, dimension);
86
+ wordEmbedding[idx] += 1;
87
+ }
88
+ // Combine: 40% character, 60% word
89
+ const combined = new Array(dimension);
90
+ for (let i = 0; i < dimension; i++) {
91
+ combined[i] = 0.4 * charEmbedding[i] + 0.6 * wordEmbedding[i];
63
92
  }
64
93
  // Normalize to unit vector
65
- const magnitude = Math.sqrt(embedding.reduce((sum, v) => sum + v * v, 0));
94
+ const magnitude = Math.sqrt(combined.reduce((sum, v) => sum + v * v, 0));
66
95
  if (magnitude > 0) {
67
96
  for (let i = 0; i < dimension; i++) {
68
- embedding[i] /= magnitude;
97
+ combined[i] /= magnitude;
69
98
  }
70
99
  }
71
- return embedding;
100
+ return combined;
101
+ }
102
+ /**
103
+ * Diversify retrieval results via round-robin across source files.
104
+ * Ensures at least one chunk per source file appears in the top-K results,
105
+ * preventing any single file from dominating retrieval.
106
+ */
107
+ function diversifyResults(results, topK) {
108
+ // Group by source file
109
+ const byFile = new Map();
110
+ for (const r of results) {
111
+ const source = r.metadata?.source || "unknown";
112
+ if (!byFile.has(source)) {
113
+ byFile.set(source, []);
114
+ }
115
+ const sourceGroup = byFile.get(source);
116
+ if (sourceGroup) {
117
+ sourceGroup.push(r);
118
+ }
119
+ }
120
+ // If only one source file, no diversification needed
121
+ if (byFile.size <= 1) {
122
+ return results.slice(0, topK);
123
+ }
124
+ // Round-robin selection from each source file group
125
+ const diversified = [];
126
+ const iterators = [...byFile.values()].map((arr) => ({ arr, idx: 0 }));
127
+ while (diversified.length < topK &&
128
+ iterators.some((it) => it.idx < it.arr.length)) {
129
+ for (const it of iterators) {
130
+ if (it.idx < it.arr.length && diversified.length < topK) {
131
+ diversified.push(it.arr[it.idx++]);
132
+ }
133
+ }
134
+ }
135
+ return diversified;
72
136
  }
73
137
  /**
74
138
  * Prepare RAG tools from the provided configuration.
@@ -85,7 +149,7 @@ function generateSimpleEmbedding(text, dimension) {
85
149
  * @returns Prepared RAG tool to inject into the tools record
86
150
  */
87
151
  export async function prepareRAGTool(ragConfig, fallbackProvider) {
88
- const { files, strategy: userStrategy, chunkSize = 1000, chunkOverlap = 200, topK = 5, toolName = "search_knowledge_base", toolDescription = "REQUIRED: Search through pre-loaded local documents to find relevant information. Use this tool FIRST before any web search or other tools. This searches an indexed knowledge base of documents the user has provided.", embeddingProvider, embeddingModel, } = ragConfig;
152
+ const { files, strategy: userStrategy, chunkSize = 1000, chunkOverlap = 200, topK: userTopK = 5, toolName = "search_knowledge_base", toolDescription = "REQUIRED: Search through pre-loaded local documents to find relevant information. Use this tool FIRST before any web search or other tools. This searches an indexed knowledge base of documents the user has provided.", embeddingProvider, embeddingModel, } = ragConfig;
89
153
  if (!files || files.length === 0) {
90
154
  throw new Error("RAG config requires at least one file path in 'files'");
91
155
  }
@@ -106,6 +170,11 @@ export async function prepareRAGTool(ragConfig, fallbackProvider) {
106
170
  logger.warn(`[RAG] Failed to read file: ${resolvedPath}: ${error instanceof Error ? error.message : String(error)}`);
107
171
  }
108
172
  }
173
+ // Auto-increase topK for multi-file scenarios to ensure coverage
174
+ // (computed after loading so it reflects only files that actually exist)
175
+ const topK = fileContents.length > 1
176
+ ? Math.max(userTopK, fileContents.length * 3)
177
+ : userTopK;
109
178
  if (fileContents.length === 0) {
110
179
  throw new Error("RAG: No files could be loaded. Check that file paths exist and are readable.");
111
180
  }
@@ -114,10 +183,11 @@ export async function prepareRAGTool(ragConfig, fallbackProvider) {
114
183
  const allChunks = [];
115
184
  for (const { path, content, strategy } of fileContents) {
116
185
  try {
117
- const chunker = ChunkerRegistry.get(strategy);
118
- const chunks = await chunker.chunk(content, {
186
+ const chunker = await createChunker(strategy, {
119
187
  maxSize: chunkSize,
120
- overlap: chunkOverlap,
188
+ overlap: Math.min(chunkOverlap, Math.floor(chunkSize * 0.5)),
189
+ });
190
+ const chunks = await chunker.chunk(content, {
121
191
  metadata: { source: path },
122
192
  });
123
193
  for (const chunk of chunks) {
@@ -175,11 +245,17 @@ export async function prepareRAGTool(ragConfig, fallbackProvider) {
175
245
  // For the in-memory store with simple embeddings,
176
246
  // generate a query embedding using the same method
177
247
  const queryEmbedding = generateSimpleEmbedding(query, EMBEDDING_DIMENSION);
178
- const results = await vectorStore.query({
248
+ // Fetch more candidates than needed so diversity can select across files
249
+ const fetchK = fileContents.length > 1 ? topK * 3 : topK;
250
+ const rawResults = await vectorStore.query({
179
251
  indexName,
180
252
  queryVector: queryEmbedding,
181
- topK,
253
+ topK: fetchK,
182
254
  });
255
+ // Apply source-file diversity for multi-file RAG
256
+ const results = fileContents.length > 1
257
+ ? diversifyResults(rawResults, topK)
258
+ : rawResults.slice(0, topK);
183
259
  if (results.length === 0) {
184
260
  return {
185
261
  relevantContext: "No relevant documents found for the query.",
@@ -209,3 +285,7 @@ export async function prepareRAGTool(ragConfig, fallbackProvider) {
209
285
  filesLoaded: fileContents.length,
210
286
  };
211
287
  }
288
+ /** @internal Exported for testing only */
289
+ export { generateSimpleEmbedding as _generateSimpleEmbedding };
290
+ /** @internal Exported for testing only */
291
+ export { diversifyResults as _diversifyResults };
@@ -253,10 +253,27 @@ export class InMemoryVectorStore {
253
253
  !fieldValue.includes(ops.$contains))) {
254
254
  return false;
255
255
  }
256
- if ("$regex" in ops &&
257
- (typeof fieldValue !== "string" ||
258
- !new RegExp(ops.$regex).test(fieldValue))) {
259
- return false;
256
+ if ("$regex" in ops) {
257
+ const pattern = ops.$regex;
258
+ let regexMatches = false;
259
+ // Guard against ReDoS: reject excessively long patterns and limit
260
+ // the tested string length to prevent pathological backtracking.
261
+ if (pattern.length <= 200) {
262
+ try {
263
+ const re = new RegExp(pattern);
264
+ const testValue = typeof fieldValue === "string"
265
+ ? fieldValue.slice(0, 10_000)
266
+ : "";
267
+ regexMatches = re.test(testValue);
268
+ }
269
+ catch {
270
+ // Invalid regex pattern — treat as non-match
271
+ regexMatches = false;
272
+ }
273
+ }
274
+ if (!regexMatches) {
275
+ return false;
276
+ }
260
277
  }
261
278
  }
262
279
  else {
@@ -166,7 +166,10 @@ export class BaseServerAdapter extends EventEmitter {
166
166
  }
167
167
  // Register all routes in the group with prefix applied
168
168
  for (const route of group.routes) {
169
- const prefixedPath = this.normalizePath(`${group.prefix}${route.path}`);
169
+ // Only prepend prefix if route path doesn't already start with it
170
+ // (route definitions include full paths like /api/agent/execute)
171
+ const needsPrefix = !route.path.startsWith(group.prefix);
172
+ const prefixedPath = this.normalizePath(needsPrefix ? `${group.prefix}${route.path}` : route.path);
170
173
  const prefixedRoute = {
171
174
  ...route,
172
175
  path: prefixedPath,
@@ -5,6 +5,7 @@
5
5
  */
6
6
  import { logger } from "../../utils/logger.js";
7
7
  import { AlreadyRunningError, ServerStartError, ServerStopError, wrapError, } from "../errors.js";
8
+ import { withTimeout } from "../../utils/errorHandling.js";
8
9
  import { BaseServerAdapter } from "../abstract/baseServerAdapter.js";
9
10
  import { isErrorResponse } from "../utils/validation.js";
10
11
  /**
@@ -12,7 +13,7 @@ import { isErrorResponse } from "../utils/validation.js";
12
13
  * Provides high-performance HTTP server with schema validation
13
14
  */
14
15
  export class FastifyServerAdapter extends BaseServerAdapter {
15
- app;
16
+ app = null;
16
17
  frameworkInitialized = false;
17
18
  constructor(neurolink, config = {}) {
18
19
  super(neurolink, config);
@@ -174,7 +175,15 @@ export class FastifyServerAdapter extends BaseServerAdapter {
174
175
  * Register route with Fastify
175
176
  */
176
177
  registerFrameworkRoute(route) {
178
+ if (!this.app) {
179
+ throw new Error("Fastify app not initialized. Call initialize() before registering routes.");
180
+ }
177
181
  const method = route.method.toUpperCase();
182
+ // Fastify does not allow duplicate method+path registrations.
183
+ // Skip if route already exists (e.g., built-in health routes).
184
+ if (this.app.hasRoute({ method, url: route.path })) {
185
+ return;
186
+ }
178
187
  this.app.route({
179
188
  method,
180
189
  url: route.path,
@@ -307,6 +316,9 @@ export class FastifyServerAdapter extends BaseServerAdapter {
307
316
  * Register middleware with Fastify
308
317
  */
309
318
  registerFrameworkMiddleware(middleware) {
319
+ if (!this.app) {
320
+ throw new Error("Fastify app not initialized. Call initialize() before registering middleware.");
321
+ }
310
322
  this.app.addHook("preHandler", async (request, _reply) => {
311
323
  // Skip excluded paths
312
324
  if (middleware.excludePaths?.some((p) => request.url.startsWith(p))) {
@@ -357,50 +369,39 @@ export class FastifyServerAdapter extends BaseServerAdapter {
357
369
  if (this.isRunning) {
358
370
  throw new AlreadyRunningError(this.config.port, this.config.host);
359
371
  }
372
+ if (!this.app) {
373
+ throw new Error("Fastify app not initialized. Call initialize() before starting.");
374
+ }
375
+ // Capture non-null reference for use in closures below
376
+ const app = this.app;
360
377
  this.lifecycleState = "starting";
361
378
  const { port, host } = this.config;
362
379
  const startupTimeout = this.config.timeout || 30000;
363
- const startPromise = (async () => {
364
- await this.app.listen({ port, host });
380
+ // Track connections via Fastify hooks (must be registered before listen)
381
+ app.addHook("onRequest", async (request) => {
382
+ const connectionId = `conn-${request.id}`;
383
+ this.trackConnection(connectionId, request.raw.socket, request.id);
384
+ });
385
+ app.addHook("onResponse", async (request) => {
386
+ const connectionId = `conn-${request.id}`;
387
+ this.untrackConnection(connectionId);
388
+ });
389
+ try {
390
+ await withTimeout(app.listen({ port, host }), startupTimeout, new ServerStartError(`Fastify server startup timed out after ${startupTimeout}ms`, undefined, port, host));
365
391
  this.isRunning = true;
366
392
  this.startTime = new Date();
367
393
  this.lifecycleState = "running";
368
- // Track connections via Fastify hooks
369
- this.app.addHook("onRequest", async (request) => {
370
- const connectionId = `conn-${request.id}`;
371
- this.trackConnection(connectionId, request.raw.socket, request.id);
372
- });
373
- this.app.addHook("onResponse", async (request) => {
374
- const connectionId = `conn-${request.id}`;
375
- this.untrackConnection(connectionId);
376
- });
377
394
  logger.info(`[FastifyAdapter] Server started on ${host}:${port}`);
378
395
  this.emit("started", {
379
396
  port,
380
397
  host,
381
398
  timestamp: this.startTime,
382
399
  });
383
- })();
384
- let startupTimer;
385
- const timeoutPromise = new Promise((_, reject) => {
386
- startupTimer = setTimeout(() => {
387
- this.lifecycleState = "error";
388
- reject(new ServerStartError(`Fastify server startup timed out after ${startupTimeout}ms`, undefined, port, host));
389
- }, startupTimeout);
390
- });
391
- try {
392
- await Promise.race([startPromise, timeoutPromise]);
393
400
  }
394
401
  catch (error) {
395
402
  this.lifecycleState = "error";
396
403
  throw error;
397
404
  }
398
- finally {
399
- // Always clear the timeout to prevent memory leak
400
- if (startupTimer) {
401
- clearTimeout(startupTimer);
402
- }
403
- }
404
405
  }
405
406
  /**
406
407
  * Stop the Fastify server with graceful shutdown
@@ -421,6 +422,7 @@ export class FastifyServerAdapter extends BaseServerAdapter {
421
422
  // Reset state for restart capability
422
423
  this.resetServerState();
423
424
  this.frameworkInitialized = false;
425
+ this.app = null;
424
426
  }
425
427
  catch (error) {
426
428
  const wrappedError = wrapError(error);
@@ -441,7 +443,10 @@ export class FastifyServerAdapter extends BaseServerAdapter {
441
443
  * Close the underlying server
442
444
  */
443
445
  async closeServer() {
444
- await this.app.close();
446
+ if (this.app) {
447
+ const closeTimeout = this.shutdownConfig.gracefulShutdownTimeoutMs;
448
+ await withTimeout(this.app.close(), closeTimeout, new Error(`Fastify server close timed out after ${closeTimeout}ms`));
449
+ }
445
450
  }
446
451
  /**
447
452
  * Force close all active connections
@@ -451,7 +456,7 @@ export class FastifyServerAdapter extends BaseServerAdapter {
451
456
  count: this.activeConnections.size,
452
457
  });
453
458
  // Get the underlying server and destroy all sockets
454
- const server = this.app.server;
459
+ const server = this.app?.server;
455
460
  if (server) {
456
461
  // Force close by destroying the server
457
462
  server.closeAllConnections?.();
@@ -177,6 +177,38 @@ export declare function setLangfuseContext<T = void>(context: {
177
177
  * console.log(context?.userId, context?.sessionId);
178
178
  */
179
179
  export declare function getLangfuseContext(): LangfuseContext | undefined;
180
+ /**
181
+ * Capture the current Langfuse AsyncLocalStorage context and return a wrapper
182
+ * that re-enters that context when executing the provided callback.
183
+ *
184
+ * This is essential for preserving trace context across async boundaries that
185
+ * break the automatic ALS propagation chain, such as `setImmediate()`,
186
+ * `setTimeout()`, or event-emitter callbacks. Without this, spans created
187
+ * inside those callbacks become orphaned traces in Langfuse.
188
+ *
189
+ * **How it works:**
190
+ * 1. Captures the current ALS store at call time (synchronously).
191
+ * 2. Returns an async function that, when invoked, re-enters the captured
192
+ * context via `contextStorage.run()` before executing the callback.
193
+ * 3. If no context exists at capture time, the callback runs without
194
+ * ALS wrapping (no-op passthrough).
195
+ *
196
+ * @param fn - The async function to execute within the captured context
197
+ * @returns A new async function that preserves the Langfuse ALS context
198
+ *
199
+ * @example
200
+ * // Before (broken — setImmediate loses ALS context):
201
+ * setImmediate(async () => {
202
+ * await this.checkAndSummarize(session, threshold);
203
+ * });
204
+ *
205
+ * // After (fixed — context is captured and re-entered):
206
+ * const wrappedFn = runWithCurrentLangfuseContext(async () => {
207
+ * await this.checkAndSummarize(session, threshold);
208
+ * });
209
+ * setImmediate(wrappedFn);
210
+ */
211
+ export declare function runWithCurrentLangfuseContext<T>(fn: () => Promise<T>): () => Promise<T>;
180
212
  /**
181
213
  * Get an OpenTelemetry Tracer for creating custom spans
182
214
  *