@juspay/neurolink 13.8.17 → 13.8.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (214) hide show
  1. package/CHANGELOG.md +5 -2
  2. package/README.md +1 -1
  3. package/dist/action/actionExecutor.d.ts +33 -3
  4. package/dist/action/actionExecutor.js +71 -15
  5. package/dist/action/actionInputs.js +4 -1
  6. package/dist/action/cliVersion.d.ts +25 -0
  7. package/dist/action/cliVersion.js +26 -0
  8. package/dist/action/index.d.ts +2 -1
  9. package/dist/action/index.js +2 -1
  10. package/dist/adapters/video/directorPipeline.js +46 -36
  11. package/dist/adapters/video/ffmpegAdapter.d.ts +17 -0
  12. package/dist/adapters/video/ffmpegAdapter.js +32 -5
  13. package/dist/adapters/video/frameExtractor.js +1 -1
  14. package/dist/adapters/video/videoMerger.js +5 -2
  15. package/dist/agent/arithmeticEvaluator.d.ts +32 -0
  16. package/dist/agent/arithmeticEvaluator.js +349 -0
  17. package/dist/agent/directTools.d.ts +6 -18
  18. package/dist/agent/directTools.js +17 -56
  19. package/dist/artifacts/artifactStore.d.ts +22 -1
  20. package/dist/artifacts/artifactStore.js +66 -6
  21. package/dist/auth/anthropicOAuth.d.ts +26 -1
  22. package/dist/auth/anthropicOAuth.js +81 -5
  23. package/dist/auth/secureFiles.d.ts +37 -0
  24. package/dist/auth/secureFiles.js +95 -0
  25. package/dist/auth/tokenStore.d.ts +43 -6
  26. package/dist/auth/tokenStore.js +260 -79
  27. package/dist/avatar/providers/DIDAvatar.js +49 -45
  28. package/dist/avatar/providers/ReplicateAvatar.js +33 -40
  29. package/dist/browser/neurolink.min.js +485 -558
  30. package/dist/cli/commands/auth.js +112 -70
  31. package/dist/cli/commands/proxy.d.ts +4 -0
  32. package/dist/cli/commands/proxy.js +128 -13
  33. package/dist/cli/commands/serve.d.ts +10 -1
  34. package/dist/cli/commands/serve.js +98 -17
  35. package/dist/cli/commands/server.js +17 -5
  36. package/dist/cli/index.js +146 -5
  37. package/dist/cli/loop/optionsSchema.d.ts +1 -1
  38. package/dist/cli/utils/envManager.js +17 -3
  39. package/dist/cli/utils/serverUtils.d.ts +39 -3
  40. package/dist/cli/utils/serverUtils.js +63 -6
  41. package/dist/core/baseProvider.js +7 -1
  42. package/dist/core/modules/MessageBuilder.js +5 -0
  43. package/dist/core/modules/ToolsManager.d.ts +46 -1
  44. package/dist/core/modules/ToolsManager.js +190 -43
  45. package/dist/files/fileReferenceRegistry.d.ts +2 -23
  46. package/dist/files/fileReferenceRegistry.js +55 -5
  47. package/dist/hitl/hitlManager.js +1 -0
  48. package/dist/index.d.ts +2 -1
  49. package/dist/index.js +6 -1
  50. package/dist/mcp/auth/tokenStorage.d.ts +7 -0
  51. package/dist/mcp/auth/tokenStorage.js +23 -4
  52. package/dist/mcp/externalServerManager.d.ts +13 -3
  53. package/dist/mcp/externalServerManager.js +154 -136
  54. package/dist/mcp/index.d.ts +2 -0
  55. package/dist/mcp/index.js +3 -0
  56. package/dist/mcp/mcpClientFactory.d.ts +14 -0
  57. package/dist/mcp/mcpClientFactory.js +70 -7
  58. package/dist/mcp/projectConfigTrust.d.ts +72 -0
  59. package/dist/mcp/projectConfigTrust.js +237 -0
  60. package/dist/mcp/transportUrlPolicy.d.ts +39 -0
  61. package/dist/mcp/transportUrlPolicy.js +172 -0
  62. package/dist/music/providers/BeatovenMusic.js +35 -46
  63. package/dist/music/providers/ReplicateMusic.js +33 -40
  64. package/dist/neurolink.d.ts +16 -1
  65. package/dist/neurolink.js +64 -11
  66. package/dist/processors/archive/ArchiveProcessor.js +15 -1
  67. package/dist/processors/archive/zipEntryIndex.d.ts +22 -0
  68. package/dist/processors/archive/zipEntryIndex.js +33 -0
  69. package/dist/processors/base/BaseFileProcessor.d.ts +9 -21
  70. package/dist/processors/base/BaseFileProcessor.js +40 -111
  71. package/dist/processors/data/YamlProcessor.d.ts +13 -1
  72. package/dist/processors/data/YamlProcessor.js +93 -2
  73. package/dist/processors/document/OpenDocumentProcessor.js +7 -2
  74. package/dist/processors/document/PptxProcessor.js +12 -7
  75. package/dist/processors/media/VideoProcessor.js +93 -15
  76. package/dist/providers/openaiChatCompletionsClient.js +12 -3
  77. package/dist/proxy/accountLedger.js +6 -8
  78. package/dist/proxy/geminiFormat.d.ts +3 -11
  79. package/dist/proxy/geminiFormat.js +11 -2
  80. package/dist/proxy/inboundAuth.d.ts +89 -0
  81. package/dist/proxy/inboundAuth.js +219 -0
  82. package/dist/proxy/openaiFormat.d.ts +3 -9
  83. package/dist/proxy/openaiFormat.js +21 -15
  84. package/dist/proxy/proxyTranslationEngine.d.ts +2 -6
  85. package/dist/proxy/proxyTranslationEngine.js +25 -14
  86. package/dist/proxy/routingPolicy.d.ts +3 -1
  87. package/dist/proxy/routingPolicy.js +7 -8
  88. package/dist/proxy/tokenRefresh.d.ts +6 -0
  89. package/dist/proxy/tokenRefresh.js +13 -1
  90. package/dist/proxy/updateState.js +4 -4
  91. package/dist/rag/document/loaders.d.ts +0 -6
  92. package/dist/rag/document/loaders.js +16 -10
  93. package/dist/rag/ragIntegration.d.ts +2 -2
  94. package/dist/rag/ragIntegration.js +35 -10
  95. package/dist/rag/retrieval/vectorQueryTool.d.ts +7 -0
  96. package/dist/rag/retrieval/vectorQueryTool.js +54 -17
  97. package/dist/server/abstract/baseServerAdapter.d.ts +43 -0
  98. package/dist/server/abstract/baseServerAdapter.js +119 -6
  99. package/dist/server/adapters/expressAdapter.d.ts +3 -0
  100. package/dist/server/adapters/expressAdapter.js +78 -6
  101. package/dist/server/adapters/fastifyAdapter.d.ts +3 -0
  102. package/dist/server/adapters/fastifyAdapter.js +165 -70
  103. package/dist/server/adapters/honoAdapter.d.ts +9 -1
  104. package/dist/server/adapters/honoAdapter.js +99 -21
  105. package/dist/server/adapters/koaAdapter.d.ts +1 -0
  106. package/dist/server/adapters/koaAdapter.js +153 -74
  107. package/dist/server/index.d.ts +2 -0
  108. package/dist/server/index.js +4 -0
  109. package/dist/server/middleware/auth.js +6 -2
  110. package/dist/server/middleware/cache.js +6 -1
  111. package/dist/server/middleware/rateLimit.js +29 -4
  112. package/dist/server/routes/agentRoutes.js +24 -11
  113. package/dist/server/routes/claudeProxyRoutes.d.ts +1 -0
  114. package/dist/server/routes/claudeProxyRoutes.js +273 -84
  115. package/dist/server/routes/codexProxyRoutes.d.ts +2 -2
  116. package/dist/server/routes/codexProxyRoutes.js +125 -40
  117. package/dist/server/routes/index.d.ts +4 -0
  118. package/dist/server/routes/index.js +35 -4
  119. package/dist/server/routes/mcpRoutes.d.ts +6 -2
  120. package/dist/server/routes/mcpRoutes.js +21 -7
  121. package/dist/server/routes/memoryRoutes.js +7 -6
  122. package/dist/server/routes/toolRoutes.d.ts +12 -2
  123. package/dist/server/routes/toolRoutes.js +33 -3
  124. package/dist/server/routes/transcribeRoutes.js +60 -13
  125. package/dist/server/utils/clientAddress.d.ts +23 -0
  126. package/dist/server/utils/clientAddress.js +65 -0
  127. package/dist/server/utils/clientError.d.ts +35 -0
  128. package/dist/server/utils/clientError.js +129 -0
  129. package/dist/server/utils/pathMatch.d.ts +21 -0
  130. package/dist/server/utils/pathMatch.js +41 -0
  131. package/dist/server/utils/validation.d.ts +9 -0
  132. package/dist/server/utils/validation.js +9 -0
  133. package/dist/server/websocket/WebSocketHandler.js +22 -3
  134. package/dist/server/websocket/frameQueue.d.ts +19 -0
  135. package/dist/server/websocket/frameQueue.js +41 -3
  136. package/dist/server/websocket/transcribeWebSocket.d.ts +8 -1
  137. package/dist/server/websocket/transcribeWebSocket.js +223 -34
  138. package/dist/skills/skillStores.js +5 -0
  139. package/dist/tasks/store/fileTaskStore.d.ts +2 -0
  140. package/dist/tasks/store/fileTaskStore.js +11 -4
  141. package/dist/types/attachments.d.ts +117 -0
  142. package/dist/types/attachments.js +15 -0
  143. package/dist/types/auth.d.ts +6 -0
  144. package/dist/types/cli.d.ts +4 -0
  145. package/dist/types/config.d.ts +56 -0
  146. package/dist/types/file.d.ts +8 -0
  147. package/dist/types/hitl.d.ts +8 -0
  148. package/dist/types/index.d.ts +1 -0
  149. package/dist/types/index.js +1 -0
  150. package/dist/types/mcp.d.ts +98 -0
  151. package/dist/types/multimodal.js +0 -41
  152. package/dist/types/openaiCompatible.d.ts +1 -1
  153. package/dist/types/processor.d.ts +14 -1
  154. package/dist/types/proxy.d.ts +71 -16
  155. package/dist/types/proxyClient.d.ts +2 -0
  156. package/dist/types/rag.d.ts +9 -0
  157. package/dist/types/safeFetch.d.ts +40 -0
  158. package/dist/types/server.d.ts +206 -10
  159. package/dist/types/tools.d.ts +57 -0
  160. package/dist/types/transcribe.d.ts +6 -0
  161. package/dist/utils/attachmentPolicy.d.ts +89 -0
  162. package/dist/utils/attachmentPolicy.js +327 -0
  163. package/dist/utils/dotenvBootstrap.d.ts +48 -19
  164. package/dist/utils/dotenvBootstrap.js +172 -31
  165. package/dist/utils/fileDetector.js +115 -65
  166. package/dist/utils/json/coerce.js +11 -1
  167. package/dist/utils/logSanitize.d.ts +5 -0
  168. package/dist/utils/logSanitize.js +8 -0
  169. package/dist/utils/logger.js +73 -1
  170. package/dist/utils/messageBuilder.d.ts +2 -2
  171. package/dist/utils/messageBuilder.js +118 -105
  172. package/dist/utils/multimodalOptionsBuilder.d.ts +1 -0
  173. package/dist/utils/multimodalOptionsBuilder.js +2 -0
  174. package/dist/utils/redis.d.ts +12 -0
  175. package/dist/utils/redis.js +72 -0
  176. package/dist/utils/retryAfter.d.ts +32 -0
  177. package/dist/utils/retryAfter.js +135 -11
  178. package/dist/utils/safeFetch.d.ts +22 -1
  179. package/dist/utils/safeFetch.js +132 -38
  180. package/dist/utils/safeStoreId.d.ts +19 -0
  181. package/dist/utils/safeStoreId.js +29 -0
  182. package/dist/utils/schemaConversion.d.ts +30 -3
  183. package/dist/utils/schemaConversion.js +457 -84
  184. package/dist/utils/sizeGuard.js +2 -2
  185. package/dist/utils/ssrfGuard.d.ts +3 -3
  186. package/dist/utils/ssrfGuard.js +71 -39
  187. package/dist/voice/eval/sttEvalStore.d.ts +26 -3
  188. package/dist/voice/eval/sttEvalStore.js +77 -12
  189. package/dist/voice/livekit/agentDefinition.d.ts +18 -0
  190. package/dist/voice/livekit/agentDefinition.js +22 -0
  191. package/dist/voice/livekit/realtimeMcpTools.d.ts +1 -1
  192. package/dist/voice/livekit/realtimeMcpTools.js +8 -7
  193. package/dist/voice/livekit/realtimeVoiceAgent.d.ts +2 -4
  194. package/dist/voice/livekit/realtimeVoiceAgent.js +18 -7
  195. package/dist/voice/livekit/voiceAgent.d.ts +2 -2
  196. package/dist/voice/livekit/voiceAgent.js +2 -1
  197. package/dist/voice/livekit/xaiSessionWatch.d.ts +0 -5
  198. package/dist/voice/livekit/xaiSessionWatch.js +30 -0
  199. package/dist/voice/providers/GeminiLive.js +21 -6
  200. package/dist/voice/providers/GoogleSTT.js +8 -5
  201. package/dist/voice/transcribe.d.ts +2 -2
  202. package/dist/voice/transcribe.js +34 -18
  203. package/dist/voice/whistle/audio.js +11 -5
  204. package/dist/workflow/workflows/adaptiveWorkflow.d.ts +2 -2
  205. package/dist/workflow/workflows/adaptiveWorkflow.js +10 -10
  206. package/dist/workflow/workflows/consensusWorkflow.d.ts +4 -4
  207. package/dist/workflow/workflows/consensusWorkflow.js +12 -12
  208. package/dist/workflow/workflows/fallbackWorkflow.d.ts +3 -3
  209. package/dist/workflow/workflows/fallbackWorkflow.js +7 -7
  210. package/dist/workflow/workflows/multiJudgeWorkflow.d.ts +7 -7
  211. package/dist/workflow/workflows/multiJudgeWorkflow.js +31 -31
  212. package/docs-site/static/search-index.json +74 -52
  213. package/package.json +173 -156
  214. package/scripts/observability/import-openobserve-dashboard.mjs +11 -3
@@ -1,4 +1,10 @@
1
1
  import type { RefreshableAccount, RefreshResult, TokenPersistTarget } from "../types/index.js";
2
+ /**
3
+ * An upstream refresh error body is logged for diagnosis, never verbatim: a
4
+ * token endpoint can echo the grant it rejected, and anything that looks like
5
+ * a credential is masked before the line is written.
6
+ */
7
+ export declare function redactRefreshErrorBody(body: string): string;
2
8
  export declare function needsRefresh(account: RefreshableAccount): boolean;
3
9
  export declare function needsBackgroundRefresh(account: RefreshableAccount): boolean;
4
10
  export declare function needsInlineRefresh(account: RefreshableAccount): boolean;
@@ -17,6 +17,18 @@ const SUCCESS_CACHE_MS = 60_000;
17
17
  // slow token-store decrypt can never stall (or abort) a live token refresh.
18
18
  const PEEK_TOKENS_TIMEOUT_MS = 2_000;
19
19
  const refreshesInFlight = new Map();
20
+ /**
21
+ * An upstream refresh error body is logged for diagnosis, never verbatim: a
22
+ * token endpoint can echo the grant it rejected, and anything that looks like
23
+ * a credential is masked before the line is written.
24
+ */
25
+ export function redactRefreshErrorBody(body) {
26
+ return body
27
+ .replace(/("?(?:access_token|refresh_token|id_token|client_secret|code)"?\s*[:=]\s*"?)[^",\s]+/gi, "$1[REDACTED]")
28
+ .replace(/\bsk-ant-[A-Za-z0-9_-]+/g, "sk-ant-[REDACTED]")
29
+ .replace(/\b[A-Za-z0-9_-]{64,}\b/g, "[REDACTED]")
30
+ .slice(0, 300);
31
+ }
20
32
  function expiresWithin(account, bufferMs) {
21
33
  return !!(account.expiresAt &&
22
34
  account.expiresAt <= Date.now() + bufferMs &&
@@ -69,7 +81,7 @@ async function performTokenRefresh(account) {
69
81
  const errorBody = await resp.text();
70
82
  logger.warn(`[token-refresh] failed for ${account.label} at ${redactUrlForError(url)}`, {
71
83
  status: resp.status,
72
- error: errorBody.slice(0, 500),
84
+ error: redactRefreshErrorBody(errorBody),
73
85
  });
74
86
  const failure = {
75
87
  success: false,
@@ -7,6 +7,7 @@
7
7
  * Suppressed versions expire after 24 hours.
8
8
  */
9
9
  import fs from "fs";
10
+ import { ensurePrivateDirectorySync } from "../auth/secureFiles.js";
10
11
  import os from "os";
11
12
  import path from "path";
12
13
  import { randomUUID } from "node:crypto";
@@ -39,10 +40,9 @@ function resolveStatePath(overridePath) {
39
40
  * Ensure the parent directory of the given file path exists.
40
41
  */
41
42
  function ensureParentDir(filePath) {
42
- const dir = path.dirname(filePath);
43
- if (!fs.existsSync(dir)) {
44
- fs.mkdirSync(dir, { recursive: true });
45
- }
43
+ // `~/.neurolink` holds credential material; whoever creates it first must
44
+ // create it owner-only, and re-assert the mode when it already exists.
45
+ ensurePrivateDirectorySync(path.dirname(filePath));
46
46
  }
47
47
  /** Reject malformed persisted failure metadata before it reaches status output. */
48
48
  function isValidLastFailure(value) {
@@ -88,12 +88,6 @@ export declare class PDFLoader implements DocumentLoader {
88
88
  private loadPdfParser;
89
89
  private parsePageRange;
90
90
  }
91
- /**
92
- * Web page loader
93
- *
94
- * Fetches and extracts content from web pages.
95
- * Supports basic HTML parsing without external dependencies.
96
- */
97
91
  export declare class WebLoader implements DocumentLoader {
98
92
  private defaultUserAgent;
99
93
  load(source: string, options?: WebLoaderOptions): Promise<MDocument>;
@@ -27,6 +27,8 @@ import { existsSync } from "fs";
27
27
  import { readFile } from "fs/promises";
28
28
  import { basename, extname } from "path";
29
29
  import { logger } from "../../utils/logger.js";
30
+ import { downloadAttachment } from "../../utils/attachmentPolicy.js";
31
+ import { redactUrlForError } from "../../utils/logSanitize.js";
30
32
  import { decodeHtmlTextEntitiesOnce } from "../../utils/htmlEntities.js";
31
33
  import { removeHtmlMarkup } from "../../utils/htmlText.js";
32
34
  import { CSVProcessor, stripBom, stripMetadataLine, splitCsvLines, dedupeColumnNames, sanitizeColumnName, } from "../../utils/csvProcessor.js";
@@ -351,6 +353,8 @@ export class PDFLoader {
351
353
  * Fetches and extracts content from web pages.
352
354
  * Supports basic HTML parsing without external dependencies.
353
355
  */
356
+ /** 25 MiB — the default cap on a fetched page. */
357
+ const WEB_LOADER_MAX_BYTES = 25 * 1024 * 1024;
354
358
  export class WebLoader {
355
359
  defaultUserAgent = "Mozilla/5.0 (compatible; NeuroLink/1.0; +https://github.com/juspay/neurolink)";
356
360
  async load(source, options) {
@@ -358,23 +362,25 @@ export class WebLoader {
358
362
  throw new Error(`Invalid URL: ${source}`);
359
363
  }
360
364
  logger.debug("[WebLoader] Fetching URL", {
361
- url: source,
365
+ url: redactUrlForError(source),
362
366
  timeout: options?.timeout,
363
367
  });
364
- const response = await fetch(source, {
365
- signal: options?.timeout
366
- ? AbortSignal.timeout(options.timeout)
367
- : undefined,
368
+ // Through the attachment policy: HTTPS to a public host unless the policy
369
+ // says otherwise, DNS-pinned, redirects re-validated per hop with the
370
+ // caller's headers dropped, and the body bounded while it streams rather
371
+ // than read whole with `text()`.
372
+ const downloaded = await downloadAttachment(source, {
373
+ policy: options?.attachmentPolicy,
374
+ maxBytes: options?.maxBytes ?? WEB_LOADER_MAX_BYTES,
375
+ label: "web page",
376
+ timeoutMs: options?.timeout,
368
377
  headers: {
369
378
  "User-Agent": options?.userAgent || this.defaultUserAgent,
370
379
  Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
371
380
  ...options?.headers,
372
381
  },
373
382
  });
374
- if (!response.ok) {
375
- throw new Error(`HTTP error ${response.status}: ${response.statusText}`);
376
- }
377
- const html = await response.text();
383
+ const html = downloaded.buffer.toString("utf-8");
378
384
  let content = html;
379
385
  // Extract main content if requested
380
386
  if (options?.extractMainContent) {
@@ -388,7 +394,7 @@ export class WebLoader {
388
394
  source,
389
395
  url: source,
390
396
  fetchedAt: new Date().toISOString(),
391
- contentType: response.headers.get("content-type") || "text/html",
397
+ contentType: downloaded.contentType || "text/html",
392
398
  ...options?.metadata,
393
399
  },
394
400
  });
@@ -1,4 +1,4 @@
1
- import type { RAGConfig, VectorQueryResult, RAGPreparedTool } from "../types/index.js";
1
+ import type { AttachmentPolicy, RAGConfig, VectorQueryResult, RAGPreparedTool } from "../types/index.js";
2
2
  /**
3
3
  * Generate deterministic embeddings for chunks.
4
4
  * Combines character-frequency (40%) with word-level hash features (60%)
@@ -30,5 +30,5 @@ declare function diversifyResults(results: VectorQueryResult[], topK: number): V
30
30
  * is the default, and the fallback when a configured embedding provider fails).
31
31
  * The returned tool rejects a query over the same length the same way.
32
32
  */
33
- export declare function prepareRAGTool(ragConfig: RAGConfig, fallbackProvider?: string): Promise<RAGPreparedTool>;
33
+ export declare function prepareRAGTool(ragConfig: RAGConfig, fallbackProvider?: string, attachmentPolicy?: AttachmentPolicy): Promise<RAGPreparedTool>;
34
34
  export {};
@@ -1,4 +1,3 @@
1
- import { existsSync, readFileSync } from "fs";
2
1
  import { extname, resolve } from "path";
3
2
  import { z } from "zod";
4
3
  import { SpanSerializer, SpanType, SpanStatus, getMetricsAggregator, } from "../observability/index.js";
@@ -6,12 +5,19 @@ import { logger } from "../utils/logger.js";
6
5
  import { redactUrlForError } from "../utils/logSanitize.js";
7
6
  import { ErrorFactory } from "../utils/errorHandling.js";
8
7
  import { ImageProcessor } from "../utils/imageProcessor.js";
8
+ import { readAttachmentFile, resolveAttachmentPath, resolveAttachmentPolicy, } from "../utils/attachmentPolicy.js";
9
9
  import { createChunker } from "./ChunkerFactory.js";
10
10
  import { createVectorQueryTool, InMemoryVectorStore, } from "./retrieval/vectorQueryTool.js";
11
11
  import { ImageLoader } from "./document/imageLoader.js";
12
12
  import { IMAGE_EXTENSIONS } from "../processors/config/index.js";
13
13
  import { withSpan } from "../telemetry/withSpan.js";
14
14
  import { tracers } from "../telemetry/tracers.js";
15
+ /**
16
+ * Ceiling for one `rag.files` text source: 50 MiB. A file is chunked and
17
+ * embedded in full, so a larger one is a mistake rather than a document;
18
+ * `attachments.maxBytes` lowers it.
19
+ */
20
+ const RAG_FILE_MAX_BYTES = 50 * 1024 * 1024;
15
21
  /**
16
22
  * Maps file extensions to recommended chunking strategies
17
23
  */
@@ -173,7 +179,7 @@ function diversifyResults(results, topK) {
173
179
  * is the default, and the fallback when a configured embedding provider fails).
174
180
  * The returned tool rejects a query over the same length the same way.
175
181
  */
176
- export async function prepareRAGTool(ragConfig, fallbackProvider) {
182
+ export async function prepareRAGTool(ragConfig, fallbackProvider, attachmentPolicy = resolveAttachmentPolicy()) {
177
183
  const span = SpanSerializer.createSpan(SpanType.RAG, "rag.prepare", {
178
184
  "rag.operation": "prepare",
179
185
  "rag.files_count": ragConfig.files?.length ?? 0,
@@ -182,7 +188,7 @@ export async function prepareRAGTool(ragConfig, fallbackProvider) {
182
188
  });
183
189
  const startTime = Date.now();
184
190
  try {
185
- const result = await _prepareRAGToolInner(ragConfig, fallbackProvider);
191
+ const result = await _prepareRAGToolInner(ragConfig, fallbackProvider, attachmentPolicy);
186
192
  span.durationMs = Date.now() - startTime;
187
193
  const endedSpan = SpanSerializer.endSpan(span, SpanStatus.OK);
188
194
  endedSpan.attributes = {
@@ -202,7 +208,7 @@ export async function prepareRAGTool(ragConfig, fallbackProvider) {
202
208
  throw error;
203
209
  }
204
210
  }
205
- async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
211
+ async function _prepareRAGToolInner(ragConfig, fallbackProvider, attachmentPolicy) {
206
212
  const { files, strategy: userStrategy, chunkSize = 1000, chunkOverlap = 200, topK: userTopK = 5, toolName = "search_knowledge_base", toolDescription = "REQUIRED: Search through pre-loaded local documents to find relevant information. Use this tool FIRST before any web search or other tools. This searches an indexed knowledge base of documents the user has provided.", embeddingProvider, embeddingModel, } = ragConfig;
207
213
  if (!files || files.length === 0) {
208
214
  throw new Error("RAG config requires at least one file path in 'files'");
@@ -248,10 +254,6 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
248
254
  continue;
249
255
  }
250
256
  const resolvedPath = resolve(filePath);
251
- if (!existsSync(resolvedPath)) {
252
- logger.warn(`[RAG] File not found, skipping: ${resolvedPath}`);
253
- continue;
254
- }
255
257
  // Detect image files by extension
256
258
  const ext = extname(resolvedPath).toLowerCase();
257
259
  if (ext === ".svg") {
@@ -260,15 +262,38 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
260
262
  continue;
261
263
  }
262
264
  if (IMAGE_EXTENSIONS.includes(ext)) {
263
- imageFiles.push({ path: resolvedPath, ext });
265
+ // The same containment as the text branch below; ImageLoader then reads
266
+ // the real path this returned. A path outside the roots is skipped with
267
+ // the same warning as an unreadable one, so the tool keeps working on
268
+ // the files it may read.
269
+ try {
270
+ const real = await resolveAttachmentPath(resolvedPath, attachmentPolicy, "RAG image");
271
+ imageFiles.push({ path: real, ext });
272
+ }
273
+ catch (error) {
274
+ logger.warn(`[RAG] Failed to read image: ${resolvedPath}: ${error instanceof Error ? error.message : String(error)}`);
275
+ }
264
276
  continue;
265
277
  }
266
278
  try {
267
- const content = readFileSync(resolvedPath, "utf-8");
279
+ // Roots containment when configured, O_NOFOLLOW open of the real path,
280
+ // regular files only, and a bounded read — where this was a bare
281
+ // `readFileSync` of whatever string arrived.
282
+ const read = await readAttachmentFile(resolvedPath, {
283
+ policy: attachmentPolicy,
284
+ maxBytes: RAG_FILE_MAX_BYTES,
285
+ label: "RAG file",
286
+ });
287
+ const content = read.buffer.toString("utf-8");
268
288
  const strategy = userStrategy || detectStrategy(resolvedPath);
269
289
  fileContents.push({ path: resolvedPath, content, strategy });
270
290
  }
271
291
  catch (error) {
292
+ const code = error?.code;
293
+ if (code === "ENOENT") {
294
+ logger.warn(`[RAG] File not found, skipping: ${resolvedPath}`);
295
+ continue;
296
+ }
272
297
  logger.warn(`[RAG] Failed to read file: ${resolvedPath}: ${error instanceof Error ? error.message : String(error)}`);
273
298
  }
274
299
  }
@@ -39,6 +39,8 @@ export declare function createVectorQueryTool(config: VectorQueryToolConfig, vec
39
39
  */
40
40
  export declare class InMemoryVectorStore implements VectorStore {
41
41
  private vectors;
42
+ /** Vetted `$regex` patterns: the compiled expression, or null when refused. */
43
+ private readonly regexFilterCache;
42
44
  /**
43
45
  * Add vectors to an index
44
46
  */
@@ -65,6 +67,11 @@ export declare class InMemoryVectorStore implements VectorStore {
65
67
  * Check if metadata matches filter
66
68
  */
67
69
  private matchesFilter;
70
+ /**
71
+ * Evaluate a `$regex` filter operator. Patterns are vetted once per store
72
+ * and the verdict cached, so a refused pattern costs one analysis.
73
+ */
74
+ private matchesRegexFilter;
68
75
  /**
69
76
  * Calculate cosine similarity between two vectors
70
77
  */
@@ -10,6 +10,7 @@ import { ProviderFactory } from "../../factories/providerFactory.js";
10
10
  import { withSpan } from "../../telemetry/withSpan.js";
11
11
  import { tracers } from "../../telemetry/tracers.js";
12
12
  import { logger } from "../../utils/logger.js";
13
+ import { vetRegexPattern } from "../../middleware/utils/regexSafety.js";
13
14
  import { rerank } from "../reranker/reranker.js";
14
15
  /**
15
16
  * Creates a vector query tool for semantic search
@@ -132,11 +133,19 @@ export function createVectorQueryTool(config, vectorStore) {
132
133
  },
133
134
  };
134
135
  }
136
+ /** Longest `$regex` filter pattern accepted; a secondary cap behind vetting. */
137
+ const MAX_FILTER_REGEX_LENGTH = 200;
138
+ /** Longest metadata value a `$regex` filter is run against. */
139
+ const MAX_FILTER_REGEX_SUBJECT = 10_000;
140
+ /** Distinct patterns remembered per store before the verdict cache is reset. */
141
+ const MAX_FILTER_REGEX_CACHE = 256;
135
142
  /**
136
143
  * In-memory vector store implementation for testing and development
137
144
  */
138
145
  export class InMemoryVectorStore {
139
146
  vectors = new Map();
147
+ /** Vetted `$regex` patterns: the compiled expression, or null when refused. */
148
+ regexFilterCache = new Map();
140
149
  /**
141
150
  * Add vectors to an index
142
151
  */
@@ -272,23 +281,14 @@ export class InMemoryVectorStore {
272
281
  return false;
273
282
  }
274
283
  if ("$regex" in ops) {
275
- const pattern = ops.$regex;
276
- let regexMatches = false;
277
- // Guard against ReDoS: reject excessively long patterns and limit
278
- // the tested string length to prevent pathological backtracking.
279
- if (pattern.length <= 200) {
280
- try {
281
- const re = new RegExp(pattern);
282
- const testValue = typeof fieldValue === "string"
283
- ? fieldValue.slice(0, 10_000)
284
- : "";
285
- regexMatches = re.test(testValue);
286
- }
287
- catch {
288
- // Invalid regex pattern — treat as non-match
289
- regexMatches = false;
290
- }
291
- }
284
+ // The pattern is chosen by the model and the value it runs on
285
+ // comes from ingested documents, so both halves of a ReDoS are
286
+ // attacker-influenced. A length cap alone does not bound
287
+ // backtracking (`^(a+)+$` is 7 characters); `vetRegexPattern`
288
+ // reads the pattern's structure and refuses the ones that can
289
+ // backtrack catastrophically. A refused or invalid pattern is a
290
+ // non-match, as an invalid one always was.
291
+ const regexMatches = this.matchesRegexFilter(ops.$regex, fieldValue);
292
292
  if (!regexMatches) {
293
293
  return false;
294
294
  }
@@ -304,6 +304,43 @@ export class InMemoryVectorStore {
304
304
  }
305
305
  return true;
306
306
  }
307
+ /**
308
+ * Evaluate a `$regex` filter operator. Patterns are vetted once per store
309
+ * and the verdict cached, so a refused pattern costs one analysis.
310
+ */
311
+ matchesRegexFilter(pattern, fieldValue) {
312
+ if (typeof pattern !== "string" ||
313
+ pattern.length > MAX_FILTER_REGEX_LENGTH) {
314
+ return false;
315
+ }
316
+ let compiled = this.regexFilterCache.get(pattern);
317
+ if (compiled === undefined) {
318
+ const verdict = vetRegexPattern(pattern);
319
+ if (verdict.safe) {
320
+ try {
321
+ compiled = new RegExp(pattern);
322
+ }
323
+ catch {
324
+ compiled = null;
325
+ }
326
+ }
327
+ else {
328
+ logger.debug("[InMemoryVectorStore] Refused a $regex filter pattern that could stall the process", { reason: verdict.reason });
329
+ compiled = null;
330
+ }
331
+ if (this.regexFilterCache.size >= MAX_FILTER_REGEX_CACHE) {
332
+ this.regexFilterCache.clear();
333
+ }
334
+ this.regexFilterCache.set(pattern, compiled);
335
+ }
336
+ if (compiled === null) {
337
+ return false;
338
+ }
339
+ const testValue = typeof fieldValue === "string"
340
+ ? fieldValue.slice(0, MAX_FILTER_REGEX_SUBJECT)
341
+ : "";
342
+ return compiled.test(testValue);
343
+ }
307
344
  /**
308
345
  * Calculate cosine similarity between two vectors
309
346
  */
@@ -7,6 +7,8 @@ import { EventEmitter } from "events";
7
7
  import type { ExternalServerManager } from "../../mcp/externalServerManager.js";
8
8
  import type { MCPToolRegistry } from "../../mcp/toolRegistry.js";
9
9
  import type { NeuroLink } from "../../neurolink.js";
10
+ /** The interface an adapter binds unless told otherwise: this host only. */
11
+ export declare const DEFAULT_SERVER_HOST = "127.0.0.1";
10
12
  import type { MiddlewareDefinition, RedactionConfig, RequiredServerAdapterConfig, RequiredShutdownConfig, RouteDefinition, ServerAdapterConfig, ServerAdapterEvents, ServerCarriedContext, ServerContext, ServerLifecycleState, ServerStatus, TrackedConnection } from "../../types/index.js";
11
13
  /**
12
14
  * Abstract base class for server adapters
@@ -29,7 +31,47 @@ export declare abstract class BaseServerAdapter extends EventEmitter {
29
31
  protected activeConnections: Map<string, TrackedConnection>;
30
32
  protected readonly shutdownConfig: RequiredShutdownConfig;
31
33
  private carryRequestedByRoutes;
34
+ private sensitiveRoutesRegistered;
35
+ private sensitiveRoutesAllowedOpen;
32
36
  constructor(neurolink: NeuroLink, config?: ServerAdapterConfig);
37
+ /**
38
+ * Whether middleware marked `authenticates: true` is registered (the
39
+ * built-in authentication middleware marks itself). The tool and MCP
40
+ * execution routes ask on every request; `registerAllRoutes()` wires them.
41
+ */
42
+ hasAuthenticationMiddleware(): boolean;
43
+ /**
44
+ * Called by `registerAllRoutes()`: routes that act on the host (tool and
45
+ * MCP execution) are mounted, so `start()` warns when no authentication
46
+ * middleware guards them. The check runs at start, when the middleware set
47
+ * is final, so registering authentication after the routes is fine.
48
+ */
49
+ noteSensitiveRoutes(options?: {
50
+ allowUnauthenticated?: boolean;
51
+ }): void;
52
+ /**
53
+ * The warning promised by `noteSensitiveRoutes()`, always on stderr: a
54
+ * `logger.warn` is dropped unless debug logging is on, which would hide a
55
+ * notice meant for the operator who just started the server.
56
+ */
57
+ protected warnIfSensitiveRoutesUnauthenticated(): void;
58
+ /**
59
+ * Give the handler a way to notice that the client is gone or the request
60
+ * is over time. The controller is this request's own unless middleware
61
+ * already attached one that the adapter carried; `applyTimeout` arms the
62
+ * configured request timeout, which the caller leaves off for a streaming
63
+ * route (a stream legitimately outlives it, as under Hono's own timeout
64
+ * middleware). `release()` disarms the timer once the response is written.
65
+ */
66
+ protected attachRequestAbort(ctx: ServerContext, options: {
67
+ onClientClose: (listener: () => void) => void;
68
+ applyTimeout: boolean;
69
+ }): {
70
+ controller: AbortController;
71
+ deadline: Promise<never> | undefined;
72
+ timedOut: () => boolean;
73
+ release: () => void;
74
+ };
33
75
  /**
34
76
  * Initialize the underlying server framework
35
77
  * May be async: adapters that dynamically import their framework package
@@ -146,6 +188,7 @@ export declare abstract class BaseServerAdapter extends EventEmitter {
146
188
  query?: Record<string, string>;
147
189
  params?: Record<string, string>;
148
190
  body?: unknown;
191
+ remoteAddress?: string;
149
192
  }): ServerContext;
150
193
  /**
151
194
  * Register built-in middleware
@@ -6,10 +6,15 @@
6
6
  import { EventEmitter } from "events";
7
7
  import { getMetricsAggregator, SpanSerializer, SpanStatus, SpanType, } from "../../observability/index.js";
8
8
  import { withTimeout } from "../../utils/errorHandling.js";
9
+ import { MAX_TIMER_MS } from "../../constants/timeouts.js";
9
10
  import { logger } from "../../utils/logger.js";
10
11
  import { markContextCarried } from "../utils/carriedContextMark.js";
12
+ import { isLoopbackAddress } from "../utils/clientAddress.js";
13
+ import { pathStartsWithSegment } from "../utils/pathMatch.js";
11
14
  import { tryImport } from "../../utils/tryImport.js";
12
- import { DrainTimeoutError, InvalidLifecycleStateError, MissingDependencyError, ServerAdapterError, ShutdownTimeoutError, } from "../errors.js";
15
+ import { ConfigurationError, DrainTimeoutError, InvalidLifecycleStateError, MissingDependencyError, ServerAdapterError, ServerTimeoutError, ShutdownTimeoutError, } from "../errors.js";
16
+ /** The interface an adapter binds unless told otherwise: this host only. */
17
+ export const DEFAULT_SERVER_HOST = "127.0.0.1";
13
18
  /**
14
19
  * Abstract base class for server adapters
15
20
  * Provides common functionality and defines the interface for framework-specific implementations
@@ -37,6 +42,9 @@ export class BaseServerAdapter extends EventEmitter {
37
42
  // Set by enableMiddlewareStateCarry(); the carryMiddlewareState option is the
38
43
  // other way in.
39
44
  carryRequestedByRoutes = false;
45
+ // Set by noteSensitiveRoutes(); read when the server starts.
46
+ sensitiveRoutesRegistered = false;
47
+ sensitiveRoutesAllowedOpen = false;
40
48
  constructor(neurolink, config = {}) {
41
49
  super();
42
50
  this.neurolink = neurolink;
@@ -53,7 +61,9 @@ export class BaseServerAdapter extends EventEmitter {
53
61
  // Apply defaults
54
62
  this.config = {
55
63
  port: config.port ?? 3000,
56
- host: config.host ?? "0.0.0.0",
64
+ // Loopback unless the operator names an interface: the routes run tools
65
+ // and remove MCP servers, so reachability is opt-in.
66
+ host: config.host ?? DEFAULT_SERVER_HOST,
57
67
  basePath: config.basePath ?? "/api",
58
68
  cors: {
59
69
  enabled: config.cors?.enabled ?? true,
@@ -85,6 +95,8 @@ export class BaseServerAdapter extends EventEmitter {
85
95
  "Too many requests, please try again later",
86
96
  skipPaths: config.rateLimit?.skipPaths,
87
97
  keyGenerator: config.rateLimit?.keyGenerator,
98
+ trustProxy: config.rateLimit?.trustProxy ?? false,
99
+ trustedProxies: config.rateLimit?.trustedProxies ?? [],
88
100
  },
89
101
  bodyParser: {
90
102
  enabled: config.bodyParser?.enabled ?? true,
@@ -107,6 +119,100 @@ export class BaseServerAdapter extends EventEmitter {
107
119
  shutdown: this.shutdownConfig,
108
120
  carryMiddlewareState: config.carryMiddlewareState ?? false,
109
121
  };
122
+ // A wildcard origin with credentials is what the CORS specification
123
+ // forbids and what an adapter reflecting the request's Origin would grant:
124
+ // any page could make credentialed calls. Refuse it for every adapter.
125
+ if (this.config.cors.enabled &&
126
+ this.config.cors.credentials &&
127
+ this.config.cors.origins.includes("*")) {
128
+ throw new ConfigurationError('cors.credentials cannot be combined with the wildcard origin "*"; list the origins that may send credentials', { origins: this.config.cors.origins });
129
+ }
130
+ }
131
+ /**
132
+ * Whether middleware marked `authenticates: true` is registered (the
133
+ * built-in authentication middleware marks itself). The tool and MCP
134
+ * execution routes ask on every request; `registerAllRoutes()` wires them.
135
+ */
136
+ hasAuthenticationMiddleware() {
137
+ return this.middlewares.some((middleware) => middleware.authenticates === true);
138
+ }
139
+ /**
140
+ * Called by `registerAllRoutes()`: routes that act on the host (tool and
141
+ * MCP execution) are mounted, so `start()` warns when no authentication
142
+ * middleware guards them. The check runs at start, when the middleware set
143
+ * is final, so registering authentication after the routes is fine.
144
+ */
145
+ noteSensitiveRoutes(options) {
146
+ this.sensitiveRoutesRegistered = true;
147
+ if (options?.allowUnauthenticated) {
148
+ this.sensitiveRoutesAllowedOpen = true;
149
+ }
150
+ }
151
+ /**
152
+ * The warning promised by `noteSensitiveRoutes()`, always on stderr: a
153
+ * `logger.warn` is dropped unless debug logging is on, which would hide a
154
+ * notice meant for the operator who just started the server.
155
+ */
156
+ warnIfSensitiveRoutesUnauthenticated() {
157
+ if (!this.sensitiveRoutesRegistered ||
158
+ this.sensitiveRoutesAllowedOpen ||
159
+ this.hasAuthenticationMiddleware()) {
160
+ return;
161
+ }
162
+ const where = isLoopbackAddress(this.config.host)
163
+ ? `loopback only (${this.config.host})`
164
+ : `${this.config.host}, reachable from the network`;
165
+ logger.alwaysStderr(`[ServerAdapter] WARNING: no authentication middleware is registered and the server listens on ${where}. ` +
166
+ "The tool and MCP execution routes answer 403 until authentication middleware (marked authenticates: true) is registered, " +
167
+ "or allowUnauthenticated: true / NEUROLINK_SERVER_ALLOW_UNAUTHENTICATED=1 opts out of that gate.");
168
+ }
169
+ /**
170
+ * Give the handler a way to notice that the client is gone or the request
171
+ * is over time. The controller is this request's own unless middleware
172
+ * already attached one that the adapter carried; `applyTimeout` arms the
173
+ * configured request timeout, which the caller leaves off for a streaming
174
+ * route (a stream legitimately outlives it, as under Hono's own timeout
175
+ * middleware). `release()` disarms the timer once the response is written.
176
+ */
177
+ attachRequestAbort(ctx, options) {
178
+ const controller = ctx.abortController ?? new AbortController();
179
+ ctx.abortController = controller;
180
+ ctx.abortSignal = controller.signal;
181
+ let timedOut = false;
182
+ let timer;
183
+ const timeoutMs = this.config.timeout;
184
+ // Node turns an over-range delay into 1 ms; keep long request budgets
185
+ // at the timer limit, as the readiness deadline does through withTimeout.
186
+ const delayMs = timeoutMs > MAX_TIMER_MS ? MAX_TIMER_MS : timeoutMs;
187
+ const deadline = options.applyTimeout && timeoutMs > 0
188
+ ? new Promise((_, reject) => {
189
+ timer = setTimeout(() => {
190
+ timedOut = true;
191
+ const error = new ServerTimeoutError(timeoutMs, `${ctx.method} ${ctx.path}`, ctx.requestId);
192
+ controller.abort(error);
193
+ reject(error);
194
+ }, delayMs);
195
+ })
196
+ : undefined;
197
+ // A rejection nobody awaits (the handler won on the race) must not
198
+ // surface as unhandled.
199
+ deadline?.catch(() => undefined);
200
+ options.onClientClose(() => {
201
+ if (!controller.signal.aborted) {
202
+ controller.abort(new Error("Client disconnected"));
203
+ }
204
+ });
205
+ return {
206
+ controller,
207
+ deadline,
208
+ timedOut: () => timedOut,
209
+ release: () => {
210
+ if (timer) {
211
+ clearTimeout(timer);
212
+ timer = undefined;
213
+ }
214
+ },
215
+ };
110
216
  }
111
217
  /**
112
218
  * The listening Node `http.Server`, once `start()` has run and the runtime
@@ -237,11 +343,17 @@ export class BaseServerAdapter extends EventEmitter {
237
343
  }
238
344
  }
239
345
  // Register all routes in the group with prefix applied
346
+ const prefix = this.normalizePath(group.prefix);
347
+ // The group's root segment ("/api" of "/api/health"): a route whose path
348
+ // already lives under it is a full path, as every built-in route is.
349
+ // Testing only the whole prefix put `/api/version`, declared in the health
350
+ // group, at `/api/health/api/version`.
351
+ const root = `/${prefix.split("/").filter(Boolean)[0] ?? ""}`;
240
352
  for (const route of group.routes) {
241
- // Only prepend prefix if route path doesn't already start with it
242
- // (route definitions include full paths like /api/agent/execute)
243
- const needsPrefix = !route.path.startsWith(group.prefix);
244
- const prefixedPath = this.normalizePath(needsPrefix ? `${group.prefix}${route.path}` : route.path);
353
+ const needsPrefix = prefix !== "/" &&
354
+ !pathStartsWithSegment(route.path, prefix) &&
355
+ !pathStartsWithSegment(route.path, root);
356
+ const prefixedPath = this.normalizePath(needsPrefix ? `${prefix}${route.path}` : route.path);
245
357
  const prefixedRoute = {
246
358
  ...route,
247
359
  path: prefixedPath,
@@ -336,6 +448,7 @@ export class BaseServerAdapter extends EventEmitter {
336
448
  query: options.query ?? {},
337
449
  params: options.params ?? {},
338
450
  body: options.body,
451
+ remoteAddress: options.remoteAddress,
339
452
  neurolink: this.neurolink,
340
453
  toolRegistry: this.toolRegistry,
341
454
  externalServerManager: this.externalServerManager,
@@ -13,6 +13,7 @@ export declare class ExpressServerAdapter extends BaseServerAdapter {
13
13
  private server?;
14
14
  private frameworkInitialized;
15
15
  private sockets;
16
+ private warnedNoClientAddress;
16
17
  constructor(neurolink: NeuroLink, config?: ServerAdapterConfig);
17
18
  /**
18
19
  * Initialize Express framework asynchronously
@@ -22,6 +23,8 @@ export declare class ExpressServerAdapter extends BaseServerAdapter {
22
23
  * Initialize Express framework with async imports
23
24
  */
24
25
  private initializeFrameworkAsync;
26
+ /** The address a request is counted against; `undefined` when there is none. */
27
+ private clientKeyFor;
25
28
  /**
26
29
  * Override initialize to ensure async framework setup
27
30
  */