@juspay/neurolink 13.8.16 → 13.8.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (194) hide show
  1. package/CHANGELOG.md +2 -2
  2. package/README.md +1 -1
  3. package/dist/action/actionExecutor.d.ts +33 -3
  4. package/dist/action/actionExecutor.js +71 -15
  5. package/dist/action/actionInputs.js +4 -1
  6. package/dist/action/cliVersion.d.ts +25 -0
  7. package/dist/action/cliVersion.js +26 -0
  8. package/dist/action/index.d.ts +2 -1
  9. package/dist/action/index.js +2 -1
  10. package/dist/adapters/video/directorPipeline.js +46 -36
  11. package/dist/adapters/video/ffmpegAdapter.d.ts +17 -0
  12. package/dist/adapters/video/ffmpegAdapter.js +32 -5
  13. package/dist/adapters/video/frameExtractor.js +1 -1
  14. package/dist/adapters/video/videoMerger.js +5 -2
  15. package/dist/agent/arithmeticEvaluator.d.ts +32 -0
  16. package/dist/agent/arithmeticEvaluator.js +349 -0
  17. package/dist/agent/directTools.d.ts +6 -18
  18. package/dist/agent/directTools.js +17 -56
  19. package/dist/artifacts/artifactStore.d.ts +22 -1
  20. package/dist/artifacts/artifactStore.js +66 -6
  21. package/dist/auth/anthropicOAuth.d.ts +26 -1
  22. package/dist/auth/anthropicOAuth.js +81 -5
  23. package/dist/auth/secureFiles.d.ts +37 -0
  24. package/dist/auth/secureFiles.js +95 -0
  25. package/dist/auth/tokenStore.d.ts +43 -6
  26. package/dist/auth/tokenStore.js +260 -79
  27. package/dist/avatar/providers/DIDAvatar.js +49 -45
  28. package/dist/avatar/providers/ReplicateAvatar.js +33 -40
  29. package/dist/browser/neurolink.min.js +485 -558
  30. package/dist/cli/commands/auth.js +112 -70
  31. package/dist/cli/commands/proxy.d.ts +4 -0
  32. package/dist/cli/commands/proxy.js +128 -13
  33. package/dist/cli/commands/serve.d.ts +10 -1
  34. package/dist/cli/commands/serve.js +98 -17
  35. package/dist/cli/commands/server.js +17 -5
  36. package/dist/cli/index.js +146 -5
  37. package/dist/cli/loop/optionsSchema.d.ts +1 -1
  38. package/dist/cli/utils/envManager.js +17 -3
  39. package/dist/cli/utils/serverUtils.d.ts +39 -3
  40. package/dist/cli/utils/serverUtils.js +63 -6
  41. package/dist/core/baseProvider.js +7 -1
  42. package/dist/core/modules/MessageBuilder.js +5 -0
  43. package/dist/core/modules/ToolsManager.d.ts +46 -1
  44. package/dist/core/modules/ToolsManager.js +190 -43
  45. package/dist/files/fileReferenceRegistry.d.ts +2 -23
  46. package/dist/files/fileReferenceRegistry.js +55 -5
  47. package/dist/hitl/hitlManager.js +1 -0
  48. package/dist/index.d.ts +2 -1
  49. package/dist/index.js +6 -1
  50. package/dist/mcp/auth/tokenStorage.d.ts +7 -0
  51. package/dist/mcp/auth/tokenStorage.js +23 -4
  52. package/dist/mcp/externalServerManager.d.ts +13 -3
  53. package/dist/mcp/externalServerManager.js +154 -136
  54. package/dist/mcp/index.d.ts +2 -0
  55. package/dist/mcp/index.js +3 -0
  56. package/dist/mcp/mcpClientFactory.d.ts +14 -0
  57. package/dist/mcp/mcpClientFactory.js +70 -7
  58. package/dist/mcp/projectConfigTrust.d.ts +72 -0
  59. package/dist/mcp/projectConfigTrust.js +237 -0
  60. package/dist/mcp/transportUrlPolicy.d.ts +39 -0
  61. package/dist/mcp/transportUrlPolicy.js +172 -0
  62. package/dist/music/providers/BeatovenMusic.js +35 -46
  63. package/dist/music/providers/ReplicateMusic.js +33 -40
  64. package/dist/neurolink.d.ts +16 -1
  65. package/dist/neurolink.js +64 -11
  66. package/dist/processors/archive/ArchiveProcessor.js +15 -1
  67. package/dist/processors/archive/zipEntryIndex.d.ts +22 -0
  68. package/dist/processors/archive/zipEntryIndex.js +33 -0
  69. package/dist/processors/base/BaseFileProcessor.d.ts +9 -21
  70. package/dist/processors/base/BaseFileProcessor.js +40 -111
  71. package/dist/processors/data/YamlProcessor.d.ts +13 -1
  72. package/dist/processors/data/YamlProcessor.js +93 -2
  73. package/dist/processors/document/OpenDocumentProcessor.js +7 -2
  74. package/dist/processors/document/PptxProcessor.js +12 -7
  75. package/dist/processors/media/VideoProcessor.js +93 -15
  76. package/dist/proxy/inboundAuth.d.ts +89 -0
  77. package/dist/proxy/inboundAuth.js +219 -0
  78. package/dist/proxy/tokenRefresh.d.ts +6 -0
  79. package/dist/proxy/tokenRefresh.js +13 -1
  80. package/dist/proxy/updateState.js +4 -4
  81. package/dist/rag/document/loaders.d.ts +0 -6
  82. package/dist/rag/document/loaders.js +16 -10
  83. package/dist/rag/ragIntegration.d.ts +2 -2
  84. package/dist/rag/ragIntegration.js +35 -10
  85. package/dist/rag/retrieval/vectorQueryTool.d.ts +7 -0
  86. package/dist/rag/retrieval/vectorQueryTool.js +54 -17
  87. package/dist/server/abstract/baseServerAdapter.d.ts +43 -0
  88. package/dist/server/abstract/baseServerAdapter.js +119 -6
  89. package/dist/server/adapters/expressAdapter.d.ts +3 -0
  90. package/dist/server/adapters/expressAdapter.js +83 -8
  91. package/dist/server/adapters/fastifyAdapter.d.ts +3 -0
  92. package/dist/server/adapters/fastifyAdapter.js +165 -70
  93. package/dist/server/adapters/honoAdapter.d.ts +9 -1
  94. package/dist/server/adapters/honoAdapter.js +99 -21
  95. package/dist/server/adapters/koaAdapter.d.ts +1 -0
  96. package/dist/server/adapters/koaAdapter.js +153 -74
  97. package/dist/server/index.d.ts +2 -0
  98. package/dist/server/index.js +4 -0
  99. package/dist/server/middleware/auth.js +6 -2
  100. package/dist/server/middleware/cache.js +6 -1
  101. package/dist/server/middleware/rateLimit.js +29 -4
  102. package/dist/server/routes/agentRoutes.js +24 -11
  103. package/dist/server/routes/claudeProxyRoutes.d.ts +1 -0
  104. package/dist/server/routes/claudeProxyRoutes.js +273 -84
  105. package/dist/server/routes/codexProxyRoutes.d.ts +2 -2
  106. package/dist/server/routes/codexProxyRoutes.js +125 -40
  107. package/dist/server/routes/index.d.ts +4 -0
  108. package/dist/server/routes/index.js +35 -4
  109. package/dist/server/routes/mcpRoutes.d.ts +6 -2
  110. package/dist/server/routes/mcpRoutes.js +21 -7
  111. package/dist/server/routes/memoryRoutes.js +7 -6
  112. package/dist/server/routes/toolRoutes.d.ts +12 -2
  113. package/dist/server/routes/toolRoutes.js +33 -3
  114. package/dist/server/routes/transcribeRoutes.js +60 -13
  115. package/dist/server/utils/clientAddress.d.ts +23 -0
  116. package/dist/server/utils/clientAddress.js +65 -0
  117. package/dist/server/utils/clientError.d.ts +35 -0
  118. package/dist/server/utils/clientError.js +129 -0
  119. package/dist/server/utils/expressBody.d.ts +33 -0
  120. package/dist/server/utils/expressBody.js +353 -0
  121. package/dist/server/utils/pathMatch.d.ts +21 -0
  122. package/dist/server/utils/pathMatch.js +41 -0
  123. package/dist/server/utils/validation.d.ts +9 -0
  124. package/dist/server/utils/validation.js +9 -0
  125. package/dist/server/websocket/WebSocketHandler.js +22 -3
  126. package/dist/server/websocket/frameQueue.d.ts +19 -0
  127. package/dist/server/websocket/frameQueue.js +41 -3
  128. package/dist/server/websocket/transcribeWebSocket.d.ts +8 -1
  129. package/dist/server/websocket/transcribeWebSocket.js +223 -34
  130. package/dist/skills/skillStores.js +5 -0
  131. package/dist/tasks/store/fileTaskStore.d.ts +2 -0
  132. package/dist/tasks/store/fileTaskStore.js +11 -4
  133. package/dist/types/attachments.d.ts +117 -0
  134. package/dist/types/attachments.js +15 -0
  135. package/dist/types/auth.d.ts +6 -0
  136. package/dist/types/cli.d.ts +4 -0
  137. package/dist/types/config.d.ts +56 -0
  138. package/dist/types/file.d.ts +8 -0
  139. package/dist/types/hitl.d.ts +8 -0
  140. package/dist/types/index.d.ts +1 -0
  141. package/dist/types/index.js +1 -0
  142. package/dist/types/mcp.d.ts +98 -0
  143. package/dist/types/multimodal.js +0 -41
  144. package/dist/types/processor.d.ts +14 -1
  145. package/dist/types/proxy.d.ts +62 -3
  146. package/dist/types/rag.d.ts +9 -0
  147. package/dist/types/safeFetch.d.ts +40 -0
  148. package/dist/types/server.d.ts +255 -10
  149. package/dist/types/tools.d.ts +57 -0
  150. package/dist/types/transcribe.d.ts +6 -0
  151. package/dist/utils/attachmentPolicy.d.ts +89 -0
  152. package/dist/utils/attachmentPolicy.js +327 -0
  153. package/dist/utils/dotenvBootstrap.d.ts +48 -19
  154. package/dist/utils/dotenvBootstrap.js +172 -31
  155. package/dist/utils/fileDetector.js +115 -65
  156. package/dist/utils/json/coerce.js +11 -1
  157. package/dist/utils/logSanitize.d.ts +5 -0
  158. package/dist/utils/logSanitize.js +8 -0
  159. package/dist/utils/logger.js +73 -1
  160. package/dist/utils/messageBuilder.d.ts +2 -2
  161. package/dist/utils/messageBuilder.js +118 -105
  162. package/dist/utils/multimodalOptionsBuilder.d.ts +1 -0
  163. package/dist/utils/multimodalOptionsBuilder.js +2 -0
  164. package/dist/utils/redis.d.ts +12 -0
  165. package/dist/utils/redis.js +72 -0
  166. package/dist/utils/safeFetch.d.ts +22 -1
  167. package/dist/utils/safeFetch.js +132 -38
  168. package/dist/utils/safeStoreId.d.ts +19 -0
  169. package/dist/utils/safeStoreId.js +29 -0
  170. package/dist/utils/schemaConversion.d.ts +30 -3
  171. package/dist/utils/schemaConversion.js +457 -84
  172. package/dist/utils/sizeGuard.js +2 -2
  173. package/dist/utils/ssrfGuard.d.ts +3 -3
  174. package/dist/utils/ssrfGuard.js +71 -39
  175. package/dist/voice/eval/sttEvalStore.d.ts +26 -3
  176. package/dist/voice/eval/sttEvalStore.js +77 -12
  177. package/dist/voice/livekit/agentDefinition.d.ts +18 -0
  178. package/dist/voice/livekit/agentDefinition.js +22 -0
  179. package/dist/voice/livekit/realtimeMcpTools.d.ts +1 -1
  180. package/dist/voice/livekit/realtimeMcpTools.js +8 -7
  181. package/dist/voice/livekit/realtimeVoiceAgent.d.ts +2 -4
  182. package/dist/voice/livekit/realtimeVoiceAgent.js +18 -7
  183. package/dist/voice/livekit/voiceAgent.d.ts +2 -2
  184. package/dist/voice/livekit/voiceAgent.js +2 -1
  185. package/dist/voice/livekit/xaiSessionWatch.d.ts +0 -5
  186. package/dist/voice/livekit/xaiSessionWatch.js +30 -0
  187. package/dist/voice/providers/GeminiLive.js +21 -6
  188. package/dist/voice/providers/GoogleSTT.js +8 -5
  189. package/dist/voice/transcribe.d.ts +2 -2
  190. package/dist/voice/transcribe.js +34 -18
  191. package/dist/voice/whistle/audio.js +11 -5
  192. package/docs-site/static/search-index.json +75 -52
  193. package/package.json +171 -156
  194. package/scripts/observability/import-openobserve-dashboard.mjs +11 -3
@@ -47,6 +47,7 @@ import { promisify } from "util";
47
47
  import { gunzip } from "zlib";
48
48
  import { SIZE_LIMITS } from "../config/index.js";
49
49
  import { isAbortError } from "../../utils/errorHandling.js";
50
+ import { downloadAttachment } from "../../utils/attachmentPolicy.js";
50
51
  import { withSpan } from "../../telemetry/withSpan.js";
51
52
  import { tracers } from "../../telemetry/tracers.js";
52
53
  // Starts tracing at the first span, as this module always did: the shared
@@ -425,126 +426,54 @@ export class BaseFileProcessor {
425
426
  /**
426
427
  * Download file from URL with authentication.
427
428
  *
429
+ * Goes through `downloadAttachment`, which applies the attachment policy
430
+ * (HTTPS and public hosts unless the instance says otherwise), pins the
431
+ * connection to the vetted addresses, re-validates every redirect hop and
432
+ * bounds the body while it streams. `authHeaders` ride on the first request
433
+ * only — they are never replayed to a redirect target.
434
+ *
428
435
  * @param url - URL to download from
429
436
  * @param authHeaders - Optional authentication headers
430
437
  * @param timeout - Optional timeout override
438
+ * @param attachmentPolicy - Policy for the fetch; the environment default when omitted
431
439
  * @returns Downloaded file content as Buffer
432
440
  * @throws Error if download fails
433
441
  */
434
- async downloadFile(url, authHeaders, timeout) {
435
- // Note: We intentionally use AbortController + setTimeout here rather than the shared
436
- // withTimeout utility. AbortController.signal cancels the actual HTTP request via
437
- // fetch's signal option, while withTimeout only races promises and would leave
438
- // the fetch running in the background, consuming network resources.
439
- const controller = new AbortController();
440
- const effectiveTimeout = timeout ?? this.config.timeoutMs;
441
- const timeoutId = setTimeout(() => {
442
- controller.abort();
443
- }, effectiveTimeout);
444
- try {
445
- const headers = {
446
- ...authHeaders,
447
- };
448
- const response = await fetch(url, {
449
- headers,
450
- signal: controller.signal,
451
- });
452
- if (!response.ok) {
453
- throw new Error(`HTTP ${response.status}: ${response.statusText}`);
454
- }
455
- // Reject HTML responses - likely an error page or redirect
456
- const contentType = response.headers.get("Content-Type");
457
- if (contentType && contentType.includes("text/html")) {
458
- throw new Error(`Received HTML response instead of file content (Content-Type: ${contentType}). This usually means the download URL returned an error page.`);
459
- }
460
- const maxBytes = this.config.maxSizeMB * 1024 * 1024;
461
- let buffer = await this.readBodyWithinLimit(response, maxBytes);
462
- // Check for gzip encoding and decompress if needed
463
- // Only decompress if the data actually starts with gzip magic bytes (0x1f 0x8b)
464
- const contentEncoding = response.headers.get("Content-Encoding");
465
- const isActuallyGzipped = buffer.length >= 2 && buffer[0] === 0x1f && buffer[1] === 0x8b;
466
- if (contentEncoding?.toLowerCase().includes("gzip") &&
467
- isActuallyGzipped) {
468
- try {
469
- // Bounded at the decoder. The size check that runs after the download
470
- // only ever sees what already fits in memory, so a few KB of gzip
471
- // declaring gigabytes was inflated in full before anything objected —
472
- // the response passed the "is it small" test on its way in.
473
- buffer = Buffer.from(await gunzipAsync(buffer, { maxOutputLength: maxBytes }));
474
- }
475
- catch (gzipError) {
476
- if (isBufferTooLargeError(gzipError)) {
477
- throw downloadTooLargeError(this.config.maxSizeMB, this.config.fileTypeName);
478
- }
479
- throw new Error(`Failed to decompress gzip response: ${gzipError instanceof Error ? gzipError.message : String(gzipError)}`, { cause: gzipError });
480
- }
481
- }
482
- return buffer;
483
- }
484
- finally {
485
- clearTimeout(timeoutId);
486
- // Release the connection on every early exit. Rejecting a response for
487
- // its status or its Content-Type leaves the body unread, which keeps the
488
- // socket out of the pool until the response is garbage-collected. After
489
- // a successful read there is nothing left in flight, so this is a no-op.
490
- controller.abort();
491
- }
492
- }
493
- /**
494
- * Read a response body, refusing it the moment it passes `maxBytes`.
495
- *
496
- * `response.arrayBuffer()` buffers whatever the server sends before anyone
497
- * can object, so the size check further up only ever ran against a body that
498
- * had already been paid for in full. Metering the stream makes the ceiling
499
- * the configured limit instead of the server's choice, and cancelling the
500
- * reader stops the transfer rather than merely stopping our interest in it.
501
- *
502
- * Content-Length is deliberately not trusted as the bound — it is a hint that
503
- * can be absent or a lie — but it is worth honouring when present, because it
504
- * refuses the honest oversized case without reading a byte.
505
- *
506
- * @param response - Fetch response whose body should be read
507
- * @param maxBytes - Ceiling for the body, in bytes
508
- * @returns The body as a Buffer
509
- * @throws When the body exceeds `maxBytes`
510
- */
511
- async readBodyWithinLimit(response, maxBytes) {
512
- const declared = Number(response.headers.get("Content-Length"));
513
- if (Number.isFinite(declared) && declared > maxBytes) {
514
- // Release the socket. An unread body keeps the connection out of the
515
- // pool until the response is collected, so a run of oversized downloads
516
- // would hold one socket open each.
517
- await response.body?.cancel();
518
- throw downloadTooLargeError(this.config.maxSizeMB, this.config.fileTypeName);
519
- }
520
- // Some responses carry no readable stream (mocked fetch, polyfills). The
521
- // buffered read is the fallback, still checked — just after the fact.
522
- if (!response.body) {
523
- const buffered = Buffer.from(await response.arrayBuffer());
524
- if (buffered.length > maxBytes) {
525
- throw downloadTooLargeError(this.config.maxSizeMB, this.config.fileTypeName);
526
- }
527
- return buffered;
442
+ async downloadFile(url, authHeaders, timeout, attachmentPolicy) {
443
+ const maxBytes = this.config.maxSizeMB * 1024 * 1024;
444
+ const downloaded = await downloadAttachment(url, {
445
+ policy: attachmentPolicy,
446
+ maxBytes,
447
+ label: `${this.config.fileTypeName} download`,
448
+ headers: authHeaders,
449
+ timeoutMs: timeout ?? this.config.timeoutMs,
450
+ });
451
+ // Reject HTML responses - likely an error page or redirect
452
+ const contentType = downloaded.contentType;
453
+ if (contentType && contentType.includes("text/html")) {
454
+ throw new Error(`Received HTML response instead of file content (Content-Type: ${contentType}). This usually means the download URL returned an error page.`);
528
455
  }
529
- const reader = response.body.getReader();
530
- const chunks = [];
531
- let total = 0;
532
- for (;;) {
533
- const { done, value } = await reader.read();
534
- if (done) {
535
- break;
536
- }
537
- if (!value) {
538
- continue;
456
+ let buffer = downloaded.buffer;
457
+ // Check for gzip encoding and decompress if needed
458
+ // Only decompress if the data actually starts with gzip magic bytes (0x1f 0x8b)
459
+ const contentEncoding = downloaded.contentEncoding;
460
+ const isActuallyGzipped = buffer.length >= 2 && buffer[0] === 0x1f && buffer[1] === 0x8b;
461
+ if (contentEncoding.toLowerCase().includes("gzip") && isActuallyGzipped) {
462
+ try {
463
+ // Bounded at the decoder. The size check that runs after the download
464
+ // only ever sees what already fits in memory, so a few KB of gzip
465
+ // declaring gigabytes was inflated in full before anything objected —
466
+ // the response passed the "is it small" test on its way in.
467
+ buffer = Buffer.from(await gunzipAsync(buffer, { maxOutputLength: maxBytes }));
539
468
  }
540
- total += value.byteLength;
541
- if (total > maxBytes) {
542
- await reader.cancel();
543
- throw downloadTooLargeError(this.config.maxSizeMB, this.config.fileTypeName);
469
+ catch (gzipError) {
470
+ if (isBufferTooLargeError(gzipError)) {
471
+ throw downloadTooLargeError(this.config.maxSizeMB, this.config.fileTypeName);
472
+ }
473
+ throw new Error(`Failed to decompress gzip response: ${gzipError instanceof Error ? gzipError.message : String(gzipError)}`, { cause: gzipError });
544
474
  }
545
- chunks.push(Buffer.from(value));
546
475
  }
547
- return Buffer.concat(chunks, total);
476
+ return buffer;
548
477
  }
549
478
  /**
550
479
  * Download file with retry logic for transient failures.
@@ -567,7 +496,7 @@ export class BaseFileProcessor {
567
496
  let lastError;
568
497
  for (let attempt = 0; attempt <= retryConfig.maxRetries; attempt++) {
569
498
  try {
570
- const buffer = await this.downloadFile(url, options?.authHeaders, options?.timeout);
499
+ const buffer = await this.downloadFile(url, options?.authHeaders, options?.timeout, options?.attachmentPolicy);
571
500
  return { success: true, data: buffer };
572
501
  }
573
502
  catch (error) {
@@ -15,7 +15,10 @@
15
15
  * 2. **Billion Laughs Attack (Entity Expansion)**: YAML supports anchors (&) and aliases (*)
16
16
  * for referencing content. Malicious YAML can use nested aliases to create exponential
17
17
  * expansion (e.g., 10 levels of 10x expansion = 10^10 entities from a small file).
18
- * We limit `maxAliasCount` to 100 to prevent memory exhaustion.
18
+ * js-yaml parses aliases as SHARED references, so the parsed tree stays small — the
19
+ * expansion happens at serialization. `asJson` is therefore produced by a bounded
20
+ * serializer that stops at the processor's byte limit and reports `null` instead of
21
+ * materialising the expanded document.
19
22
  *
20
23
  * 3. **Denial of Service**: Large or deeply nested YAML files can exhaust memory/CPU.
21
24
  * Size limits are enforced by the base processor's maxSizeMB configuration.
@@ -109,6 +112,15 @@ export declare class YamlProcessor extends BaseFileProcessor<ProcessedYaml> {
109
112
  */
110
113
  protected buildProcessedResult(buffer: Buffer, fileInfo: FileInfo): ProcessedYaml;
111
114
  }
115
+ /**
116
+ * `JSON.stringify(value, null, 2)` that stops once the output would pass
117
+ * `maxChars`, returning `null` instead. Walks the value iteratively enough to
118
+ * visit shared references once per path (which is what expands an alias
119
+ * bomb) but counts output as it goes, so the work is bounded by the limit
120
+ * rather than by the document's expanded size. Cycles — which YAML anchors
121
+ * can also express — serialize as `"[Circular]"`.
122
+ */
123
+ export declare function boundedJsonStringify(value: unknown, maxChars: number): string | null;
112
124
  /** Singleton YAML processor instance */
113
125
  export declare const yamlProcessor: YamlProcessor;
114
126
  /**
@@ -15,7 +15,10 @@
15
15
  * 2. **Billion Laughs Attack (Entity Expansion)**: YAML supports anchors (&) and aliases (*)
16
16
  * for referencing content. Malicious YAML can use nested aliases to create exponential
17
17
  * expansion (e.g., 10 levels of 10x expansion = 10^10 entities from a small file).
18
- * We limit `maxAliasCount` to 100 to prevent memory exhaustion.
18
+ * js-yaml parses aliases as SHARED references, so the parsed tree stays small — the
19
+ * expansion happens at serialization. `asJson` is therefore produced by a bounded
20
+ * serializer that stops at the processor's byte limit and reports `null` instead of
21
+ * materialising the expanded document.
19
22
  *
20
23
  * 3. **Denial of Service**: Large or deeply nested YAML files can exhaust memory/CPU.
21
24
  * Size limits are enforced by the base processor's maxSizeMB configuration.
@@ -45,6 +48,7 @@ import { createRequire } from "node:module";
45
48
  import { BaseFileProcessor } from "../base/BaseFileProcessor.js";
46
49
  import { SIZE_LIMITS_MB } from "../config/index.js";
47
50
  import { createFileError, FileErrorCode } from "../errors/index.js";
51
+ import { logger } from "../../utils/logger.js";
48
52
  // Re-export for consumers who import from this module
49
53
  // Import for local use
50
54
  const require = createRequire(import.meta.url);
@@ -211,7 +215,14 @@ export class YamlProcessor extends BaseFileProcessor {
211
215
  try {
212
216
  // Use secure parsing - validation already passed, but maintain consistent security
213
217
  parsed = this.parseYamlSecurely(content);
214
- asJson = JSON.stringify(parsed, null, 2);
218
+ // Bounded: a 30-level alias chain is a few hundred bytes of YAML and a
219
+ // small shared-reference graph in memory, but 2^30 nodes once
220
+ // stringified. The serializer gives up at the processor's own byte
221
+ // limit rather than letting `JSON.stringify` expand it.
222
+ asJson = boundedJsonStringify(parsed, this.config.maxSizeMB * 1024 * 1024);
223
+ if (asJson === null) {
224
+ logger.warn(`[YamlProcessor] "${this.getFilename(fileInfo)}" expands past the ${this.config.maxSizeMB}MB limit when serialized (alias expansion); asJson omitted`);
225
+ }
215
226
  }
216
227
  catch (error) {
217
228
  // This shouldn't happen since we validate, but handle gracefully
@@ -232,6 +243,86 @@ export class YamlProcessor extends BaseFileProcessor {
232
243
  }
233
244
  }
234
245
  // =============================================================================
246
+ // BOUNDED SERIALIZATION
247
+ // =============================================================================
248
+ /**
249
+ * `JSON.stringify(value, null, 2)` that stops once the output would pass
250
+ * `maxChars`, returning `null` instead. Walks the value iteratively enough to
251
+ * visit shared references once per path (which is what expands an alias
252
+ * bomb) but counts output as it goes, so the work is bounded by the limit
253
+ * rather than by the document's expanded size. Cycles — which YAML anchors
254
+ * can also express — serialize as `"[Circular]"`.
255
+ */
256
+ export function boundedJsonStringify(value, maxChars) {
257
+ const parts = [];
258
+ let length = 0;
259
+ const ancestors = [];
260
+ const push = (text) => {
261
+ length += text.length;
262
+ if (length > maxChars) {
263
+ return false;
264
+ }
265
+ parts.push(text);
266
+ return true;
267
+ };
268
+ const walk = (node, indent) => {
269
+ if (node === null || typeof node !== "object") {
270
+ if (node === undefined || typeof node === "function") {
271
+ return push("null");
272
+ }
273
+ if (typeof node === "number" && !Number.isFinite(node)) {
274
+ return push("null");
275
+ }
276
+ return push(JSON.stringify(node));
277
+ }
278
+ if (ancestors.includes(node)) {
279
+ return push('"[Circular]"');
280
+ }
281
+ ancestors.push(node);
282
+ try {
283
+ const inner = `${indent} `;
284
+ if (Array.isArray(node)) {
285
+ if (node.length === 0) {
286
+ return push("[]");
287
+ }
288
+ if (!push("[\n")) {
289
+ return false;
290
+ }
291
+ for (let i = 0; i < node.length; i++) {
292
+ if (!push(inner) || !walk(node[i], inner)) {
293
+ return false;
294
+ }
295
+ if (!push(i < node.length - 1 ? ",\n" : "\n")) {
296
+ return false;
297
+ }
298
+ }
299
+ return push(`${indent}]`);
300
+ }
301
+ const entries = Object.entries(node).filter(([, v]) => v !== undefined && typeof v !== "function");
302
+ if (entries.length === 0) {
303
+ return push("{}");
304
+ }
305
+ if (!push("{\n")) {
306
+ return false;
307
+ }
308
+ for (let i = 0; i < entries.length; i++) {
309
+ const [key, child] = entries[i];
310
+ if (!push(`${inner}${JSON.stringify(key)}: `) || !walk(child, inner)) {
311
+ return false;
312
+ }
313
+ if (!push(i < entries.length - 1 ? ",\n" : "\n")) {
314
+ return false;
315
+ }
316
+ }
317
+ return push(`${indent}}`);
318
+ }
319
+ finally {
320
+ ancestors.pop();
321
+ }
322
+ };
323
+ return walk(value, "") ? parts.join("") : null;
324
+ }
325
+ // =============================================================================
235
326
  // SINGLETON INSTANCE
236
327
  // =============================================================================
237
328
  /** Singleton YAML processor instance */
@@ -10,6 +10,7 @@ import { createRequire } from "node:module";
10
10
  import * as zlib from "node:zlib";
11
11
  import { BaseFileProcessor, contentTooLargeError, isContentTooLarge, } from "../base/BaseFileProcessor.js";
12
12
  import { readZipEntryWithinLimit } from "../archive/zipEntryReader.js";
13
+ import { indexZipEntriesByName } from "../archive/zipEntryIndex.js";
13
14
  import { SIZE_LIMITS } from "../config/index.js";
14
15
  const require = createRequire(import.meta.url);
15
16
  // Re-import for local use within this file
@@ -63,8 +64,12 @@ export class OpenDocumentProcessor extends BaseFileProcessor {
63
64
  // Dynamically import adm-zip to avoid issues if not available
64
65
  const AdmZip = require("adm-zip");
65
66
  const zip = new AdmZip(buffer);
66
- // Try to get content.xml
67
- const contentEntry = zip.getEntry("content.xml");
67
+ // Try to get content.xml — the FIRST member of that name, which is the
68
+ // one a listing shows; `getEntry` would resolve the last duplicate.
69
+ // `require("adm-zip")` is untyped; name the entry shape the bounded
70
+ // reader needs so the index is typed rather than `any`.
71
+ const zipEntries = zip.getEntries();
72
+ const contentEntry = indexZipEntriesByName(zipEntries).byName.get("content.xml");
68
73
  if (contentEntry) {
69
74
  // Bounded rather than `getData()`: an ODF file is a ZIP, so its
70
75
  // content.xml can declare any uncompressed size it likes, and
@@ -41,6 +41,7 @@
41
41
  import AdmZip from "adm-zip";
42
42
  import * as zlib from "node:zlib";
43
43
  import { readZipEntryWithinLimit } from "../archive/zipEntryReader.js";
44
+ import { indexZipEntriesByName } from "../archive/zipEntryIndex.js";
44
45
  import { SIZE_LIMITS } from "../config/index.js";
45
46
  /**
46
47
  * Total budget for everything read out of one presentation.
@@ -182,9 +183,12 @@ export class PptxProcessor {
182
183
  const zip = new AdmZip(content);
183
184
  const entries = zip.getEntries();
184
185
  const readEntry = createBoundedEntryReader();
186
+ // First occurrence per name: a deck with two `slide1.xml` members shows
187
+ // the slide the listing found first, and only once.
188
+ const { byName } = indexZipEntriesByName(entries);
185
189
  // Collect slide entries with their slide numbers for sorting
186
190
  const slides = [];
187
- for (const entry of entries) {
191
+ for (const entry of byName.values()) {
188
192
  const match = entry.entryName.match(SLIDE_ENTRY_REGEX);
189
193
  if (match) {
190
194
  const slideNumber = parseInt(match[1], 10);
@@ -203,7 +207,7 @@ export class PptxProcessor {
203
207
  parts.push(`Presentation: ${slides.length} slide(s)\n`);
204
208
  for (const slide of slides) {
205
209
  const texts = PptxProcessor.extractTextFromXml(slide.xml);
206
- const notes = PptxProcessor.extractNotesForSlide(zip, slide.slideNumber, readEntry);
210
+ const notes = PptxProcessor.extractNotesForSlide(byName, slide.slideNumber, readEntry);
207
211
  // Emit a slide section when it has either body text or speaker notes.
208
212
  if (texts.length > 0 || notes) {
209
213
  parts.push(`### Slide ${slide.slideNumber}`);
@@ -232,8 +236,8 @@ export class PptxProcessor {
232
236
  * @param slideNumber - 1-indexed slide number
233
237
  * @returns The notes text (runs joined by a space), or null when absent
234
238
  */
235
- static extractNotesForSlide(zip, slideNumber, readEntry) {
236
- const relsXml = readEntry(zip.getEntry(SLIDE_RELS_NAME(slideNumber)));
239
+ static extractNotesForSlide(byName, slideNumber, readEntry) {
240
+ const relsXml = readEntry(byName.get(SLIDE_RELS_NAME(slideNumber)));
237
241
  if (relsXml === null) {
238
242
  return null;
239
243
  }
@@ -254,7 +258,7 @@ export class PptxProcessor {
254
258
  const normalized = notesTarget
255
259
  .replace(/^\.\.\//, "ppt/")
256
260
  .replace(/^\/+/, "");
257
- const notesXml = readEntry(zip.getEntry(normalized));
261
+ const notesXml = readEntry(byName.get(normalized));
258
262
  if (notesXml === null) {
259
263
  return null;
260
264
  }
@@ -352,9 +356,10 @@ export class PptxProcessor {
352
356
  const zip = new AdmZip(content);
353
357
  const entries = zip.getEntries();
354
358
  const readEntry = createBoundedEntryReader();
359
+ const { byName } = indexZipEntriesByName(entries);
355
360
  // Collect all slides
356
361
  const slides = [];
357
- for (const entry of entries) {
362
+ for (const entry of byName.values()) {
358
363
  const match = entry.entryName.match(SLIDE_ENTRY_REGEX);
359
364
  if (match) {
360
365
  const slideNumber = parseInt(match[1], 10);
@@ -380,7 +385,7 @@ export class PptxProcessor {
380
385
  const parts = [];
381
386
  for (const slide of slides) {
382
387
  const texts = PptxProcessor.extractTextFromXml(slide.xml);
383
- const notes = PptxProcessor.extractNotesForSlide(zip, slide.slideNumber, readEntry);
388
+ const notes = PptxProcessor.extractNotesForSlide(byName, slide.slideNumber, readEntry);
384
389
  parts.push(`### Slide ${slide.slideNumber}`);
385
390
  if (texts.length > 0) {
386
391
  parts.push(texts.join("\n"));
@@ -43,7 +43,6 @@
43
43
  * }
44
44
  * ```
45
45
  */
46
- import { randomUUID } from "crypto";
47
46
  import { createWriteStream, existsSync, promises as fs } from "fs";
48
47
  import { tmpdir } from "os";
49
48
  import { join } from "path";
@@ -135,6 +134,44 @@ async function loadSharp() {
135
134
  // =============================================================================
136
135
  // FFMPEG PATH INITIALIZATION
137
136
  // =============================================================================
137
+ /**
138
+ * Extensions a caller-supplied filename may contribute to a temp file name.
139
+ * Anything else is replaced by the MIME-derived or `.mp4` default, so the
140
+ * suffix ffmpeg sees as a format hint is always one of ours.
141
+ */
142
+ const ALLOWED_TEMP_VIDEO_EXTENSIONS = new Set([
143
+ ".mp4",
144
+ ".m4v",
145
+ ".mkv",
146
+ ".mov",
147
+ ".webm",
148
+ ".avi",
149
+ ".wmv",
150
+ ".flv",
151
+ ".3gp",
152
+ ".3g2",
153
+ ".ts",
154
+ ".mts",
155
+ ".m2ts",
156
+ ".ogv",
157
+ ".ogg",
158
+ ".mpg",
159
+ ".mpeg",
160
+ ]);
161
+ /**
162
+ * Input-side ffmpeg/ffprobe options applied to every invocation here.
163
+ *
164
+ * `-protocol_whitelist file,crypto,data` pins the demuxer to local files: a
165
+ * playlist or container that references `http://`, `concat:` or `subfile:`
166
+ * cannot make ffmpeg fetch or read anything else, whatever the installed
167
+ * build's defaults are. `-nostdin` keeps a child from ever waiting on a
168
+ * terminal.
169
+ */
170
+ const FFMPEG_INPUT_HARDENING = [
171
+ "-nostdin",
172
+ "-protocol_whitelist",
173
+ "file,crypto,data",
174
+ ];
138
175
  /**
139
176
  * Whether ffmpeg/ffprobe paths have been initialized.
140
177
  * We only attempt path resolution once to avoid repeated dynamic import overhead.
@@ -361,8 +398,9 @@ export class VideoProcessor extends BaseFileProcessor {
361
398
  logger.info(`[NEUROLINK] Video processing started: ${filename} (${(sizeBytes / (1024 * 1024)).toFixed(2)} MB, ${fileInfo.mimetype || "video/mp4"})`);
362
399
  // Ensure ffmpeg paths are initialized before any processing
363
400
  await initFfmpegPaths();
364
- // Temp directory for this processing run
365
- const tempDir = join(tmpdir(), `neurolink-video-${randomUUID()}`);
401
+ // Temp directory for this processing run — minted with mkdtemp
402
+ // (0o700) once there are bytes to write, see Step 3.
403
+ let tempDir = "";
366
404
  let tempCreated = false;
367
405
  try {
368
406
  // Step 1: Validate file type and size
@@ -430,7 +468,7 @@ export class VideoProcessor extends BaseFileProcessor {
430
468
  };
431
469
  }
432
470
  // Step 3: Write buffer to temp file (ffmpeg needs a file path)
433
- await fs.mkdir(tempDir, { recursive: true });
471
+ tempDir = await fs.mkdtemp(join(tmpdir(), "neurolink-video-"));
434
472
  tempCreated = true;
435
473
  const extension = this.getExtensionFromFileInfo(fileInfo);
436
474
  const tempVideoPath = join(tempDir, `input${extension}`);
@@ -589,7 +627,7 @@ export class VideoProcessor extends BaseFileProcessor {
589
627
  error: `ffprobe timed out after ${VIDEO_CONFIG.FFPROBE_TIMEOUT_MS}ms`,
590
628
  });
591
629
  }, VIDEO_CONFIG.FFPROBE_TIMEOUT_MS);
592
- ffmpeg.ffprobe(filePath, (err, data) => {
630
+ ffmpeg.ffprobe(filePath, ["-protocol_whitelist", "file,crypto,data"], (err, data) => {
593
631
  clearTimeout(timeoutId);
594
632
  if (err) {
595
633
  resolve({
@@ -671,7 +709,17 @@ export class VideoProcessor extends BaseFileProcessor {
671
709
  let report;
672
710
  let exitFailure;
673
711
  try {
674
- const { stderr } = await runFfmpeg(["-hide_banner", "-i", filePath, "-t", "0", "-f", "null", "-"], { timeoutMs: VIDEO_CONFIG.FFPROBE_TIMEOUT_MS });
712
+ const { stderr } = await runFfmpeg([
713
+ ...FFMPEG_INPUT_HARDENING,
714
+ "-hide_banner",
715
+ "-i",
716
+ filePath,
717
+ "-t",
718
+ "0",
719
+ "-f",
720
+ "null",
721
+ "-",
722
+ ], { timeoutMs: VIDEO_CONFIG.FFPROBE_TIMEOUT_MS });
675
723
  report = stderr;
676
724
  }
677
725
  catch (error) {
@@ -923,6 +971,7 @@ export class VideoProcessor extends BaseFileProcessor {
923
971
  const audioPath = join(tempDir, `audio.${audioFormat}`);
924
972
  try {
925
973
  await runFfmpeg([
974
+ ...FFMPEG_INPUT_HARDENING,
926
975
  "-y",
927
976
  "-v",
928
977
  "error",
@@ -1045,7 +1094,7 @@ export class VideoProcessor extends BaseFileProcessor {
1045
1094
  }
1046
1095
  // Extract frames using ffmpeg
1047
1096
  const framesDir = join(tempDir, "frames");
1048
- await fs.mkdir(framesDir, { recursive: true });
1097
+ await fs.mkdir(framesDir, { recursive: true, mode: 0o700 });
1049
1098
  // The requested `timestamps` are an idealized, evenly-spaced schedule.
1050
1099
  // ffmpeg's `-vf select` only guarantees a MINIMUM gap since the last
1051
1100
  // frame it actually selected — on a source whose real frame spacing
@@ -1131,10 +1180,20 @@ export class VideoProcessor extends BaseFileProcessor {
1131
1180
  const selectExpr = `isnan(prev_selected_t)+gte(t-prev_selected_t,${intervalSec}-0.001)`;
1132
1181
  const PTS_TIME_RE = /pts_time:\s*(-?\d+(?:\.\d+)?)/;
1133
1182
  const realTimestamps = [];
1183
+ // Keep the command so the timeout can kill the child rather than only
1184
+ // abandon it: a rejected promise used to leave ffmpeg running to
1185
+ // completion on a file nobody would read.
1186
+ const command = ff(videoPath).inputOptions([...FFMPEG_INPUT_HARDENING]);
1134
1187
  const timeoutId = setTimeout(() => {
1188
+ try {
1189
+ command.kill("SIGKILL");
1190
+ }
1191
+ catch {
1192
+ // Already exited.
1193
+ }
1135
1194
  reject(new Error(`ffmpeg frame extraction timed out after ${VIDEO_CONFIG.FFMPEG_TIMEOUT_MS}ms`));
1136
1195
  }, VIDEO_CONFIG.FFMPEG_TIMEOUT_MS);
1137
- ff(videoPath)
1196
+ command
1138
1197
  .outputOptions([
1139
1198
  "-vf",
1140
1199
  `select='${selectExpr}',showinfo,scale='min(${VIDEO_CONFIG.FRAME_MAX_DIMENSION}\\,iw):-2'`,
@@ -1205,10 +1264,19 @@ export class VideoProcessor extends BaseFileProcessor {
1205
1264
  const subtitlePath = join(tempDir, "subtitles.srt");
1206
1265
  const ffSub = await loadFluentFfmpeg();
1207
1266
  await new Promise((resolve, reject) => {
1267
+ const command = ffSub(videoPath).inputOptions([
1268
+ ...FFMPEG_INPUT_HARDENING,
1269
+ ]);
1208
1270
  const timeoutId = setTimeout(() => {
1271
+ try {
1272
+ command.kill("SIGKILL");
1273
+ }
1274
+ catch {
1275
+ // Already exited.
1276
+ }
1209
1277
  reject(new Error(`ffmpeg subtitle extraction timed out after ${VIDEO_CONFIG.FFMPEG_TIMEOUT_MS}ms`));
1210
1278
  }, VIDEO_CONFIG.FFMPEG_TIMEOUT_MS);
1211
- ffSub(videoPath)
1279
+ command
1212
1280
  .outputOptions(["-map", "0:s:0", "-c:s", "srt"])
1213
1281
  .output(subtitlePath)
1214
1282
  .on("end", () => {
@@ -1383,7 +1451,13 @@ export class VideoProcessor extends BaseFileProcessor {
1383
1451
  const name = fileInfo.name || "";
1384
1452
  const dotIndex = name.lastIndexOf(".");
1385
1453
  if (dotIndex >= 0) {
1386
- return name.substring(dotIndex).toLowerCase();
1454
+ const fromName = name.substring(dotIndex).toLowerCase();
1455
+ // The tail of a caller-supplied name becomes a temp file suffix that
1456
+ // ffmpeg reads as a format hint; only extensions this processor knows
1457
+ // are allowed through, anything else falls back to the MIME type.
1458
+ if (ALLOWED_TEMP_VIDEO_EXTENSIONS.has(fromName)) {
1459
+ return fromName;
1460
+ }
1387
1461
  }
1388
1462
  // Fallback: derive from MIME type
1389
1463
  const mimeExtMap = {
@@ -1409,7 +1483,9 @@ export class VideoProcessor extends BaseFileProcessor {
1409
1483
  */
1410
1484
  async writeBufferToFile(buffer, filePath) {
1411
1485
  const readable = Readable.from(buffer);
1412
- const writable = createWriteStream(filePath);
1486
+ // Owner-only, and never over an existing file: the directory is fresh
1487
+ // from mkdtemp, so a path that already exists is something planted.
1488
+ const writable = createWriteStream(filePath, { mode: 0o600, flags: "wx" });
1413
1489
  await pipeline(readable, writable);
1414
1490
  }
1415
1491
  // ===========================================================================
@@ -1431,9 +1507,8 @@ export class VideoProcessor extends BaseFileProcessor {
1431
1507
  */
1432
1508
  async extractFrameRange(buffer, filename, startSec, endSec, frameCount = 5) {
1433
1509
  await initFfmpegPaths();
1434
- const tempDir = join(tmpdir(), `neurolink-video-extract-${randomUUID()}`);
1510
+ const tempDir = await fs.mkdtemp(join(tmpdir(), "neurolink-video-extract-"));
1435
1511
  try {
1436
- await fs.mkdir(tempDir, { recursive: true });
1437
1512
  // Write buffer to temp file
1438
1513
  const ext = this.guessExtensionFromName(filename);
1439
1514
  const tempVideoPath = join(tempDir, `input${ext}`);
@@ -1457,7 +1532,7 @@ export class VideoProcessor extends BaseFileProcessor {
1457
1532
  }
1458
1533
  // Extract frames
1459
1534
  const framesDir = join(tempDir, "frames");
1460
- await fs.mkdir(framesDir, { recursive: true });
1535
+ await fs.mkdir(framesDir, { recursive: true, mode: 0o700 });
1461
1536
  await this.runFfmpegFrameExtraction(tempVideoPath, framesDir, timestamps, interval);
1462
1537
  // Read and resize frames
1463
1538
  const keyframes = [];
@@ -1494,7 +1569,10 @@ export class VideoProcessor extends BaseFileProcessor {
1494
1569
  guessExtensionFromName(filename) {
1495
1570
  const dotIndex = filename.lastIndexOf(".");
1496
1571
  if (dotIndex >= 0) {
1497
- return filename.substring(dotIndex).toLowerCase();
1572
+ const ext = filename.substring(dotIndex).toLowerCase();
1573
+ if (ALLOWED_TEMP_VIDEO_EXTENSIONS.has(ext)) {
1574
+ return ext;
1575
+ }
1498
1576
  }
1499
1577
  return ".mp4";
1500
1578
  }