@juspay/neurolink 13.8.17 → 13.8.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (214) hide show
  1. package/CHANGELOG.md +5 -2
  2. package/README.md +1 -1
  3. package/dist/action/actionExecutor.d.ts +33 -3
  4. package/dist/action/actionExecutor.js +71 -15
  5. package/dist/action/actionInputs.js +4 -1
  6. package/dist/action/cliVersion.d.ts +25 -0
  7. package/dist/action/cliVersion.js +26 -0
  8. package/dist/action/index.d.ts +2 -1
  9. package/dist/action/index.js +2 -1
  10. package/dist/adapters/video/directorPipeline.js +46 -36
  11. package/dist/adapters/video/ffmpegAdapter.d.ts +17 -0
  12. package/dist/adapters/video/ffmpegAdapter.js +32 -5
  13. package/dist/adapters/video/frameExtractor.js +1 -1
  14. package/dist/adapters/video/videoMerger.js +5 -2
  15. package/dist/agent/arithmeticEvaluator.d.ts +32 -0
  16. package/dist/agent/arithmeticEvaluator.js +349 -0
  17. package/dist/agent/directTools.d.ts +6 -18
  18. package/dist/agent/directTools.js +17 -56
  19. package/dist/artifacts/artifactStore.d.ts +22 -1
  20. package/dist/artifacts/artifactStore.js +66 -6
  21. package/dist/auth/anthropicOAuth.d.ts +26 -1
  22. package/dist/auth/anthropicOAuth.js +81 -5
  23. package/dist/auth/secureFiles.d.ts +37 -0
  24. package/dist/auth/secureFiles.js +95 -0
  25. package/dist/auth/tokenStore.d.ts +43 -6
  26. package/dist/auth/tokenStore.js +260 -79
  27. package/dist/avatar/providers/DIDAvatar.js +49 -45
  28. package/dist/avatar/providers/ReplicateAvatar.js +33 -40
  29. package/dist/browser/neurolink.min.js +485 -558
  30. package/dist/cli/commands/auth.js +112 -70
  31. package/dist/cli/commands/proxy.d.ts +4 -0
  32. package/dist/cli/commands/proxy.js +128 -13
  33. package/dist/cli/commands/serve.d.ts +10 -1
  34. package/dist/cli/commands/serve.js +98 -17
  35. package/dist/cli/commands/server.js +17 -5
  36. package/dist/cli/index.js +146 -5
  37. package/dist/cli/loop/optionsSchema.d.ts +1 -1
  38. package/dist/cli/utils/envManager.js +17 -3
  39. package/dist/cli/utils/serverUtils.d.ts +39 -3
  40. package/dist/cli/utils/serverUtils.js +63 -6
  41. package/dist/core/baseProvider.js +7 -1
  42. package/dist/core/modules/MessageBuilder.js +5 -0
  43. package/dist/core/modules/ToolsManager.d.ts +46 -1
  44. package/dist/core/modules/ToolsManager.js +190 -43
  45. package/dist/files/fileReferenceRegistry.d.ts +2 -23
  46. package/dist/files/fileReferenceRegistry.js +55 -5
  47. package/dist/hitl/hitlManager.js +1 -0
  48. package/dist/index.d.ts +2 -1
  49. package/dist/index.js +6 -1
  50. package/dist/mcp/auth/tokenStorage.d.ts +7 -0
  51. package/dist/mcp/auth/tokenStorage.js +23 -4
  52. package/dist/mcp/externalServerManager.d.ts +13 -3
  53. package/dist/mcp/externalServerManager.js +154 -136
  54. package/dist/mcp/index.d.ts +2 -0
  55. package/dist/mcp/index.js +3 -0
  56. package/dist/mcp/mcpClientFactory.d.ts +14 -0
  57. package/dist/mcp/mcpClientFactory.js +70 -7
  58. package/dist/mcp/projectConfigTrust.d.ts +72 -0
  59. package/dist/mcp/projectConfigTrust.js +237 -0
  60. package/dist/mcp/transportUrlPolicy.d.ts +39 -0
  61. package/dist/mcp/transportUrlPolicy.js +172 -0
  62. package/dist/music/providers/BeatovenMusic.js +35 -46
  63. package/dist/music/providers/ReplicateMusic.js +33 -40
  64. package/dist/neurolink.d.ts +16 -1
  65. package/dist/neurolink.js +64 -11
  66. package/dist/processors/archive/ArchiveProcessor.js +15 -1
  67. package/dist/processors/archive/zipEntryIndex.d.ts +22 -0
  68. package/dist/processors/archive/zipEntryIndex.js +33 -0
  69. package/dist/processors/base/BaseFileProcessor.d.ts +9 -21
  70. package/dist/processors/base/BaseFileProcessor.js +40 -111
  71. package/dist/processors/data/YamlProcessor.d.ts +13 -1
  72. package/dist/processors/data/YamlProcessor.js +93 -2
  73. package/dist/processors/document/OpenDocumentProcessor.js +7 -2
  74. package/dist/processors/document/PptxProcessor.js +12 -7
  75. package/dist/processors/media/VideoProcessor.js +93 -15
  76. package/dist/providers/openaiChatCompletionsClient.js +12 -3
  77. package/dist/proxy/accountLedger.js +6 -8
  78. package/dist/proxy/geminiFormat.d.ts +3 -11
  79. package/dist/proxy/geminiFormat.js +11 -2
  80. package/dist/proxy/inboundAuth.d.ts +89 -0
  81. package/dist/proxy/inboundAuth.js +219 -0
  82. package/dist/proxy/openaiFormat.d.ts +3 -9
  83. package/dist/proxy/openaiFormat.js +21 -15
  84. package/dist/proxy/proxyTranslationEngine.d.ts +2 -6
  85. package/dist/proxy/proxyTranslationEngine.js +25 -14
  86. package/dist/proxy/routingPolicy.d.ts +3 -1
  87. package/dist/proxy/routingPolicy.js +7 -8
  88. package/dist/proxy/tokenRefresh.d.ts +6 -0
  89. package/dist/proxy/tokenRefresh.js +13 -1
  90. package/dist/proxy/updateState.js +4 -4
  91. package/dist/rag/document/loaders.d.ts +0 -6
  92. package/dist/rag/document/loaders.js +16 -10
  93. package/dist/rag/ragIntegration.d.ts +2 -2
  94. package/dist/rag/ragIntegration.js +35 -10
  95. package/dist/rag/retrieval/vectorQueryTool.d.ts +7 -0
  96. package/dist/rag/retrieval/vectorQueryTool.js +54 -17
  97. package/dist/server/abstract/baseServerAdapter.d.ts +43 -0
  98. package/dist/server/abstract/baseServerAdapter.js +119 -6
  99. package/dist/server/adapters/expressAdapter.d.ts +3 -0
  100. package/dist/server/adapters/expressAdapter.js +78 -6
  101. package/dist/server/adapters/fastifyAdapter.d.ts +3 -0
  102. package/dist/server/adapters/fastifyAdapter.js +165 -70
  103. package/dist/server/adapters/honoAdapter.d.ts +9 -1
  104. package/dist/server/adapters/honoAdapter.js +99 -21
  105. package/dist/server/adapters/koaAdapter.d.ts +1 -0
  106. package/dist/server/adapters/koaAdapter.js +153 -74
  107. package/dist/server/index.d.ts +2 -0
  108. package/dist/server/index.js +4 -0
  109. package/dist/server/middleware/auth.js +6 -2
  110. package/dist/server/middleware/cache.js +6 -1
  111. package/dist/server/middleware/rateLimit.js +29 -4
  112. package/dist/server/routes/agentRoutes.js +24 -11
  113. package/dist/server/routes/claudeProxyRoutes.d.ts +1 -0
  114. package/dist/server/routes/claudeProxyRoutes.js +273 -84
  115. package/dist/server/routes/codexProxyRoutes.d.ts +2 -2
  116. package/dist/server/routes/codexProxyRoutes.js +125 -40
  117. package/dist/server/routes/index.d.ts +4 -0
  118. package/dist/server/routes/index.js +35 -4
  119. package/dist/server/routes/mcpRoutes.d.ts +6 -2
  120. package/dist/server/routes/mcpRoutes.js +21 -7
  121. package/dist/server/routes/memoryRoutes.js +7 -6
  122. package/dist/server/routes/toolRoutes.d.ts +12 -2
  123. package/dist/server/routes/toolRoutes.js +33 -3
  124. package/dist/server/routes/transcribeRoutes.js +60 -13
  125. package/dist/server/utils/clientAddress.d.ts +23 -0
  126. package/dist/server/utils/clientAddress.js +65 -0
  127. package/dist/server/utils/clientError.d.ts +35 -0
  128. package/dist/server/utils/clientError.js +129 -0
  129. package/dist/server/utils/pathMatch.d.ts +21 -0
  130. package/dist/server/utils/pathMatch.js +41 -0
  131. package/dist/server/utils/validation.d.ts +9 -0
  132. package/dist/server/utils/validation.js +9 -0
  133. package/dist/server/websocket/WebSocketHandler.js +22 -3
  134. package/dist/server/websocket/frameQueue.d.ts +19 -0
  135. package/dist/server/websocket/frameQueue.js +41 -3
  136. package/dist/server/websocket/transcribeWebSocket.d.ts +8 -1
  137. package/dist/server/websocket/transcribeWebSocket.js +223 -34
  138. package/dist/skills/skillStores.js +5 -0
  139. package/dist/tasks/store/fileTaskStore.d.ts +2 -0
  140. package/dist/tasks/store/fileTaskStore.js +11 -4
  141. package/dist/types/attachments.d.ts +117 -0
  142. package/dist/types/attachments.js +15 -0
  143. package/dist/types/auth.d.ts +6 -0
  144. package/dist/types/cli.d.ts +4 -0
  145. package/dist/types/config.d.ts +56 -0
  146. package/dist/types/file.d.ts +8 -0
  147. package/dist/types/hitl.d.ts +8 -0
  148. package/dist/types/index.d.ts +1 -0
  149. package/dist/types/index.js +1 -0
  150. package/dist/types/mcp.d.ts +98 -0
  151. package/dist/types/multimodal.js +0 -41
  152. package/dist/types/openaiCompatible.d.ts +1 -1
  153. package/dist/types/processor.d.ts +14 -1
  154. package/dist/types/proxy.d.ts +71 -16
  155. package/dist/types/proxyClient.d.ts +2 -0
  156. package/dist/types/rag.d.ts +9 -0
  157. package/dist/types/safeFetch.d.ts +40 -0
  158. package/dist/types/server.d.ts +206 -10
  159. package/dist/types/tools.d.ts +57 -0
  160. package/dist/types/transcribe.d.ts +6 -0
  161. package/dist/utils/attachmentPolicy.d.ts +89 -0
  162. package/dist/utils/attachmentPolicy.js +327 -0
  163. package/dist/utils/dotenvBootstrap.d.ts +48 -19
  164. package/dist/utils/dotenvBootstrap.js +172 -31
  165. package/dist/utils/fileDetector.js +115 -65
  166. package/dist/utils/json/coerce.js +11 -1
  167. package/dist/utils/logSanitize.d.ts +5 -0
  168. package/dist/utils/logSanitize.js +8 -0
  169. package/dist/utils/logger.js +73 -1
  170. package/dist/utils/messageBuilder.d.ts +2 -2
  171. package/dist/utils/messageBuilder.js +118 -105
  172. package/dist/utils/multimodalOptionsBuilder.d.ts +1 -0
  173. package/dist/utils/multimodalOptionsBuilder.js +2 -0
  174. package/dist/utils/redis.d.ts +12 -0
  175. package/dist/utils/redis.js +72 -0
  176. package/dist/utils/retryAfter.d.ts +32 -0
  177. package/dist/utils/retryAfter.js +135 -11
  178. package/dist/utils/safeFetch.d.ts +22 -1
  179. package/dist/utils/safeFetch.js +132 -38
  180. package/dist/utils/safeStoreId.d.ts +19 -0
  181. package/dist/utils/safeStoreId.js +29 -0
  182. package/dist/utils/schemaConversion.d.ts +30 -3
  183. package/dist/utils/schemaConversion.js +457 -84
  184. package/dist/utils/sizeGuard.js +2 -2
  185. package/dist/utils/ssrfGuard.d.ts +3 -3
  186. package/dist/utils/ssrfGuard.js +71 -39
  187. package/dist/voice/eval/sttEvalStore.d.ts +26 -3
  188. package/dist/voice/eval/sttEvalStore.js +77 -12
  189. package/dist/voice/livekit/agentDefinition.d.ts +18 -0
  190. package/dist/voice/livekit/agentDefinition.js +22 -0
  191. package/dist/voice/livekit/realtimeMcpTools.d.ts +1 -1
  192. package/dist/voice/livekit/realtimeMcpTools.js +8 -7
  193. package/dist/voice/livekit/realtimeVoiceAgent.d.ts +2 -4
  194. package/dist/voice/livekit/realtimeVoiceAgent.js +18 -7
  195. package/dist/voice/livekit/voiceAgent.d.ts +2 -2
  196. package/dist/voice/livekit/voiceAgent.js +2 -1
  197. package/dist/voice/livekit/xaiSessionWatch.d.ts +0 -5
  198. package/dist/voice/livekit/xaiSessionWatch.js +30 -0
  199. package/dist/voice/providers/GeminiLive.js +21 -6
  200. package/dist/voice/providers/GoogleSTT.js +8 -5
  201. package/dist/voice/transcribe.d.ts +2 -2
  202. package/dist/voice/transcribe.js +34 -18
  203. package/dist/voice/whistle/audio.js +11 -5
  204. package/dist/workflow/workflows/adaptiveWorkflow.d.ts +2 -2
  205. package/dist/workflow/workflows/adaptiveWorkflow.js +10 -10
  206. package/dist/workflow/workflows/consensusWorkflow.d.ts +4 -4
  207. package/dist/workflow/workflows/consensusWorkflow.js +12 -12
  208. package/dist/workflow/workflows/fallbackWorkflow.d.ts +3 -3
  209. package/dist/workflow/workflows/fallbackWorkflow.js +7 -7
  210. package/dist/workflow/workflows/multiJudgeWorkflow.d.ts +7 -7
  211. package/dist/workflow/workflows/multiJudgeWorkflow.js +31 -31
  212. package/docs-site/static/search-index.json +74 -52
  213. package/package.json +173 -156
  214. package/scripts/observability/import-openobserve-dashboard.mjs +11 -3
@@ -16,6 +16,7 @@
16
16
  */
17
17
  import { ReadableStream } from "node:stream/web";
18
18
  import { logger } from "../../utils/logger.js";
19
+ import { markAgentDefinition } from "./agentDefinition.js";
19
20
  import { createVoiceBrain } from "./brain.js";
20
21
  import { resolveBrainDefaults, resolveEouTurnDetection } from "./config.js";
21
22
  import { attachEventBridge } from "./eventBridge.js";
@@ -458,5 +459,5 @@ export function defineVoiceAgent(config) {
458
459
  });
459
460
  }
460
461
  }
461
- return { entry };
462
+ return markAgentDefinition({ entry });
462
463
  }
@@ -18,9 +18,4 @@
18
18
  * third.
19
19
  */
20
20
  import type { XaiObservedSession } from "../../types/index.js";
21
- /**
22
- * Log the usage of every response, warn when the service runs another model
23
- * than the one asked for, and report a refused upgrade as a session error.
24
- * `requestedModel` is the id the session was configured with.
25
- */
26
21
  export declare function observeXaiSession(session: XaiObservedSession, requestedModel: string): void;
@@ -129,8 +129,38 @@ function watchForRefusedUpgrade(session) {
129
129
  * than the one asked for, and report a refused upgrade as a session error.
130
130
  * `requestedModel` is the id the session was configured with.
131
131
  */
132
+ /** The plugin's own wording for a refused upgrade (agents-plugin-xai >= 1.9). */
133
+ const PLUGIN_REFUSAL = /refused the connection: (\d{3})/;
132
134
  export function observeXaiSession(session, requestedModel) {
133
135
  const stopWatching = watchForRefusedUpgrade(session);
136
+ // Since @livekit/agents-plugin-xai 1.9 a refused upgrade no longer escapes
137
+ // as an uncaught exception: the plugin reports it as its own `error` event,
138
+ // worded for OpenAI and naming no key. This listener is registered before
139
+ // the agent's, so it can end the crash watch and relabel the event in place
140
+ // with the xAI-specific diagnosis (HTTP status + XAI_API_KEY hint).
141
+ session.on("error", (event) => {
142
+ if (!isRecord(event)) {
143
+ return;
144
+ }
145
+ const inner = event.error;
146
+ const text = inner instanceof Error
147
+ ? inner.message
148
+ : typeof event.message === "string"
149
+ ? event.message
150
+ : "";
151
+ const status = Number(PLUGIN_REFUSAL.exec(text)?.[1]);
152
+ if (Number.isNaN(status)) {
153
+ return;
154
+ }
155
+ stopWatching();
156
+ const message = describeRefusal(status);
157
+ logger.error("realtime.xai.connectRefused", { status, error: message });
158
+ event.error = new Error(message);
159
+ event.recoverable = false;
160
+ if (typeof event.label !== "string" || event.label.length === 0) {
161
+ event.label = "xai.RealtimeModel";
162
+ }
163
+ });
134
164
  session.on("openai_server_event_received", (event) => {
135
165
  if (isRecord(event) && event.type === "session.created") {
136
166
  stopWatching();
@@ -6,8 +6,21 @@
6
6
  * @module voice/providers/GeminiLive
7
7
  */
8
8
  import { logger } from "../../utils/logger.js";
9
+ import { redactUrlsInText } from "../../utils/logSanitize.js";
9
10
  import { RealtimeError } from "../errors.js";
10
11
  import { BaseRealtimeHandler } from "../RealtimeVoiceAPI.js";
12
+ /**
13
+ * A copy of `err` whose message has every URL — and so the `?key=` the Live
14
+ * endpoint requires — reduced to origin + path. The original is left alone
15
+ * for anything still holding it; `cause` is not carried, since it would hold
16
+ * the unscrubbed text.
17
+ */
18
+ function scrubUrlFromError(err) {
19
+ const scrubbed = new Error(redactUrlsInText(err.message));
20
+ scrubbed.name = err.name;
21
+ scrubbed.stack = err.stack ? redactUrlsInText(err.stack) : undefined;
22
+ return scrubbed;
23
+ }
11
24
  /**
12
25
  * Google Gemini Live Voice API Handler
13
26
  *
@@ -52,7 +65,9 @@ export class GeminiLive extends BaseRealtimeHandler {
52
65
  const { default: WebSocket } = await import("ws");
53
66
  // Determine model
54
67
  const model = config.model ?? "gemini-2.5-flash-native-audio-preview-09-2025";
55
- // Connect to Gemini Live API
68
+ // Connect to Gemini Live API. The Live endpoint authenticates with the
69
+ // key as a query parameter; every error below is scrubbed with
70
+ // `redactUrlsInText` so that URL never reaches a log or a caller.
56
71
  const wsUrl = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent?key=${this.apiKey}`;
57
72
  this.ws = new WebSocket(wsUrl);
58
73
  // Issue 9: capture a local reference so the closure below doesn't need
@@ -70,7 +85,7 @@ export class GeminiLive extends BaseRealtimeHandler {
70
85
  const errorHandler = (err) => {
71
86
  clearTimeout(timeout);
72
87
  ws.off("open", openHandler);
73
- reject(err);
88
+ reject(scrubUrlFromError(err));
74
89
  };
75
90
  const timeout = setTimeout(() => {
76
91
  // C1: close the half-opened socket and detach temp listeners so we
@@ -92,7 +107,7 @@ export class GeminiLive extends BaseRealtimeHandler {
92
107
  this.session = null;
93
108
  });
94
109
  this.ws.on("error", (err) => {
95
- this.emitError(err);
110
+ this.emitError(scrubUrlFromError(err));
96
111
  });
97
112
  // Send setup message
98
113
  await this.sendSetup(config, model);
@@ -114,8 +129,8 @@ export class GeminiLive extends BaseRealtimeHandler {
114
129
  }
115
130
  catch (err) {
116
131
  this.emitStateChange("error");
117
- const errorMessage = err instanceof Error ? err.message : String(err || "Unknown error");
118
- throw RealtimeError.connectionFailed(errorMessage, "gemini-live", err instanceof Error ? err : undefined);
132
+ const errorMessage = redactUrlsInText(err instanceof Error ? err.message : String(err || "Unknown error"));
133
+ throw RealtimeError.connectionFailed(errorMessage, "gemini-live", err instanceof Error ? scrubUrlFromError(err) : undefined);
119
134
  }
120
135
  }
121
136
  async disconnect() {
@@ -133,7 +148,7 @@ export class GeminiLive extends BaseRealtimeHandler {
133
148
  logger.info("[GeminiLiveHandler] Disconnected");
134
149
  }
135
150
  catch (err) {
136
- const errorMessage = err instanceof Error ? err.message : String(err || "Unknown error");
151
+ const errorMessage = redactUrlsInText(err instanceof Error ? err.message : String(err || "Unknown error"));
137
152
  throw RealtimeError.protocolError(`Disconnect failed: ${errorMessage}`, "gemini-live", err instanceof Error ? err : undefined);
138
153
  }
139
154
  }
@@ -6,6 +6,7 @@
6
6
  * @module voice/providers/GoogleSTT
7
7
  */
8
8
  import { logger } from "../../utils/logger.js";
9
+ import { redactUrlsInText } from "../../utils/logSanitize.js";
9
10
  import { STTError } from "../errors.js";
10
11
  /**
11
12
  * Google Cloud Speech-to-Text Handler
@@ -213,10 +214,9 @@ export class GoogleSTT {
213
214
  content: audioBuffer.toString("base64"),
214
215
  },
215
216
  };
216
- // Build URL with API key
217
- const url = this.apiKey
218
- ? `${this.baseUrl}/speech:recognize?key=${this.apiKey}`
219
- : `${this.baseUrl}/speech:recognize`;
217
+ // The key travels in the `x-goog-api-key` header, never in the URL:
218
+ // a URL is what proxies, access logs and `fetch failed` causes record.
219
+ const url = `${this.baseUrl}/speech:recognize`;
220
220
  const controller = new AbortController();
221
221
  const timeoutId = setTimeout(() => controller.abort(), 30000);
222
222
  let response;
@@ -225,6 +225,7 @@ export class GoogleSTT {
225
225
  method: "POST",
226
226
  headers: {
227
227
  "Content-Type": "application/json",
228
+ ...(this.apiKey ? { "x-goog-api-key": this.apiKey } : {}),
228
229
  ...(this.credentialsPath && !this.apiKey
229
230
  ? { Authorization: `Bearer ${await this.getAccessToken()}` }
230
231
  : {}),
@@ -237,7 +238,9 @@ export class GoogleSTT {
237
238
  if (fetchErr instanceof Error && fetchErr.name === "AbortError") {
238
239
  throw STTError.transcriptionFailed("Google STT request timed out after 30 seconds", "google-stt", fetchErr);
239
240
  }
240
- throw fetchErr;
241
+ // A transport error embeds the request URL; scrub any URL (and its
242
+ // query) before the message reaches a log or a caller.
243
+ throw STTError.transcriptionFailed(redactUrlsInText(fetchErr instanceof Error ? fetchErr.message : String(fetchErr)), "google-stt", fetchErr instanceof Error ? fetchErr : undefined);
241
244
  }
242
245
  finally {
243
246
  clearTimeout(timeoutId);
@@ -12,7 +12,7 @@
12
12
  *
13
13
  * @module voice/transcribe
14
14
  */
15
- import type { STTOptions, STTStreamingOptions, TranscribeAudioInput, TranscribeDeps, TranscribeOptions, TranscribeResult, TTSAudioFormat } from "../types/index.js";
15
+ import type { AttachmentPolicy, STTOptions, STTStreamingOptions, TranscribeAudioInput, TranscribeDeps, TranscribeOptions, TranscribeResult, TTSAudioFormat } from "../types/index.js";
16
16
  /**
17
17
  * The first `seconds` of a PCM WAV as a smaller WAV, so a language probe
18
18
  * uploads kilobytes instead of the whole recording. Anything that is not a
@@ -33,7 +33,7 @@ export declare function audioFormatFromBytes(buffer: Buffer): TTSAudioFormat | u
33
33
  *
34
34
  * @throws STTError when the input is a frame stream, or cannot be read
35
35
  */
36
- export declare function loadTranscribeAudio(input: TranscribeAudioInput, timeoutMs?: number): Promise<{
36
+ export declare function loadTranscribeAudio(input: TranscribeAudioInput, timeoutMs?: number, attachmentPolicy?: AttachmentPolicy): Promise<{
37
37
  buffer: Buffer;
38
38
  formatHint?: TTSAudioFormat;
39
39
  }>;
@@ -12,9 +12,9 @@
12
12
  *
13
13
  * @module voice/transcribe
14
14
  */
15
- import { promises as fs } from "fs";
16
15
  import { extname } from "path";
17
16
  import { fileURLToPath } from "url";
17
+ import { downloadAttachment, readAttachmentFile, } from "../utils/attachmentPolicy.js";
18
18
  import { guardEngineResult, promptTermsOf, runTranscribeStream, } from "./transcribeStream.js";
19
19
  import { ENGINE_SAMPLE_RATE } from "./streaming/endpointer.js";
20
20
  import { alignWordsToText } from "./correction/align.js";
@@ -74,6 +74,12 @@ function correctionLinesDisabled() {
74
74
  }
75
75
  /** Default timeout for fetching an `http(s)://` audio input. */
76
76
  const URL_FETCH_TIMEOUT_MS = 60_000;
77
+ /**
78
+ * Ceiling for a recording named by path or URL: 512 MiB, the same budget the
79
+ * transcription WebSocket allows a file-mode upload. `attachments.maxBytes`
80
+ * lowers it.
81
+ */
82
+ const TRANSCRIBE_AUDIO_MAX_BYTES = 512 * 1024 * 1024;
77
83
  /** Container names an extension or magic number can map to. */
78
84
  const KNOWN_FORMATS = [
79
85
  "mp3",
@@ -140,7 +146,7 @@ export function audioFormatFromBytes(buffer) {
140
146
  *
141
147
  * @throws STTError when the input is a frame stream, or cannot be read
142
148
  */
143
- export async function loadTranscribeAudio(input, timeoutMs) {
149
+ export async function loadTranscribeAudio(input, timeoutMs, attachmentPolicy) {
144
150
  if (Buffer.isBuffer(input)) {
145
151
  return { buffer: input };
146
152
  }
@@ -160,17 +166,20 @@ export async function loadTranscribeAudio(input, timeoutMs) {
160
166
  }
161
167
  const source = input.trim();
162
168
  if (/^https?:\/\//i.test(source)) {
163
- const controller = new AbortController();
164
169
  const limit = timeoutMs ?? URL_FETCH_TIMEOUT_MS;
165
- const timer = setTimeout(() => controller.abort(), limit);
166
170
  try {
167
- const response = await fetch(source, { signal: controller.signal });
168
- if (!response.ok) {
169
- throw STTError.audioLoadFailed(`fetching the audio URL returned HTTP ${response.status}`);
170
- }
171
- const buffer = Buffer.from(await response.arrayBuffer());
171
+ // The instance's attachment policy: HTTPS to a public host unless it
172
+ // says otherwise, the connection pinned to the vetted addresses,
173
+ // redirects re-validated per hop, and the body bounded while it
174
+ // streams rather than read whole with `arrayBuffer()`.
175
+ const downloaded = await downloadAttachment(source, {
176
+ policy: attachmentPolicy,
177
+ maxBytes: TRANSCRIBE_AUDIO_MAX_BYTES,
178
+ label: "audio URL",
179
+ timeoutMs: limit,
180
+ });
172
181
  return {
173
- buffer,
182
+ buffer: downloaded.buffer,
174
183
  formatHint: audioFormatFromName(new URL(source).pathname),
175
184
  };
176
185
  }
@@ -183,9 +192,6 @@ export async function loadTranscribeAudio(input, timeoutMs) {
183
192
  ? `fetching the audio URL timed out after ${limit}ms`
184
193
  : `fetching the audio URL failed: ${error instanceof Error ? error.message : String(error)}`, error instanceof Error ? error : undefined);
185
194
  }
186
- finally {
187
- clearTimeout(timer);
188
- }
189
195
  }
190
196
  let path;
191
197
  try {
@@ -195,16 +201,26 @@ export async function loadTranscribeAudio(input, timeoutMs) {
195
201
  throw STTError.audioLoadFailed("the file: URL is not a valid local path", error instanceof Error ? error : undefined);
196
202
  }
197
203
  try {
204
+ // Roots containment when configured, O_NOFOLLOW open of the real path,
205
+ // regular files only, bounded read.
206
+ const read = await readAttachmentFile(path, {
207
+ policy: attachmentPolicy,
208
+ maxBytes: TRANSCRIBE_AUDIO_MAX_BYTES,
209
+ label: "audio file",
210
+ });
198
211
  return {
199
- buffer: await fs.readFile(path),
212
+ buffer: read.buffer,
200
213
  formatHint: audioFormatFromName(path),
201
214
  };
202
215
  }
203
216
  catch (error) {
204
217
  const code = error?.code;
205
- throw STTError.audioLoadFailed(code === "ENOENT"
206
- ? "no file at the given path"
207
- : `reading the file failed${code ? ` (${code})` : ""}`, error instanceof Error ? error : undefined);
218
+ const detail = typeof code === "string" && code.startsWith("ATTACHMENT_")
219
+ ? error instanceof Error
220
+ ? error.message
221
+ : String(error)
222
+ : `reading the file failed${code ? ` (${String(code)})` : ""}`;
223
+ throw STTError.audioLoadFailed(code === "ENOENT" ? "no file at the given path" : detail, error instanceof Error ? error : undefined);
208
224
  }
209
225
  }
210
226
  // ============================================================================
@@ -575,7 +591,7 @@ function sumTimings(corrections, key) {
575
591
  async function transcribeLayers(options, deps, startedAt) {
576
592
  const steps = [];
577
593
  // 1. Audio, and its container when the caller did not name one.
578
- const { buffer, formatHint } = await loadTranscribeAudio(options.audio, options.timeoutMs);
594
+ const { buffer, formatHint } = await loadTranscribeAudio(options.audio, options.timeoutMs, deps.attachmentPolicy);
579
595
  if (buffer.length === 0) {
580
596
  throw STTError.audioEmpty(options.provider);
581
597
  }
@@ -12,8 +12,7 @@
12
12
  * @module voice/whistle/audio
13
13
  */
14
14
  import { spawn } from "node:child_process";
15
- import { randomUUID } from "node:crypto";
16
- import { rm, writeFile } from "node:fs/promises";
15
+ import { mkdtemp, rm, writeFile } from "node:fs/promises";
17
16
  import { tmpdir } from "node:os";
18
17
  import { join } from "node:path";
19
18
  import { STT_ERROR_CODES } from "../../types/index.js";
@@ -209,8 +208,11 @@ function ffmpegUnavailable(format, cause) {
209
208
  async function decodeWithFfmpeg(audio, format) {
210
209
  const binary = await resolveFfmpeg();
211
210
  // A file, not a pipe: MP4/M4A often keep their index at the end, which a pipe cannot seek to.
212
- const input = join(tmpdir(), `neurolink-whistle-${randomUUID()}.${format ?? "bin"}`);
213
- await writeFile(input, audio);
211
+ // Its own mkdtemp directory (0o700) and an owner-only file: the recording is
212
+ // the caller's, and the OS temp directory is shared.
213
+ const dir = await mkdtemp(join(tmpdir(), "neurolink-whistle-"));
214
+ const input = join(dir, `input.${format ?? "bin"}`);
215
+ await writeFile(input, audio, { mode: 0o600, flag: "wx" });
214
216
  try {
215
217
  const raw = await new Promise((resolvePromise, reject) => {
216
218
  const proc = spawn(binary, [
@@ -218,6 +220,10 @@ async function decodeWithFfmpeg(audio, format) {
218
220
  "-hide_banner",
219
221
  "-loglevel",
220
222
  "error",
223
+ // Pin the demuxer to local files: a container that references
224
+ // another URL or file cannot make ffmpeg open it.
225
+ "-protocol_whitelist",
226
+ "file,crypto,data",
221
227
  "-i",
222
228
  input,
223
229
  "-f",
@@ -262,6 +268,6 @@ async function decodeWithFfmpeg(audio, format) {
262
268
  throw STTError.invalidFormat(`${format ?? "unknown"} (ffmpeg could not decode it: ${error instanceof Error ? error.message : String(error)})`, [...WHISTLE_NATIVE_FORMATS, ...WHISTLE_FFMPEG_FORMATS], "whistle");
263
269
  }
264
270
  finally {
265
- await rm(input, { force: true }).catch(() => undefined);
271
+ await rm(dir, { recursive: true, force: true }).catch(() => undefined);
266
272
  }
267
273
  }
@@ -42,7 +42,7 @@ export declare const QUALITY_MAX_WORKFLOW: WorkflowConfig;
42
42
  *
43
43
  * Optimizes for speed with quality fallback:
44
44
  * 1. Fast tier: Single fast model (GPT-4o-mini)
45
- * 2. Balanced tier: If fast fails, use Gemini 2.0
45
+ * 2. Balanced tier: If fast fails, use Gemini 3.8 Flash
46
46
  * 3. Quality tier: If both fail, use GPT-4o
47
47
  */
48
48
  export declare const SPEED_FIRST_WORKFLOW: WorkflowConfig;
@@ -51,7 +51,7 @@ export declare const SPEED_FIRST_WORKFLOW: WorkflowConfig;
51
51
  *
52
52
  * Balances speed, cost, and quality:
53
53
  * 1. Standard tier (parallel): GPT-4o-mini + Gemini Flash
54
- * 2. Premium tier (parallel): GPT-4o + Claude 3.5 if standard uncertain
54
+ * 2. Premium tier (parallel): GPT-4o + Claude Sonnet 4.6 if standard uncertain
55
55
  */
56
56
  export declare const BALANCED_ADAPTIVE_WORKFLOW: WorkflowConfig;
57
57
  /**
@@ -67,7 +67,7 @@ export const QUALITY_MAX_WORKFLOW = {
67
67
  },
68
68
  {
69
69
  provider: AIProviderName.GOOGLE_AI,
70
- model: "gemini-2.0-flash",
70
+ model: "gemini-3.8-flash",
71
71
  label: "Gemini Flash",
72
72
  temperature: 0.7,
73
73
  timeout: 10000,
@@ -93,8 +93,8 @@ export const QUALITY_MAX_WORKFLOW = {
93
93
  },
94
94
  {
95
95
  provider: AIProviderName.ANTHROPIC,
96
- model: "claude-3-5-sonnet-20241022",
97
- label: "Claude 3.5 Sonnet",
96
+ model: "claude-sonnet-4-6",
97
+ label: "Claude Sonnet 4.6",
98
98
  temperature: 0.7,
99
99
  systemPrompt: "Think deeply and provide nuanced, well-reasoned responses.",
100
100
  timeout: 20000,
@@ -112,8 +112,8 @@ export const QUALITY_MAX_WORKFLOW = {
112
112
  models: [
113
113
  {
114
114
  provider: AIProviderName.ANTHROPIC,
115
- model: "claude-3-5-sonnet-20241022",
116
- label: "Claude 3.5 Sonnet Expert",
115
+ model: "claude-sonnet-4-6",
116
+ label: "Claude Sonnet 4.6 Expert",
117
117
  temperature: 0.6, // Lower temp for consistency
118
118
  systemPrompt: "You are an expert. Provide the highest quality, most accurate response possible. Be thorough, precise, and authoritative.",
119
119
  timeout: 30000,
@@ -161,7 +161,7 @@ export const QUALITY_MAX_WORKFLOW = {
161
161
  *
162
162
  * Optimizes for speed with quality fallback:
163
163
  * 1. Fast tier: Single fast model (GPT-4o-mini)
164
- * 2. Balanced tier: If fast fails, use Gemini 2.0
164
+ * 2. Balanced tier: If fast fails, use Gemini 3.8 Flash
165
165
  * 3. Quality tier: If both fail, use GPT-4o
166
166
  */
167
167
  export const SPEED_FIRST_WORKFLOW = {
@@ -198,7 +198,7 @@ export const SPEED_FIRST_WORKFLOW = {
198
198
  models: [
199
199
  {
200
200
  provider: AIProviderName.GOOGLE_AI,
201
- model: "gemini-2.0-flash",
201
+ model: "gemini-3.8-flash",
202
202
  temperature: 0.7,
203
203
  timeout: 10000,
204
204
  },
@@ -251,7 +251,7 @@ export const SPEED_FIRST_WORKFLOW = {
251
251
  *
252
252
  * Balances speed, cost, and quality:
253
253
  * 1. Standard tier (parallel): GPT-4o-mini + Gemini Flash
254
- * 2. Premium tier (parallel): GPT-4o + Claude 3.5 if standard uncertain
254
+ * 2. Premium tier (parallel): GPT-4o + Claude Sonnet 4.6 if standard uncertain
255
255
  */
256
256
  export const BALANCED_ADAPTIVE_WORKFLOW = {
257
257
  id: "balanced-adaptive",
@@ -278,7 +278,7 @@ export const BALANCED_ADAPTIVE_WORKFLOW = {
278
278
  },
279
279
  {
280
280
  provider: AIProviderName.GOOGLE_AI,
281
- model: "gemini-2.0-flash",
281
+ model: "gemini-3.8-flash",
282
282
  temperature: 0.7,
283
283
  },
284
284
  ],
@@ -299,7 +299,7 @@ export const BALANCED_ADAPTIVE_WORKFLOW = {
299
299
  },
300
300
  {
301
301
  provider: AIProviderName.ANTHROPIC,
302
- model: "claude-3-5-sonnet-20241022",
302
+ model: "claude-sonnet-4-6",
303
303
  temperature: 0.7,
304
304
  },
305
305
  ],
@@ -17,8 +17,8 @@ import type { WorkflowConfig } from "../../types/index.js";
17
17
  *
18
18
  * Uses 3 high-quality models in parallel:
19
19
  * - GPT-4o (OpenAI) - Strong reasoning
20
- * - Claude 3.5 Sonnet (Anthropic) - Thoughtful analysis
21
- * - Gemini 2.0 Flash (Google) - Fast and capable
20
+ * - Claude Sonnet 4.6 (Anthropic) - Thoughtful analysis
21
+ * - Gemini 3.8 Flash (Google) - Fast and capable
22
22
  *
23
23
  * Judge: GPT-4o evaluates on accuracy, clarity, and completeness
24
24
  *
@@ -63,7 +63,7 @@ export declare function createConsensus3WithPrompt(systemPrompt: string): Workfl
63
63
  *
64
64
  * Uses faster/cheaper models with same consensus approach:
65
65
  * - GPT-4o-mini
66
- * - Claude 3 Haiku
67
- * - Gemini 2.0 Flash
66
+ * - Claude Haiku 4.5
67
+ * - Gemini 3.8 Flash
68
68
  */
69
69
  export declare const CONSENSUS_3_FAST_WORKFLOW: WorkflowConfig;
@@ -18,8 +18,8 @@ import { WORKFLOW_CREATION_DATE } from "../config.js";
18
18
  *
19
19
  * Uses 3 high-quality models in parallel:
20
20
  * - GPT-4o (OpenAI) - Strong reasoning
21
- * - Claude 3.5 Sonnet (Anthropic) - Thoughtful analysis
22
- * - Gemini 2.0 Flash (Google) - Fast and capable
21
+ * - Claude Sonnet 4.6 (Anthropic) - Thoughtful analysis
22
+ * - Gemini 3.8 Flash (Google) - Fast and capable
23
23
  *
24
24
  * Judge: GPT-4o evaluates on accuracy, clarity, and completeness
25
25
  *
@@ -55,15 +55,15 @@ export const CONSENSUS_3_WORKFLOW = {
55
55
  },
56
56
  {
57
57
  provider: AIProviderName.ANTHROPIC,
58
- model: "claude-3-5-sonnet-20241022",
59
- label: "Claude 3.5 Sonnet",
58
+ model: "claude-sonnet-4-6",
59
+ label: "Claude Sonnet 4.6",
60
60
  weight: 1.0,
61
61
  temperature: 0.7,
62
62
  },
63
63
  {
64
64
  provider: AIProviderName.GOOGLE_AI,
65
- model: "gemini-2.0-flash",
66
- label: "Gemini 2.0 Flash",
65
+ model: "gemini-3.8-flash",
66
+ label: "Gemini 3.8 Flash",
67
67
  weight: 1.0,
68
68
  temperature: 0.7,
69
69
  },
@@ -130,8 +130,8 @@ export function createConsensus3WithPrompt(systemPrompt) {
130
130
  *
131
131
  * Uses faster/cheaper models with same consensus approach:
132
132
  * - GPT-4o-mini
133
- * - Claude 3 Haiku
134
- * - Gemini 2.0 Flash
133
+ * - Claude Haiku 4.5
134
+ * - Gemini 3.8 Flash
135
135
  */
136
136
  export const CONSENSUS_3_FAST_WORKFLOW = {
137
137
  id: "consensus-3-fast",
@@ -149,15 +149,15 @@ export const CONSENSUS_3_FAST_WORKFLOW = {
149
149
  },
150
150
  {
151
151
  provider: AIProviderName.ANTHROPIC,
152
- model: "claude-3-haiku-20240307",
153
- label: "Claude 3 Haiku",
152
+ model: "claude-haiku-4-5-20251001",
153
+ label: "Claude Haiku 4.5",
154
154
  weight: 1.0,
155
155
  temperature: 0.7,
156
156
  },
157
157
  {
158
158
  provider: AIProviderName.GOOGLE_AI,
159
- model: "gemini-2.0-flash",
160
- label: "Gemini 2.0 Flash",
159
+ model: "gemini-3.8-flash",
160
+ label: "Gemini 3.8 Flash",
161
161
  weight: 1.0,
162
162
  temperature: 0.7,
163
163
  },
@@ -17,8 +17,8 @@ import type { WorkflowConfig } from "../../types/index.js";
17
17
  *
18
18
  * Uses layer-based execution with sequential groups:
19
19
  * 1. Fast tier: GPT-4o-mini (try first)
20
- * 2. Mid tier: Gemini 2.0 Flash (if fast fails)
21
- * 3. Premium tier: GPT-4o or Claude 3.5 Sonnet (last resort)
20
+ * 2. Mid tier: Gemini 3.8 Flash (if fast fails)
21
+ * 3. Premium tier: GPT-4o or Claude Sonnet 4.6 (last resort)
22
22
  *
23
23
  * Each group runs sequentially - only proceeds if previous fails
24
24
  *
@@ -42,7 +42,7 @@ export declare const FAST_FALLBACK_WORKFLOW: WorkflowConfig;
42
42
  *
43
43
  * More aggressive fallback with parallel premium tier:
44
44
  * 1. Fast tier: GPT-4o-mini (sequential)
45
- * 2. Premium tier: GPT-4o + Claude 3.5 (parallel, both execute)
45
+ * 2. Premium tier: GPT-4o + Claude Sonnet 4.6 (parallel, both execute)
46
46
  *
47
47
  * Guarantees high quality if fast tier fails
48
48
  */
@@ -18,8 +18,8 @@ import { WORKFLOW_CREATION_DATE } from "../config.js";
18
18
  *
19
19
  * Uses layer-based execution with sequential groups:
20
20
  * 1. Fast tier: GPT-4o-mini (try first)
21
- * 2. Mid tier: Gemini 2.0 Flash (if fast fails)
22
- * 3. Premium tier: GPT-4o or Claude 3.5 Sonnet (last resort)
21
+ * 2. Mid tier: Gemini 3.8 Flash (if fast fails)
22
+ * 3. Premium tier: GPT-4o or Claude Sonnet 4.6 (last resort)
23
23
  *
24
24
  * Each group runs sequentially - only proceeds if previous fails
25
25
  *
@@ -76,8 +76,8 @@ export const FAST_FALLBACK_WORKFLOW = {
76
76
  models: [
77
77
  {
78
78
  provider: AIProviderName.GOOGLE_AI,
79
- model: "gemini-2.0-flash",
80
- label: "Gemini 2.0 Flash",
79
+ model: "gemini-3.8-flash",
80
+ label: "Gemini 3.8 Flash",
81
81
  temperature: 0.7,
82
82
  timeout: 15000, // 15 second timeout
83
83
  },
@@ -141,7 +141,7 @@ export const FAST_FALLBACK_WORKFLOW = {
141
141
  *
142
142
  * More aggressive fallback with parallel premium tier:
143
143
  * 1. Fast tier: GPT-4o-mini (sequential)
144
- * 2. Premium tier: GPT-4o + Claude 3.5 (parallel, both execute)
144
+ * 2. Premium tier: GPT-4o + Claude Sonnet 4.6 (parallel, both execute)
145
145
  *
146
146
  * Guarantees high quality if fast tier fails
147
147
  */
@@ -186,8 +186,8 @@ export const AGGRESSIVE_FALLBACK_WORKFLOW = {
186
186
  },
187
187
  {
188
188
  provider: AIProviderName.ANTHROPIC,
189
- model: "claude-3-5-sonnet-20241022",
190
- label: "Claude 3.5 Sonnet",
189
+ model: "claude-sonnet-4-6",
190
+ label: "Claude Sonnet 4.6",
191
191
  temperature: 0.7,
192
192
  },
193
193
  ],
@@ -18,14 +18,14 @@ import type { WorkflowConfig } from "../../types/index.js";
18
18
  * Uses 5 models across different providers:
19
19
  * - GPT-4o (OpenAI)
20
20
  * - GPT-4o-mini (OpenAI)
21
- * - Claude 3.5 Sonnet (Anthropic)
22
- * - Claude 3 Haiku (Anthropic)
23
- * - Gemini 2.0 Flash (Google)
21
+ * - Claude Sonnet 4.6 (Anthropic)
22
+ * - Claude Haiku 4.5 (Anthropic)
23
+ * - Gemini 3.8 Flash (Google)
24
24
  *
25
25
  * 3 independent judges vote:
26
26
  * - GPT-4o evaluates accuracy & clarity
27
- * - Claude 3.5 Sonnet evaluates reasoning & depth
28
- * - Gemini 2.0 Flash evaluates completeness & coherence
27
+ * - Claude Sonnet 4.6 evaluates reasoning & depth
28
+ * - Gemini 3.8 Flash evaluates completeness & coherence
29
29
  *
30
30
  * Scores are averaged across all judges for final selection
31
31
  *
@@ -48,8 +48,8 @@ export declare const MULTI_JUDGE_5_WORKFLOW: WorkflowConfig;
48
48
  * Multi-Judge-3 Workflow (Lighter Version)
49
49
  *
50
50
  * 3 models with 2 judges (more cost-effective):
51
- * - GPT-4o, Claude 3.5, Gemini 2.0
52
- * - Judged by GPT-4o and Claude 3.5
51
+ * - GPT-4o, Claude Sonnet 4.6, Gemini 3.8
52
+ * - Judged by GPT-4o and Claude Sonnet 4.6
53
53
  */
54
54
  export declare const MULTI_JUDGE_3_WORKFLOW: WorkflowConfig;
55
55
  /**