@juspay/neurolink 13.8.16 → 13.8.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (194) hide show
  1. package/CHANGELOG.md +2 -2
  2. package/README.md +1 -1
  3. package/dist/action/actionExecutor.d.ts +33 -3
  4. package/dist/action/actionExecutor.js +71 -15
  5. package/dist/action/actionInputs.js +4 -1
  6. package/dist/action/cliVersion.d.ts +25 -0
  7. package/dist/action/cliVersion.js +26 -0
  8. package/dist/action/index.d.ts +2 -1
  9. package/dist/action/index.js +2 -1
  10. package/dist/adapters/video/directorPipeline.js +46 -36
  11. package/dist/adapters/video/ffmpegAdapter.d.ts +17 -0
  12. package/dist/adapters/video/ffmpegAdapter.js +32 -5
  13. package/dist/adapters/video/frameExtractor.js +1 -1
  14. package/dist/adapters/video/videoMerger.js +5 -2
  15. package/dist/agent/arithmeticEvaluator.d.ts +32 -0
  16. package/dist/agent/arithmeticEvaluator.js +349 -0
  17. package/dist/agent/directTools.d.ts +6 -18
  18. package/dist/agent/directTools.js +17 -56
  19. package/dist/artifacts/artifactStore.d.ts +22 -1
  20. package/dist/artifacts/artifactStore.js +66 -6
  21. package/dist/auth/anthropicOAuth.d.ts +26 -1
  22. package/dist/auth/anthropicOAuth.js +81 -5
  23. package/dist/auth/secureFiles.d.ts +37 -0
  24. package/dist/auth/secureFiles.js +95 -0
  25. package/dist/auth/tokenStore.d.ts +43 -6
  26. package/dist/auth/tokenStore.js +260 -79
  27. package/dist/avatar/providers/DIDAvatar.js +49 -45
  28. package/dist/avatar/providers/ReplicateAvatar.js +33 -40
  29. package/dist/browser/neurolink.min.js +485 -558
  30. package/dist/cli/commands/auth.js +112 -70
  31. package/dist/cli/commands/proxy.d.ts +4 -0
  32. package/dist/cli/commands/proxy.js +128 -13
  33. package/dist/cli/commands/serve.d.ts +10 -1
  34. package/dist/cli/commands/serve.js +98 -17
  35. package/dist/cli/commands/server.js +17 -5
  36. package/dist/cli/index.js +146 -5
  37. package/dist/cli/loop/optionsSchema.d.ts +1 -1
  38. package/dist/cli/utils/envManager.js +17 -3
  39. package/dist/cli/utils/serverUtils.d.ts +39 -3
  40. package/dist/cli/utils/serverUtils.js +63 -6
  41. package/dist/core/baseProvider.js +7 -1
  42. package/dist/core/modules/MessageBuilder.js +5 -0
  43. package/dist/core/modules/ToolsManager.d.ts +46 -1
  44. package/dist/core/modules/ToolsManager.js +190 -43
  45. package/dist/files/fileReferenceRegistry.d.ts +2 -23
  46. package/dist/files/fileReferenceRegistry.js +55 -5
  47. package/dist/hitl/hitlManager.js +1 -0
  48. package/dist/index.d.ts +2 -1
  49. package/dist/index.js +6 -1
  50. package/dist/mcp/auth/tokenStorage.d.ts +7 -0
  51. package/dist/mcp/auth/tokenStorage.js +23 -4
  52. package/dist/mcp/externalServerManager.d.ts +13 -3
  53. package/dist/mcp/externalServerManager.js +154 -136
  54. package/dist/mcp/index.d.ts +2 -0
  55. package/dist/mcp/index.js +3 -0
  56. package/dist/mcp/mcpClientFactory.d.ts +14 -0
  57. package/dist/mcp/mcpClientFactory.js +70 -7
  58. package/dist/mcp/projectConfigTrust.d.ts +72 -0
  59. package/dist/mcp/projectConfigTrust.js +237 -0
  60. package/dist/mcp/transportUrlPolicy.d.ts +39 -0
  61. package/dist/mcp/transportUrlPolicy.js +172 -0
  62. package/dist/music/providers/BeatovenMusic.js +35 -46
  63. package/dist/music/providers/ReplicateMusic.js +33 -40
  64. package/dist/neurolink.d.ts +16 -1
  65. package/dist/neurolink.js +64 -11
  66. package/dist/processors/archive/ArchiveProcessor.js +15 -1
  67. package/dist/processors/archive/zipEntryIndex.d.ts +22 -0
  68. package/dist/processors/archive/zipEntryIndex.js +33 -0
  69. package/dist/processors/base/BaseFileProcessor.d.ts +9 -21
  70. package/dist/processors/base/BaseFileProcessor.js +40 -111
  71. package/dist/processors/data/YamlProcessor.d.ts +13 -1
  72. package/dist/processors/data/YamlProcessor.js +93 -2
  73. package/dist/processors/document/OpenDocumentProcessor.js +7 -2
  74. package/dist/processors/document/PptxProcessor.js +12 -7
  75. package/dist/processors/media/VideoProcessor.js +93 -15
  76. package/dist/proxy/inboundAuth.d.ts +89 -0
  77. package/dist/proxy/inboundAuth.js +219 -0
  78. package/dist/proxy/tokenRefresh.d.ts +6 -0
  79. package/dist/proxy/tokenRefresh.js +13 -1
  80. package/dist/proxy/updateState.js +4 -4
  81. package/dist/rag/document/loaders.d.ts +0 -6
  82. package/dist/rag/document/loaders.js +16 -10
  83. package/dist/rag/ragIntegration.d.ts +2 -2
  84. package/dist/rag/ragIntegration.js +35 -10
  85. package/dist/rag/retrieval/vectorQueryTool.d.ts +7 -0
  86. package/dist/rag/retrieval/vectorQueryTool.js +54 -17
  87. package/dist/server/abstract/baseServerAdapter.d.ts +43 -0
  88. package/dist/server/abstract/baseServerAdapter.js +119 -6
  89. package/dist/server/adapters/expressAdapter.d.ts +3 -0
  90. package/dist/server/adapters/expressAdapter.js +83 -8
  91. package/dist/server/adapters/fastifyAdapter.d.ts +3 -0
  92. package/dist/server/adapters/fastifyAdapter.js +165 -70
  93. package/dist/server/adapters/honoAdapter.d.ts +9 -1
  94. package/dist/server/adapters/honoAdapter.js +99 -21
  95. package/dist/server/adapters/koaAdapter.d.ts +1 -0
  96. package/dist/server/adapters/koaAdapter.js +153 -74
  97. package/dist/server/index.d.ts +2 -0
  98. package/dist/server/index.js +4 -0
  99. package/dist/server/middleware/auth.js +6 -2
  100. package/dist/server/middleware/cache.js +6 -1
  101. package/dist/server/middleware/rateLimit.js +29 -4
  102. package/dist/server/routes/agentRoutes.js +24 -11
  103. package/dist/server/routes/claudeProxyRoutes.d.ts +1 -0
  104. package/dist/server/routes/claudeProxyRoutes.js +273 -84
  105. package/dist/server/routes/codexProxyRoutes.d.ts +2 -2
  106. package/dist/server/routes/codexProxyRoutes.js +125 -40
  107. package/dist/server/routes/index.d.ts +4 -0
  108. package/dist/server/routes/index.js +35 -4
  109. package/dist/server/routes/mcpRoutes.d.ts +6 -2
  110. package/dist/server/routes/mcpRoutes.js +21 -7
  111. package/dist/server/routes/memoryRoutes.js +7 -6
  112. package/dist/server/routes/toolRoutes.d.ts +12 -2
  113. package/dist/server/routes/toolRoutes.js +33 -3
  114. package/dist/server/routes/transcribeRoutes.js +60 -13
  115. package/dist/server/utils/clientAddress.d.ts +23 -0
  116. package/dist/server/utils/clientAddress.js +65 -0
  117. package/dist/server/utils/clientError.d.ts +35 -0
  118. package/dist/server/utils/clientError.js +129 -0
  119. package/dist/server/utils/expressBody.d.ts +33 -0
  120. package/dist/server/utils/expressBody.js +353 -0
  121. package/dist/server/utils/pathMatch.d.ts +21 -0
  122. package/dist/server/utils/pathMatch.js +41 -0
  123. package/dist/server/utils/validation.d.ts +9 -0
  124. package/dist/server/utils/validation.js +9 -0
  125. package/dist/server/websocket/WebSocketHandler.js +22 -3
  126. package/dist/server/websocket/frameQueue.d.ts +19 -0
  127. package/dist/server/websocket/frameQueue.js +41 -3
  128. package/dist/server/websocket/transcribeWebSocket.d.ts +8 -1
  129. package/dist/server/websocket/transcribeWebSocket.js +223 -34
  130. package/dist/skills/skillStores.js +5 -0
  131. package/dist/tasks/store/fileTaskStore.d.ts +2 -0
  132. package/dist/tasks/store/fileTaskStore.js +11 -4
  133. package/dist/types/attachments.d.ts +117 -0
  134. package/dist/types/attachments.js +15 -0
  135. package/dist/types/auth.d.ts +6 -0
  136. package/dist/types/cli.d.ts +4 -0
  137. package/dist/types/config.d.ts +56 -0
  138. package/dist/types/file.d.ts +8 -0
  139. package/dist/types/hitl.d.ts +8 -0
  140. package/dist/types/index.d.ts +1 -0
  141. package/dist/types/index.js +1 -0
  142. package/dist/types/mcp.d.ts +98 -0
  143. package/dist/types/multimodal.js +0 -41
  144. package/dist/types/processor.d.ts +14 -1
  145. package/dist/types/proxy.d.ts +62 -3
  146. package/dist/types/rag.d.ts +9 -0
  147. package/dist/types/safeFetch.d.ts +40 -0
  148. package/dist/types/server.d.ts +255 -10
  149. package/dist/types/tools.d.ts +57 -0
  150. package/dist/types/transcribe.d.ts +6 -0
  151. package/dist/utils/attachmentPolicy.d.ts +89 -0
  152. package/dist/utils/attachmentPolicy.js +327 -0
  153. package/dist/utils/dotenvBootstrap.d.ts +48 -19
  154. package/dist/utils/dotenvBootstrap.js +172 -31
  155. package/dist/utils/fileDetector.js +115 -65
  156. package/dist/utils/json/coerce.js +11 -1
  157. package/dist/utils/logSanitize.d.ts +5 -0
  158. package/dist/utils/logSanitize.js +8 -0
  159. package/dist/utils/logger.js +73 -1
  160. package/dist/utils/messageBuilder.d.ts +2 -2
  161. package/dist/utils/messageBuilder.js +118 -105
  162. package/dist/utils/multimodalOptionsBuilder.d.ts +1 -0
  163. package/dist/utils/multimodalOptionsBuilder.js +2 -0
  164. package/dist/utils/redis.d.ts +12 -0
  165. package/dist/utils/redis.js +72 -0
  166. package/dist/utils/safeFetch.d.ts +22 -1
  167. package/dist/utils/safeFetch.js +132 -38
  168. package/dist/utils/safeStoreId.d.ts +19 -0
  169. package/dist/utils/safeStoreId.js +29 -0
  170. package/dist/utils/schemaConversion.d.ts +30 -3
  171. package/dist/utils/schemaConversion.js +457 -84
  172. package/dist/utils/sizeGuard.js +2 -2
  173. package/dist/utils/ssrfGuard.d.ts +3 -3
  174. package/dist/utils/ssrfGuard.js +71 -39
  175. package/dist/voice/eval/sttEvalStore.d.ts +26 -3
  176. package/dist/voice/eval/sttEvalStore.js +77 -12
  177. package/dist/voice/livekit/agentDefinition.d.ts +18 -0
  178. package/dist/voice/livekit/agentDefinition.js +22 -0
  179. package/dist/voice/livekit/realtimeMcpTools.d.ts +1 -1
  180. package/dist/voice/livekit/realtimeMcpTools.js +8 -7
  181. package/dist/voice/livekit/realtimeVoiceAgent.d.ts +2 -4
  182. package/dist/voice/livekit/realtimeVoiceAgent.js +18 -7
  183. package/dist/voice/livekit/voiceAgent.d.ts +2 -2
  184. package/dist/voice/livekit/voiceAgent.js +2 -1
  185. package/dist/voice/livekit/xaiSessionWatch.d.ts +0 -5
  186. package/dist/voice/livekit/xaiSessionWatch.js +30 -0
  187. package/dist/voice/providers/GeminiLive.js +21 -6
  188. package/dist/voice/providers/GoogleSTT.js +8 -5
  189. package/dist/voice/transcribe.d.ts +2 -2
  190. package/dist/voice/transcribe.js +34 -18
  191. package/dist/voice/whistle/audio.js +11 -5
  192. package/docs-site/static/search-index.json +75 -52
  193. package/package.json +171 -156
  194. package/scripts/observability/import-openobserve-dashboard.mjs +11 -3
@@ -26,6 +26,29 @@
26
26
  */
27
27
  import { lookup } from "node:dns/promises";
28
28
  import { isIP } from "node:net";
29
+ import { redactUrlForError } from "./logSanitize.js";
30
+ /**
31
+ * Parse `url` and enforce the scheme rule. HTTPS only unless the caller
32
+ * explicitly allowed `http:`; nothing else is ever a download URL.
33
+ */
34
+ function parseGuardedUrl(url, options) {
35
+ let parsed;
36
+ try {
37
+ parsed = new URL(url);
38
+ }
39
+ catch {
40
+ throw new Error(`Invalid URL: "${url}"`);
41
+ }
42
+ if (parsed.protocol === "https:") {
43
+ return parsed;
44
+ }
45
+ if (parsed.protocol === "http:" && options?.allowInsecure === true) {
46
+ return parsed;
47
+ }
48
+ throw new Error(options?.allowInsecure === true
49
+ ? `Only HTTP(S) URLs are permitted; got "${parsed.protocol}//" in "${url}"`
50
+ : `Only HTTPS URLs are permitted; got "${parsed.protocol}//" in "${url}"`);
51
+ }
29
52
  /**
30
53
  * Blocked IPv4 CIDRs.
31
54
  *
@@ -302,7 +325,7 @@ export async function assertSafeUrl(url) {
302
325
  return; // routable IP literal — safe
303
326
  }
304
327
  if (literalCheck !== "not-an-ip") {
305
- throw new Error(`URL "${url}" rejected: ${literalCheck}`);
328
+ throw new Error(`URL "${redactUrlForError(url)}" rejected: ${literalCheck}`);
306
329
  }
307
330
  // Hostname — resolve BOTH A and AAAA. Reject if either family yields a
308
331
  // blocked address (closes off the "publish AAAA public, A private" attack).
@@ -316,7 +339,7 @@ export async function assertSafeUrl(url) {
316
339
  * @throws {Error} when resolution fails entirely or any address is blocked
317
340
  * (all-must-pass — the caller may connect to any address in the set).
318
341
  */
319
- async function resolveAndValidateHost(url, host) {
342
+ async function resolveAndValidateHost(url, host, allowPrivate = false) {
320
343
  const [a, aaaa] = await Promise.allSettled([
321
344
  lookup(host, { family: 4, all: true }),
322
345
  lookup(host, { family: 6, all: true }),
@@ -350,11 +373,28 @@ async function resolveAndValidateHost(url, host) {
350
373
  ? aaaa.reason.message
351
374
  : String(aaaa.reason)
352
375
  : "ok";
353
- throw new Error(`URL "${url}" rejected: hostname ${host} could not be resolved (A: ${aErr}; AAAA: ${aaaaErr})`);
376
+ // Keep the resolver's own error on the chain (and its code, e.g.
377
+ // ENOTFOUND) so a caller's retry classification and cause-aware redaction
378
+ // see the real failure rather than this wrapper.
379
+ const resolverError = a.status === "rejected" && a.reason instanceof Error
380
+ ? a.reason
381
+ : aaaa.status === "rejected" && aaaa.reason instanceof Error
382
+ ? aaaa.reason
383
+ : undefined;
384
+ const unresolved = new Error(`URL "${redactUrlForError(url)}" rejected: hostname ${host} could not be resolved (A: ${aErr}; AAAA: ${aaaaErr})`, resolverError ? { cause: resolverError } : undefined);
385
+ const resolverCode = resolverError
386
+ ?.code;
387
+ if (resolverCode !== undefined) {
388
+ unresolved.code = resolverCode;
389
+ }
390
+ throw unresolved;
391
+ }
392
+ if (allowPrivate) {
393
+ return { v4: v4Addresses, v6: v6Addresses };
354
394
  }
355
395
  for (const addr of v4Addresses) {
356
396
  if (isBlockedIPv4(addr)) {
357
- throw new Error(`URL "${url}" rejected: hostname ${host} resolves to ${addr} (IPv4 in blocked range)`);
397
+ throw new Error(`URL "${redactUrlForError(url)}" rejected: hostname ${host} resolves to ${addr} (IPv4 in blocked range)`);
358
398
  }
359
399
  }
360
400
  for (const addr of v6Addresses) {
@@ -362,7 +402,7 @@ async function resolveAndValidateHost(url, host) {
362
402
  // addresses are caught.
363
403
  const reason = checkHostLiteral(addr.toLowerCase());
364
404
  if (reason && reason !== "not-an-ip") {
365
- throw new Error(`URL "${url}" rejected: hostname ${host} resolves to ${addr} (IPv6 ${reason})`);
405
+ throw new Error(`URL "${redactUrlForError(url)}" rejected: hostname ${host} resolves to ${addr} (IPv6 ${reason})`);
366
406
  }
367
407
  }
368
408
  return { v4: v4Addresses, v6: v6Addresses };
@@ -395,31 +435,28 @@ const INTERNAL_HOST_SUFFIXES = [
395
435
  *
396
436
  * Same throw semantics as {@link assertSafeUrl}.
397
437
  */
398
- export function assertSafeUrlForProxy(url) {
399
- let parsed;
400
- try {
401
- parsed = new URL(url);
402
- }
403
- catch {
404
- throw new Error(`Invalid URL: "${url}"`);
405
- }
406
- if (parsed.protocol !== "https:") {
407
- throw new Error(`Only HTTPS URLs are permitted; got "${parsed.protocol}//" in "${url}"`);
408
- }
438
+ export function assertSafeUrlForProxy(url, options) {
439
+ const parsed = parseGuardedUrl(url, options);
409
440
  const host = stripBrackets(parsed.hostname).toLowerCase();
410
441
  const literalCheck = checkHostLiteral(host);
411
442
  if (literalCheck === null) {
412
443
  return; // routable IP literal
413
444
  }
414
445
  if (literalCheck !== "not-an-ip") {
415
- throw new Error(`URL "${url}" rejected: ${literalCheck}`);
446
+ if (options?.allowPrivateNetwork === true) {
447
+ return; // a blocked range the caller explicitly opened
448
+ }
449
+ throw new Error(`URL "${redactUrlForError(url)}" rejected: ${literalCheck}`);
450
+ }
451
+ if (options?.allowPrivateNetwork === true) {
452
+ return; // internal names are exactly what the caller asked to reach
416
453
  }
417
454
  const name = host.endsWith(".") ? host.slice(0, -1) : host;
418
455
  if (!name.includes(".")) {
419
- throw new Error(`URL "${url}" rejected: "${name}" is a single-label host name`);
456
+ throw new Error(`URL "${redactUrlForError(url)}" rejected: "${name}" is a single-label host name`);
420
457
  }
421
458
  if (INTERNAL_HOST_SUFFIXES.some((suffix) => name === suffix || name.endsWith(`.${suffix}`))) {
422
- throw new Error(`URL "${url}" rejected: "${name}" is an internal host name`);
459
+ throw new Error(`URL "${redactUrlForError(url)}" rejected: "${name}" is an internal host name`);
423
460
  }
424
461
  }
425
462
  /**
@@ -439,31 +476,26 @@ export function assertSafeUrlForProxy(url) {
439
476
  * This is the canonical entry point for binary downloads where DNS-rebinding
440
477
  * pinning matters — see `safeFetch.ts`.
441
478
  */
442
- export async function validateAndResolveUrl(url) {
443
- let parsed;
444
- try {
445
- parsed = new URL(url);
446
- }
447
- catch {
448
- throw new Error(`Invalid URL: "${url}"`);
449
- }
450
- if (parsed.protocol !== "https:") {
451
- throw new Error(`Only HTTPS URLs are permitted; got "${parsed.protocol}//" in "${url}"`);
452
- }
479
+ export async function validateAndResolveUrl(url, options) {
480
+ const parsed = parseGuardedUrl(url, options);
453
481
  const host = stripBrackets(parsed.hostname).toLowerCase();
482
+ // Only an explicit opt-in relaxes the range check; the default stays the
483
+ // server path's deny list. The resolve-and-pin step runs either way, so an
484
+ // opened-up caller still gets rebinding protection.
485
+ const allowPrivate = options?.allowPrivateNetwork === true;
454
486
  // IP literal — normalise + check, return canonical form
455
487
  const v4 = normalizeIPv4(host);
456
488
  if (v4) {
457
- if (isBlockedIPv4(v4)) {
458
- throw new Error(`URL "${url}" rejected: IPv4 ${host} → ${v4} is in a blocked range`);
489
+ if (!allowPrivate && isBlockedIPv4(v4)) {
490
+ throw new Error(`URL "${redactUrlForError(url)}" rejected: IPv4 ${host} → ${v4} is in a blocked range`);
459
491
  }
460
492
  return { url, ip: v4, family: 4, addresses: [{ ip: v4, family: 4 }] };
461
493
  }
462
494
  if (host.includes(":")) {
463
495
  const v4FromMapped = extractIPv4FromMapped(host);
464
496
  if (v4FromMapped) {
465
- if (isBlockedIPv4(v4FromMapped)) {
466
- throw new Error(`URL "${url}" rejected: IPv4-mapped IPv6 ${host} → ${v4FromMapped} is in a blocked range`);
497
+ if (!allowPrivate && isBlockedIPv4(v4FromMapped)) {
498
+ throw new Error(`URL "${redactUrlForError(url)}" rejected: IPv4-mapped IPv6 ${host} → ${v4FromMapped} is in a blocked range`);
467
499
  }
468
500
  return {
469
501
  url,
@@ -474,10 +506,10 @@ export async function validateAndResolveUrl(url) {
474
506
  }
475
507
  const expanded = expandIPv6(host);
476
508
  if (!expanded) {
477
- throw new Error(`URL "${url}" rejected: IPv6 ${host} could not be parsed`);
509
+ throw new Error(`URL "${redactUrlForError(url)}" rejected: IPv6 ${host} could not be parsed`);
478
510
  }
479
- if (isBlockedIPv6(expanded)) {
480
- throw new Error(`URL "${url}" rejected: IPv6 ${host} is in a blocked range`);
511
+ if (!allowPrivate && isBlockedIPv6(expanded)) {
512
+ throw new Error(`URL "${redactUrlForError(url)}" rejected: IPv6 ${host} is in a blocked range`);
481
513
  }
482
514
  return { url, ip: host, family: 6, addresses: [{ ip: host, family: 6 }] };
483
515
  }
@@ -485,7 +517,7 @@ export async function validateAndResolveUrl(url) {
485
517
  // set. Validating the same answer we hand to the connect layer also
486
518
  // removes the rebinding window the previous separate `lookup()` left
487
519
  // between validation and pinning.
488
- const { v4: v4Addresses, v6: v6Addresses } = await resolveAndValidateHost(url, host);
520
+ const { v4: v4Addresses, v6: v6Addresses } = await resolveAndValidateHost(url, host, allowPrivate);
489
521
  const addresses = [
490
522
  ...v4Addresses.map((ip) => ({ ip, family: 4 })),
491
523
  ...v6Addresses.map((ip) => ({ ip, family: 6 })),
@@ -494,7 +526,7 @@ export async function validateAndResolveUrl(url) {
494
526
  if (!first) {
495
527
  // Both lookups "succeeded" but returned zero addresses — treat exactly
496
528
  // like a resolution failure rather than pinning nothing.
497
- throw new Error(`URL "${url}" rejected: hostname ${host} resolved to an empty address set`);
529
+ throw new Error(`URL "${redactUrlForError(url)}" rejected: hostname ${host} resolved to an empty address set`);
498
530
  }
499
531
  return { url, ip: first.ip, family: first.family, addresses };
500
532
  }
@@ -3,21 +3,44 @@ export declare class STTEvalStore {
3
3
  private readonly _dir;
4
4
  private readonly _maxCalls;
5
5
  private readonly _maxSegments;
6
+ private readonly _maxBytes;
7
+ private _totalBytes;
6
8
  private readonly _calls;
7
9
  /** Per call: the tail of its accept queue, so batches apply and append in arrival order. */
8
10
  private readonly _queues;
9
11
  private _ready;
10
12
  private readonly _langfuse;
11
13
  constructor(options?: ServerSTTEvalOptions);
12
- accept(batch: ServerSTTEvalBatchRequest): Promise<ServerSTTEvalBatchResponse>;
13
- list(): Promise<ServerSTTEvalCallSummary[]>;
14
- report(callId: string, includeSegments?: boolean): Promise<ServerSTTEvalReport | undefined>;
14
+ /** Bytes of segments held in memory right now, across every call. */
15
+ get heldBytes(): number;
16
+ /**
17
+ * Take a batch. `owner` is the authenticated caller, when the server
18
+ * identified one: a call created with an owner accepts batches only from
19
+ * that owner, and `undefined` is returned for anyone else's.
20
+ */
21
+ accept(batch: ServerSTTEvalBatchRequest, owner?: string): Promise<ServerSTTEvalBatchResponse | undefined>;
22
+ /** Calls visible to `owner`: theirs and the unowned ones (see `_visibleTo`). */
23
+ list(owner?: string): Promise<ServerSTTEvalCallSummary[]>;
24
+ report(callId: string, includeSegments?: boolean, owner?: string): Promise<ServerSTTEvalReport | undefined>;
25
+ /**
26
+ * A call an identified caller created is theirs alone. A call with no owner
27
+ * (made under no authentication or a shared key) is visible to everyone, as
28
+ * every call was before owners existed.
29
+ */
30
+ private _visibleTo;
15
31
  private _reportOf;
16
32
  /** Runs `task` after every earlier task for the same call has settled. */
17
33
  private _serial;
18
34
  private _create;
19
35
  private _apply;
20
36
  private _callFor;
37
+ private _charge;
38
+ /**
39
+ * Drop the least recently updated calls while there are more than
40
+ * `maxCalls`, or while the held segments exceed `maxBytes`; a dropped call
41
+ * is still on disk when there is a directory. The newest call is never
42
+ * dropped, so a single call larger than the budget stays.
43
+ */
21
44
  private _evict;
22
45
  private _fileFor;
23
46
  /** Replays the directory once; every caller awaits the same load. */
@@ -21,13 +21,21 @@ import { phoneticKey } from "../codeMix.js";
21
21
  import { STTEvalLangfuseExporter, sttEvalLangfuseConfigFromEnv, } from "./sttEvalLangfuse.js";
22
22
  const DEFAULT_MAX_CALLS = 1000;
23
23
  const DEFAULT_MAX_SEGMENTS = 50_000;
24
+ /** Bytes of segments held in memory across every call. */
25
+ const DEFAULT_MAX_BYTES = 64 * 1024 * 1024;
24
26
  /** Words per side above which a speaker is aligned in time windows instead of whole. */
25
27
  const FULL_ALIGN_MAX_WORDS = 4000;
26
28
  const WINDOW_S = 300;
29
+ /** Roughly what a segment costs in memory: its JSON, which is what is kept. */
30
+ function segmentBytes(segment) {
31
+ return JSON.stringify(segment).length;
32
+ }
27
33
  export class STTEvalStore {
28
34
  _dir;
29
35
  _maxCalls;
30
36
  _maxSegments;
37
+ _maxBytes;
38
+ _totalBytes = 0;
31
39
  _calls = new Map();
32
40
  /** Per call: the tail of its accept queue, so batches apply and append in arrival order. */
33
41
  _queues = new Map();
@@ -38,19 +46,36 @@ export class STTEvalStore {
38
46
  this._dir = dir ? dir : undefined;
39
47
  this._maxCalls = Math.max(1, options.maxCalls ?? DEFAULT_MAX_CALLS);
40
48
  this._maxSegments = Math.max(1, options.maxSegmentsPerCall ?? DEFAULT_MAX_SEGMENTS);
49
+ this._maxBytes = Math.max(1, options.maxBytes ?? DEFAULT_MAX_BYTES);
41
50
  const langfuse = sttEvalLangfuseConfigFromEnv();
42
51
  this._langfuse = langfuse
43
52
  ? new STTEvalLangfuseExporter(langfuse)
44
53
  : undefined;
45
54
  }
46
- async accept(batch) {
55
+ /** Bytes of segments held in memory right now, across every call. */
56
+ get heldBytes() {
57
+ return this._totalBytes;
58
+ }
59
+ /**
60
+ * Take a batch. `owner` is the authenticated caller, when the server
61
+ * identified one: a call created with an owner accepts batches only from
62
+ * that owner, and `undefined` is returned for anyone else's.
63
+ */
64
+ async accept(batch, owner) {
47
65
  await this._load();
48
66
  return this._serial(batch.call_id, async () => {
49
- const call = (await this._callFor(batch.call_id)) ?? this._create(batch);
67
+ const held = await this._callFor(batch.call_id);
68
+ if (held && !this._visibleTo(held, owner)) {
69
+ return undefined;
70
+ }
71
+ const call = held ?? this._create(batch, owner);
50
72
  const counts = this._apply(call, batch);
51
73
  if (this._dir) {
52
74
  try {
53
- await appendFile(this._fileFor(batch.call_id), `${JSON.stringify(batch)}\n`, "utf8");
75
+ const stored = owner
76
+ ? { ...batch, owner }
77
+ : batch;
78
+ await appendFile(this._fileFor(batch.call_id), `${JSON.stringify(stored)}\n`, "utf8");
54
79
  }
55
80
  catch (error) {
56
81
  logger.warn("[stt-eval] could not persist batch", {
@@ -73,16 +98,28 @@ export class STTEvalStore {
73
98
  };
74
99
  });
75
100
  }
76
- async list() {
101
+ /** Calls visible to `owner`: theirs and the unowned ones (see `_visibleTo`). */
102
+ async list(owner) {
77
103
  await this._load();
78
104
  return [...this._calls.values()]
105
+ .filter((call) => this._visibleTo(call, owner))
79
106
  .sort((a, b) => b.updatedAt - a.updatedAt)
80
107
  .map(summarize);
81
108
  }
82
- async report(callId, includeSegments = false) {
109
+ async report(callId, includeSegments = false, owner) {
83
110
  await this._load();
84
111
  const call = await this._callFor(callId);
85
- return call ? this._reportOf(call, includeSegments) : undefined;
112
+ return call && this._visibleTo(call, owner)
113
+ ? this._reportOf(call, includeSegments)
114
+ : undefined;
115
+ }
116
+ /**
117
+ * A call an identified caller created is theirs alone. A call with no owner
118
+ * (made under no authentication or a shared key) is visible to everyone, as
119
+ * every call was before owners existed.
120
+ */
121
+ _visibleTo(call, owner) {
122
+ return call.owner === undefined || call.owner === owner;
86
123
  }
87
124
  _reportOf(call, includeSegments) {
88
125
  const ordered = [...call.segments.values()].sort((a, b) => timeOf(a) - timeOf(b));
@@ -128,7 +165,7 @@ export class STTEvalStore {
128
165
  });
129
166
  return run;
130
167
  }
131
- _create(batch) {
168
+ _create(batch, owner) {
132
169
  const call = {
133
170
  callId: batch.call_id,
134
171
  room: batch.room ?? null,
@@ -137,6 +174,8 @@ export class STTEvalStore {
137
174
  batches: new Set(),
138
175
  segments: new Map(),
139
176
  updatedAt: Date.now(),
177
+ bytes: 0,
178
+ ...(owner ? { owner } : {}),
140
179
  };
141
180
  this._calls.set(batch.call_id, call);
142
181
  return call;
@@ -146,8 +185,10 @@ export class STTEvalStore {
146
185
  let updated = 0;
147
186
  let ignored = 0;
148
187
  for (const segment of batch.segments) {
149
- if (call.segments.has(segment.id)) {
188
+ const previous = call.segments.get(segment.id);
189
+ if (previous) {
150
190
  updated++;
191
+ this._charge(call, -segmentBytes(previous));
151
192
  }
152
193
  else if (call.segments.size >= this._maxSegments) {
153
194
  ignored++;
@@ -157,6 +198,7 @@ export class STTEvalStore {
157
198
  inserted++;
158
199
  }
159
200
  call.segments.set(segment.id, { ...segment });
201
+ this._charge(call, segmentBytes(segment));
160
202
  }
161
203
  call.batches.add(batch.batch);
162
204
  call.final = call.final || batch.final === true;
@@ -173,17 +215,38 @@ export class STTEvalStore {
173
215
  if (held || !this._dir) {
174
216
  return held;
175
217
  }
176
- // Evicted from memory but still on disk.
218
+ // Evicted from memory but still on disk. Replaying re-registers the call
219
+ // and its bytes; the next eviction pass trims the budget again.
177
220
  const text = await readFile(this._fileFor(callId), "utf8").catch(() => undefined);
178
- return text === undefined ? undefined : this._replay(text);
221
+ if (text === undefined) {
222
+ return undefined;
223
+ }
224
+ const call = this._replay(text);
225
+ this._evict();
226
+ return call;
179
227
  }
228
+ _charge(call, delta) {
229
+ call.bytes += delta;
230
+ this._totalBytes = Math.max(0, this._totalBytes + delta);
231
+ }
232
+ /**
233
+ * Drop the least recently updated calls while there are more than
234
+ * `maxCalls`, or while the held segments exceed `maxBytes`; a dropped call
235
+ * is still on disk when there is a directory. The newest call is never
236
+ * dropped, so a single call larger than the budget stays.
237
+ */
180
238
  _evict() {
181
- while (this._calls.size > this._maxCalls) {
239
+ while (this._calls.size > 1 &&
240
+ (this._calls.size > this._maxCalls || this._totalBytes > this._maxBytes)) {
182
241
  const oldest = this._calls.keys().next().value;
183
242
  if (oldest === undefined) {
184
243
  return;
185
244
  }
245
+ const call = this._calls.get(oldest);
186
246
  this._calls.delete(oldest);
247
+ if (call) {
248
+ this._totalBytes = Math.max(0, this._totalBytes - call.bytes);
249
+ }
187
250
  }
188
251
  }
189
252
  _fileFor(callId) {
@@ -210,8 +273,10 @@ export class STTEvalStore {
210
273
  // Replay order is filesystem order; keep the most recent calls.
211
274
  const recent = [...this._calls.values()].sort((a, b) => a.updatedAt - b.updatedAt);
212
275
  this._calls.clear();
276
+ this._totalBytes = 0;
213
277
  for (const call of recent) {
214
278
  this._calls.set(call.callId, call);
279
+ this._totalBytes += call.bytes;
215
280
  }
216
281
  this._evict();
217
282
  }
@@ -231,7 +296,7 @@ export class STTEvalStore {
231
296
  }
232
297
  try {
233
298
  const batch = JSON.parse(line);
234
- call = call ?? this._create(batch);
299
+ call = call ?? this._create(batch, batch.owner);
235
300
  this._apply(call, batch);
236
301
  for (const segment of batch.segments) {
237
302
  lastSeen = Math.max(lastSeen, segment.received_at * 1000);
@@ -0,0 +1,18 @@
1
+ /**
2
+ * Marks an agent object so the `@livekit/agents` job process accepts it.
3
+ *
4
+ * Since 1.9 the framework's `isAgent` gate (what the job process runs on the
5
+ * entry file's default export) no longer looks for an `entry` function: it
6
+ * checks for the registry symbol `Symbol.for("livekit.agents.AgentDefinition")`
7
+ * that `defineAgent` stamps, and an unmarked object fails with "Unable to load
8
+ * agent: Missing or invalid default export". `defineVoiceAgent` and
9
+ * `defineRealtimeVoiceAgent` return their object through this helper so an
10
+ * entry file's `export default defineVoiceAgent({...})` keeps working.
11
+ *
12
+ * The symbol is read from the global registry rather than imported, because
13
+ * `@livekit/agents` is an optional dependency this module only ever imports
14
+ * dynamically. The property is defined exactly as `defineAgent` defines it, so a
15
+ * consumer that additionally wraps the result in `defineAgent` redefines an
16
+ * identical property, which is permitted.
17
+ */
18
+ export declare function markAgentDefinition<T extends object>(agent: T): T;
@@ -0,0 +1,22 @@
1
+ /**
2
+ * Marks an agent object so the `@livekit/agents` job process accepts it.
3
+ *
4
+ * Since 1.9 the framework's `isAgent` gate (what the job process runs on the
5
+ * entry file's default export) no longer looks for an `entry` function: it
6
+ * checks for the registry symbol `Symbol.for("livekit.agents.AgentDefinition")`
7
+ * that `defineAgent` stamps, and an unmarked object fails with "Unable to load
8
+ * agent: Missing or invalid default export". `defineVoiceAgent` and
9
+ * `defineRealtimeVoiceAgent` return their object through this helper so an
10
+ * entry file's `export default defineVoiceAgent({...})` keeps working.
11
+ *
12
+ * The symbol is read from the global registry rather than imported, because
13
+ * `@livekit/agents` is an optional dependency this module only ever imports
14
+ * dynamically. The property is defined exactly as `defineAgent` defines it, so a
15
+ * consumer that additionally wraps the result in `defineAgent` redefines an
16
+ * identical property, which is permitted.
17
+ */
18
+ const AGENT_DEFINITION_SYMBOL = Symbol.for("livekit.agents.AgentDefinition");
19
+ export function markAgentDefinition(agent) {
20
+ Object.defineProperty(agent, AGENT_DEFINITION_SYMBOL, { value: true });
21
+ return agent;
22
+ }
@@ -20,7 +20,7 @@ import type { BuildRealtimeMcpToolsParams } from "../../types/index.js";
20
20
  export declare function mcpResultToText(result: unknown): string;
21
21
  /**
22
22
  * Connect to the MCP server as the calling user and bridge every tool into a
23
- * LiveKit `ToolContext`. Returns the tool map + the client (close on shutdown).
23
+ * LiveKit `ToolContext`. Returns that context + the client (close on shutdown).
24
24
  * The server already scopes the tool list by `x-context`, so no client-side
25
25
  * filtering is needed.
26
26
  */
@@ -73,7 +73,7 @@ function buildHitlDeclineMessage(toolName) {
73
73
  * First registration wins, so a unique full name is never shadowed by a generic
74
74
  * short name. Returns the aliases actually registered.
75
75
  */
76
- function registerToolAliases(toolContext, mcpToolName, handler) {
76
+ function registerToolAliases(registry, mcpToolName, handler) {
77
77
  const safeFullName = mcpToolName.replace(/[^a-zA-Z0-9_]/g, "_");
78
78
  const shortName = safeFullName.includes("_")
79
79
  ? safeFullName.slice(safeFullName.lastIndexOf("_") + 1)
@@ -81,17 +81,17 @@ function registerToolAliases(toolContext, mcpToolName, handler) {
81
81
  const candidates = [safeFullName, shortName].filter((name, index, all) => name.length > 0 && all.indexOf(name) === index);
82
82
  const registered = [];
83
83
  for (const alias of candidates) {
84
- if (alias in toolContext) {
84
+ if (alias in registry) {
85
85
  continue; // taken by another tool — don't clobber
86
86
  }
87
- toolContext[alias] = handler;
87
+ registry[alias] = handler;
88
88
  registered.push(alias);
89
89
  }
90
90
  return registered;
91
91
  }
92
92
  /**
93
93
  * Connect to the MCP server as the calling user and bridge every tool into a
94
- * LiveKit `ToolContext`. Returns the tool map + the client (close on shutdown).
94
+ * LiveKit `ToolContext`. Returns that context + the client (close on shutdown).
95
95
  * The server already scopes the tool list by `x-context`, so no client-side
96
96
  * filtering is needed.
97
97
  */
@@ -126,7 +126,7 @@ export async function buildRealtimeMcpTools(params) {
126
126
  try {
127
127
  const { tools: mcpTools } = await client.listTools();
128
128
  logger.info("realtime.mcp.toolsListed", { count: mcpTools.length });
129
- const toolContext = {};
129
+ const registry = {};
130
130
  const skipped = [];
131
131
  for (const mcpTool of mcpTools) {
132
132
  const parameters = sanitizeToolParameters(mcpTool.inputSchema);
@@ -209,15 +209,16 @@ export async function buildRealtimeMcpTools(params) {
209
209
  }
210
210
  },
211
211
  });
212
- const aliases = registerToolAliases(toolContext, mcpTool.name, handler);
212
+ const aliases = registerToolAliases(registry, mcpTool.name, handler);
213
213
  logger.info("realtime.mcp.toolRegistered", {
214
214
  tool: mcpTool.name,
215
215
  aliases,
216
216
  hitl: requiresConfirmation,
217
217
  });
218
218
  }
219
+ const toolContext = new llm.ToolContext(registry);
219
220
  logger.info("realtime.mcp.connected", {
220
- registered: Object.keys(toolContext).length,
221
+ registered: Object.keys(registry).length,
221
222
  skipped: skipped.length,
222
223
  skippedTools: skipped,
223
224
  });
@@ -15,7 +15,7 @@
15
15
  *
16
16
  * See docs/features/livekit-voice-agent.md.
17
17
  */
18
- import type { JobContext } from "@livekit/agents";
18
+ import type { AgentDefinition } from "@livekit/agents";
19
19
  import type { RealtimeVoiceAgentConfig } from "../../types/index.js";
20
20
  /**
21
21
  * Define a realtime (speech-to-speech) LiveKit voice agent: Gemini Live by
@@ -25,6 +25,4 @@ import type { RealtimeVoiceAgentConfig } from "../../types/index.js";
25
25
  * with `startRealtimeVoiceAgentWorker`. With no `config` everything is resolved
26
26
  * from the environment (see `resolveRealtimeVoiceConfig`).
27
27
  */
28
- export declare function defineRealtimeVoiceAgent(config?: RealtimeVoiceAgentConfig): {
29
- entry: (ctx: JobContext) => Promise<void>;
30
- };
28
+ export declare function defineRealtimeVoiceAgent(config?: RealtimeVoiceAgentConfig): AgentDefinition;
@@ -15,8 +15,10 @@
15
15
  *
16
16
  * See docs/features/livekit-voice-agent.md.
17
17
  */
18
+ import { ReadableStream } from "node:stream/web";
18
19
  import { z } from "zod";
19
20
  import { logger } from "../../utils/logger.js";
21
+ import { markAgentDefinition } from "./agentDefinition.js";
20
22
  import { resolveRealtimeVoiceConfig } from "./config.js";
21
23
  import { ensureVertexAdc, clearGeminiApiKeyEnv } from "./vertexAuth.js";
22
24
  import { createRealtimeLlm, requireXaiApiKey } from "./realtimeModel.js";
@@ -232,7 +234,7 @@ export function defineRealtimeVoiceAgent(config = {}) {
232
234
  : {}),
233
235
  });
234
236
  await installRealtimeJobLifecycle(ctx, cfg);
235
- const { voice } = await import("@livekit/agents");
237
+ const { voice, toStream } = await import("@livekit/agents");
236
238
  const realtimeLlm = await createRealtimeLlm(cfg);
237
239
  const bridge = await attachRealtimeEventBridge({
238
240
  room: ctx.room,
@@ -271,7 +273,7 @@ export function defineRealtimeVoiceAgent(config = {}) {
271
273
  agentTools = toolset.tools;
272
274
  logger.info("realtime.mcp.enabled", {
273
275
  mcpUrl: cfg.mcpUrl,
274
- toolCount: Object.keys(toolset.tools).length,
276
+ toolCount: Object.keys(toolset.tools.functionTools).length,
275
277
  hasAuthToken: authToken !== "",
276
278
  });
277
279
  ctx.addShutdownCallback(async () => {
@@ -311,7 +313,12 @@ export function defineRealtimeVoiceAgent(config = {}) {
311
313
  ...(seededChatCtx ? { chatCtx: seededChatCtx } : {}),
312
314
  });
313
315
  agent.transcriptionNode = async (textStream, _modelSettings) => {
314
- const [forUi, forDownstream] = textStream.tee();
316
+ // 1.9 hands the node a ReadableStream or a bare AsyncIterable; only a
317
+ // stream has tee(), so mirror the framework default and wrap the latter.
318
+ const source = textStream instanceof ReadableStream
319
+ ? textStream
320
+ : toStream(textStream);
321
+ const [forUi, forDownstream] = source.tee();
315
322
  void (async () => {
316
323
  const reader = forUi.getReader();
317
324
  let chunkCount = 0;
@@ -363,18 +370,22 @@ export function defineRealtimeVoiceAgent(config = {}) {
363
370
  });
364
371
  // --- Session events → browser ------------------------------------------
365
372
  session.on(voice.AgentSessionEventTypes.UserInputTranscribed, (ev) => {
373
+ // agents-plugin-xai >= 1.9 emits only the *completed* transcription and
374
+ // flags it `isFinal: false`; without this every xAI user turn would be
375
+ // published as interim and the recorder would never see a final line.
376
+ const final = ev.isFinal || cfg.backend === "xai";
366
377
  logger.debug("realtime.session.userTranscript", {
367
378
  ms: sinceConnect(),
368
- final: ev.isFinal,
379
+ final,
369
380
  transcript: ev.transcript,
370
381
  });
371
382
  bridge.publishEvent("user-text", {
372
383
  text: ev.transcript,
373
- final: ev.isFinal,
384
+ final,
374
385
  });
375
386
  recorder?.userTranscript({
376
387
  text: ev.transcript,
377
- final: ev.isFinal,
388
+ final,
378
389
  });
379
390
  });
380
391
  session.on(voice.AgentSessionEventTypes.AgentStateChanged, (ev) => {
@@ -416,5 +427,5 @@ export function defineRealtimeVoiceAgent(config = {}) {
416
427
  session.generateReply({ instructions: cfg.greeting });
417
428
  }
418
429
  }
419
- return { entry };
430
+ return markAgentDefinition({ entry });
420
431
  }
@@ -14,7 +14,7 @@
14
14
  *
15
15
  * See docs/features/livekit-voice-agent.md.
16
16
  */
17
- import type { Agent as JobAgent } from "@livekit/agents";
17
+ import type { AgentDefinition } from "@livekit/agents";
18
18
  import type { LiveKitVoiceAgentConfig } from "../../types/index.js";
19
19
  /**
20
20
  * Define a LiveKit voice agent backed by NeuroLink.
@@ -29,4 +29,4 @@ import type { LiveKitVoiceAgentConfig } from "../../types/index.js";
29
29
  * });
30
30
  * ```
31
31
  */
32
- export declare function defineVoiceAgent(config: LiveKitVoiceAgentConfig): JobAgent;
32
+ export declare function defineVoiceAgent(config: LiveKitVoiceAgentConfig): AgentDefinition;