@juspay/neurolink 13.8.17 → 13.8.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -2
- package/README.md +1 -1
- package/dist/action/actionExecutor.d.ts +33 -3
- package/dist/action/actionExecutor.js +71 -15
- package/dist/action/actionInputs.js +4 -1
- package/dist/action/cliVersion.d.ts +25 -0
- package/dist/action/cliVersion.js +26 -0
- package/dist/action/index.d.ts +2 -1
- package/dist/action/index.js +2 -1
- package/dist/adapters/video/directorPipeline.js +46 -36
- package/dist/adapters/video/ffmpegAdapter.d.ts +17 -0
- package/dist/adapters/video/ffmpegAdapter.js +32 -5
- package/dist/adapters/video/frameExtractor.js +1 -1
- package/dist/adapters/video/videoMerger.js +5 -2
- package/dist/agent/arithmeticEvaluator.d.ts +32 -0
- package/dist/agent/arithmeticEvaluator.js +349 -0
- package/dist/agent/directTools.d.ts +6 -18
- package/dist/agent/directTools.js +17 -56
- package/dist/artifacts/artifactStore.d.ts +22 -1
- package/dist/artifacts/artifactStore.js +66 -6
- package/dist/auth/anthropicOAuth.d.ts +26 -1
- package/dist/auth/anthropicOAuth.js +81 -5
- package/dist/auth/secureFiles.d.ts +37 -0
- package/dist/auth/secureFiles.js +95 -0
- package/dist/auth/tokenStore.d.ts +43 -6
- package/dist/auth/tokenStore.js +260 -79
- package/dist/avatar/providers/DIDAvatar.js +49 -45
- package/dist/avatar/providers/ReplicateAvatar.js +33 -40
- package/dist/browser/neurolink.min.js +485 -558
- package/dist/cli/commands/auth.js +112 -70
- package/dist/cli/commands/proxy.d.ts +4 -0
- package/dist/cli/commands/proxy.js +128 -13
- package/dist/cli/commands/serve.d.ts +10 -1
- package/dist/cli/commands/serve.js +98 -17
- package/dist/cli/commands/server.js +17 -5
- package/dist/cli/index.js +146 -5
- package/dist/cli/loop/optionsSchema.d.ts +1 -1
- package/dist/cli/utils/envManager.js +17 -3
- package/dist/cli/utils/serverUtils.d.ts +39 -3
- package/dist/cli/utils/serverUtils.js +63 -6
- package/dist/core/baseProvider.js +7 -1
- package/dist/core/modules/MessageBuilder.js +5 -0
- package/dist/core/modules/ToolsManager.d.ts +46 -1
- package/dist/core/modules/ToolsManager.js +190 -43
- package/dist/files/fileReferenceRegistry.d.ts +2 -23
- package/dist/files/fileReferenceRegistry.js +55 -5
- package/dist/hitl/hitlManager.js +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +6 -1
- package/dist/mcp/auth/tokenStorage.d.ts +7 -0
- package/dist/mcp/auth/tokenStorage.js +23 -4
- package/dist/mcp/externalServerManager.d.ts +13 -3
- package/dist/mcp/externalServerManager.js +154 -136
- package/dist/mcp/index.d.ts +2 -0
- package/dist/mcp/index.js +3 -0
- package/dist/mcp/mcpClientFactory.d.ts +14 -0
- package/dist/mcp/mcpClientFactory.js +70 -7
- package/dist/mcp/projectConfigTrust.d.ts +72 -0
- package/dist/mcp/projectConfigTrust.js +237 -0
- package/dist/mcp/transportUrlPolicy.d.ts +39 -0
- package/dist/mcp/transportUrlPolicy.js +172 -0
- package/dist/music/providers/BeatovenMusic.js +35 -46
- package/dist/music/providers/ReplicateMusic.js +33 -40
- package/dist/neurolink.d.ts +16 -1
- package/dist/neurolink.js +64 -11
- package/dist/processors/archive/ArchiveProcessor.js +15 -1
- package/dist/processors/archive/zipEntryIndex.d.ts +22 -0
- package/dist/processors/archive/zipEntryIndex.js +33 -0
- package/dist/processors/base/BaseFileProcessor.d.ts +9 -21
- package/dist/processors/base/BaseFileProcessor.js +40 -111
- package/dist/processors/data/YamlProcessor.d.ts +13 -1
- package/dist/processors/data/YamlProcessor.js +93 -2
- package/dist/processors/document/OpenDocumentProcessor.js +7 -2
- package/dist/processors/document/PptxProcessor.js +12 -7
- package/dist/processors/media/VideoProcessor.js +93 -15
- package/dist/providers/openaiChatCompletionsClient.js +12 -3
- package/dist/proxy/accountLedger.js +6 -8
- package/dist/proxy/geminiFormat.d.ts +3 -11
- package/dist/proxy/geminiFormat.js +11 -2
- package/dist/proxy/inboundAuth.d.ts +89 -0
- package/dist/proxy/inboundAuth.js +219 -0
- package/dist/proxy/openaiFormat.d.ts +3 -9
- package/dist/proxy/openaiFormat.js +21 -15
- package/dist/proxy/proxyTranslationEngine.d.ts +2 -6
- package/dist/proxy/proxyTranslationEngine.js +25 -14
- package/dist/proxy/routingPolicy.d.ts +3 -1
- package/dist/proxy/routingPolicy.js +7 -8
- package/dist/proxy/tokenRefresh.d.ts +6 -0
- package/dist/proxy/tokenRefresh.js +13 -1
- package/dist/proxy/updateState.js +4 -4
- package/dist/rag/document/loaders.d.ts +0 -6
- package/dist/rag/document/loaders.js +16 -10
- package/dist/rag/ragIntegration.d.ts +2 -2
- package/dist/rag/ragIntegration.js +35 -10
- package/dist/rag/retrieval/vectorQueryTool.d.ts +7 -0
- package/dist/rag/retrieval/vectorQueryTool.js +54 -17
- package/dist/server/abstract/baseServerAdapter.d.ts +43 -0
- package/dist/server/abstract/baseServerAdapter.js +119 -6
- package/dist/server/adapters/expressAdapter.d.ts +3 -0
- package/dist/server/adapters/expressAdapter.js +78 -6
- package/dist/server/adapters/fastifyAdapter.d.ts +3 -0
- package/dist/server/adapters/fastifyAdapter.js +165 -70
- package/dist/server/adapters/honoAdapter.d.ts +9 -1
- package/dist/server/adapters/honoAdapter.js +99 -21
- package/dist/server/adapters/koaAdapter.d.ts +1 -0
- package/dist/server/adapters/koaAdapter.js +153 -74
- package/dist/server/index.d.ts +2 -0
- package/dist/server/index.js +4 -0
- package/dist/server/middleware/auth.js +6 -2
- package/dist/server/middleware/cache.js +6 -1
- package/dist/server/middleware/rateLimit.js +29 -4
- package/dist/server/routes/agentRoutes.js +24 -11
- package/dist/server/routes/claudeProxyRoutes.d.ts +1 -0
- package/dist/server/routes/claudeProxyRoutes.js +273 -84
- package/dist/server/routes/codexProxyRoutes.d.ts +2 -2
- package/dist/server/routes/codexProxyRoutes.js +125 -40
- package/dist/server/routes/index.d.ts +4 -0
- package/dist/server/routes/index.js +35 -4
- package/dist/server/routes/mcpRoutes.d.ts +6 -2
- package/dist/server/routes/mcpRoutes.js +21 -7
- package/dist/server/routes/memoryRoutes.js +7 -6
- package/dist/server/routes/toolRoutes.d.ts +12 -2
- package/dist/server/routes/toolRoutes.js +33 -3
- package/dist/server/routes/transcribeRoutes.js +60 -13
- package/dist/server/utils/clientAddress.d.ts +23 -0
- package/dist/server/utils/clientAddress.js +65 -0
- package/dist/server/utils/clientError.d.ts +35 -0
- package/dist/server/utils/clientError.js +129 -0
- package/dist/server/utils/pathMatch.d.ts +21 -0
- package/dist/server/utils/pathMatch.js +41 -0
- package/dist/server/utils/validation.d.ts +9 -0
- package/dist/server/utils/validation.js +9 -0
- package/dist/server/websocket/WebSocketHandler.js +22 -3
- package/dist/server/websocket/frameQueue.d.ts +19 -0
- package/dist/server/websocket/frameQueue.js +41 -3
- package/dist/server/websocket/transcribeWebSocket.d.ts +8 -1
- package/dist/server/websocket/transcribeWebSocket.js +223 -34
- package/dist/skills/skillStores.js +5 -0
- package/dist/tasks/store/fileTaskStore.d.ts +2 -0
- package/dist/tasks/store/fileTaskStore.js +11 -4
- package/dist/types/attachments.d.ts +117 -0
- package/dist/types/attachments.js +15 -0
- package/dist/types/auth.d.ts +6 -0
- package/dist/types/cli.d.ts +4 -0
- package/dist/types/config.d.ts +56 -0
- package/dist/types/file.d.ts +8 -0
- package/dist/types/hitl.d.ts +8 -0
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.js +1 -0
- package/dist/types/mcp.d.ts +98 -0
- package/dist/types/multimodal.js +0 -41
- package/dist/types/openaiCompatible.d.ts +1 -1
- package/dist/types/processor.d.ts +14 -1
- package/dist/types/proxy.d.ts +71 -16
- package/dist/types/proxyClient.d.ts +2 -0
- package/dist/types/rag.d.ts +9 -0
- package/dist/types/safeFetch.d.ts +40 -0
- package/dist/types/server.d.ts +206 -10
- package/dist/types/tools.d.ts +57 -0
- package/dist/types/transcribe.d.ts +6 -0
- package/dist/utils/attachmentPolicy.d.ts +89 -0
- package/dist/utils/attachmentPolicy.js +327 -0
- package/dist/utils/dotenvBootstrap.d.ts +48 -19
- package/dist/utils/dotenvBootstrap.js +172 -31
- package/dist/utils/fileDetector.js +115 -65
- package/dist/utils/json/coerce.js +11 -1
- package/dist/utils/logSanitize.d.ts +5 -0
- package/dist/utils/logSanitize.js +8 -0
- package/dist/utils/logger.js +73 -1
- package/dist/utils/messageBuilder.d.ts +2 -2
- package/dist/utils/messageBuilder.js +118 -105
- package/dist/utils/multimodalOptionsBuilder.d.ts +1 -0
- package/dist/utils/multimodalOptionsBuilder.js +2 -0
- package/dist/utils/redis.d.ts +12 -0
- package/dist/utils/redis.js +72 -0
- package/dist/utils/retryAfter.d.ts +32 -0
- package/dist/utils/retryAfter.js +135 -11
- package/dist/utils/safeFetch.d.ts +22 -1
- package/dist/utils/safeFetch.js +132 -38
- package/dist/utils/safeStoreId.d.ts +19 -0
- package/dist/utils/safeStoreId.js +29 -0
- package/dist/utils/schemaConversion.d.ts +30 -3
- package/dist/utils/schemaConversion.js +457 -84
- package/dist/utils/sizeGuard.js +2 -2
- package/dist/utils/ssrfGuard.d.ts +3 -3
- package/dist/utils/ssrfGuard.js +71 -39
- package/dist/voice/eval/sttEvalStore.d.ts +26 -3
- package/dist/voice/eval/sttEvalStore.js +77 -12
- package/dist/voice/livekit/agentDefinition.d.ts +18 -0
- package/dist/voice/livekit/agentDefinition.js +22 -0
- package/dist/voice/livekit/realtimeMcpTools.d.ts +1 -1
- package/dist/voice/livekit/realtimeMcpTools.js +8 -7
- package/dist/voice/livekit/realtimeVoiceAgent.d.ts +2 -4
- package/dist/voice/livekit/realtimeVoiceAgent.js +18 -7
- package/dist/voice/livekit/voiceAgent.d.ts +2 -2
- package/dist/voice/livekit/voiceAgent.js +2 -1
- package/dist/voice/livekit/xaiSessionWatch.d.ts +0 -5
- package/dist/voice/livekit/xaiSessionWatch.js +30 -0
- package/dist/voice/providers/GeminiLive.js +21 -6
- package/dist/voice/providers/GoogleSTT.js +8 -5
- package/dist/voice/transcribe.d.ts +2 -2
- package/dist/voice/transcribe.js +34 -18
- package/dist/voice/whistle/audio.js +11 -5
- package/dist/workflow/workflows/adaptiveWorkflow.d.ts +2 -2
- package/dist/workflow/workflows/adaptiveWorkflow.js +10 -10
- package/dist/workflow/workflows/consensusWorkflow.d.ts +4 -4
- package/dist/workflow/workflows/consensusWorkflow.js +12 -12
- package/dist/workflow/workflows/fallbackWorkflow.d.ts +3 -3
- package/dist/workflow/workflows/fallbackWorkflow.js +7 -7
- package/dist/workflow/workflows/multiJudgeWorkflow.d.ts +7 -7
- package/dist/workflow/workflows/multiJudgeWorkflow.js +31 -31
- package/docs-site/static/search-index.json +74 -52
- package/package.json +173 -156
- package/scripts/observability/import-openobserve-dashboard.mjs +11 -3
package/dist/utils/ssrfGuard.js
CHANGED
|
@@ -26,6 +26,29 @@
|
|
|
26
26
|
*/
|
|
27
27
|
import { lookup } from "node:dns/promises";
|
|
28
28
|
import { isIP } from "node:net";
|
|
29
|
+
import { redactUrlForError } from "./logSanitize.js";
|
|
30
|
+
/**
|
|
31
|
+
* Parse `url` and enforce the scheme rule. HTTPS only unless the caller
|
|
32
|
+
* explicitly allowed `http:`; nothing else is ever a download URL.
|
|
33
|
+
*/
|
|
34
|
+
function parseGuardedUrl(url, options) {
|
|
35
|
+
let parsed;
|
|
36
|
+
try {
|
|
37
|
+
parsed = new URL(url);
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
throw new Error(`Invalid URL: "${url}"`);
|
|
41
|
+
}
|
|
42
|
+
if (parsed.protocol === "https:") {
|
|
43
|
+
return parsed;
|
|
44
|
+
}
|
|
45
|
+
if (parsed.protocol === "http:" && options?.allowInsecure === true) {
|
|
46
|
+
return parsed;
|
|
47
|
+
}
|
|
48
|
+
throw new Error(options?.allowInsecure === true
|
|
49
|
+
? `Only HTTP(S) URLs are permitted; got "${parsed.protocol}//" in "${url}"`
|
|
50
|
+
: `Only HTTPS URLs are permitted; got "${parsed.protocol}//" in "${url}"`);
|
|
51
|
+
}
|
|
29
52
|
/**
|
|
30
53
|
* Blocked IPv4 CIDRs.
|
|
31
54
|
*
|
|
@@ -302,7 +325,7 @@ export async function assertSafeUrl(url) {
|
|
|
302
325
|
return; // routable IP literal — safe
|
|
303
326
|
}
|
|
304
327
|
if (literalCheck !== "not-an-ip") {
|
|
305
|
-
throw new Error(`URL "${url}" rejected: ${literalCheck}`);
|
|
328
|
+
throw new Error(`URL "${redactUrlForError(url)}" rejected: ${literalCheck}`);
|
|
306
329
|
}
|
|
307
330
|
// Hostname — resolve BOTH A and AAAA. Reject if either family yields a
|
|
308
331
|
// blocked address (closes off the "publish AAAA public, A private" attack).
|
|
@@ -316,7 +339,7 @@ export async function assertSafeUrl(url) {
|
|
|
316
339
|
* @throws {Error} when resolution fails entirely or any address is blocked
|
|
317
340
|
* (all-must-pass — the caller may connect to any address in the set).
|
|
318
341
|
*/
|
|
319
|
-
async function resolveAndValidateHost(url, host) {
|
|
342
|
+
async function resolveAndValidateHost(url, host, allowPrivate = false) {
|
|
320
343
|
const [a, aaaa] = await Promise.allSettled([
|
|
321
344
|
lookup(host, { family: 4, all: true }),
|
|
322
345
|
lookup(host, { family: 6, all: true }),
|
|
@@ -350,11 +373,28 @@ async function resolveAndValidateHost(url, host) {
|
|
|
350
373
|
? aaaa.reason.message
|
|
351
374
|
: String(aaaa.reason)
|
|
352
375
|
: "ok";
|
|
353
|
-
|
|
376
|
+
// Keep the resolver's own error on the chain (and its code, e.g.
|
|
377
|
+
// ENOTFOUND) so a caller's retry classification and cause-aware redaction
|
|
378
|
+
// see the real failure rather than this wrapper.
|
|
379
|
+
const resolverError = a.status === "rejected" && a.reason instanceof Error
|
|
380
|
+
? a.reason
|
|
381
|
+
: aaaa.status === "rejected" && aaaa.reason instanceof Error
|
|
382
|
+
? aaaa.reason
|
|
383
|
+
: undefined;
|
|
384
|
+
const unresolved = new Error(`URL "${redactUrlForError(url)}" rejected: hostname ${host} could not be resolved (A: ${aErr}; AAAA: ${aaaaErr})`, resolverError ? { cause: resolverError } : undefined);
|
|
385
|
+
const resolverCode = resolverError
|
|
386
|
+
?.code;
|
|
387
|
+
if (resolverCode !== undefined) {
|
|
388
|
+
unresolved.code = resolverCode;
|
|
389
|
+
}
|
|
390
|
+
throw unresolved;
|
|
391
|
+
}
|
|
392
|
+
if (allowPrivate) {
|
|
393
|
+
return { v4: v4Addresses, v6: v6Addresses };
|
|
354
394
|
}
|
|
355
395
|
for (const addr of v4Addresses) {
|
|
356
396
|
if (isBlockedIPv4(addr)) {
|
|
357
|
-
throw new Error(`URL "${url}" rejected: hostname ${host} resolves to ${addr} (IPv4 in blocked range)`);
|
|
397
|
+
throw new Error(`URL "${redactUrlForError(url)}" rejected: hostname ${host} resolves to ${addr} (IPv4 in blocked range)`);
|
|
358
398
|
}
|
|
359
399
|
}
|
|
360
400
|
for (const addr of v6Addresses) {
|
|
@@ -362,7 +402,7 @@ async function resolveAndValidateHost(url, host) {
|
|
|
362
402
|
// addresses are caught.
|
|
363
403
|
const reason = checkHostLiteral(addr.toLowerCase());
|
|
364
404
|
if (reason && reason !== "not-an-ip") {
|
|
365
|
-
throw new Error(`URL "${url}" rejected: hostname ${host} resolves to ${addr} (IPv6 ${reason})`);
|
|
405
|
+
throw new Error(`URL "${redactUrlForError(url)}" rejected: hostname ${host} resolves to ${addr} (IPv6 ${reason})`);
|
|
366
406
|
}
|
|
367
407
|
}
|
|
368
408
|
return { v4: v4Addresses, v6: v6Addresses };
|
|
@@ -395,31 +435,28 @@ const INTERNAL_HOST_SUFFIXES = [
|
|
|
395
435
|
*
|
|
396
436
|
* Same throw semantics as {@link assertSafeUrl}.
|
|
397
437
|
*/
|
|
398
|
-
export function assertSafeUrlForProxy(url) {
|
|
399
|
-
|
|
400
|
-
try {
|
|
401
|
-
parsed = new URL(url);
|
|
402
|
-
}
|
|
403
|
-
catch {
|
|
404
|
-
throw new Error(`Invalid URL: "${url}"`);
|
|
405
|
-
}
|
|
406
|
-
if (parsed.protocol !== "https:") {
|
|
407
|
-
throw new Error(`Only HTTPS URLs are permitted; got "${parsed.protocol}//" in "${url}"`);
|
|
408
|
-
}
|
|
438
|
+
export function assertSafeUrlForProxy(url, options) {
|
|
439
|
+
const parsed = parseGuardedUrl(url, options);
|
|
409
440
|
const host = stripBrackets(parsed.hostname).toLowerCase();
|
|
410
441
|
const literalCheck = checkHostLiteral(host);
|
|
411
442
|
if (literalCheck === null) {
|
|
412
443
|
return; // routable IP literal
|
|
413
444
|
}
|
|
414
445
|
if (literalCheck !== "not-an-ip") {
|
|
415
|
-
|
|
446
|
+
if (options?.allowPrivateNetwork === true) {
|
|
447
|
+
return; // a blocked range the caller explicitly opened
|
|
448
|
+
}
|
|
449
|
+
throw new Error(`URL "${redactUrlForError(url)}" rejected: ${literalCheck}`);
|
|
450
|
+
}
|
|
451
|
+
if (options?.allowPrivateNetwork === true) {
|
|
452
|
+
return; // internal names are exactly what the caller asked to reach
|
|
416
453
|
}
|
|
417
454
|
const name = host.endsWith(".") ? host.slice(0, -1) : host;
|
|
418
455
|
if (!name.includes(".")) {
|
|
419
|
-
throw new Error(`URL "${url}" rejected: "${name}" is a single-label host name`);
|
|
456
|
+
throw new Error(`URL "${redactUrlForError(url)}" rejected: "${name}" is a single-label host name`);
|
|
420
457
|
}
|
|
421
458
|
if (INTERNAL_HOST_SUFFIXES.some((suffix) => name === suffix || name.endsWith(`.${suffix}`))) {
|
|
422
|
-
throw new Error(`URL "${url}" rejected: "${name}" is an internal host name`);
|
|
459
|
+
throw new Error(`URL "${redactUrlForError(url)}" rejected: "${name}" is an internal host name`);
|
|
423
460
|
}
|
|
424
461
|
}
|
|
425
462
|
/**
|
|
@@ -439,31 +476,26 @@ export function assertSafeUrlForProxy(url) {
|
|
|
439
476
|
* This is the canonical entry point for binary downloads where DNS-rebinding
|
|
440
477
|
* pinning matters — see `safeFetch.ts`.
|
|
441
478
|
*/
|
|
442
|
-
export async function validateAndResolveUrl(url) {
|
|
443
|
-
|
|
444
|
-
try {
|
|
445
|
-
parsed = new URL(url);
|
|
446
|
-
}
|
|
447
|
-
catch {
|
|
448
|
-
throw new Error(`Invalid URL: "${url}"`);
|
|
449
|
-
}
|
|
450
|
-
if (parsed.protocol !== "https:") {
|
|
451
|
-
throw new Error(`Only HTTPS URLs are permitted; got "${parsed.protocol}//" in "${url}"`);
|
|
452
|
-
}
|
|
479
|
+
export async function validateAndResolveUrl(url, options) {
|
|
480
|
+
const parsed = parseGuardedUrl(url, options);
|
|
453
481
|
const host = stripBrackets(parsed.hostname).toLowerCase();
|
|
482
|
+
// Only an explicit opt-in relaxes the range check; the default stays the
|
|
483
|
+
// server path's deny list. The resolve-and-pin step runs either way, so an
|
|
484
|
+
// opened-up caller still gets rebinding protection.
|
|
485
|
+
const allowPrivate = options?.allowPrivateNetwork === true;
|
|
454
486
|
// IP literal — normalise + check, return canonical form
|
|
455
487
|
const v4 = normalizeIPv4(host);
|
|
456
488
|
if (v4) {
|
|
457
|
-
if (isBlockedIPv4(v4)) {
|
|
458
|
-
throw new Error(`URL "${url}" rejected: IPv4 ${host} → ${v4} is in a blocked range`);
|
|
489
|
+
if (!allowPrivate && isBlockedIPv4(v4)) {
|
|
490
|
+
throw new Error(`URL "${redactUrlForError(url)}" rejected: IPv4 ${host} → ${v4} is in a blocked range`);
|
|
459
491
|
}
|
|
460
492
|
return { url, ip: v4, family: 4, addresses: [{ ip: v4, family: 4 }] };
|
|
461
493
|
}
|
|
462
494
|
if (host.includes(":")) {
|
|
463
495
|
const v4FromMapped = extractIPv4FromMapped(host);
|
|
464
496
|
if (v4FromMapped) {
|
|
465
|
-
if (isBlockedIPv4(v4FromMapped)) {
|
|
466
|
-
throw new Error(`URL "${url}" rejected: IPv4-mapped IPv6 ${host} → ${v4FromMapped} is in a blocked range`);
|
|
497
|
+
if (!allowPrivate && isBlockedIPv4(v4FromMapped)) {
|
|
498
|
+
throw new Error(`URL "${redactUrlForError(url)}" rejected: IPv4-mapped IPv6 ${host} → ${v4FromMapped} is in a blocked range`);
|
|
467
499
|
}
|
|
468
500
|
return {
|
|
469
501
|
url,
|
|
@@ -474,10 +506,10 @@ export async function validateAndResolveUrl(url) {
|
|
|
474
506
|
}
|
|
475
507
|
const expanded = expandIPv6(host);
|
|
476
508
|
if (!expanded) {
|
|
477
|
-
throw new Error(`URL "${url}" rejected: IPv6 ${host} could not be parsed`);
|
|
509
|
+
throw new Error(`URL "${redactUrlForError(url)}" rejected: IPv6 ${host} could not be parsed`);
|
|
478
510
|
}
|
|
479
|
-
if (isBlockedIPv6(expanded)) {
|
|
480
|
-
throw new Error(`URL "${url}" rejected: IPv6 ${host} is in a blocked range`);
|
|
511
|
+
if (!allowPrivate && isBlockedIPv6(expanded)) {
|
|
512
|
+
throw new Error(`URL "${redactUrlForError(url)}" rejected: IPv6 ${host} is in a blocked range`);
|
|
481
513
|
}
|
|
482
514
|
return { url, ip: host, family: 6, addresses: [{ ip: host, family: 6 }] };
|
|
483
515
|
}
|
|
@@ -485,7 +517,7 @@ export async function validateAndResolveUrl(url) {
|
|
|
485
517
|
// set. Validating the same answer we hand to the connect layer also
|
|
486
518
|
// removes the rebinding window the previous separate `lookup()` left
|
|
487
519
|
// between validation and pinning.
|
|
488
|
-
const { v4: v4Addresses, v6: v6Addresses } = await resolveAndValidateHost(url, host);
|
|
520
|
+
const { v4: v4Addresses, v6: v6Addresses } = await resolveAndValidateHost(url, host, allowPrivate);
|
|
489
521
|
const addresses = [
|
|
490
522
|
...v4Addresses.map((ip) => ({ ip, family: 4 })),
|
|
491
523
|
...v6Addresses.map((ip) => ({ ip, family: 6 })),
|
|
@@ -494,7 +526,7 @@ export async function validateAndResolveUrl(url) {
|
|
|
494
526
|
if (!first) {
|
|
495
527
|
// Both lookups "succeeded" but returned zero addresses — treat exactly
|
|
496
528
|
// like a resolution failure rather than pinning nothing.
|
|
497
|
-
throw new Error(`URL "${url}" rejected: hostname ${host} resolved to an empty address set`);
|
|
529
|
+
throw new Error(`URL "${redactUrlForError(url)}" rejected: hostname ${host} resolved to an empty address set`);
|
|
498
530
|
}
|
|
499
531
|
return { url, ip: first.ip, family: first.family, addresses };
|
|
500
532
|
}
|
|
@@ -3,21 +3,44 @@ export declare class STTEvalStore {
|
|
|
3
3
|
private readonly _dir;
|
|
4
4
|
private readonly _maxCalls;
|
|
5
5
|
private readonly _maxSegments;
|
|
6
|
+
private readonly _maxBytes;
|
|
7
|
+
private _totalBytes;
|
|
6
8
|
private readonly _calls;
|
|
7
9
|
/** Per call: the tail of its accept queue, so batches apply and append in arrival order. */
|
|
8
10
|
private readonly _queues;
|
|
9
11
|
private _ready;
|
|
10
12
|
private readonly _langfuse;
|
|
11
13
|
constructor(options?: ServerSTTEvalOptions);
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
14
|
+
/** Bytes of segments held in memory right now, across every call. */
|
|
15
|
+
get heldBytes(): number;
|
|
16
|
+
/**
|
|
17
|
+
* Take a batch. `owner` is the authenticated caller, when the server
|
|
18
|
+
* identified one: a call created with an owner accepts batches only from
|
|
19
|
+
* that owner, and `undefined` is returned for anyone else's.
|
|
20
|
+
*/
|
|
21
|
+
accept(batch: ServerSTTEvalBatchRequest, owner?: string): Promise<ServerSTTEvalBatchResponse | undefined>;
|
|
22
|
+
/** Calls visible to `owner`: theirs and the unowned ones (see `_visibleTo`). */
|
|
23
|
+
list(owner?: string): Promise<ServerSTTEvalCallSummary[]>;
|
|
24
|
+
report(callId: string, includeSegments?: boolean, owner?: string): Promise<ServerSTTEvalReport | undefined>;
|
|
25
|
+
/**
|
|
26
|
+
* A call an identified caller created is theirs alone. A call with no owner
|
|
27
|
+
* (made under no authentication or a shared key) is visible to everyone, as
|
|
28
|
+
* every call was before owners existed.
|
|
29
|
+
*/
|
|
30
|
+
private _visibleTo;
|
|
15
31
|
private _reportOf;
|
|
16
32
|
/** Runs `task` after every earlier task for the same call has settled. */
|
|
17
33
|
private _serial;
|
|
18
34
|
private _create;
|
|
19
35
|
private _apply;
|
|
20
36
|
private _callFor;
|
|
37
|
+
private _charge;
|
|
38
|
+
/**
|
|
39
|
+
* Drop the least recently updated calls while there are more than
|
|
40
|
+
* `maxCalls`, or while the held segments exceed `maxBytes`; a dropped call
|
|
41
|
+
* is still on disk when there is a directory. The newest call is never
|
|
42
|
+
* dropped, so a single call larger than the budget stays.
|
|
43
|
+
*/
|
|
21
44
|
private _evict;
|
|
22
45
|
private _fileFor;
|
|
23
46
|
/** Replays the directory once; every caller awaits the same load. */
|
|
@@ -21,13 +21,21 @@ import { phoneticKey } from "../codeMix.js";
|
|
|
21
21
|
import { STTEvalLangfuseExporter, sttEvalLangfuseConfigFromEnv, } from "./sttEvalLangfuse.js";
|
|
22
22
|
const DEFAULT_MAX_CALLS = 1000;
|
|
23
23
|
const DEFAULT_MAX_SEGMENTS = 50_000;
|
|
24
|
+
/** Bytes of segments held in memory across every call. */
|
|
25
|
+
const DEFAULT_MAX_BYTES = 64 * 1024 * 1024;
|
|
24
26
|
/** Words per side above which a speaker is aligned in time windows instead of whole. */
|
|
25
27
|
const FULL_ALIGN_MAX_WORDS = 4000;
|
|
26
28
|
const WINDOW_S = 300;
|
|
29
|
+
/** Roughly what a segment costs in memory: its JSON, which is what is kept. */
|
|
30
|
+
function segmentBytes(segment) {
|
|
31
|
+
return JSON.stringify(segment).length;
|
|
32
|
+
}
|
|
27
33
|
export class STTEvalStore {
|
|
28
34
|
_dir;
|
|
29
35
|
_maxCalls;
|
|
30
36
|
_maxSegments;
|
|
37
|
+
_maxBytes;
|
|
38
|
+
_totalBytes = 0;
|
|
31
39
|
_calls = new Map();
|
|
32
40
|
/** Per call: the tail of its accept queue, so batches apply and append in arrival order. */
|
|
33
41
|
_queues = new Map();
|
|
@@ -38,19 +46,36 @@ export class STTEvalStore {
|
|
|
38
46
|
this._dir = dir ? dir : undefined;
|
|
39
47
|
this._maxCalls = Math.max(1, options.maxCalls ?? DEFAULT_MAX_CALLS);
|
|
40
48
|
this._maxSegments = Math.max(1, options.maxSegmentsPerCall ?? DEFAULT_MAX_SEGMENTS);
|
|
49
|
+
this._maxBytes = Math.max(1, options.maxBytes ?? DEFAULT_MAX_BYTES);
|
|
41
50
|
const langfuse = sttEvalLangfuseConfigFromEnv();
|
|
42
51
|
this._langfuse = langfuse
|
|
43
52
|
? new STTEvalLangfuseExporter(langfuse)
|
|
44
53
|
: undefined;
|
|
45
54
|
}
|
|
46
|
-
|
|
55
|
+
/** Bytes of segments held in memory right now, across every call. */
|
|
56
|
+
get heldBytes() {
|
|
57
|
+
return this._totalBytes;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Take a batch. `owner` is the authenticated caller, when the server
|
|
61
|
+
* identified one: a call created with an owner accepts batches only from
|
|
62
|
+
* that owner, and `undefined` is returned for anyone else's.
|
|
63
|
+
*/
|
|
64
|
+
async accept(batch, owner) {
|
|
47
65
|
await this._load();
|
|
48
66
|
return this._serial(batch.call_id, async () => {
|
|
49
|
-
const
|
|
67
|
+
const held = await this._callFor(batch.call_id);
|
|
68
|
+
if (held && !this._visibleTo(held, owner)) {
|
|
69
|
+
return undefined;
|
|
70
|
+
}
|
|
71
|
+
const call = held ?? this._create(batch, owner);
|
|
50
72
|
const counts = this._apply(call, batch);
|
|
51
73
|
if (this._dir) {
|
|
52
74
|
try {
|
|
53
|
-
|
|
75
|
+
const stored = owner
|
|
76
|
+
? { ...batch, owner }
|
|
77
|
+
: batch;
|
|
78
|
+
await appendFile(this._fileFor(batch.call_id), `${JSON.stringify(stored)}\n`, "utf8");
|
|
54
79
|
}
|
|
55
80
|
catch (error) {
|
|
56
81
|
logger.warn("[stt-eval] could not persist batch", {
|
|
@@ -73,16 +98,28 @@ export class STTEvalStore {
|
|
|
73
98
|
};
|
|
74
99
|
});
|
|
75
100
|
}
|
|
76
|
-
|
|
101
|
+
/** Calls visible to `owner`: theirs and the unowned ones (see `_visibleTo`). */
|
|
102
|
+
async list(owner) {
|
|
77
103
|
await this._load();
|
|
78
104
|
return [...this._calls.values()]
|
|
105
|
+
.filter((call) => this._visibleTo(call, owner))
|
|
79
106
|
.sort((a, b) => b.updatedAt - a.updatedAt)
|
|
80
107
|
.map(summarize);
|
|
81
108
|
}
|
|
82
|
-
async report(callId, includeSegments = false) {
|
|
109
|
+
async report(callId, includeSegments = false, owner) {
|
|
83
110
|
await this._load();
|
|
84
111
|
const call = await this._callFor(callId);
|
|
85
|
-
return call
|
|
112
|
+
return call && this._visibleTo(call, owner)
|
|
113
|
+
? this._reportOf(call, includeSegments)
|
|
114
|
+
: undefined;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* A call an identified caller created is theirs alone. A call with no owner
|
|
118
|
+
* (made under no authentication or a shared key) is visible to everyone, as
|
|
119
|
+
* every call was before owners existed.
|
|
120
|
+
*/
|
|
121
|
+
_visibleTo(call, owner) {
|
|
122
|
+
return call.owner === undefined || call.owner === owner;
|
|
86
123
|
}
|
|
87
124
|
_reportOf(call, includeSegments) {
|
|
88
125
|
const ordered = [...call.segments.values()].sort((a, b) => timeOf(a) - timeOf(b));
|
|
@@ -128,7 +165,7 @@ export class STTEvalStore {
|
|
|
128
165
|
});
|
|
129
166
|
return run;
|
|
130
167
|
}
|
|
131
|
-
_create(batch) {
|
|
168
|
+
_create(batch, owner) {
|
|
132
169
|
const call = {
|
|
133
170
|
callId: batch.call_id,
|
|
134
171
|
room: batch.room ?? null,
|
|
@@ -137,6 +174,8 @@ export class STTEvalStore {
|
|
|
137
174
|
batches: new Set(),
|
|
138
175
|
segments: new Map(),
|
|
139
176
|
updatedAt: Date.now(),
|
|
177
|
+
bytes: 0,
|
|
178
|
+
...(owner ? { owner } : {}),
|
|
140
179
|
};
|
|
141
180
|
this._calls.set(batch.call_id, call);
|
|
142
181
|
return call;
|
|
@@ -146,8 +185,10 @@ export class STTEvalStore {
|
|
|
146
185
|
let updated = 0;
|
|
147
186
|
let ignored = 0;
|
|
148
187
|
for (const segment of batch.segments) {
|
|
149
|
-
|
|
188
|
+
const previous = call.segments.get(segment.id);
|
|
189
|
+
if (previous) {
|
|
150
190
|
updated++;
|
|
191
|
+
this._charge(call, -segmentBytes(previous));
|
|
151
192
|
}
|
|
152
193
|
else if (call.segments.size >= this._maxSegments) {
|
|
153
194
|
ignored++;
|
|
@@ -157,6 +198,7 @@ export class STTEvalStore {
|
|
|
157
198
|
inserted++;
|
|
158
199
|
}
|
|
159
200
|
call.segments.set(segment.id, { ...segment });
|
|
201
|
+
this._charge(call, segmentBytes(segment));
|
|
160
202
|
}
|
|
161
203
|
call.batches.add(batch.batch);
|
|
162
204
|
call.final = call.final || batch.final === true;
|
|
@@ -173,17 +215,38 @@ export class STTEvalStore {
|
|
|
173
215
|
if (held || !this._dir) {
|
|
174
216
|
return held;
|
|
175
217
|
}
|
|
176
|
-
// Evicted from memory but still on disk.
|
|
218
|
+
// Evicted from memory but still on disk. Replaying re-registers the call
|
|
219
|
+
// and its bytes; the next eviction pass trims the budget again.
|
|
177
220
|
const text = await readFile(this._fileFor(callId), "utf8").catch(() => undefined);
|
|
178
|
-
|
|
221
|
+
if (text === undefined) {
|
|
222
|
+
return undefined;
|
|
223
|
+
}
|
|
224
|
+
const call = this._replay(text);
|
|
225
|
+
this._evict();
|
|
226
|
+
return call;
|
|
179
227
|
}
|
|
228
|
+
_charge(call, delta) {
|
|
229
|
+
call.bytes += delta;
|
|
230
|
+
this._totalBytes = Math.max(0, this._totalBytes + delta);
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* Drop the least recently updated calls while there are more than
|
|
234
|
+
* `maxCalls`, or while the held segments exceed `maxBytes`; a dropped call
|
|
235
|
+
* is still on disk when there is a directory. The newest call is never
|
|
236
|
+
* dropped, so a single call larger than the budget stays.
|
|
237
|
+
*/
|
|
180
238
|
_evict() {
|
|
181
|
-
while (this._calls.size >
|
|
239
|
+
while (this._calls.size > 1 &&
|
|
240
|
+
(this._calls.size > this._maxCalls || this._totalBytes > this._maxBytes)) {
|
|
182
241
|
const oldest = this._calls.keys().next().value;
|
|
183
242
|
if (oldest === undefined) {
|
|
184
243
|
return;
|
|
185
244
|
}
|
|
245
|
+
const call = this._calls.get(oldest);
|
|
186
246
|
this._calls.delete(oldest);
|
|
247
|
+
if (call) {
|
|
248
|
+
this._totalBytes = Math.max(0, this._totalBytes - call.bytes);
|
|
249
|
+
}
|
|
187
250
|
}
|
|
188
251
|
}
|
|
189
252
|
_fileFor(callId) {
|
|
@@ -210,8 +273,10 @@ export class STTEvalStore {
|
|
|
210
273
|
// Replay order is filesystem order; keep the most recent calls.
|
|
211
274
|
const recent = [...this._calls.values()].sort((a, b) => a.updatedAt - b.updatedAt);
|
|
212
275
|
this._calls.clear();
|
|
276
|
+
this._totalBytes = 0;
|
|
213
277
|
for (const call of recent) {
|
|
214
278
|
this._calls.set(call.callId, call);
|
|
279
|
+
this._totalBytes += call.bytes;
|
|
215
280
|
}
|
|
216
281
|
this._evict();
|
|
217
282
|
}
|
|
@@ -231,7 +296,7 @@ export class STTEvalStore {
|
|
|
231
296
|
}
|
|
232
297
|
try {
|
|
233
298
|
const batch = JSON.parse(line);
|
|
234
|
-
call = call ?? this._create(batch);
|
|
299
|
+
call = call ?? this._create(batch, batch.owner);
|
|
235
300
|
this._apply(call, batch);
|
|
236
301
|
for (const segment of batch.segments) {
|
|
237
302
|
lastSeen = Math.max(lastSeen, segment.received_at * 1000);
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Marks an agent object so the `@livekit/agents` job process accepts it.
|
|
3
|
+
*
|
|
4
|
+
* Since 1.9 the framework's `isAgent` gate (what the job process runs on the
|
|
5
|
+
* entry file's default export) no longer looks for an `entry` function: it
|
|
6
|
+
* checks for the registry symbol `Symbol.for("livekit.agents.AgentDefinition")`
|
|
7
|
+
* that `defineAgent` stamps, and an unmarked object fails with "Unable to load
|
|
8
|
+
* agent: Missing or invalid default export". `defineVoiceAgent` and
|
|
9
|
+
* `defineRealtimeVoiceAgent` return their object through this helper so an
|
|
10
|
+
* entry file's `export default defineVoiceAgent({...})` keeps working.
|
|
11
|
+
*
|
|
12
|
+
* The symbol is read from the global registry rather than imported, because
|
|
13
|
+
* `@livekit/agents` is an optional dependency this module only ever imports
|
|
14
|
+
* dynamically. The property is defined exactly as `defineAgent` defines it, so a
|
|
15
|
+
* consumer that additionally wraps the result in `defineAgent` redefines an
|
|
16
|
+
* identical property, which is permitted.
|
|
17
|
+
*/
|
|
18
|
+
export declare function markAgentDefinition<T extends object>(agent: T): T;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Marks an agent object so the `@livekit/agents` job process accepts it.
|
|
3
|
+
*
|
|
4
|
+
* Since 1.9 the framework's `isAgent` gate (what the job process runs on the
|
|
5
|
+
* entry file's default export) no longer looks for an `entry` function: it
|
|
6
|
+
* checks for the registry symbol `Symbol.for("livekit.agents.AgentDefinition")`
|
|
7
|
+
* that `defineAgent` stamps, and an unmarked object fails with "Unable to load
|
|
8
|
+
* agent: Missing or invalid default export". `defineVoiceAgent` and
|
|
9
|
+
* `defineRealtimeVoiceAgent` return their object through this helper so an
|
|
10
|
+
* entry file's `export default defineVoiceAgent({...})` keeps working.
|
|
11
|
+
*
|
|
12
|
+
* The symbol is read from the global registry rather than imported, because
|
|
13
|
+
* `@livekit/agents` is an optional dependency this module only ever imports
|
|
14
|
+
* dynamically. The property is defined exactly as `defineAgent` defines it, so a
|
|
15
|
+
* consumer that additionally wraps the result in `defineAgent` redefines an
|
|
16
|
+
* identical property, which is permitted.
|
|
17
|
+
*/
|
|
18
|
+
const AGENT_DEFINITION_SYMBOL = Symbol.for("livekit.agents.AgentDefinition");
|
|
19
|
+
export function markAgentDefinition(agent) {
|
|
20
|
+
Object.defineProperty(agent, AGENT_DEFINITION_SYMBOL, { value: true });
|
|
21
|
+
return agent;
|
|
22
|
+
}
|
|
@@ -20,7 +20,7 @@ import type { BuildRealtimeMcpToolsParams } from "../../types/index.js";
|
|
|
20
20
|
export declare function mcpResultToText(result: unknown): string;
|
|
21
21
|
/**
|
|
22
22
|
* Connect to the MCP server as the calling user and bridge every tool into a
|
|
23
|
-
* LiveKit `ToolContext`. Returns
|
|
23
|
+
* LiveKit `ToolContext`. Returns that context + the client (close on shutdown).
|
|
24
24
|
* The server already scopes the tool list by `x-context`, so no client-side
|
|
25
25
|
* filtering is needed.
|
|
26
26
|
*/
|
|
@@ -73,7 +73,7 @@ function buildHitlDeclineMessage(toolName) {
|
|
|
73
73
|
* First registration wins, so a unique full name is never shadowed by a generic
|
|
74
74
|
* short name. Returns the aliases actually registered.
|
|
75
75
|
*/
|
|
76
|
-
function registerToolAliases(
|
|
76
|
+
function registerToolAliases(registry, mcpToolName, handler) {
|
|
77
77
|
const safeFullName = mcpToolName.replace(/[^a-zA-Z0-9_]/g, "_");
|
|
78
78
|
const shortName = safeFullName.includes("_")
|
|
79
79
|
? safeFullName.slice(safeFullName.lastIndexOf("_") + 1)
|
|
@@ -81,17 +81,17 @@ function registerToolAliases(toolContext, mcpToolName, handler) {
|
|
|
81
81
|
const candidates = [safeFullName, shortName].filter((name, index, all) => name.length > 0 && all.indexOf(name) === index);
|
|
82
82
|
const registered = [];
|
|
83
83
|
for (const alias of candidates) {
|
|
84
|
-
if (alias in
|
|
84
|
+
if (alias in registry) {
|
|
85
85
|
continue; // taken by another tool — don't clobber
|
|
86
86
|
}
|
|
87
|
-
|
|
87
|
+
registry[alias] = handler;
|
|
88
88
|
registered.push(alias);
|
|
89
89
|
}
|
|
90
90
|
return registered;
|
|
91
91
|
}
|
|
92
92
|
/**
|
|
93
93
|
* Connect to the MCP server as the calling user and bridge every tool into a
|
|
94
|
-
* LiveKit `ToolContext`. Returns
|
|
94
|
+
* LiveKit `ToolContext`. Returns that context + the client (close on shutdown).
|
|
95
95
|
* The server already scopes the tool list by `x-context`, so no client-side
|
|
96
96
|
* filtering is needed.
|
|
97
97
|
*/
|
|
@@ -126,7 +126,7 @@ export async function buildRealtimeMcpTools(params) {
|
|
|
126
126
|
try {
|
|
127
127
|
const { tools: mcpTools } = await client.listTools();
|
|
128
128
|
logger.info("realtime.mcp.toolsListed", { count: mcpTools.length });
|
|
129
|
-
const
|
|
129
|
+
const registry = {};
|
|
130
130
|
const skipped = [];
|
|
131
131
|
for (const mcpTool of mcpTools) {
|
|
132
132
|
const parameters = sanitizeToolParameters(mcpTool.inputSchema);
|
|
@@ -209,15 +209,16 @@ export async function buildRealtimeMcpTools(params) {
|
|
|
209
209
|
}
|
|
210
210
|
},
|
|
211
211
|
});
|
|
212
|
-
const aliases = registerToolAliases(
|
|
212
|
+
const aliases = registerToolAliases(registry, mcpTool.name, handler);
|
|
213
213
|
logger.info("realtime.mcp.toolRegistered", {
|
|
214
214
|
tool: mcpTool.name,
|
|
215
215
|
aliases,
|
|
216
216
|
hitl: requiresConfirmation,
|
|
217
217
|
});
|
|
218
218
|
}
|
|
219
|
+
const toolContext = new llm.ToolContext(registry);
|
|
219
220
|
logger.info("realtime.mcp.connected", {
|
|
220
|
-
registered: Object.keys(
|
|
221
|
+
registered: Object.keys(registry).length,
|
|
221
222
|
skipped: skipped.length,
|
|
222
223
|
skippedTools: skipped,
|
|
223
224
|
});
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
*
|
|
16
16
|
* See docs/features/livekit-voice-agent.md.
|
|
17
17
|
*/
|
|
18
|
-
import type {
|
|
18
|
+
import type { AgentDefinition } from "@livekit/agents";
|
|
19
19
|
import type { RealtimeVoiceAgentConfig } from "../../types/index.js";
|
|
20
20
|
/**
|
|
21
21
|
* Define a realtime (speech-to-speech) LiveKit voice agent: Gemini Live by
|
|
@@ -25,6 +25,4 @@ import type { RealtimeVoiceAgentConfig } from "../../types/index.js";
|
|
|
25
25
|
* with `startRealtimeVoiceAgentWorker`. With no `config` everything is resolved
|
|
26
26
|
* from the environment (see `resolveRealtimeVoiceConfig`).
|
|
27
27
|
*/
|
|
28
|
-
export declare function defineRealtimeVoiceAgent(config?: RealtimeVoiceAgentConfig):
|
|
29
|
-
entry: (ctx: JobContext) => Promise<void>;
|
|
30
|
-
};
|
|
28
|
+
export declare function defineRealtimeVoiceAgent(config?: RealtimeVoiceAgentConfig): AgentDefinition;
|
|
@@ -15,8 +15,10 @@
|
|
|
15
15
|
*
|
|
16
16
|
* See docs/features/livekit-voice-agent.md.
|
|
17
17
|
*/
|
|
18
|
+
import { ReadableStream } from "node:stream/web";
|
|
18
19
|
import { z } from "zod";
|
|
19
20
|
import { logger } from "../../utils/logger.js";
|
|
21
|
+
import { markAgentDefinition } from "./agentDefinition.js";
|
|
20
22
|
import { resolveRealtimeVoiceConfig } from "./config.js";
|
|
21
23
|
import { ensureVertexAdc, clearGeminiApiKeyEnv } from "./vertexAuth.js";
|
|
22
24
|
import { createRealtimeLlm, requireXaiApiKey } from "./realtimeModel.js";
|
|
@@ -232,7 +234,7 @@ export function defineRealtimeVoiceAgent(config = {}) {
|
|
|
232
234
|
: {}),
|
|
233
235
|
});
|
|
234
236
|
await installRealtimeJobLifecycle(ctx, cfg);
|
|
235
|
-
const { voice } = await import("@livekit/agents");
|
|
237
|
+
const { voice, toStream } = await import("@livekit/agents");
|
|
236
238
|
const realtimeLlm = await createRealtimeLlm(cfg);
|
|
237
239
|
const bridge = await attachRealtimeEventBridge({
|
|
238
240
|
room: ctx.room,
|
|
@@ -271,7 +273,7 @@ export function defineRealtimeVoiceAgent(config = {}) {
|
|
|
271
273
|
agentTools = toolset.tools;
|
|
272
274
|
logger.info("realtime.mcp.enabled", {
|
|
273
275
|
mcpUrl: cfg.mcpUrl,
|
|
274
|
-
toolCount: Object.keys(toolset.tools).length,
|
|
276
|
+
toolCount: Object.keys(toolset.tools.functionTools).length,
|
|
275
277
|
hasAuthToken: authToken !== "",
|
|
276
278
|
});
|
|
277
279
|
ctx.addShutdownCallback(async () => {
|
|
@@ -311,7 +313,12 @@ export function defineRealtimeVoiceAgent(config = {}) {
|
|
|
311
313
|
...(seededChatCtx ? { chatCtx: seededChatCtx } : {}),
|
|
312
314
|
});
|
|
313
315
|
agent.transcriptionNode = async (textStream, _modelSettings) => {
|
|
314
|
-
|
|
316
|
+
// 1.9 hands the node a ReadableStream or a bare AsyncIterable; only a
|
|
317
|
+
// stream has tee(), so mirror the framework default and wrap the latter.
|
|
318
|
+
const source = textStream instanceof ReadableStream
|
|
319
|
+
? textStream
|
|
320
|
+
: toStream(textStream);
|
|
321
|
+
const [forUi, forDownstream] = source.tee();
|
|
315
322
|
void (async () => {
|
|
316
323
|
const reader = forUi.getReader();
|
|
317
324
|
let chunkCount = 0;
|
|
@@ -363,18 +370,22 @@ export function defineRealtimeVoiceAgent(config = {}) {
|
|
|
363
370
|
});
|
|
364
371
|
// --- Session events → browser ------------------------------------------
|
|
365
372
|
session.on(voice.AgentSessionEventTypes.UserInputTranscribed, (ev) => {
|
|
373
|
+
// agents-plugin-xai >= 1.9 emits only the *completed* transcription and
|
|
374
|
+
// flags it `isFinal: false`; without this every xAI user turn would be
|
|
375
|
+
// published as interim and the recorder would never see a final line.
|
|
376
|
+
const final = ev.isFinal || cfg.backend === "xai";
|
|
366
377
|
logger.debug("realtime.session.userTranscript", {
|
|
367
378
|
ms: sinceConnect(),
|
|
368
|
-
final
|
|
379
|
+
final,
|
|
369
380
|
transcript: ev.transcript,
|
|
370
381
|
});
|
|
371
382
|
bridge.publishEvent("user-text", {
|
|
372
383
|
text: ev.transcript,
|
|
373
|
-
final
|
|
384
|
+
final,
|
|
374
385
|
});
|
|
375
386
|
recorder?.userTranscript({
|
|
376
387
|
text: ev.transcript,
|
|
377
|
-
final
|
|
388
|
+
final,
|
|
378
389
|
});
|
|
379
390
|
});
|
|
380
391
|
session.on(voice.AgentSessionEventTypes.AgentStateChanged, (ev) => {
|
|
@@ -416,5 +427,5 @@ export function defineRealtimeVoiceAgent(config = {}) {
|
|
|
416
427
|
session.generateReply({ instructions: cfg.greeting });
|
|
417
428
|
}
|
|
418
429
|
}
|
|
419
|
-
return { entry };
|
|
430
|
+
return markAgentDefinition({ entry });
|
|
420
431
|
}
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
*
|
|
15
15
|
* See docs/features/livekit-voice-agent.md.
|
|
16
16
|
*/
|
|
17
|
-
import type {
|
|
17
|
+
import type { AgentDefinition } from "@livekit/agents";
|
|
18
18
|
import type { LiveKitVoiceAgentConfig } from "../../types/index.js";
|
|
19
19
|
/**
|
|
20
20
|
* Define a LiveKit voice agent backed by NeuroLink.
|
|
@@ -29,4 +29,4 @@ import type { LiveKitVoiceAgentConfig } from "../../types/index.js";
|
|
|
29
29
|
* });
|
|
30
30
|
* ```
|
|
31
31
|
*/
|
|
32
|
-
export declare function defineVoiceAgent(config: LiveKitVoiceAgentConfig):
|
|
32
|
+
export declare function defineVoiceAgent(config: LiveKitVoiceAgentConfig): AgentDefinition;
|