@juspay/neurolink 13.6.0 → 13.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -3
- package/dist/browser/neurolink.min.js +379 -379
- package/dist/core/modules/ToolsManager.js +2 -3
- package/dist/core/nativeGenerateLoop.js +1 -0
- package/dist/neurolink.js +2 -1
- package/dist/providers/catalog/provider-catalog.schema.json +7 -1
- package/dist/providers/catalog/scaledown.json +17 -3
- package/dist/providers/catalog/schema.js +8 -0
- package/dist/providers/configuredDecision.js +1 -0
- package/dist/providers/decide/errors.d.ts +1 -0
- package/dist/providers/decide/errors.js +9 -2
- package/dist/types/config.d.ts +13 -1
- package/dist/types/generate.d.ts +9 -2
- package/dist/types/livekit.d.ts +19 -0
- package/dist/types/providerCatalog.d.ts +8 -0
- package/dist/utils/toolExecutionId.d.ts +7 -0
- package/dist/utils/toolExecutionId.js +9 -0
- package/dist/voice/livekit/config.js +3 -2
- package/dist/voice/livekit/realtimeModel.d.ts +2 -2
- package/dist/voice/livekit/realtimeModel.js +25 -6
- package/dist/voice/livekit/realtimeVoiceAgent.js +3 -1
- package/dist/voice/livekit/xaiSessionWatch.d.ts +26 -0
- package/dist/voice/livekit/xaiSessionWatch.js +157 -0
- package/docs-site/static/search-index.json +5 -5
- package/package.json +1 -1
|
@@ -6,6 +6,7 @@ import { logger } from "../../utils/logger.js";
|
|
|
6
6
|
import { TOOL_EVENTS_WRAPPED } from "../../types/index.js";
|
|
7
7
|
import { getKeyCount } from "../../utils/transformationUtils.js";
|
|
8
8
|
import { extractIsErrorText } from "../../utils/toolResultStatus.js";
|
|
9
|
+
import { createToolExecutionId } from "../../utils/toolExecutionId.js";
|
|
9
10
|
import { convertJsonSchemaToZod } from "../../utils/schemaConversion.js";
|
|
10
11
|
import { generateToolOutputPreview, DEFAULT_MAX_PREVIEW_BYTES, RETRIEVE_CONTEXT_TOOL_NAME, } from "../../context/toolOutputLimits.js";
|
|
11
12
|
/**
|
|
@@ -516,9 +517,7 @@ export class ToolsManager {
|
|
|
516
517
|
// listener can pair this start with its end.
|
|
517
518
|
const eventIds = {
|
|
518
519
|
...idField,
|
|
519
|
-
executionId:
|
|
520
|
-
.toString(36)
|
|
521
|
-
.slice(2, 11)}`,
|
|
520
|
+
executionId: createToolExecutionId(toolName),
|
|
522
521
|
};
|
|
523
522
|
return withSpan({
|
|
524
523
|
name: toolSpanName(toolName),
|
|
@@ -353,6 +353,7 @@ const validateToolInput = (tool, input) => {
|
|
|
353
353
|
? resolveToolInputSchema(tool.inputSchema)
|
|
354
354
|
: undefined;
|
|
355
355
|
if (!schema ||
|
|
356
|
+
typeof schema !== "object" ||
|
|
356
357
|
!("safeParse" in schema) ||
|
|
357
358
|
typeof schema.safeParse !== "function") {
|
|
358
359
|
return undefined;
|
package/dist/neurolink.js
CHANGED
|
@@ -76,6 +76,7 @@ import { SpanSerializer } from "./observability/utils/spanSerializer.js";
|
|
|
76
76
|
import { endSpanWhenDropped, withSpan } from "./telemetry/withSpan.js";
|
|
77
77
|
import { getActiveTraceContext } from "./telemetry/traceContext.js";
|
|
78
78
|
import { calculateCost } from "./utils/pricing.js";
|
|
79
|
+
import { createToolExecutionId } from "./utils/toolExecutionId.js";
|
|
79
80
|
import { acquireTelemetry, ensureTelemetry, flushOpenTelemetry, getLangfuseHealthStatus, isOpenTelemetryInitialized, releaseTelemetry, setLangfuseContext, traceMetadataFromContext, } from "./services/server/ai/observability/instrumentation.js";
|
|
80
81
|
import { clearChecklistState, createChecklistTools, getChecklistState, resolveChecklistSessionId, } from "./agent/taskChecklist.js";
|
|
81
82
|
import { cancelDelegates as cancelBackgroundDelegates, collectDelegates as collectBackgroundDelegates, configureDelegation, createDelegationTools, spawnDelegate as spawnBackgroundDelegate, } from "./agent/backgroundDelegation.js";
|
|
@@ -10599,7 +10600,7 @@ Current user's request: ${currentInput}`;
|
|
|
10599
10600
|
* @returns executionId for tracking this specific execution
|
|
10600
10601
|
*/
|
|
10601
10602
|
emitToolStart(toolName, input, startTime = Date.now()) {
|
|
10602
|
-
const executionId =
|
|
10603
|
+
const executionId = createToolExecutionId(toolName, startTime);
|
|
10603
10604
|
// Create execution context for tracking
|
|
10604
10605
|
const context = {
|
|
10605
10606
|
executionId,
|
|
@@ -311,7 +311,13 @@
|
|
|
311
311
|
"description": "The rule also needs the error message to match this case-insensitive regular expression (first 2,000 characters, key removed). Bounded: outside a character class no unescaped *, + or {n,}, no quantifier on a group, no count above 255, and the counts side by side multiply to at most 4,096 ways (each ? is 2, each {n,m} is m-n+1)."
|
|
312
312
|
},
|
|
313
313
|
"kind": { "$ref": "#/definitions/decisionErrorKind" },
|
|
314
|
-
"retryable": { "type": "boolean" }
|
|
314
|
+
"retryable": { "type": "boolean" },
|
|
315
|
+
"hint": {
|
|
316
|
+
"type": "string",
|
|
317
|
+
"minLength": 1,
|
|
318
|
+
"maxLength": 300,
|
|
319
|
+
"description": "Text added after the error message, behind a dash, when this rule matches. For a status the vendor uses for more than one cause. {apiKeyEnvVar} is the only placeholder."
|
|
320
|
+
}
|
|
315
321
|
}
|
|
316
322
|
}
|
|
317
323
|
},
|
|
@@ -24,6 +24,7 @@
|
|
|
24
24
|
}
|
|
25
25
|
}
|
|
26
26
|
},
|
|
27
|
+
"timeouts": { "decideMs": 30000 },
|
|
27
28
|
"decide": {
|
|
28
29
|
"dialect": "system-one",
|
|
29
30
|
"order": 8,
|
|
@@ -35,14 +36,27 @@
|
|
|
35
36
|
"errors": {
|
|
36
37
|
"rules": [
|
|
37
38
|
{ "status": 401, "kind": "authentication" },
|
|
38
|
-
{
|
|
39
|
+
{
|
|
40
|
+
"status": 402,
|
|
41
|
+
"kind": "invalid_request",
|
|
42
|
+
"hint": "ScaleDown answers HTTP 402 both when the account has no credit left and when it does not recognise the API key; check {apiKeyEnvVar} first."
|
|
43
|
+
},
|
|
44
|
+
{ "status": 422, "kind": "invalid_request" },
|
|
39
45
|
{ "status": 429, "kind": "rate_limit" },
|
|
40
46
|
{ "status": 502, "kind": "server" },
|
|
41
47
|
{ "status": 504, "kind": "timeout" },
|
|
42
48
|
{ "statusMin": 500, "kind": "server" }
|
|
43
49
|
],
|
|
44
50
|
"default": "invalid_request",
|
|
45
|
-
"bodyShapes": [
|
|
51
|
+
"bodyShapes": [
|
|
52
|
+
{ "path": "detail.message" },
|
|
53
|
+
{ "path": "detail" },
|
|
54
|
+
{
|
|
55
|
+
"path": "detail",
|
|
56
|
+
"isArray": true,
|
|
57
|
+
"formatter": "fastapi-validation"
|
|
58
|
+
}
|
|
59
|
+
]
|
|
46
60
|
}
|
|
47
61
|
},
|
|
48
62
|
"setup": {
|
|
@@ -54,7 +68,7 @@
|
|
|
54
68
|
"1. Create an account and an API key, following https://docs.scaledown.ai",
|
|
55
69
|
"2. Set {apiKeyEnvVar} in your .env file",
|
|
56
70
|
"3. ScaleDown's documentation says the key goes in an x-api-key header rather than Authorization: Bearer, and NeuroLink sends it that way",
|
|
57
|
-
"4. ScaleDown's documentation says a request made with no credit left returns HTTP 402"
|
|
71
|
+
"4. ScaleDown's documentation says a request made with no credit left returns HTTP 402; a live test on 2026-10-20 saw it return 402 for an unrecognised key as well, so check {apiKeyEnvVar} before adding credit"
|
|
58
72
|
]
|
|
59
73
|
},
|
|
60
74
|
"evidence": {
|
|
@@ -482,6 +482,13 @@ const decideMessagePatternSchema = z
|
|
|
482
482
|
.refine((pattern) => !hasRiskyQuantifier(pattern), {
|
|
483
483
|
message: "messagePattern must be bounded: an unescaped * or +, an open-ended {n,}, a {n,m} count above 255, or any quantifier on a group, outside a character class, can backtrack catastrophically on an error message. Put ?, or {n,m} up to 255, only on a plain character, a character class or an escape, and keep the counts side by side to 4096 ways in all (each ? is 2, each {n,m} is m-n+1)",
|
|
484
484
|
});
|
|
485
|
+
const decideErrorHintSchema = z
|
|
486
|
+
.string()
|
|
487
|
+
.min(1)
|
|
488
|
+
.max(300)
|
|
489
|
+
.refine((hint) => !/[{}]/.test(hint.split("{apiKeyEnvVar}").join("")), {
|
|
490
|
+
message: "hint may only use the {apiKeyEnvVar} placeholder",
|
|
491
|
+
});
|
|
485
492
|
const decideErrorRuleSchema = z.strictObject({
|
|
486
493
|
status: z
|
|
487
494
|
.union([decideStatusSchema, z.array(decideStatusSchema).min(1)])
|
|
@@ -490,6 +497,7 @@ const decideErrorRuleSchema = z.strictObject({
|
|
|
490
497
|
messagePattern: decideMessagePatternSchema.optional(),
|
|
491
498
|
kind: decideErrorKindSchema,
|
|
492
499
|
retryable: z.boolean().optional(),
|
|
500
|
+
hint: decideErrorHintSchema.optional(),
|
|
493
501
|
});
|
|
494
502
|
const UNSAFE_PATH_SEGMENTS = new Set(["__proto__", "constructor", "prototype"]);
|
|
495
503
|
const HTTP_TOKEN_PATTERN = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/;
|
|
@@ -52,7 +52,7 @@ function firstShapeText(shapes, payload) {
|
|
|
52
52
|
return undefined;
|
|
53
53
|
}
|
|
54
54
|
export function classifyDecisionError(spec, input) {
|
|
55
|
-
const { status, payload, requestId, label, apiKey } = input;
|
|
55
|
+
const { status, payload, requestId, label, apiKey, apiKeyEnvVar } = input;
|
|
56
56
|
// Redacted once, here: it stands in for a shape's text that redaction
|
|
57
57
|
// empties, and the label is catalog data a key can match. Redaction is not
|
|
58
58
|
// idempotent for a key such as "e" (it matches inside "[redacted]"), so the
|
|
@@ -65,9 +65,16 @@ export function classifyDecisionError(spec, input) {
|
|
|
65
65
|
const rule = spec.rules.find((candidate) => ruleMatches(candidate, status, message));
|
|
66
66
|
const ruleKind = rule?.kind ?? spec.default;
|
|
67
67
|
const ruleRetryable = rule?.retryable ?? isRetryableByDefault(ruleKind);
|
|
68
|
+
// Added after the rule is chosen, so a pattern never has to allow for it,
|
|
69
|
+
// and redacted on its own, once, for the same reason as the fallback above.
|
|
70
|
+
const hinted = rule?.hint
|
|
71
|
+
? `${message} — ${redactCredentials(rule.hint
|
|
72
|
+
.split("{apiKeyEnvVar}")
|
|
73
|
+
.join(apiKeyEnvVar ?? "the API key variable"), apiKey)}`
|
|
74
|
+
: message;
|
|
68
75
|
const build = (kind, retryable) => ({
|
|
69
76
|
kind,
|
|
70
|
-
message,
|
|
77
|
+
message: hinted,
|
|
71
78
|
status,
|
|
72
79
|
requestId,
|
|
73
80
|
retryable,
|
package/dist/types/config.d.ts
CHANGED
|
@@ -443,6 +443,13 @@ export type ToolConfig = {
|
|
|
443
443
|
* full. Note: the instance-level `include` list deliberately does NOT
|
|
444
444
|
* force tools hot — it scopes the catalog, and discovery defers within
|
|
445
445
|
* that scope (a scoped-but-large catalog is exactly where deferral pays).
|
|
446
|
+
*
|
|
447
|
+
* What the model and your listeners see once discovery is on: the
|
|
448
|
+
* `search_tools` description ends with an instruction to check the catalog
|
|
449
|
+
* before saying a task cannot be done; each `search_tools` call emits
|
|
450
|
+
* `tool:start` and `tool:end` (paired by `executionId`) and is traced and
|
|
451
|
+
* observed like any other tool call; and the events of every wrapped
|
|
452
|
+
* external tool carry an `executionId`.
|
|
446
453
|
* Default: false.
|
|
447
454
|
*/
|
|
448
455
|
discovery?: boolean;
|
|
@@ -451,7 +458,12 @@ export type ToolConfig = {
|
|
|
451
458
|
* - `"external"`: tools of servers attached with `addExternalMCPServer`.
|
|
452
459
|
* - `"all"`: every tool in the request's record except the built-in tools,
|
|
453
460
|
* so tools registered with `registerTool`/`registerTools` and in-process
|
|
454
|
-
* MCP tools are deferred as well.
|
|
461
|
+
* MCP tools are deferred as well. That includes the tools NeuroLink
|
|
462
|
+
* registers for its own features through `registerTool()`, such as
|
|
463
|
+
* `retrieve_context` (memory retrieval), the skills tools, and the tools
|
|
464
|
+
* of `registerTaskTools`, `registerDelegationTools`,
|
|
465
|
+
* `registerBackgroundCommandTools` and `registerGitTools`. Name them in
|
|
466
|
+
* `discoveryAlwaysVisible` to keep them in full.
|
|
455
467
|
* Per-call tools, per-call whitelists, forced `toolChoice` tools and
|
|
456
468
|
* already-discovered tools stay hot in both scopes. Ignored unless
|
|
457
469
|
* `discovery` is true.
|
package/dist/types/generate.d.ts
CHANGED
|
@@ -1838,12 +1838,19 @@ export type NativeGenerateLoopArgs = {
|
|
|
1838
1838
|
conversation: Array<Record<string, unknown>>;
|
|
1839
1839
|
/** Tool declarations in the v3 shape doGenerate already knows how to convert. */
|
|
1840
1840
|
tools?: Array<Record<string, unknown>>;
|
|
1841
|
-
/**
|
|
1841
|
+
/**
|
|
1842
|
+
* Registered tools, used to execute a call the model asks for. Typed
|
|
1843
|
+
* `Record<string, Tool>`; it was `Record<string, unknown>` before discovery
|
|
1844
|
+
* learned to declare tools that join it mid-turn, so a caller that builds
|
|
1845
|
+
* this record from loosely typed tools now needs a cast to `Tool`.
|
|
1846
|
+
*/
|
|
1842
1847
|
toolsRecord: Record<string, Tool>;
|
|
1843
1848
|
/**
|
|
1844
1849
|
* Builds the v3 declaration for one tool. When set, every step declares any
|
|
1845
1850
|
* tool that joined `toolsRecord` mid-turn (discovery's search_tools loads
|
|
1846
|
-
* tools into it), so the model can call what it just loaded.
|
|
1851
|
+
* tools into it), so the model can call what it just loaded. Only the
|
|
1852
|
+
* OpenAI-compatible `generate()` path supplies it; the other providers that
|
|
1853
|
+
* run this loop do not declare a tool `search_tools` loaded mid-turn.
|
|
1847
1854
|
*/
|
|
1848
1855
|
declareTool?: (name: string, tool: Tool) => Record<string, unknown>;
|
|
1849
1856
|
/** The turn's resolved tool choice, applied per step via `resolveStepToolChoice`. */
|
package/dist/types/livekit.d.ts
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
*
|
|
10
10
|
* See docs/features/livekit-voice-agent.md.
|
|
11
11
|
*/
|
|
12
|
+
import type { EventEmitter } from "node:events";
|
|
12
13
|
import type { NeuroLinkEvents, TypedEventEmitter } from "./common.js";
|
|
13
14
|
import type { StreamOptions, StreamResult } from "./stream.js";
|
|
14
15
|
import type { ConversationSeedMessage, ConversationSeedOptions, RecordConversationTurnOptions, RecordConversationTurnResult, StreamEventSequence } from "./conversation.js";
|
|
@@ -437,6 +438,24 @@ export type RealtimeLlm = {
|
|
|
437
438
|
label: () => string;
|
|
438
439
|
close: () => Promise<void>;
|
|
439
440
|
};
|
|
441
|
+
/**
|
|
442
|
+
* A session of the xAI plugin as far as the xai model's watcher needs it: an
|
|
443
|
+
* event emitter (the plugin republishes every server event as
|
|
444
|
+
* `openai_server_event_received`) that can be closed.
|
|
445
|
+
*/
|
|
446
|
+
export type XaiObservedSession = EventEmitter & {
|
|
447
|
+
close: () => Promise<void>;
|
|
448
|
+
};
|
|
449
|
+
/**
|
|
450
|
+
* Token counts of one xAI response. xAI sends them at the top level of
|
|
451
|
+
* `response.done`; the plugin reads `response.usage`, which xAI leaves empty.
|
|
452
|
+
*/
|
|
453
|
+
export type XaiResponseUsage = {
|
|
454
|
+
inputTokens: number;
|
|
455
|
+
inputAudioTokens: number;
|
|
456
|
+
outputTokens: number;
|
|
457
|
+
outputAudioTokens: number;
|
|
458
|
+
};
|
|
440
459
|
/**
|
|
441
460
|
* Realtime voice configuration resolved from the environment.
|
|
442
461
|
*
|
|
@@ -290,6 +290,14 @@ export type DecideErrorRule = {
|
|
|
290
290
|
messagePattern?: string;
|
|
291
291
|
kind: DecisionErrorKind;
|
|
292
292
|
retryable?: boolean;
|
|
293
|
+
/**
|
|
294
|
+
* Text added after the error message, behind a dash, when this rule is the
|
|
295
|
+
* one that matches. For a status the vendor uses for more than one cause,
|
|
296
|
+
* where the body's own words name only one of them. `{apiKeyEnvVar}` is the
|
|
297
|
+
* only placeholder. A `messagePattern` is judged on the message without it,
|
|
298
|
+
* and the key is removed from it as from the message.
|
|
299
|
+
*/
|
|
300
|
+
hint?: string;
|
|
293
301
|
};
|
|
294
302
|
export type DecideBodyShape = {
|
|
295
303
|
/** Dot path into the parsed error body. */
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @description Builds the id that pairs one tool call's `tool:start` and `tool:end` events.
|
|
3
|
+
* @param toolName {string} - Name of the tool being run.
|
|
4
|
+
* @param startTime {number} - When the call started, in epoch milliseconds.
|
|
5
|
+
* @returns {string} `<toolName>-<startTime>-<nine random base-36 characters>`.
|
|
6
|
+
*/
|
|
7
|
+
export declare function createToolExecutionId(toolName: string, startTime?: number): string;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @description Builds the id that pairs one tool call's `tool:start` and `tool:end` events.
|
|
3
|
+
* @param toolName {string} - Name of the tool being run.
|
|
4
|
+
* @param startTime {number} - When the call started, in epoch milliseconds.
|
|
5
|
+
* @returns {string} `<toolName>-<startTime>-<nine random base-36 characters>`.
|
|
6
|
+
*/
|
|
7
|
+
export function createToolExecutionId(toolName, startTime = Date.now()) {
|
|
8
|
+
return `${toolName}-${startTime}-${Math.random().toString(36).slice(2, 11)}`;
|
|
9
|
+
}
|
|
@@ -68,8 +68,9 @@ export function resolveBrainDefaults() {
|
|
|
68
68
|
};
|
|
69
69
|
}
|
|
70
70
|
const DEFAULT_REALTIME_MODEL = "gemini-live-2.5-flash";
|
|
71
|
-
// xAI's documented flagship Grok Voice id
|
|
72
|
-
// service
|
|
71
|
+
// xAI's documented flagship Grok Voice id, accepted and honoured by the live
|
|
72
|
+
// service (2026-10-20). The plugin's own default, `grok-voice-think-fast-1.0`,
|
|
73
|
+
// ran as `grok-voice-3.0` there.
|
|
73
74
|
const DEFAULT_XAI_REALTIME_MODEL = "grok-voice-3.0";
|
|
74
75
|
const DEFAULT_REALTIME_LOCATION = "global";
|
|
75
76
|
const DEFAULT_RESPONSE_MODALITY = "AUDIO";
|
|
@@ -8,8 +8,8 @@
|
|
|
8
8
|
* `@google/genai` are imported dynamically so the core package does not require
|
|
9
9
|
* them unless the realtime agent is used.
|
|
10
10
|
*
|
|
11
|
-
* The xai backend
|
|
12
|
-
*
|
|
11
|
+
* The xai backend was run against the real xAI service on 2026-10-20; what that
|
|
12
|
+
* showed, and what is still unmeasured, is in the document below.
|
|
13
13
|
*
|
|
14
14
|
* See docs/features/livekit-voice-agent.md.
|
|
15
15
|
*/
|
|
@@ -8,12 +8,13 @@
|
|
|
8
8
|
* `@google/genai` are imported dynamically so the core package does not require
|
|
9
9
|
* them unless the realtime agent is used.
|
|
10
10
|
*
|
|
11
|
-
* The xai backend
|
|
12
|
-
*
|
|
11
|
+
* The xai backend was run against the real xAI service on 2026-10-20; what that
|
|
12
|
+
* showed, and what is still unmeasured, is in the document below.
|
|
13
13
|
*
|
|
14
14
|
* See docs/features/livekit-voice-agent.md.
|
|
15
15
|
*/
|
|
16
16
|
import { requireWebSocketUrl } from "./config.js";
|
|
17
|
+
import { observeXaiSession } from "./xaiSessionWatch.js";
|
|
17
18
|
/**
|
|
18
19
|
* Fail early, naming the variable, when `XAI_API_KEY` is blank or absent.
|
|
19
20
|
*
|
|
@@ -66,16 +67,34 @@ async function createXaiRealtimeLlm(cfg) {
|
|
|
66
67
|
// own defaults, so a key holding `undefined` would erase the default: a
|
|
67
68
|
// missing `baseURL` would send the xAI key to OPENAI_BASE_URL or
|
|
68
69
|
// api.openai.com, and no `apiKey` option is passed at all because the plugin
|
|
69
|
-
// reads XAI_API_KEY itself. The plugin forwards `baseURL`
|
|
70
|
-
// declared option type leaves
|
|
71
|
-
|
|
70
|
+
// reads XAI_API_KEY itself. The plugin forwards `baseURL` and
|
|
71
|
+
// `inputAudioTranscription` although its declared option type leaves them out.
|
|
72
|
+
//
|
|
73
|
+
// `inputAudioTranscription` is the one key set to `undefined` on purpose, to
|
|
74
|
+
// erase the plugin's OpenAI default of `gpt-4o-mini-transcribe`. With it xAI
|
|
75
|
+
// answered one spoken utterance with five "completed" transcripts, which the
|
|
76
|
+
// turn recorder stored as one user message with the utterance repeated;
|
|
77
|
+
// without it xAI sends one. `undefined`, unlike `null`, keeps the plugin's
|
|
78
|
+
// user-transcription capability on, and JSON.stringify leaves the key out of
|
|
79
|
+
// the `session.update` it sends.
|
|
80
|
+
const modelOptions = {
|
|
81
|
+
model: cfg.model,
|
|
82
|
+
inputAudioTranscription: undefined,
|
|
83
|
+
};
|
|
72
84
|
if (cfg.voice) {
|
|
73
85
|
modelOptions.voice = cfg.voice;
|
|
74
86
|
}
|
|
75
87
|
if (cfg.xaiBaseURL) {
|
|
76
88
|
modelOptions.baseURL = cfg.xaiBaseURL;
|
|
77
89
|
}
|
|
78
|
-
|
|
90
|
+
class ObservedXaiRealtimeModel extends xai.realtime.RealtimeModel {
|
|
91
|
+
session() {
|
|
92
|
+
const session = super.session();
|
|
93
|
+
observeXaiSession(session, cfg.model);
|
|
94
|
+
return session;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
return new ObservedXaiRealtimeModel(modelOptions);
|
|
79
98
|
}
|
|
80
99
|
/**
|
|
81
100
|
* Build the realtime model for `cfg.backend`.
|
|
@@ -343,7 +343,9 @@ export function defineRealtimeVoiceAgent(config = {}) {
|
|
|
343
343
|
};
|
|
344
344
|
session.on(voice.AgentSessionEventTypes.MetricsCollected, (ev) => {
|
|
345
345
|
const metrics = ev.metrics;
|
|
346
|
-
|
|
346
|
+
// xAI leaves `response.usage` empty, which is where the plugin reads, so
|
|
347
|
+
// its metrics are zero; the xai model logs `realtime.usage` itself.
|
|
348
|
+
if (!isGemini || metrics.type !== "realtime_model_metrics") {
|
|
347
349
|
return;
|
|
348
350
|
}
|
|
349
351
|
const inAudio = metrics.inputTokenDetails.audioTokens;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What this package adds around an xAI realtime session.
|
|
3
|
+
*
|
|
4
|
+
* The LiveKit xAI plugin is the OpenAI realtime plugin pointed at xAI, and the
|
|
5
|
+
* live service differs from OpenAI in three ways the plugin does not cover
|
|
6
|
+
* (measured 2026-10-20, see docs/features/livekit-voice-agent.md):
|
|
7
|
+
*
|
|
8
|
+
* - xAI sends usage at the top level of `response.done` and leaves
|
|
9
|
+
* `response.usage` an empty object, which is what the plugin reads, so its
|
|
10
|
+
* metrics are zero for every response;
|
|
11
|
+
* - an unknown model id is accepted and runs a default model, with no error;
|
|
12
|
+
* - a wrong key is refused with HTTP 400 at the WebSocket upgrade, and the
|
|
13
|
+
* plugin opens its socket without an 'error' listener, so the refusal is an
|
|
14
|
+
* uncaught exception.
|
|
15
|
+
*
|
|
16
|
+
* `observeXaiSession` reads the stream of server events the plugin republishes
|
|
17
|
+
* as `openai_server_event_received` for the first two, and watches for the
|
|
18
|
+
* third.
|
|
19
|
+
*/
|
|
20
|
+
import type { XaiObservedSession } from "../../types/index.js";
|
|
21
|
+
/**
|
|
22
|
+
* Log the usage of every response, warn when the service runs another model
|
|
23
|
+
* than the one asked for, and report a refused upgrade as a session error.
|
|
24
|
+
* `requestedModel` is the id the session was configured with.
|
|
25
|
+
*/
|
|
26
|
+
export declare function observeXaiSession(session: XaiObservedSession, requestedModel: string): void;
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What this package adds around an xAI realtime session.
|
|
3
|
+
*
|
|
4
|
+
* The LiveKit xAI plugin is the OpenAI realtime plugin pointed at xAI, and the
|
|
5
|
+
* live service differs from OpenAI in three ways the plugin does not cover
|
|
6
|
+
* (measured 2026-10-20, see docs/features/livekit-voice-agent.md):
|
|
7
|
+
*
|
|
8
|
+
* - xAI sends usage at the top level of `response.done` and leaves
|
|
9
|
+
* `response.usage` an empty object, which is what the plugin reads, so its
|
|
10
|
+
* metrics are zero for every response;
|
|
11
|
+
* - an unknown model id is accepted and runs a default model, with no error;
|
|
12
|
+
* - a wrong key is refused with HTTP 400 at the WebSocket upgrade, and the
|
|
13
|
+
* plugin opens its socket without an 'error' listener, so the refusal is an
|
|
14
|
+
* uncaught exception.
|
|
15
|
+
*
|
|
16
|
+
* `observeXaiSession` reads the stream of server events the plugin republishes
|
|
17
|
+
* as `openai_server_event_received` for the first two, and watches for the
|
|
18
|
+
* third.
|
|
19
|
+
*/
|
|
20
|
+
import { logger } from "../../utils/logger.js";
|
|
21
|
+
const TOKEN_COUNT_KEYS = ["input_tokens", "output_tokens"];
|
|
22
|
+
const REFUSED_UPGRADE = /^Unexpected server response: (\d{3})$/;
|
|
23
|
+
const KEY_REFUSAL_STATUSES = new Set([400, 401, 403]);
|
|
24
|
+
function isRecord(value) {
|
|
25
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
26
|
+
}
|
|
27
|
+
function countOf(source, key) {
|
|
28
|
+
const value = source[key];
|
|
29
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0
|
|
30
|
+
? value
|
|
31
|
+
: 0;
|
|
32
|
+
}
|
|
33
|
+
function detailOf(source, group, key) {
|
|
34
|
+
const details = source[group];
|
|
35
|
+
return isRecord(details) ? countOf(details, key) : 0;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* The token counts of a `response.done` event, or `undefined` when it reports
|
|
39
|
+
* none. xAI's own place is the top level of the event; `response.usage`, where
|
|
40
|
+
* OpenAI puts them, is the fallback, so a service that fills it still works.
|
|
41
|
+
* A report with seconds but no token count (a cancelled response) is not usage
|
|
42
|
+
* and would log as zeros.
|
|
43
|
+
*/
|
|
44
|
+
function readXaiResponseUsage(event) {
|
|
45
|
+
if (!isRecord(event) || event.type !== "response.done") {
|
|
46
|
+
return undefined;
|
|
47
|
+
}
|
|
48
|
+
const nested = isRecord(event.response) ? event.response.usage : undefined;
|
|
49
|
+
const usage = [event.usage, nested].find((candidate) => isRecord(candidate) &&
|
|
50
|
+
TOKEN_COUNT_KEYS.some((key) => typeof candidate[key] === "number"));
|
|
51
|
+
if (usage === undefined) {
|
|
52
|
+
return undefined;
|
|
53
|
+
}
|
|
54
|
+
return {
|
|
55
|
+
inputTokens: countOf(usage, "input_tokens"),
|
|
56
|
+
inputAudioTokens: detailOf(usage, "input_token_details", "audio_tokens"),
|
|
57
|
+
outputTokens: countOf(usage, "output_tokens"),
|
|
58
|
+
outputAudioTokens: detailOf(usage, "output_token_details", "audio_tokens"),
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
/** The same record the agent logs for the Gemini backend, built from the same three counts. */
|
|
62
|
+
function logUsage(usage) {
|
|
63
|
+
logger.info("realtime.usage", {
|
|
64
|
+
inputTokens: usage.inputTokens,
|
|
65
|
+
inputAudioTokens: usage.inputAudioTokens,
|
|
66
|
+
inputTextTokens: Math.max(0, usage.inputTokens - usage.inputAudioTokens),
|
|
67
|
+
outputTokens: usage.outputTokens,
|
|
68
|
+
outputAudioTokens: usage.outputAudioTokens,
|
|
69
|
+
outputTextTokens: Math.max(0, usage.outputTokens - usage.outputAudioTokens),
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
function describeRefusal(status) {
|
|
73
|
+
const keyHint = KEY_REFUSAL_STATUSES.has(status)
|
|
74
|
+
? " xAI answers a wrong XAI_API_KEY with HTTP 400, not 401, so check the key first."
|
|
75
|
+
: "";
|
|
76
|
+
return `xAI refused the realtime connection with HTTP ${status}.${keyHint}`;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Turn the plugin's uncaught socket error into the error event the plugin
|
|
80
|
+
* meant to emit. Returns the function that ends the watch.
|
|
81
|
+
*
|
|
82
|
+
* `uncaughtExceptionMonitor` observes a crash without handling it, so the
|
|
83
|
+
* process still fails exactly as it would have; what changes is that the
|
|
84
|
+
* session reports a classified, unrecoverable error first, which the agent
|
|
85
|
+
* entry already logs and publishes. The watch is on only while the connection
|
|
86
|
+
* is being made, so it cannot take a crash elsewhere in the process for a
|
|
87
|
+
* refused upgrade once the call is running.
|
|
88
|
+
*
|
|
89
|
+
* A crash event carries no socket, so a refusal cannot be tied to one session:
|
|
90
|
+
* every session still connecting is told. That is accurate, because the crash
|
|
91
|
+
* ends the process for all of them, and the sessions of one process share
|
|
92
|
+
* `XAI_API_KEY`, so a refused key is refused for each.
|
|
93
|
+
*/
|
|
94
|
+
function watchForRefusedUpgrade(session) {
|
|
95
|
+
const onCrash = (error) => {
|
|
96
|
+
const status = error instanceof Error
|
|
97
|
+
? Number(REFUSED_UPGRADE.exec(error.message)?.[1])
|
|
98
|
+
: Number.NaN;
|
|
99
|
+
if (Number.isNaN(status)) {
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
stop();
|
|
103
|
+
const message = describeRefusal(status);
|
|
104
|
+
logger.error("realtime.xai.connectRefused", { status, error: message });
|
|
105
|
+
if (session.listenerCount("error") === 0) {
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
try {
|
|
109
|
+
session.emit("error", {
|
|
110
|
+
type: "realtime_model_error",
|
|
111
|
+
timestamp: Date.now(),
|
|
112
|
+
label: "xai.RealtimeModel",
|
|
113
|
+
error: new Error(message),
|
|
114
|
+
recoverable: false,
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
/* the process is already failing; a listener's error must not mask it */
|
|
119
|
+
}
|
|
120
|
+
};
|
|
121
|
+
const stop = () => {
|
|
122
|
+
process.off("uncaughtExceptionMonitor", onCrash);
|
|
123
|
+
};
|
|
124
|
+
process.on("uncaughtExceptionMonitor", onCrash);
|
|
125
|
+
return stop;
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* Log the usage of every response, warn when the service runs another model
|
|
129
|
+
* than the one asked for, and report a refused upgrade as a session error.
|
|
130
|
+
* `requestedModel` is the id the session was configured with.
|
|
131
|
+
*/
|
|
132
|
+
export function observeXaiSession(session, requestedModel) {
|
|
133
|
+
const stopWatching = watchForRefusedUpgrade(session);
|
|
134
|
+
session.on("openai_server_event_received", (event) => {
|
|
135
|
+
if (isRecord(event) && event.type === "session.created") {
|
|
136
|
+
stopWatching();
|
|
137
|
+
const served = isRecord(event.session) ? event.session.model : undefined;
|
|
138
|
+
if (typeof served === "string" && served !== requestedModel) {
|
|
139
|
+
logger.warn("realtime.xai.modelMismatch", {
|
|
140
|
+
requested: requestedModel,
|
|
141
|
+
served,
|
|
142
|
+
note: "xAI runs a default model for an id it does not know, without an error; an alias comes back as the model it points to",
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
const usage = readXaiResponseUsage(event);
|
|
148
|
+
if (usage !== undefined) {
|
|
149
|
+
logUsage(usage);
|
|
150
|
+
}
|
|
151
|
+
});
|
|
152
|
+
const close = session.close.bind(session);
|
|
153
|
+
session.close = async () => {
|
|
154
|
+
stopWatching();
|
|
155
|
+
await close();
|
|
156
|
+
};
|
|
157
|
+
}
|