@stackbone/cli 0.9.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/main.js +343 -84
- package/migrations/0122_model_routing_object.sql +16 -0
- package/package.json +1 -1
- package/stackbone-cli-0.9.2.tgz +0 -0
- package/voice-worker/main.js +215 -58
- package/voice-worker/main.js.map +4 -4
- package/stackbone-cli-0.9.0.tgz +0 -0
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
-- 0122_model_routing_object.sql — a provider routing is OpenRouter's whole
|
|
2
|
+
-- `provider` object (`{ "only": ["deepinfra/fp8"], "allow_fallbacks": false }`),
|
|
3
|
+
-- not only its `sort`.
|
|
4
|
+
--
|
|
5
|
+
-- `routing` holds the object. `sort` stays, now nullable, and carries the routing
|
|
6
|
+
-- only when it is nothing but a plain sort: an SDK older than the object reads
|
|
7
|
+
-- `sort` alone, and it must see a routing it cannot apply as no routing, never
|
|
8
|
+
-- apply half of it. Rows written before 0122 keep `routing` NULL; every reader
|
|
9
|
+
-- falls back to `{ "sort": sort }`.
|
|
10
|
+
--
|
|
11
|
+
-- No CHECK on `routing`, for the reason 0121 has none on `sort`.
|
|
12
|
+
ALTER TABLE stackbone_platform.model_routing
|
|
13
|
+
ADD COLUMN IF NOT EXISTS routing jsonb;
|
|
14
|
+
|
|
15
|
+
ALTER TABLE stackbone_platform.model_routing
|
|
16
|
+
ALTER COLUMN sort DROP NOT NULL;
|
package/package.json
CHANGED
|
Binary file
|
package/voice-worker/main.js
CHANGED
|
@@ -149,7 +149,7 @@ var loadVad = async () => {
|
|
|
149
149
|
};
|
|
150
150
|
|
|
151
151
|
// libs/runtime/voice-worker/src/worker.ts
|
|
152
|
-
import { log as
|
|
152
|
+
import { log as log5, voice as voice4 } from "@livekit/agents";
|
|
153
153
|
import {
|
|
154
154
|
BackgroundVoiceCancellation,
|
|
155
155
|
TelephonyBackgroundVoiceCancellation
|
|
@@ -975,8 +975,12 @@ var turnSchema = z8.strictObject({
|
|
|
975
975
|
mode: z8.enum(["stt", "livekit"]),
|
|
976
976
|
minEndpointingDelayMs: millisecondsSchema,
|
|
977
977
|
maxEndpointingDelayMs: millisecondsSchema,
|
|
978
|
-
detectorThreshold: thresholdSchema
|
|
978
|
+
detectorThreshold: thresholdSchema,
|
|
979
|
+
// Flux refuses under 0.3 and above its own end-of-turn threshold (0.7 by default,
|
|
980
|
+
// not a setting), so the bound is the range Flux takes with that default.
|
|
981
|
+
eagerThreshold: z8.number().min(0.3).max(0.7)
|
|
979
982
|
});
|
|
983
|
+
var VOICE_EAGER_THRESHOLD_DEFAULT = 0.3;
|
|
980
984
|
var interruptionsSchema = z8.strictObject({
|
|
981
985
|
enabled: z8.boolean(),
|
|
982
986
|
minDurationMs: millisecondsSchema,
|
|
@@ -995,13 +999,21 @@ var backgroundAudioSchema = z8.strictObject({
|
|
|
995
999
|
ambientSound: z8.enum(VOICE_AMBIENT_SOUNDS),
|
|
996
1000
|
ambientVolume: volumeSchema,
|
|
997
1001
|
thinkingSound: z8.enum(VOICE_THINKING_SOUNDS),
|
|
998
|
-
thinkingVolume: volumeSchema
|
|
1002
|
+
thinkingVolume: volumeSchema,
|
|
1003
|
+
/**
|
|
1004
|
+
* How long a reply's wait runs before the thinking sound starts, when no box tool
|
|
1005
|
+
* call is open (a tool call starts it at once). Every turn waits for the knowledge
|
|
1006
|
+
* search and the model's first token, about 1.4 s at the median on a voice box
|
|
1007
|
+
* (2026-11-09), so a delay under that plays the sound on nearly every reply.
|
|
1008
|
+
*/
|
|
1009
|
+
thinkingDelayMs: millisecondsSchema
|
|
999
1010
|
});
|
|
1000
1011
|
var VOICE_BACKGROUND_AUDIO_DEFAULTS = Object.freeze({
|
|
1001
1012
|
ambientSound: "office-ambience",
|
|
1002
1013
|
ambientVolume: 0.7,
|
|
1003
1014
|
thinkingSound: "keyboard-typing",
|
|
1004
|
-
thinkingVolume: 0.6
|
|
1015
|
+
thinkingVolume: 0.6,
|
|
1016
|
+
thinkingDelayMs: 2e3
|
|
1005
1017
|
});
|
|
1006
1018
|
var effectiveVoiceSettingsObjectSchema = z8.strictObject({
|
|
1007
1019
|
/** BCP-47-ish tag the STT is hinted with and the TTS speaks (`es`, `en-US`), or `multi`. */
|
|
@@ -1009,7 +1021,12 @@ var effectiveVoiceSettingsObjectSchema = z8.strictObject({
|
|
|
1009
1021
|
sttModel: z8.enum(VOICE_STT_MODELS),
|
|
1010
1022
|
keyterms: keytermsSchema.default(() => []),
|
|
1011
1023
|
vad: vadSchema,
|
|
1012
|
-
|
|
1024
|
+
// Defaulted here, on the read side only: a box older than STA3-94 answers, and
|
|
1025
|
+
// dispatches, a `turn` without it. Not on `turnSchema`, whose `.partial()` layers
|
|
1026
|
+
// would then all declare it.
|
|
1027
|
+
turn: turnSchema.extend({
|
|
1028
|
+
eagerThreshold: turnSchema.shape.eagerThreshold.default(VOICE_EAGER_THRESHOLD_DEFAULT)
|
|
1029
|
+
}),
|
|
1013
1030
|
interruptions: interruptionsSchema,
|
|
1014
1031
|
ttsModel: z8.enum(VOICE_TTS_MODELS),
|
|
1015
1032
|
/**
|
|
@@ -1078,7 +1095,12 @@ var callVoiceSettingsObjectSchema = effectiveVoiceSettingsObjectSchema.omit({
|
|
|
1078
1095
|
rag: true,
|
|
1079
1096
|
refusalPhrase: true
|
|
1080
1097
|
}).extend({
|
|
1081
|
-
|
|
1098
|
+
// A box older than `thinkingDelayMs` dispatches the group without it.
|
|
1099
|
+
backgroundAudio: backgroundAudioSchema.extend({
|
|
1100
|
+
thinkingDelayMs: millisecondsSchema.default(
|
|
1101
|
+
VOICE_BACKGROUND_AUDIO_DEFAULTS.thinkingDelayMs
|
|
1102
|
+
)
|
|
1103
|
+
}).default(() => ({ ...VOICE_BACKGROUND_AUDIO_DEFAULTS }))
|
|
1082
1104
|
});
|
|
1083
1105
|
var callVoiceSettingsSchema = callVoiceSettingsObjectSchema.superRefine(checkCrossFieldRules);
|
|
1084
1106
|
var VOICE_SETTINGS_LEAF_PATHS = [
|
|
@@ -1092,6 +1114,7 @@ var VOICE_SETTINGS_LEAF_PATHS = [
|
|
|
1092
1114
|
"turn.minEndpointingDelayMs",
|
|
1093
1115
|
"turn.maxEndpointingDelayMs",
|
|
1094
1116
|
"turn.detectorThreshold",
|
|
1117
|
+
"turn.eagerThreshold",
|
|
1095
1118
|
"interruptions.enabled",
|
|
1096
1119
|
"interruptions.minDurationMs",
|
|
1097
1120
|
"interruptions.minWords",
|
|
@@ -1105,6 +1128,7 @@ var VOICE_SETTINGS_LEAF_PATHS = [
|
|
|
1105
1128
|
"backgroundAudio.ambientVolume",
|
|
1106
1129
|
"backgroundAudio.thinkingSound",
|
|
1107
1130
|
"backgroundAudio.thinkingVolume",
|
|
1131
|
+
"backgroundAudio.thinkingDelayMs",
|
|
1108
1132
|
"recording",
|
|
1109
1133
|
"rag.collection",
|
|
1110
1134
|
"rag.minScore",
|
|
@@ -1115,11 +1139,19 @@ var VOICE_SETTINGS_LEAF_PATHS = [
|
|
|
1115
1139
|
"refusalPhrase"
|
|
1116
1140
|
];
|
|
1117
1141
|
var voiceSettingOriginSchema = z8.enum(["studio", "code", "default"]);
|
|
1142
|
+
var LEAVES_NEWER_THAN_SOME_BOXES = /* @__PURE__ */ new Set([
|
|
1143
|
+
"turn.eagerThreshold"
|
|
1144
|
+
]);
|
|
1118
1145
|
var effectiveVoiceSettingsViewSchema = z8.strictObject({
|
|
1119
1146
|
value: effectiveVoiceSettingsObjectSchema,
|
|
1120
1147
|
origins: z8.strictObject(
|
|
1121
1148
|
Object.fromEntries(
|
|
1122
|
-
|
|
1149
|
+
// A leaf newer than the box answering does not come back: that box runs its default.
|
|
1150
|
+
// Every other leaf stays required, so a box that drops one is still caught.
|
|
1151
|
+
VOICE_SETTINGS_LEAF_PATHS.map((path) => [
|
|
1152
|
+
path,
|
|
1153
|
+
LEAVES_NEWER_THAN_SOME_BOXES.has(path) ? voiceSettingOriginSchema.default("default") : voiceSettingOriginSchema
|
|
1154
|
+
])
|
|
1123
1155
|
)
|
|
1124
1156
|
)
|
|
1125
1157
|
});
|
|
@@ -1233,6 +1265,9 @@ var VOICE_CALLS_PATH = "/voice/v1/calls";
|
|
|
1233
1265
|
var voiceCallBootstrapSchema = z8.strictObject({
|
|
1234
1266
|
identities: integrationIdentitiesSchema.optional()
|
|
1235
1267
|
});
|
|
1268
|
+
var voiceCallKnowledgeSchema = z8.strictObject({
|
|
1269
|
+
text: z8.string().min(1).max(HEARD_TEXT_MAX)
|
|
1270
|
+
});
|
|
1236
1271
|
var voiceCallTranscriptSchema = z8.array(
|
|
1237
1272
|
z8.strictObject({
|
|
1238
1273
|
role: z8.enum(["user", "assistant"]),
|
|
@@ -1252,7 +1287,9 @@ var voiceCallTurnMetricsSchema = z8.strictObject({
|
|
|
1252
1287
|
transcriptionMs: turnLatencyMs,
|
|
1253
1288
|
llmTtftMs: turnLatencyMs,
|
|
1254
1289
|
ttsTtfbMs: turnLatencyMs,
|
|
1255
|
-
e2eMs: turnLatencyMs
|
|
1290
|
+
e2eMs: turnLatencyMs,
|
|
1291
|
+
/** How long before the final transcript the turn's last eager end of turn left for the box (STA3-94). */
|
|
1292
|
+
preflightLeadMs: turnLatencyMs
|
|
1256
1293
|
});
|
|
1257
1294
|
var voiceCallTurnsSchema = z8.array(voiceCallTurnMetricsSchema).max(VOICE_CALL_TURNS_MAX);
|
|
1258
1295
|
var voiceCallCompleteSchema = z8.strictObject({
|
|
@@ -1302,6 +1339,7 @@ async function* readSseData(body) {
|
|
|
1302
1339
|
// libs/runtime/voice-worker/src/box-wire.ts
|
|
1303
1340
|
var BODY_EXCERPT_CHARS = 300;
|
|
1304
1341
|
var TURN_IDLE_TIMEOUT_MS = 3e4;
|
|
1342
|
+
var KNOWLEDGE_TIMEOUT_MS = 2e3;
|
|
1305
1343
|
var NON_ASCII = /[\u007f-\uffff]/g;
|
|
1306
1344
|
var runErrorOf = ({ message, code }) => new Error(`box run failed${code ? ` (${code})` : ""}: ${message}`);
|
|
1307
1345
|
var BoxTurnError = class extends Error {
|
|
@@ -1410,23 +1448,39 @@ var toWireEvent = (ev) => {
|
|
|
1410
1448
|
return { kind: "other", type };
|
|
1411
1449
|
}
|
|
1412
1450
|
};
|
|
1451
|
+
var postCallOperation = (wire, operation, payload, timeoutMs) => fetch(
|
|
1452
|
+
boxEndpoint(
|
|
1453
|
+
wire.boxUrl,
|
|
1454
|
+
`${VOICE_CALLS_PATH}/${encodeURIComponent(wire.sessionKey)}/${operation}`
|
|
1455
|
+
),
|
|
1456
|
+
{
|
|
1457
|
+
method: "POST",
|
|
1458
|
+
headers: { "content-type": "application/json", authorization: `Bearer ${wire.token}` },
|
|
1459
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
1460
|
+
body: JSON.stringify(payload)
|
|
1461
|
+
}
|
|
1462
|
+
);
|
|
1413
1463
|
var voiceCallRequest = async (wire, operation, payload) => {
|
|
1414
|
-
const response = await
|
|
1415
|
-
boxEndpoint(
|
|
1416
|
-
wire.boxUrl,
|
|
1417
|
-
`${VOICE_CALLS_PATH}/${encodeURIComponent(wire.sessionKey)}/${operation}`
|
|
1418
|
-
),
|
|
1419
|
-
{
|
|
1420
|
-
method: "POST",
|
|
1421
|
-
headers: { "content-type": "application/json", authorization: `Bearer ${wire.token}` },
|
|
1422
|
-
signal: AbortSignal.timeout(TURN_IDLE_TIMEOUT_MS),
|
|
1423
|
-
body: JSON.stringify(payload)
|
|
1424
|
-
}
|
|
1425
|
-
);
|
|
1464
|
+
const response = await postCallOperation(wire, operation, payload, TURN_IDLE_TIMEOUT_MS);
|
|
1426
1465
|
if (!response.ok)
|
|
1427
1466
|
throw new BoxTurnError(wire.agent, response.status, await readExcerpt(response));
|
|
1428
1467
|
voiceCallLifecycleResponseSchema.parse(await response.json());
|
|
1429
1468
|
};
|
|
1469
|
+
var voiceCallKnowledge = async (wire, text) => {
|
|
1470
|
+
try {
|
|
1471
|
+
const response = await postCallOperation(
|
|
1472
|
+
wire,
|
|
1473
|
+
"knowledge",
|
|
1474
|
+
{ text },
|
|
1475
|
+
KNOWLEDGE_TIMEOUT_MS
|
|
1476
|
+
);
|
|
1477
|
+
await response.body?.cancel().catch(() => void 0);
|
|
1478
|
+
if (response.ok) return "sent";
|
|
1479
|
+
return response.status === 404 ? "gone" : "failed";
|
|
1480
|
+
} catch {
|
|
1481
|
+
return "failed";
|
|
1482
|
+
}
|
|
1483
|
+
};
|
|
1430
1484
|
|
|
1431
1485
|
// libs/runtime/voice-worker/src/heard.ts
|
|
1432
1486
|
var VOICE_RESPONSE_MARKER = "stackboneVoiceResponse";
|
|
@@ -1607,11 +1661,13 @@ var defined = (metrics) => Object.fromEntries(Object.entries(metrics).filter(([,
|
|
|
1607
1661
|
var watchTurnMetrics = (session, write, options = {}) => {
|
|
1608
1662
|
let caller;
|
|
1609
1663
|
let delivered;
|
|
1664
|
+
let lead;
|
|
1610
1665
|
session.on(voice2.AgentSessionEventTypes.ConversationItemAdded, ({ item }) => {
|
|
1611
1666
|
if (!(item instanceof llm.ChatMessage)) return;
|
|
1612
1667
|
if (item.role === "user") {
|
|
1613
1668
|
caller = item.metrics;
|
|
1614
1669
|
delivered = options.endOfTurnMs?.();
|
|
1670
|
+
lead = options.preflightLeadMs?.();
|
|
1615
1671
|
return;
|
|
1616
1672
|
}
|
|
1617
1673
|
if (item.role !== "assistant") return;
|
|
@@ -1623,10 +1679,12 @@ var watchTurnMetrics = (session, write, options = {}) => {
|
|
|
1623
1679
|
transcriptionMs: options.endOfTurnMs ? void 0 : ms(caller?.transcriptionDelay),
|
|
1624
1680
|
llmTtftMs: ms(reply.llmNodeTtft),
|
|
1625
1681
|
ttsTtfbMs: ms(reply.ttsNodeTtfb),
|
|
1626
|
-
e2eMs: e2eMs === void 0 || delivered === void 0 ? e2eMs : e2eMs + delivered
|
|
1682
|
+
e2eMs: e2eMs === void 0 || delivered === void 0 ? e2eMs : e2eMs + delivered,
|
|
1683
|
+
preflightLeadMs: lead === void 0 ? void 0 : Math.round(lead)
|
|
1627
1684
|
});
|
|
1628
1685
|
caller = void 0;
|
|
1629
1686
|
delivered = void 0;
|
|
1687
|
+
lead = void 0;
|
|
1630
1688
|
if (Object.keys(metrics).length === 0) return;
|
|
1631
1689
|
write(typeof runId === "string" ? { ...metrics, runId } : metrics);
|
|
1632
1690
|
});
|
|
@@ -1965,15 +2023,56 @@ var createHangUp = ({
|
|
|
1965
2023
|
// libs/runtime/voice-worker/src/session-options.ts
|
|
1966
2024
|
import { tts, voice as voice3 } from "@livekit/agents";
|
|
1967
2025
|
|
|
1968
|
-
// libs/runtime/voice-worker/src/
|
|
2026
|
+
// libs/runtime/voice-worker/src/knowledge-prefetch.ts
|
|
2027
|
+
import { log as log3, stt as stt2 } from "@livekit/agents";
|
|
1969
2028
|
import { TransformStream as TransformStream2 } from "node:stream/web";
|
|
2029
|
+
var knowledgeTap = (send, { onLead, now = Date.now } = {}) => {
|
|
2030
|
+
let sentThisTurn;
|
|
2031
|
+
let sentAt;
|
|
2032
|
+
let gone = false;
|
|
2033
|
+
let failureLogged = false;
|
|
2034
|
+
const tap = (event) => {
|
|
2035
|
+
if (gone || typeof event === "string") return;
|
|
2036
|
+
if (event.type === stt2.SpeechEventType.FINAL_TRANSCRIPT) {
|
|
2037
|
+
if (sentAt !== void 0) onLead?.(now() - sentAt);
|
|
2038
|
+
sentThisTurn = void 0;
|
|
2039
|
+
sentAt = void 0;
|
|
2040
|
+
return;
|
|
2041
|
+
}
|
|
2042
|
+
if (event.type !== stt2.SpeechEventType.PREFLIGHT_TRANSCRIPT) return;
|
|
2043
|
+
const text = event.alternatives?.[0]?.text.trim();
|
|
2044
|
+
if (!text || text === sentThisTurn) return;
|
|
2045
|
+
sentThisTurn = text;
|
|
2046
|
+
sentAt = now();
|
|
2047
|
+
void send(text).then((answer) => {
|
|
2048
|
+
if (answer === "gone") gone = true;
|
|
2049
|
+
if (answer === "failed" && !failureLogged) {
|
|
2050
|
+
failureLogged = true;
|
|
2051
|
+
log3().warn(
|
|
2052
|
+
"voice worker: the box refused a knowledge prefetch; turns search on their own"
|
|
2053
|
+
);
|
|
2054
|
+
}
|
|
2055
|
+
}).catch(() => void 0);
|
|
2056
|
+
};
|
|
2057
|
+
return (events) => events.pipeThrough(
|
|
2058
|
+
new TransformStream2({
|
|
2059
|
+
transform(event, controller) {
|
|
2060
|
+
tap(event);
|
|
2061
|
+
controller.enqueue(event);
|
|
2062
|
+
}
|
|
2063
|
+
})
|
|
2064
|
+
);
|
|
2065
|
+
};
|
|
2066
|
+
|
|
2067
|
+
// libs/runtime/voice-worker/src/number-text.ts
|
|
2068
|
+
import { TransformStream as TransformStream3 } from "node:stream/web";
|
|
1970
2069
|
var THOUSANDS_COMMA = /(\d),(?=\d{3}(?!\d))/g;
|
|
1971
2070
|
var TRAILING_NUMBER = /[\d.,]+$/;
|
|
1972
2071
|
var rewrite = (text) => text.replace(THOUSANDS_COMMA, "$1");
|
|
1973
2072
|
var dropThousandsComma = (input) => {
|
|
1974
2073
|
let pending = "";
|
|
1975
2074
|
return input.pipeThrough(
|
|
1976
|
-
new
|
|
2075
|
+
new TransformStream3({
|
|
1977
2076
|
transform(chunk, controller) {
|
|
1978
2077
|
const text = pending + chunk;
|
|
1979
2078
|
const tail = TRAILING_NUMBER.exec(text)?.[0] ?? "";
|
|
@@ -1990,7 +2089,7 @@ var dropThousandsComma = (input) => {
|
|
|
1990
2089
|
|
|
1991
2090
|
// libs/runtime/voice-worker/src/phone-voice.ts
|
|
1992
2091
|
import { AudioFrame as AudioFrame2 } from "@livekit/rtc-node";
|
|
1993
|
-
import { TransformStream as
|
|
2092
|
+
import { TransformStream as TransformStream4 } from "node:stream/web";
|
|
1994
2093
|
var LOW_CUT_HZ = 300;
|
|
1995
2094
|
var HIGH_CUT_HZ = 3400;
|
|
1996
2095
|
var Q = Math.SQRT1_2;
|
|
@@ -2026,7 +2125,7 @@ var phoneLine = (audio) => {
|
|
|
2026
2125
|
let chainRate = 0;
|
|
2027
2126
|
const norm = Math.tanh(DRIVE);
|
|
2028
2127
|
return audio.pipeThrough(
|
|
2029
|
-
new
|
|
2128
|
+
new TransformStream4({
|
|
2030
2129
|
transform(frame, controller) {
|
|
2031
2130
|
if (!chain || chainRate !== frame.sampleRate) {
|
|
2032
2131
|
chainRate = frame.sampleRate;
|
|
@@ -2052,11 +2151,18 @@ var phoneLine = (audio) => {
|
|
|
2052
2151
|
};
|
|
2053
2152
|
|
|
2054
2153
|
// libs/runtime/voice-worker/src/session-options.ts
|
|
2055
|
-
var sttOptions = ({ sttModel, language, keyterms }) => {
|
|
2154
|
+
var sttOptions = ({ sttModel, language, keyterms, turn }) => {
|
|
2056
2155
|
const terms = keyterms.length > 0 ? [...keyterms] : void 0;
|
|
2057
2156
|
const keyterm = terms ? { keyterm: terms } : void 0;
|
|
2058
2157
|
if (sttModel.startsWith("deepgram/flux")) {
|
|
2059
|
-
return {
|
|
2158
|
+
return {
|
|
2159
|
+
model: sttModel,
|
|
2160
|
+
modelOptions: {
|
|
2161
|
+
language_hint: language,
|
|
2162
|
+
...keyterm,
|
|
2163
|
+
eager_eot_threshold: turn.eagerThreshold
|
|
2164
|
+
}
|
|
2165
|
+
};
|
|
2060
2166
|
}
|
|
2061
2167
|
if (sttModel.startsWith("assemblyai/")) {
|
|
2062
2168
|
return {
|
|
@@ -2080,12 +2186,28 @@ var markupContext = (expressive, speech) => {
|
|
|
2080
2186
|
return guide === void 0 ? [] : [{ description: VOICE_TTS_MARKUP_CONTEXT, value: guide }];
|
|
2081
2187
|
};
|
|
2082
2188
|
var VoiceCallAgent = class extends voice3.Agent {
|
|
2083
|
-
constructor() {
|
|
2084
|
-
super(
|
|
2189
|
+
constructor({ sendKnowledge, ...options }) {
|
|
2190
|
+
super(options);
|
|
2085
2191
|
this.sttDelivery = createSttDeliveryMeter();
|
|
2192
|
+
this.#knowledge = sendKnowledge && knowledgeTap(sendKnowledge, {
|
|
2193
|
+
onLead: (ms2) => {
|
|
2194
|
+
this.#preflightLead = ms2;
|
|
2195
|
+
}
|
|
2196
|
+
});
|
|
2197
|
+
}
|
|
2198
|
+
#knowledge;
|
|
2199
|
+
#preflightLead;
|
|
2200
|
+
/** The last turn's preflight head start in ms, once: a turn never reuses another's. */
|
|
2201
|
+
takePreflightLead() {
|
|
2202
|
+
const lead = this.#preflightLead;
|
|
2203
|
+
this.#preflightLead = void 0;
|
|
2204
|
+
return lead;
|
|
2086
2205
|
}
|
|
2087
2206
|
async sttNode(audio, modelSettings) {
|
|
2088
|
-
return this.sttDelivery.wrap(audio, (counted) =>
|
|
2207
|
+
return this.sttDelivery.wrap(audio, async (counted) => {
|
|
2208
|
+
const events = await super.sttNode(counted, modelSettings);
|
|
2209
|
+
return events && this.#knowledge ? this.#knowledge(events) : events;
|
|
2210
|
+
});
|
|
2089
2211
|
}
|
|
2090
2212
|
};
|
|
2091
2213
|
var PhoneLineAgent = class extends VoiceCallAgent {
|
|
@@ -2102,6 +2224,11 @@ var voiceSessionOptions = (settings, pieces) => {
|
|
|
2102
2224
|
minSilenceDuration: vad.minSilenceMs
|
|
2103
2225
|
});
|
|
2104
2226
|
const speech = pieces.createTts(ttsOptions(settings));
|
|
2227
|
+
const agentOptions = {
|
|
2228
|
+
// The box holds the prompt: LiveKit's own chat context stays prompt-less.
|
|
2229
|
+
instructions: "",
|
|
2230
|
+
sendKnowledge: pieces.sendKnowledge
|
|
2231
|
+
};
|
|
2105
2232
|
return {
|
|
2106
2233
|
session: {
|
|
2107
2234
|
vad: pieces.vad,
|
|
@@ -2141,14 +2268,13 @@ var voiceSessionOptions = (settings, pieces) => {
|
|
|
2141
2268
|
preemptiveGeneration: { enabled: false, preemptiveTts: false }
|
|
2142
2269
|
}
|
|
2143
2270
|
},
|
|
2144
|
-
|
|
2145
|
-
agent: settings.phoneLine ? new PhoneLineAgent({ instructions: "" }) : new VoiceCallAgent({ instructions: "" }),
|
|
2271
|
+
agent: settings.phoneLine ? new PhoneLineAgent(agentOptions) : new VoiceCallAgent(agentOptions),
|
|
2146
2272
|
boxContext: markupContext(settings.expressive, speech)
|
|
2147
2273
|
};
|
|
2148
2274
|
};
|
|
2149
2275
|
|
|
2150
2276
|
// libs/runtime/voice-worker/src/telemetry/call-traces.ts
|
|
2151
|
-
import { log as
|
|
2277
|
+
import { log as log4, telemetry } from "@livekit/agents";
|
|
2152
2278
|
import { ExportResultCode } from "@opentelemetry/core";
|
|
2153
2279
|
import { defaultResource, resourceFromAttributes } from "@opentelemetry/resources";
|
|
2154
2280
|
import {
|
|
@@ -2160,7 +2286,7 @@ var warnOnRefusedBatch = (exporter, sessionKey) => ({
|
|
|
2160
2286
|
export: (spans, done) => exporter.export(spans, (result) => {
|
|
2161
2287
|
try {
|
|
2162
2288
|
if (result.code !== ExportResultCode.SUCCESS) {
|
|
2163
|
-
|
|
2289
|
+
log4().warn(
|
|
2164
2290
|
{ sessionKey, error: result.error?.message ?? "unknown" },
|
|
2165
2291
|
"voice worker: a batch of call spans failed to export to the box"
|
|
2166
2292
|
);
|
|
@@ -2220,13 +2346,14 @@ var CALL_CLOSING_PHRASE = Object.freeze({
|
|
|
2220
2346
|
});
|
|
2221
2347
|
var MAX_FAILED_TURNS_IN_A_ROW = 3;
|
|
2222
2348
|
var inCallLanguage = (phrase, language) => language.split("-")[0]?.toLowerCase() === "es" ? phrase.es : phrase.en;
|
|
2223
|
-
var thinkingCue = () => {
|
|
2349
|
+
var thinkingCue = (delayMs) => {
|
|
2224
2350
|
let cue;
|
|
2225
2351
|
let open = 0;
|
|
2226
|
-
let
|
|
2352
|
+
let waitedLong = false;
|
|
2353
|
+
let wait;
|
|
2227
2354
|
let playing = false;
|
|
2228
2355
|
const sync = () => {
|
|
2229
|
-
const play =
|
|
2356
|
+
const play = waitedLong || open > 0;
|
|
2230
2357
|
if (play === playing || !cue) return;
|
|
2231
2358
|
playing = play;
|
|
2232
2359
|
if (play) cue.start();
|
|
@@ -2239,13 +2366,32 @@ var thinkingCue = () => {
|
|
|
2239
2366
|
sync();
|
|
2240
2367
|
},
|
|
2241
2368
|
onAgentState: (state) => {
|
|
2242
|
-
|
|
2243
|
-
|
|
2369
|
+
if (state !== "thinking") {
|
|
2370
|
+
clearTimeout(wait);
|
|
2371
|
+
wait = void 0;
|
|
2372
|
+
waitedLong = false;
|
|
2373
|
+
sync();
|
|
2374
|
+
return;
|
|
2375
|
+
}
|
|
2376
|
+
if (wait !== void 0 || waitedLong) return;
|
|
2377
|
+
wait = setTimeout(() => {
|
|
2378
|
+
wait = void 0;
|
|
2379
|
+
waitedLong = true;
|
|
2380
|
+
sync();
|
|
2381
|
+
}, delayMs);
|
|
2244
2382
|
},
|
|
2245
2383
|
onToolCall: (phase) => {
|
|
2246
2384
|
if (phase === "start") open += 1;
|
|
2247
2385
|
else if (open > 0) open -= 1;
|
|
2248
2386
|
sync();
|
|
2387
|
+
},
|
|
2388
|
+
/** The call is ending: a pending delay must not start the sound in a closed room. */
|
|
2389
|
+
close: () => {
|
|
2390
|
+
clearTimeout(wait);
|
|
2391
|
+
wait = void 0;
|
|
2392
|
+
waitedLong = false;
|
|
2393
|
+
open = 0;
|
|
2394
|
+
sync();
|
|
2249
2395
|
}
|
|
2250
2396
|
};
|
|
2251
2397
|
};
|
|
@@ -2254,14 +2400,14 @@ var callerNoiseCancellation = (caller) => {
|
|
|
2254
2400
|
try {
|
|
2255
2401
|
return caller.kind === ParticipantKind.SIP ? TelephonyBackgroundVoiceCancellation() : BackgroundVoiceCancellation();
|
|
2256
2402
|
} catch (err) {
|
|
2257
|
-
|
|
2403
|
+
log5().warn({ error: messageOf(err) }, "voice worker: no noise cancellation on this call");
|
|
2258
2404
|
return void 0;
|
|
2259
2405
|
}
|
|
2260
2406
|
};
|
|
2261
2407
|
var runVoiceJob = async (ctx, deps) => {
|
|
2262
2408
|
const dispatch = readDispatch(ctx.job.metadata);
|
|
2263
2409
|
if (!dispatch) {
|
|
2264
|
-
|
|
2410
|
+
log5().warn(
|
|
2265
2411
|
{ metadataChars: ctx.job.metadata?.length ?? 0 },
|
|
2266
2412
|
"voice worker: refused a job without a valid dispatch"
|
|
2267
2413
|
);
|
|
@@ -2270,7 +2416,7 @@ var runVoiceJob = async (ctx, deps) => {
|
|
|
2270
2416
|
}
|
|
2271
2417
|
await ctx.connect();
|
|
2272
2418
|
const caller = await ctx.waitForParticipant().catch((err) => {
|
|
2273
|
-
|
|
2419
|
+
log5().info(
|
|
2274
2420
|
{ reason: messageOf(err) },
|
|
2275
2421
|
"voice worker: the room closed before the caller joined"
|
|
2276
2422
|
);
|
|
@@ -2290,7 +2436,7 @@ var runVoiceJob = async (ctx, deps) => {
|
|
|
2290
2436
|
deps
|
|
2291
2437
|
);
|
|
2292
2438
|
const settings = dispatch.voiceSettings;
|
|
2293
|
-
const cue = thinkingCue();
|
|
2439
|
+
const cue = thinkingCue(settings.backgroundAudio.thinkingDelayMs);
|
|
2294
2440
|
const fallback = inCallLanguage(TURN_FALLBACK_PHRASE, settings.language);
|
|
2295
2441
|
let failedInARow = 0;
|
|
2296
2442
|
let ending = false;
|
|
@@ -2298,22 +2444,22 @@ var runVoiceJob = async (ctx, deps) => {
|
|
|
2298
2444
|
try {
|
|
2299
2445
|
session.say(text);
|
|
2300
2446
|
} catch (err) {
|
|
2301
|
-
|
|
2447
|
+
log5().info({ error: messageOf(err) }, `voice worker: the ${what} was not spoken`);
|
|
2302
2448
|
}
|
|
2303
2449
|
};
|
|
2304
2450
|
const hangUpAfter = (played, reason) => {
|
|
2305
2451
|
ending = true;
|
|
2306
2452
|
session.input.setAudioEnabled(false);
|
|
2307
2453
|
void Promise.resolve().then(played).catch((err) => {
|
|
2308
|
-
|
|
2454
|
+
log5().info({ error: messageOf(err) }, "voice worker: the closing phrase was not spoken");
|
|
2309
2455
|
}).then(() => ctx.deleteRoom()).catch((err) => {
|
|
2310
|
-
|
|
2456
|
+
log5().warn({ error: messageOf(err) }, "voice worker: the room was not deleted");
|
|
2311
2457
|
}).then(
|
|
2312
2458
|
() => finishCall(
|
|
2313
2459
|
reason === "turns kept failing" || reason === "box refused the call token" ? "failed" : "completed"
|
|
2314
2460
|
)
|
|
2315
2461
|
).catch(() => {
|
|
2316
|
-
|
|
2462
|
+
log5().warn(
|
|
2317
2463
|
{ agent: dispatch.agent, sessionKey: dispatch.sessionKey, reason: "completion_failed" },
|
|
2318
2464
|
"voice worker: call finalization failed"
|
|
2319
2465
|
);
|
|
@@ -2333,12 +2479,12 @@ var runVoiceJob = async (ctx, deps) => {
|
|
|
2333
2479
|
});
|
|
2334
2480
|
const onTurnError = (err) => {
|
|
2335
2481
|
if (err instanceof BoxTurnError && err.status === 401) {
|
|
2336
|
-
|
|
2482
|
+
log5().error(boxFailure(err), "voice worker: the box refused the call token (401)");
|
|
2337
2483
|
if (!ending) endCall(CALL_CLOSING_PHRASE, "box refused the call token");
|
|
2338
2484
|
return;
|
|
2339
2485
|
}
|
|
2340
2486
|
failedInARow += 1;
|
|
2341
|
-
|
|
2487
|
+
log5().warn({ ...boxFailure(err), failedInARow }, "voice worker: a turn failed");
|
|
2342
2488
|
if (ending) return;
|
|
2343
2489
|
if (failedInARow >= MAX_FAILED_TURNS_IN_A_ROW) {
|
|
2344
2490
|
endCall(CALL_CLOSING_PHRASE, "turns kept failing");
|
|
@@ -2350,13 +2496,20 @@ var runVoiceJob = async (ctx, deps) => {
|
|
|
2350
2496
|
failedInARow = 0;
|
|
2351
2497
|
if (outcome === "interrupt" && !ending) endCall(PAUSED_RUN_PHRASE, "run paused for review");
|
|
2352
2498
|
};
|
|
2353
|
-
const
|
|
2354
|
-
const wire = {
|
|
2499
|
+
const call = {
|
|
2355
2500
|
boxUrl: dispatch.boxUrl,
|
|
2356
2501
|
token: dispatch.callToken,
|
|
2357
2502
|
agent: dispatch.agent,
|
|
2358
2503
|
sessionKey: dispatch.sessionKey,
|
|
2359
|
-
tags: dispatch.tags
|
|
2504
|
+
tags: dispatch.tags
|
|
2505
|
+
};
|
|
2506
|
+
const plan = voiceSessionOptions(settings, {
|
|
2507
|
+
...deps,
|
|
2508
|
+
vad: ctx.proc.userData.vad,
|
|
2509
|
+
sendKnowledge: (text) => voiceCallKnowledge(call, text)
|
|
2510
|
+
});
|
|
2511
|
+
const wire = {
|
|
2512
|
+
...call,
|
|
2360
2513
|
// With `expressive`, the TTS markup guide: LiveKit gives it only to its own chat context.
|
|
2361
2514
|
context: plan.boxContext
|
|
2362
2515
|
};
|
|
@@ -2372,7 +2525,7 @@ var runVoiceJob = async (ctx, deps) => {
|
|
|
2372
2525
|
hangUpAfter,
|
|
2373
2526
|
isEnding: () => ending,
|
|
2374
2527
|
onResumeFailed: (err) => {
|
|
2375
|
-
|
|
2528
|
+
log5().warn(boxFailure(err), "voice worker: the hang_up resume failed");
|
|
2376
2529
|
}
|
|
2377
2530
|
});
|
|
2378
2531
|
const brain = new BoxLLM({
|
|
@@ -2391,6 +2544,7 @@ var runVoiceJob = async (ctx, deps) => {
|
|
|
2391
2544
|
const finishCall = (outcome = "disconnected") => {
|
|
2392
2545
|
if (completion) return completion;
|
|
2393
2546
|
ending = true;
|
|
2547
|
+
cue.close();
|
|
2394
2548
|
completion = Promise.resolve().then(async () => {
|
|
2395
2549
|
await hangUp.pendingResume();
|
|
2396
2550
|
await session.close();
|
|
@@ -2403,14 +2557,14 @@ var runVoiceJob = async (ctx, deps) => {
|
|
|
2403
2557
|
if (!(turns.length > 0 && error instanceof BoxTurnError && error.status === 400)) {
|
|
2404
2558
|
throw error;
|
|
2405
2559
|
}
|
|
2406
|
-
|
|
2560
|
+
log5().warn(
|
|
2407
2561
|
{ agent: dispatch.agent, sessionKey: dispatch.sessionKey, status: error.status },
|
|
2408
2562
|
"voice worker: the box refused the turn metrics, completing without them"
|
|
2409
2563
|
);
|
|
2410
2564
|
await complete({ transcript, outcome });
|
|
2411
2565
|
}
|
|
2412
2566
|
}).catch((error) => {
|
|
2413
|
-
|
|
2567
|
+
log5().warn(
|
|
2414
2568
|
{ agent: dispatch.agent, sessionKey: dispatch.sessionKey, reason: "completion_failed" },
|
|
2415
2569
|
"voice worker: call finalization failed"
|
|
2416
2570
|
);
|
|
@@ -2431,12 +2585,15 @@ var runVoiceJob = async (ctx, deps) => {
|
|
|
2431
2585
|
session,
|
|
2432
2586
|
(metrics) => {
|
|
2433
2587
|
if (turns.length < VOICE_CALL_TURNS_MAX) turns.push(metrics);
|
|
2434
|
-
|
|
2588
|
+
log5().info(
|
|
2435
2589
|
{ agent: dispatch.agent, sessionKey: dispatch.sessionKey, room: ctx.room.name, ...metrics },
|
|
2436
2590
|
VOICE_TURN_METRICS_MESSAGE
|
|
2437
2591
|
);
|
|
2438
2592
|
},
|
|
2439
|
-
|
|
2593
|
+
{
|
|
2594
|
+
...deliversItsOwnEndOfTurn(settings) ? { endOfTurnMs: () => plan.agent.sttDelivery.take() } : {},
|
|
2595
|
+
preflightLeadMs: () => plan.agent.takePreflightLead()
|
|
2596
|
+
}
|
|
2440
2597
|
);
|
|
2441
2598
|
const noiseCancellation = callerNoiseCancellation(participant);
|
|
2442
2599
|
await session.start({
|
|
@@ -2446,7 +2603,7 @@ var runVoiceJob = async (ctx, deps) => {
|
|
|
2446
2603
|
});
|
|
2447
2604
|
if (settings.greeting !== null) sayIfOpen(settings.greeting, "greeting");
|
|
2448
2605
|
await deps.createCue({ room: ctx.room, session, audio: settings.backgroundAudio }).then(cue.attach, (err) => {
|
|
2449
|
-
|
|
2606
|
+
log5().warn({ error: messageOf(err) }, "voice worker: the background audio did not start");
|
|
2450
2607
|
});
|
|
2451
2608
|
};
|
|
2452
2609
|
|