@stackbone/cli 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +83 -0
- package/main.js +2410 -615
- package/migrations/0117_rag_extract_fields.sql +3 -0
- package/migrations/0118_voice_outbound_calls.sql +29 -0
- package/migrations/0119_voice_call_metrics.sql +20 -0
- package/migrations/0120_run_steps_voice.sql +19 -0
- package/migrations/0121_model_routing.sql +28 -0
- package/package.json +1 -1
- package/stackbone-cli-0.8.0.tgz +0 -0
- package/voice-worker/main.js +232 -61
- package/voice-worker/main.js.map +4 -4
- package/stackbone-cli-0.7.1.tgz +0 -0
package/voice-worker/main.js
CHANGED
|
@@ -6,7 +6,7 @@ import {
|
|
|
6
6
|
defineAgent,
|
|
7
7
|
inference,
|
|
8
8
|
telemetry as telemetry2,
|
|
9
|
-
voice as
|
|
9
|
+
voice as voice5
|
|
10
10
|
} from "@livekit/agents";
|
|
11
11
|
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
|
|
12
12
|
|
|
@@ -24,11 +24,11 @@ var PATTERN = [
|
|
|
24
24
|
[0, 900]
|
|
25
25
|
];
|
|
26
26
|
var renderPattern = () => {
|
|
27
|
-
const total = PATTERN.reduce((n, [,
|
|
27
|
+
const total = PATTERN.reduce((n, [, ms2]) => n + SAMPLE_RATE * ms2 / 1e3, 0);
|
|
28
28
|
const pcm = new Int16Array(total);
|
|
29
29
|
let offset = 0;
|
|
30
|
-
for (const [hz,
|
|
31
|
-
const length = SAMPLE_RATE *
|
|
30
|
+
for (const [hz, ms2] of PATTERN) {
|
|
31
|
+
const length = SAMPLE_RATE * ms2 / 1e3;
|
|
32
32
|
for (let i = 0; hz > 0 && i < length; i++) {
|
|
33
33
|
const envelope = 0.5 - 0.5 * Math.cos(2 * Math.PI * i / length);
|
|
34
34
|
const sample = Math.sin(2 * Math.PI * hz * i / SAMPLE_RATE) * envelope * VOLUME;
|
|
@@ -101,8 +101,11 @@ var loadVad = async () => {
|
|
|
101
101
|
};
|
|
102
102
|
|
|
103
103
|
// libs/runtime/voice-worker/src/worker.ts
|
|
104
|
-
import { log as log4, voice as
|
|
105
|
-
import {
|
|
104
|
+
import { log as log4, voice as voice4 } from "@livekit/agents";
|
|
105
|
+
import {
|
|
106
|
+
BackgroundVoiceCancellation,
|
|
107
|
+
TelephonyBackgroundVoiceCancellation
|
|
108
|
+
} from "@livekit/noise-cancellation-node";
|
|
106
109
|
import { ParticipantKind } from "@livekit/rtc-node";
|
|
107
110
|
|
|
108
111
|
// libs/shared/validators/src/lib/common/json-object.ts
|
|
@@ -558,9 +561,22 @@ var ragCollectionNameParamSchema = z7.string().min(1).max(128).regex(STUDIO_RAG_
|
|
|
558
561
|
var ragDistanceSchema = z7.enum(["cosine", "l2", "ip"]);
|
|
559
562
|
var ragIndexKindSchema = z7.enum(["hnsw", "exact"]);
|
|
560
563
|
var ragNumberFormatSchema = z7.enum(["eu", "us"]);
|
|
564
|
+
var ragRangeFieldSchema = z7.string().min(1).refine((field) => !field.includes("."), {
|
|
565
|
+
message: "field must be a literal top-level metadata key without dots"
|
|
566
|
+
});
|
|
567
|
+
var ragExtractFieldSchema = z7.object({
|
|
568
|
+
name: z7.string().trim().pipe(ragRangeFieldSchema),
|
|
569
|
+
type: z7.enum(["number", "date", "text"]),
|
|
570
|
+
removed: z7.boolean().optional()
|
|
571
|
+
});
|
|
572
|
+
var uniqueFieldNames = (fields) => new Set(fields.map((field) => field.name)).size === fields.length;
|
|
573
|
+
var ragExtractFieldsRequestSchema = z7.array(ragExtractFieldSchema).refine(uniqueFieldNames, {
|
|
574
|
+
message: "Field names must be unique"
|
|
575
|
+
});
|
|
561
576
|
var ragCollectionSchema = z7.object({
|
|
562
577
|
name: z7.string(),
|
|
563
578
|
dimensions: z7.number().int().min(1),
|
|
579
|
+
embedding_state: z7.enum(["ready", "rebuilding", "failed"]).optional(),
|
|
564
580
|
// Optional for boxes that predate collection-owned embeddings.
|
|
565
581
|
embedding_provider: z7.string().min(1).optional(),
|
|
566
582
|
embedding_model: z7.string().min(1).optional(),
|
|
@@ -574,7 +590,9 @@ var ragCollectionSchema = z7.object({
|
|
|
574
590
|
last_ingested_at: z7.iso.datetime().nullable(),
|
|
575
591
|
// STA3-57. Optional so a list from a box older than hybrid search still parses.
|
|
576
592
|
hybrid_enabled: z7.boolean().optional(),
|
|
577
|
-
number_format: ragNumberFormatSchema.optional()
|
|
593
|
+
number_format: ragNumberFormatSchema.optional(),
|
|
594
|
+
extract_enabled: z7.boolean().optional(),
|
|
595
|
+
extract_fields: z7.array(ragExtractFieldSchema).optional()
|
|
578
596
|
});
|
|
579
597
|
var ragCollectionListResponseSchema = z7.object({
|
|
580
598
|
items: z7.array(ragCollectionSchema)
|
|
@@ -621,7 +639,7 @@ var ragDocumentSchema = z7.object({
|
|
|
621
639
|
pending_version: z7.number().int().min(1).nullable().optional()
|
|
622
640
|
});
|
|
623
641
|
var ragFilterSchema = jsonObject();
|
|
624
|
-
var
|
|
642
|
+
var ragDateValueSchema = z7.string().length(10).regex(/^[0-9]{4}-[0-9]{2}-[0-9]{2}$/).refine((value) => {
|
|
625
643
|
const year = Number(value.slice(0, 4));
|
|
626
644
|
const month = Number(value.slice(5, 7));
|
|
627
645
|
const day = Number(value.slice(8, 10));
|
|
@@ -629,9 +647,6 @@ var ragDateBoundSchema = z7.string().length(10).regex(/^[0-9]{4}-[0-9]{2}-[0-9]{
|
|
|
629
647
|
date.setUTCFullYear(year, month - 1, day);
|
|
630
648
|
return year >= 1 && year <= 9999 && date.getUTCFullYear() === year && date.getUTCMonth() === month - 1 && date.getUTCDate() === day;
|
|
631
649
|
}, "date must be a real YYYY-MM-DD day in years 0001\u20139999");
|
|
632
|
-
var ragRangeFieldSchema = z7.string().min(1).refine((field) => !field.includes("."), {
|
|
633
|
-
message: "field must be a literal top-level metadata key without dots"
|
|
634
|
-
});
|
|
635
650
|
var ragRangeSchema = z7.discriminatedUnion("type", [
|
|
636
651
|
z7.strictObject({
|
|
637
652
|
field: ragRangeFieldSchema,
|
|
@@ -644,10 +659,10 @@ var ragRangeSchema = z7.discriminatedUnion("type", [
|
|
|
644
659
|
z7.strictObject({
|
|
645
660
|
field: ragRangeFieldSchema,
|
|
646
661
|
type: z7.literal("date"),
|
|
647
|
-
gt:
|
|
648
|
-
gte:
|
|
649
|
-
lt:
|
|
650
|
-
lte:
|
|
662
|
+
gt: ragDateValueSchema.optional(),
|
|
663
|
+
gte: ragDateValueSchema.optional(),
|
|
664
|
+
lt: ragDateValueSchema.optional(),
|
|
665
|
+
lte: ragDateValueSchema.optional()
|
|
651
666
|
})
|
|
652
667
|
]).superRefine((range, ctx) => {
|
|
653
668
|
const lower = range.gt ?? range.gte;
|
|
@@ -690,6 +705,32 @@ var ragDocumentVersionListResponseSchema = z7.object({
|
|
|
690
705
|
items: z7.array(ragDocumentVersionSchema)
|
|
691
706
|
});
|
|
692
707
|
var ragDocumentVersionNumberSchema = z7.number().int().min(1).max(2147483647);
|
|
708
|
+
var updateRagDocumentValuesRequestSchema = z7.strictObject({
|
|
709
|
+
version: ragDocumentVersionNumberSchema.optional(),
|
|
710
|
+
values: jsonObject(),
|
|
711
|
+
add_fields: z7.array(ragExtractFieldSchema.omit({ removed: true })).refine(uniqueFieldNames, {
|
|
712
|
+
message: "Field names must be unique"
|
|
713
|
+
}).optional()
|
|
714
|
+
});
|
|
715
|
+
var updateRagDocumentValuesResponseSchema = z7.object({
|
|
716
|
+
id: z7.string(),
|
|
717
|
+
collection: z7.string(),
|
|
718
|
+
version: ragDocumentVersionNumberSchema,
|
|
719
|
+
metadata: jsonObject()
|
|
720
|
+
});
|
|
721
|
+
var extractRagDocumentRequestSchema = z7.strictObject({
|
|
722
|
+
version: ragDocumentVersionNumberSchema.optional()
|
|
723
|
+
});
|
|
724
|
+
var extractRagDocumentResponseSchema = z7.object({
|
|
725
|
+
proposal: z7.array(
|
|
726
|
+
ragExtractFieldSchema.omit({ removed: true }).extend({
|
|
727
|
+
value: z7.union([z7.number().finite(), z7.string()]),
|
|
728
|
+
new: z7.boolean()
|
|
729
|
+
})
|
|
730
|
+
),
|
|
731
|
+
failed: z7.boolean(),
|
|
732
|
+
truncated: z7.boolean()
|
|
733
|
+
});
|
|
693
734
|
var ragDocumentVersionParamSchema = z7.string().regex(/^\d+$/, { message: "version must be a positive integer" }).transform((value) => Number.parseInt(value, 10)).pipe(ragDocumentVersionNumberSchema);
|
|
694
735
|
var activateRagDocumentVersionResponseSchema = z7.object({
|
|
695
736
|
id: z7.string(),
|
|
@@ -721,6 +762,8 @@ var ragQueryHitSchema = z7.object({
|
|
|
721
762
|
// The reciprocal rank fusion score a hybrid search ordered by (STA3-57 D10).
|
|
722
763
|
// Absent when only the search by meaning ran, and from a box older than it.
|
|
723
764
|
mixed_score: z7.number().optional(),
|
|
765
|
+
dense_rank: z7.number().int().positive().optional(),
|
|
766
|
+
lexical_rank: z7.number().int().positive().optional(),
|
|
724
767
|
content: z7.string(),
|
|
725
768
|
metadata: jsonObject()
|
|
726
769
|
});
|
|
@@ -767,7 +810,9 @@ var createRagCollectionRequestSchema = z7.object({
|
|
|
767
810
|
embedding_model: z7.string().trim().min(1).optional(),
|
|
768
811
|
// Omitted, the box creates the collection with hybrid search off and `eu`.
|
|
769
812
|
hybrid_enabled: z7.boolean().optional(),
|
|
770
|
-
number_format: ragNumberFormatSchema.optional()
|
|
813
|
+
number_format: ragNumberFormatSchema.optional(),
|
|
814
|
+
extract_enabled: z7.boolean().optional(),
|
|
815
|
+
extract_fields: ragExtractFieldsRequestSchema.optional()
|
|
771
816
|
}).refine(
|
|
772
817
|
(value) => value.embedding_provider === void 0 === (value.embedding_model === void 0),
|
|
773
818
|
{
|
|
@@ -777,14 +822,19 @@ var createRagCollectionRequestSchema = z7.object({
|
|
|
777
822
|
);
|
|
778
823
|
var updateRagCollectionRequestSchema = z7.strictObject({
|
|
779
824
|
hybrid_enabled: z7.boolean().optional(),
|
|
780
|
-
number_format: ragNumberFormatSchema.optional()
|
|
781
|
-
|
|
782
|
-
|
|
825
|
+
number_format: ragNumberFormatSchema.optional(),
|
|
826
|
+
extract_enabled: z7.boolean().optional(),
|
|
827
|
+
extract_fields: ragExtractFieldsRequestSchema.optional()
|
|
828
|
+
}).refine((value) => Object.values(value).some((setting) => setting !== void 0), {
|
|
829
|
+
message: "Send at least one collection setting"
|
|
783
830
|
});
|
|
784
831
|
var updateRagCollectionResponseSchema = z7.object({
|
|
785
832
|
name: z7.string(),
|
|
786
833
|
hybrid_enabled: z7.boolean(),
|
|
787
|
-
number_format: ragNumberFormatSchema
|
|
834
|
+
number_format: ragNumberFormatSchema,
|
|
835
|
+
extract_enabled: z7.boolean().optional(),
|
|
836
|
+
extract_fields: z7.array(ragExtractFieldSchema).optional(),
|
|
837
|
+
unconverted: z7.number().int().nonnegative().optional()
|
|
788
838
|
});
|
|
789
839
|
var deleteRagCollectionResponseSchema = z7.object({
|
|
790
840
|
name: z7.string(),
|
|
@@ -881,7 +931,8 @@ var ragSchema = z8.strictObject({
|
|
|
881
931
|
minScore: z8.number().min(0).max(1),
|
|
882
932
|
minWords: z8.int().min(0).max(20),
|
|
883
933
|
topK: z8.int().min(1).max(10),
|
|
884
|
-
timeoutMs: z8.int().min(200).max(1e4)
|
|
934
|
+
timeoutMs: z8.int().min(200).max(1e4),
|
|
935
|
+
previousExchanges: z8.int().min(0).max(5)
|
|
885
936
|
});
|
|
886
937
|
var effectiveVoiceSettingsObjectSchema = z8.strictObject({
|
|
887
938
|
/** BCP-47-ish tag the STT is hinted with and the TTS speaks (`es`, `en-US`), or `multi`. */
|
|
@@ -915,20 +966,11 @@ var effectiveVoiceSettingsObjectSchema = z8.strictObject({
|
|
|
915
966
|
* dispatch to the worker leaves it out (`callVoiceSettingsSchema`).
|
|
916
967
|
*/
|
|
917
968
|
recording: z8.boolean(),
|
|
918
|
-
/**
|
|
919
|
-
* Ask OpenRouter to route this agent's model calls to the provider with the
|
|
920
|
-
* highest throughput (`provider: { sort: 'throughput' }`) instead of its default
|
|
921
|
-
* price routing. Gemma 4 31B on price routing landed on a 13 tok/s provider (5 s
|
|
922
|
-
* a turn); throughput routing answered in about 1 s. A request field, not a
|
|
923
|
-
* `:nitro` model suffix, so the model id the box checks and prices stays the
|
|
924
|
-
* one written. The box strips it off any other gateway. The SDK applies it when
|
|
925
|
-
* it builds the graph, so the worker never reads it.
|
|
926
|
-
*/
|
|
927
|
-
fastModelRouting: z8.boolean(),
|
|
928
969
|
/** The box runs the search before the model's turn, so the worker never reads it. */
|
|
929
970
|
rag: ragSchema,
|
|
930
971
|
/**
|
|
931
|
-
* What the agent says when
|
|
972
|
+
* What the agent says when a guardrail stops the caller's sentence (FDE-66) or the
|
|
973
|
+
* agent's reply (STA3-59):
|
|
932
974
|
* a policy message written for a chat reader would leak the rule if a TTS read it.
|
|
933
975
|
* The box answers the turn itself, so the worker never reads it. Not nullable: a
|
|
934
976
|
* blocked turn always says something.
|
|
@@ -963,7 +1005,6 @@ var checkCrossFieldRules = (value, ctx) => {
|
|
|
963
1005
|
var effectiveVoiceSettingsSchema = effectiveVoiceSettingsObjectSchema.superRefine(checkCrossFieldRules);
|
|
964
1006
|
var callVoiceSettingsObjectSchema = effectiveVoiceSettingsObjectSchema.omit({
|
|
965
1007
|
recording: true,
|
|
966
|
-
fastModelRouting: true,
|
|
967
1008
|
rag: true,
|
|
968
1009
|
refusalPhrase: true
|
|
969
1010
|
});
|
|
@@ -989,12 +1030,12 @@ var VOICE_SETTINGS_LEAF_PATHS = [
|
|
|
989
1030
|
"phoneLine",
|
|
990
1031
|
"expressive",
|
|
991
1032
|
"recording",
|
|
992
|
-
"fastModelRouting",
|
|
993
1033
|
"rag.collection",
|
|
994
1034
|
"rag.minScore",
|
|
995
1035
|
"rag.minWords",
|
|
996
1036
|
"rag.topK",
|
|
997
1037
|
"rag.timeoutMs",
|
|
1038
|
+
"rag.previousExchanges",
|
|
998
1039
|
"refusalPhrase"
|
|
999
1040
|
];
|
|
1000
1041
|
var voiceSettingOriginSchema = z8.enum(["studio", "code", "default"]);
|
|
@@ -1058,6 +1099,7 @@ var VOICE_TRACES_PATH = "/voice/v1/traces";
|
|
|
1058
1099
|
var VOICE_WORKER_AGENT_NAME = "stackbone-voice";
|
|
1059
1100
|
var VOICE_WORKER_AGENT_NAME_ENV = "STACKBONE_VOICE_AGENT_NAME";
|
|
1060
1101
|
var VOICE_WORKER_PORT_ENV = "STACKBONE_VOICE_WORKER_PORT";
|
|
1102
|
+
var VOICE_TURN_METRICS_MESSAGE = "voice turn metrics";
|
|
1061
1103
|
var VOICE_CALL_TOKEN_MAX = 2048;
|
|
1062
1104
|
var voiceDispatchSchema = z8.strictObject({
|
|
1063
1105
|
agent: deepAgentNameSchema,
|
|
@@ -1122,15 +1164,34 @@ var voiceCallTranscriptSchema = z8.array(
|
|
|
1122
1164
|
).max(1e3).refine((items) => items.reduce((size, item) => size + item.text.length, 0) <= 2e5, {
|
|
1123
1165
|
message: "A call transcript may contain at most 200000 characters."
|
|
1124
1166
|
});
|
|
1167
|
+
var VOICE_CALL_TURNS_MAX = 1e3;
|
|
1168
|
+
var AGUI_RUN_ID_MAX = 200;
|
|
1169
|
+
var VOICE_TURN_LATENCY_MAX_MS = 36e5;
|
|
1170
|
+
var turnLatencyMs = z8.number().int().min(0).max(VOICE_TURN_LATENCY_MAX_MS).optional();
|
|
1171
|
+
var voiceCallTurnMetricsSchema = z8.strictObject({
|
|
1172
|
+
/** The AG-UI `runId` of the box run that answered; absent on a reply the box did not run. */
|
|
1173
|
+
runId: z8.string().min(1).max(AGUI_RUN_ID_MAX).optional(),
|
|
1174
|
+
endOfTurnMs: turnLatencyMs,
|
|
1175
|
+
transcriptionMs: turnLatencyMs,
|
|
1176
|
+
llmTtftMs: turnLatencyMs,
|
|
1177
|
+
ttsTtfbMs: turnLatencyMs,
|
|
1178
|
+
e2eMs: turnLatencyMs
|
|
1179
|
+
});
|
|
1180
|
+
var voiceCallTurnsSchema = z8.array(voiceCallTurnMetricsSchema).max(VOICE_CALL_TURNS_MAX);
|
|
1125
1181
|
var voiceCallCompleteSchema = z8.strictObject({
|
|
1126
1182
|
transcript: voiceCallTranscriptSchema,
|
|
1127
|
-
outcome: z8.enum(["completed", "disconnected", "failed"])
|
|
1183
|
+
outcome: z8.enum(["completed", "disconnected", "failed"]),
|
|
1184
|
+
// The transcript, the memory and the `onEnd` start this body carries must not be
|
|
1185
|
+
// refused over a bad latency, so `turns` is any value here and read apart through
|
|
1186
|
+
// `readVoiceCallTurns`. Not `.catch()` on the array: this schema is a documented
|
|
1187
|
+
// route's body, and the OpenAPI generator cannot describe a catch (STA3-56).
|
|
1188
|
+
turns: z8.unknown().optional()
|
|
1128
1189
|
});
|
|
1129
1190
|
var voiceCallLifecycleResponseSchema = z8.strictObject({ ok: z8.literal(true) });
|
|
1130
1191
|
|
|
1131
1192
|
// libs/runtime/voice-worker/src/box-llm.ts
|
|
1132
1193
|
import { randomUUID } from "node:crypto";
|
|
1133
|
-
import { DEFAULT_API_CONNECT_OPTIONS, llm, log } from "@livekit/agents";
|
|
1194
|
+
import { DEFAULT_API_CONNECT_OPTIONS, llm as llm2, log } from "@livekit/agents";
|
|
1134
1195
|
|
|
1135
1196
|
// libs/shared/llm-wire/src/lib/sse/read-sse-data.ts
|
|
1136
1197
|
async function* readSseData(body) {
|
|
@@ -1175,8 +1236,9 @@ var BoxTurnError = class extends Error {
|
|
|
1175
1236
|
}
|
|
1176
1237
|
};
|
|
1177
1238
|
var boxEndpoint = (boxUrl, path) => `${boxUrl.replace(/\/+$/, "")}${path}`;
|
|
1239
|
+
var mintAguiRunId = () => `run-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
1178
1240
|
async function* aguiTurn(opts) {
|
|
1179
|
-
const runId =
|
|
1241
|
+
const runId = opts.runId ?? mintAguiRunId();
|
|
1180
1242
|
const heard = "toolResult" in opts ? void 0 : opts.heard;
|
|
1181
1243
|
const message = "toolResult" in opts ? {
|
|
1182
1244
|
id: `tool-${opts.toolResult.toolCallId}`,
|
|
@@ -1456,8 +1518,45 @@ var lastNewAssistantMessage = (chatCtx, seen) => {
|
|
|
1456
1518
|
return void 0;
|
|
1457
1519
|
};
|
|
1458
1520
|
|
|
1521
|
+
// libs/runtime/voice-worker/src/turn-metrics.ts
|
|
1522
|
+
import { llm, voice as voice2 } from "@livekit/agents";
|
|
1523
|
+
var AGUI_RUN_ID_EXTRA = "stackboneAguiRunId";
|
|
1524
|
+
var ms = (seconds) => {
|
|
1525
|
+
if (typeof seconds !== "number" || !Number.isFinite(seconds)) return void 0;
|
|
1526
|
+
const value = Math.round(seconds * 1e3);
|
|
1527
|
+
return value >= 0 && value <= VOICE_TURN_LATENCY_MAX_MS ? value : void 0;
|
|
1528
|
+
};
|
|
1529
|
+
var defined = (metrics) => Object.fromEntries(Object.entries(metrics).filter(([, value]) => value !== void 0));
|
|
1530
|
+
var watchTurnMetrics = (session, write, options = {}) => {
|
|
1531
|
+
let caller;
|
|
1532
|
+
let delivered;
|
|
1533
|
+
session.on(voice2.AgentSessionEventTypes.ConversationItemAdded, ({ item }) => {
|
|
1534
|
+
if (!(item instanceof llm.ChatMessage)) return;
|
|
1535
|
+
if (item.role === "user") {
|
|
1536
|
+
caller = item.metrics;
|
|
1537
|
+
delivered = options.endOfTurnMs?.();
|
|
1538
|
+
return;
|
|
1539
|
+
}
|
|
1540
|
+
if (item.role !== "assistant") return;
|
|
1541
|
+
const reply = item.metrics;
|
|
1542
|
+
const runId = item.extra[AGUI_RUN_ID_EXTRA];
|
|
1543
|
+
const e2eMs = ms(reply.e2eLatency);
|
|
1544
|
+
const metrics = defined({
|
|
1545
|
+
endOfTurnMs: delivered ?? ms(caller?.endOfTurnDelay),
|
|
1546
|
+
transcriptionMs: options.endOfTurnMs ? void 0 : ms(caller?.transcriptionDelay),
|
|
1547
|
+
llmTtftMs: ms(reply.llmNodeTtft),
|
|
1548
|
+
ttsTtfbMs: ms(reply.ttsNodeTtfb),
|
|
1549
|
+
e2eMs: e2eMs === void 0 || delivered === void 0 ? e2eMs : e2eMs + delivered
|
|
1550
|
+
});
|
|
1551
|
+
caller = void 0;
|
|
1552
|
+
delivered = void 0;
|
|
1553
|
+
if (Object.keys(metrics).length === 0) return;
|
|
1554
|
+
write(typeof runId === "string" ? { ...metrics, runId } : metrics);
|
|
1555
|
+
});
|
|
1556
|
+
};
|
|
1557
|
+
|
|
1459
1558
|
// libs/runtime/voice-worker/src/box-llm.ts
|
|
1460
|
-
var BoxLLM = class extends
|
|
1559
|
+
var BoxLLM = class extends llm2.LLM {
|
|
1461
1560
|
#opts;
|
|
1462
1561
|
#replies = new HeardTracker();
|
|
1463
1562
|
#pending = /* @__PURE__ */ new Set();
|
|
@@ -1554,7 +1653,7 @@ var lastUserText = (chatCtx) => {
|
|
|
1554
1653
|
}
|
|
1555
1654
|
return "";
|
|
1556
1655
|
};
|
|
1557
|
-
var BoxLLMStream = class extends
|
|
1656
|
+
var BoxLLMStream = class extends llm2.LLMStream {
|
|
1558
1657
|
#opts;
|
|
1559
1658
|
#replies;
|
|
1560
1659
|
#settle;
|
|
@@ -1584,6 +1683,7 @@ var BoxLLMStream = class extends llm.LLMStream {
|
|
|
1584
1683
|
...turn
|
|
1585
1684
|
} = this.#opts;
|
|
1586
1685
|
const id = randomUUID();
|
|
1686
|
+
const runId = mintAguiRunId();
|
|
1587
1687
|
const { signal } = this.abortController;
|
|
1588
1688
|
await pendingResume?.();
|
|
1589
1689
|
if (signal.aborted || isEnding?.()) return;
|
|
@@ -1608,7 +1708,8 @@ var BoxLLMStream = class extends llm.LLMStream {
|
|
|
1608
1708
|
tools: clientTools,
|
|
1609
1709
|
text: lastUserText(this.chatCtx),
|
|
1610
1710
|
...heard ? { heard } : {},
|
|
1611
|
-
signal: turnSignal
|
|
1711
|
+
signal: turnSignal,
|
|
1712
|
+
runId
|
|
1612
1713
|
})) {
|
|
1613
1714
|
restartIdle();
|
|
1614
1715
|
reply.observe(ev);
|
|
@@ -1619,7 +1720,7 @@ var BoxLLMStream = class extends llm.LLMStream {
|
|
|
1619
1720
|
delta: {
|
|
1620
1721
|
role: "assistant",
|
|
1621
1722
|
content: ev.text,
|
|
1622
|
-
extra: { [VOICE_RESPONSE_MARKER]: id }
|
|
1723
|
+
extra: { [VOICE_RESPONSE_MARKER]: id, [AGUI_RUN_ID_EXTRA]: runId }
|
|
1623
1724
|
}
|
|
1624
1725
|
});
|
|
1625
1726
|
break;
|
|
@@ -1666,6 +1767,49 @@ var BoxLLMStream = class extends llm.LLMStream {
|
|
|
1666
1767
|
}
|
|
1667
1768
|
};
|
|
1668
1769
|
|
|
1770
|
+
// libs/runtime/voice-worker/src/stt-delivery.ts
|
|
1771
|
+
import { stt } from "@livekit/agents";
|
|
1772
|
+
import { TransformStream } from "node:stream/web";
|
|
1773
|
+
var finalDeliveryMs = (event, audioSentSec) => {
|
|
1774
|
+
const alt = event.alternatives?.[0];
|
|
1775
|
+
if (alt === void 0 || !Number.isFinite(alt.endTime) || alt.endTime <= 0) return void 0;
|
|
1776
|
+
const segmentEndSec = alt.endTime - (alt.words?.[0]?.startTimeOffset ?? 0);
|
|
1777
|
+
const ms2 = Math.round((audioSentSec - segmentEndSec) * 1e3);
|
|
1778
|
+
return ms2 >= 0 ? ms2 : void 0;
|
|
1779
|
+
};
|
|
1780
|
+
var createSttDeliveryMeter = () => {
|
|
1781
|
+
let last;
|
|
1782
|
+
return {
|
|
1783
|
+
wrap: async (audio, inner) => {
|
|
1784
|
+
let audioSentSec = 0;
|
|
1785
|
+
async function* counted() {
|
|
1786
|
+
for await (const frame of audio) {
|
|
1787
|
+
audioSentSec += frame.samplesPerChannel / frame.sampleRate;
|
|
1788
|
+
yield frame;
|
|
1789
|
+
}
|
|
1790
|
+
}
|
|
1791
|
+
const events = await inner(counted());
|
|
1792
|
+
if (events === null) return null;
|
|
1793
|
+
return events.pipeThrough(
|
|
1794
|
+
new TransformStream({
|
|
1795
|
+
transform(event, controller) {
|
|
1796
|
+
if (typeof event !== "string" && event.type === stt.SpeechEventType.FINAL_TRANSCRIPT) {
|
|
1797
|
+
last = finalDeliveryMs(event, audioSentSec) ?? last;
|
|
1798
|
+
}
|
|
1799
|
+
controller.enqueue(event);
|
|
1800
|
+
}
|
|
1801
|
+
})
|
|
1802
|
+
);
|
|
1803
|
+
},
|
|
1804
|
+
take: () => {
|
|
1805
|
+
const taken = last;
|
|
1806
|
+
last = void 0;
|
|
1807
|
+
return taken;
|
|
1808
|
+
}
|
|
1809
|
+
};
|
|
1810
|
+
};
|
|
1811
|
+
var deliversItsOwnEndOfTurn = (settings) => settings.turn.mode === "stt" && settings.sttModel.startsWith("deepgram/flux");
|
|
1812
|
+
|
|
1669
1813
|
// libs/runtime/voice-worker/src/dispatch.ts
|
|
1670
1814
|
var readDispatch = (metadata) => {
|
|
1671
1815
|
if (!metadata) return null;
|
|
@@ -1742,17 +1886,17 @@ var createHangUp = ({
|
|
|
1742
1886
|
};
|
|
1743
1887
|
|
|
1744
1888
|
// libs/runtime/voice-worker/src/session-options.ts
|
|
1745
|
-
import { tts, voice as
|
|
1889
|
+
import { tts, voice as voice3 } from "@livekit/agents";
|
|
1746
1890
|
|
|
1747
1891
|
// libs/runtime/voice-worker/src/number-text.ts
|
|
1748
|
-
import { TransformStream } from "node:stream/web";
|
|
1892
|
+
import { TransformStream as TransformStream2 } from "node:stream/web";
|
|
1749
1893
|
var THOUSANDS_COMMA = /(\d),(?=\d{3}(?!\d))/g;
|
|
1750
1894
|
var TRAILING_NUMBER = /[\d.,]+$/;
|
|
1751
1895
|
var rewrite = (text) => text.replace(THOUSANDS_COMMA, "$1");
|
|
1752
1896
|
var dropThousandsComma = (input) => {
|
|
1753
1897
|
let pending = "";
|
|
1754
1898
|
return input.pipeThrough(
|
|
1755
|
-
new
|
|
1899
|
+
new TransformStream2({
|
|
1756
1900
|
transform(chunk, controller) {
|
|
1757
1901
|
const text = pending + chunk;
|
|
1758
1902
|
const tail = TRAILING_NUMBER.exec(text)?.[0] ?? "";
|
|
@@ -1769,7 +1913,7 @@ var dropThousandsComma = (input) => {
|
|
|
1769
1913
|
|
|
1770
1914
|
// libs/runtime/voice-worker/src/phone-voice.ts
|
|
1771
1915
|
import { AudioFrame as AudioFrame2 } from "@livekit/rtc-node";
|
|
1772
|
-
import { TransformStream as
|
|
1916
|
+
import { TransformStream as TransformStream3 } from "node:stream/web";
|
|
1773
1917
|
var LOW_CUT_HZ = 300;
|
|
1774
1918
|
var HIGH_CUT_HZ = 3400;
|
|
1775
1919
|
var Q = Math.SQRT1_2;
|
|
@@ -1805,7 +1949,7 @@ var phoneLine = (audio) => {
|
|
|
1805
1949
|
let chainRate = 0;
|
|
1806
1950
|
const norm = Math.tanh(DRIVE);
|
|
1807
1951
|
return audio.pipeThrough(
|
|
1808
|
-
new
|
|
1952
|
+
new TransformStream3({
|
|
1809
1953
|
transform(frame, controller) {
|
|
1810
1954
|
if (!chain || chainRate !== frame.sampleRate) {
|
|
1811
1955
|
chainRate = frame.sampleRate;
|
|
@@ -1858,7 +2002,16 @@ var markupContext = (expressive, speech) => {
|
|
|
1858
2002
|
});
|
|
1859
2003
|
return guide === void 0 ? [] : [{ description: VOICE_TTS_MARKUP_CONTEXT, value: guide }];
|
|
1860
2004
|
};
|
|
1861
|
-
var
|
|
2005
|
+
var VoiceCallAgent = class extends voice3.Agent {
|
|
2006
|
+
constructor() {
|
|
2007
|
+
super(...arguments);
|
|
2008
|
+
this.sttDelivery = createSttDeliveryMeter();
|
|
2009
|
+
}
|
|
2010
|
+
async sttNode(audio, modelSettings) {
|
|
2011
|
+
return this.sttDelivery.wrap(audio, (counted) => super.sttNode(counted, modelSettings));
|
|
2012
|
+
}
|
|
2013
|
+
};
|
|
2014
|
+
var PhoneLineAgent = class extends VoiceCallAgent {
|
|
1862
2015
|
async ttsNode(text, modelSettings) {
|
|
1863
2016
|
const audio = await super.ttsNode(text, modelSettings);
|
|
1864
2017
|
return audio && phoneLine(audio);
|
|
@@ -1912,7 +2065,7 @@ var voiceSessionOptions = (settings, pieces) => {
|
|
|
1912
2065
|
}
|
|
1913
2066
|
},
|
|
1914
2067
|
// The box holds the prompt: LiveKit's own chat context stays prompt-less.
|
|
1915
|
-
agent: settings.phoneLine ? new PhoneLineAgent({ instructions: "" }) : new
|
|
2068
|
+
agent: settings.phoneLine ? new PhoneLineAgent({ instructions: "" }) : new VoiceCallAgent({ instructions: "" }),
|
|
1916
2069
|
boxContext: markupContext(settings.expressive, speech)
|
|
1917
2070
|
};
|
|
1918
2071
|
};
|
|
@@ -2011,9 +2164,9 @@ var toolCallCue = () => {
|
|
|
2011
2164
|
};
|
|
2012
2165
|
};
|
|
2013
2166
|
var messageOf = (err) => err instanceof Error ? err.message : String(err);
|
|
2014
|
-
var callerNoiseCancellation = () => {
|
|
2167
|
+
var callerNoiseCancellation = (caller) => {
|
|
2015
2168
|
try {
|
|
2016
|
-
return BackgroundVoiceCancellation();
|
|
2169
|
+
return caller.kind === ParticipantKind.SIP ? TelephonyBackgroundVoiceCancellation() : BackgroundVoiceCancellation();
|
|
2017
2170
|
} catch (err) {
|
|
2018
2171
|
log4().warn({ error: messageOf(err) }, "voice worker: no noise cancellation on this call");
|
|
2019
2172
|
return void 0;
|
|
@@ -2147,6 +2300,7 @@ var runVoiceJob = async (ctx, deps) => {
|
|
|
2147
2300
|
onRunFinished
|
|
2148
2301
|
});
|
|
2149
2302
|
const session = deps.createSession({ ...plan.session, llm: brain });
|
|
2303
|
+
const turns = [];
|
|
2150
2304
|
let completion;
|
|
2151
2305
|
const finishCall = (outcome = "disconnected") => {
|
|
2152
2306
|
if (completion) return completion;
|
|
@@ -2155,14 +2309,20 @@ var runVoiceJob = async (ctx, deps) => {
|
|
|
2155
2309
|
await hangUp.pendingResume();
|
|
2156
2310
|
await session.close();
|
|
2157
2311
|
await brain.settled();
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2312
|
+
const transcript = brain.heardTranscript(session.history);
|
|
2313
|
+
const complete = (body) => voiceCallRequest(wire, "complete", voiceCallCompleteSchema.parse(body));
|
|
2314
|
+
try {
|
|
2315
|
+
await complete({ transcript, outcome, ...turns.length > 0 ? { turns } : {} });
|
|
2316
|
+
} catch (error) {
|
|
2317
|
+
if (!(turns.length > 0 && error instanceof BoxTurnError && error.status === 400)) {
|
|
2318
|
+
throw error;
|
|
2319
|
+
}
|
|
2320
|
+
log4().warn(
|
|
2321
|
+
{ agent: dispatch.agent, sessionKey: dispatch.sessionKey, status: error.status },
|
|
2322
|
+
"voice worker: the box refused the turn metrics, completing without them"
|
|
2323
|
+
);
|
|
2324
|
+
await complete({ transcript, outcome });
|
|
2325
|
+
}
|
|
2166
2326
|
}).catch((error) => {
|
|
2167
2327
|
log4().warn(
|
|
2168
2328
|
{ agent: dispatch.agent, sessionKey: dispatch.sessionKey, reason: "completion_failed" },
|
|
@@ -2173,11 +2333,22 @@ var runVoiceJob = async (ctx, deps) => {
|
|
|
2173
2333
|
return completion;
|
|
2174
2334
|
};
|
|
2175
2335
|
ctx.addShutdownCallback(() => finishCall());
|
|
2176
|
-
session.on(
|
|
2336
|
+
session.on(voice4.AgentSessionEventTypes.Close, () => {
|
|
2177
2337
|
if (completion || ending) return;
|
|
2178
2338
|
void finishCall().catch(() => void 0).finally(() => ctx.shutdown("caller left"));
|
|
2179
2339
|
});
|
|
2180
|
-
|
|
2340
|
+
watchTurnMetrics(
|
|
2341
|
+
session,
|
|
2342
|
+
(metrics) => {
|
|
2343
|
+
if (turns.length < VOICE_CALL_TURNS_MAX) turns.push(metrics);
|
|
2344
|
+
log4().info(
|
|
2345
|
+
{ agent: dispatch.agent, sessionKey: dispatch.sessionKey, room: ctx.room.name, ...metrics },
|
|
2346
|
+
VOICE_TURN_METRICS_MESSAGE
|
|
2347
|
+
);
|
|
2348
|
+
},
|
|
2349
|
+
deliversItsOwnEndOfTurn(settings) ? { endOfTurnMs: () => plan.agent.sttDelivery.take() } : {}
|
|
2350
|
+
);
|
|
2351
|
+
const noiseCancellation = callerNoiseCancellation(participant);
|
|
2181
2352
|
await session.start({
|
|
2182
2353
|
agent: plan.agent,
|
|
2183
2354
|
room: ctx.room,
|
|
@@ -2191,7 +2362,7 @@ var runVoiceJob = async (ctx, deps) => {
|
|
|
2191
2362
|
|
|
2192
2363
|
// libs/runtime/voice-worker/src/main.ts
|
|
2193
2364
|
var liveDeps = () => ({
|
|
2194
|
-
createSession: (options) => new
|
|
2365
|
+
createSession: (options) => new voice5.AgentSession(options),
|
|
2195
2366
|
createStt: (options) => new inference.STT(options),
|
|
2196
2367
|
createTts: (options) => new inference.TTS(options),
|
|
2197
2368
|
createTurnDetector: (options) => new inference.TurnDetector(options),
|