@stackbone/cli 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +147 -0
- package/main.js +14724 -11224
- package/migrations/0114_memory_person_id.sql +16 -0
- package/migrations/0115_memory_erased_person_keeps_no_identifier.sql +31 -0
- package/migrations/0116_rag_hybrid_search.sql +14 -0
- package/migrations/0117_rag_extract_fields.sql +3 -0
- package/migrations/0118_voice_outbound_calls.sql +29 -0
- package/migrations/0119_voice_call_metrics.sql +20 -0
- package/migrations/0120_run_steps_voice.sql +19 -0
- package/migrations/0121_model_routing.sql +28 -0
- package/package.json +1 -1
- package/stackbone-cli-0.8.0.tgz +0 -0
- package/voice-worker/main.js +299 -51
- package/voice-worker/main.js.map +4 -4
- package/stackbone-cli-0.7.0.tgz +0 -0
package/voice-worker/main.js
CHANGED
|
@@ -6,7 +6,7 @@ import {
|
|
|
6
6
|
defineAgent,
|
|
7
7
|
inference,
|
|
8
8
|
telemetry as telemetry2,
|
|
9
|
-
voice as
|
|
9
|
+
voice as voice5
|
|
10
10
|
} from "@livekit/agents";
|
|
11
11
|
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
|
|
12
12
|
|
|
@@ -24,11 +24,11 @@ var PATTERN = [
|
|
|
24
24
|
[0, 900]
|
|
25
25
|
];
|
|
26
26
|
var renderPattern = () => {
|
|
27
|
-
const total = PATTERN.reduce((n, [,
|
|
27
|
+
const total = PATTERN.reduce((n, [, ms2]) => n + SAMPLE_RATE * ms2 / 1e3, 0);
|
|
28
28
|
const pcm = new Int16Array(total);
|
|
29
29
|
let offset = 0;
|
|
30
|
-
for (const [hz,
|
|
31
|
-
const length = SAMPLE_RATE *
|
|
30
|
+
for (const [hz, ms2] of PATTERN) {
|
|
31
|
+
const length = SAMPLE_RATE * ms2 / 1e3;
|
|
32
32
|
for (let i = 0; hz > 0 && i < length; i++) {
|
|
33
33
|
const envelope = 0.5 - 0.5 * Math.cos(2 * Math.PI * i / length);
|
|
34
34
|
const sample = Math.sin(2 * Math.PI * hz * i / SAMPLE_RATE) * envelope * VOLUME;
|
|
@@ -101,8 +101,11 @@ var loadVad = async () => {
|
|
|
101
101
|
};
|
|
102
102
|
|
|
103
103
|
// libs/runtime/voice-worker/src/worker.ts
|
|
104
|
-
import { log as log4, voice as
|
|
105
|
-
import {
|
|
104
|
+
import { log as log4, voice as voice4 } from "@livekit/agents";
|
|
105
|
+
import {
|
|
106
|
+
BackgroundVoiceCancellation,
|
|
107
|
+
TelephonyBackgroundVoiceCancellation
|
|
108
|
+
} from "@livekit/noise-cancellation-node";
|
|
106
109
|
import { ParticipantKind } from "@livekit/rtc-node";
|
|
107
110
|
|
|
108
111
|
// libs/shared/validators/src/lib/common/json-object.ts
|
|
@@ -557,9 +560,23 @@ var STUDIO_RAG_COLLECTION_REGEX = /^[A-Za-z0-9][A-Za-z0-9._\-/:]{0,127}$/;
|
|
|
557
560
|
var ragCollectionNameParamSchema = z7.string().min(1).max(128).regex(STUDIO_RAG_COLLECTION_REGEX, "collection must be 1..128 ASCII identifier-safe chars");
|
|
558
561
|
var ragDistanceSchema = z7.enum(["cosine", "l2", "ip"]);
|
|
559
562
|
var ragIndexKindSchema = z7.enum(["hnsw", "exact"]);
|
|
563
|
+
var ragNumberFormatSchema = z7.enum(["eu", "us"]);
|
|
564
|
+
var ragRangeFieldSchema = z7.string().min(1).refine((field) => !field.includes("."), {
|
|
565
|
+
message: "field must be a literal top-level metadata key without dots"
|
|
566
|
+
});
|
|
567
|
+
var ragExtractFieldSchema = z7.object({
|
|
568
|
+
name: z7.string().trim().pipe(ragRangeFieldSchema),
|
|
569
|
+
type: z7.enum(["number", "date", "text"]),
|
|
570
|
+
removed: z7.boolean().optional()
|
|
571
|
+
});
|
|
572
|
+
var uniqueFieldNames = (fields) => new Set(fields.map((field) => field.name)).size === fields.length;
|
|
573
|
+
var ragExtractFieldsRequestSchema = z7.array(ragExtractFieldSchema).refine(uniqueFieldNames, {
|
|
574
|
+
message: "Field names must be unique"
|
|
575
|
+
});
|
|
560
576
|
var ragCollectionSchema = z7.object({
|
|
561
577
|
name: z7.string(),
|
|
562
578
|
dimensions: z7.number().int().min(1),
|
|
579
|
+
embedding_state: z7.enum(["ready", "rebuilding", "failed"]).optional(),
|
|
563
580
|
// Optional for boxes that predate collection-owned embeddings.
|
|
564
581
|
embedding_provider: z7.string().min(1).optional(),
|
|
565
582
|
embedding_model: z7.string().min(1).optional(),
|
|
@@ -570,7 +587,12 @@ var ragCollectionSchema = z7.object({
|
|
|
570
587
|
// Maximum `created_at` across the collection's documents. Null when the
|
|
571
588
|
// collection has zero documents (an empty collection row created before its
|
|
572
589
|
// first ingest).
|
|
573
|
-
last_ingested_at: z7.iso.datetime().nullable()
|
|
590
|
+
last_ingested_at: z7.iso.datetime().nullable(),
|
|
591
|
+
// STA3-57. Optional so a list from a box older than hybrid search still parses.
|
|
592
|
+
hybrid_enabled: z7.boolean().optional(),
|
|
593
|
+
number_format: ragNumberFormatSchema.optional(),
|
|
594
|
+
extract_enabled: z7.boolean().optional(),
|
|
595
|
+
extract_fields: z7.array(ragExtractFieldSchema).optional()
|
|
574
596
|
});
|
|
575
597
|
var ragCollectionListResponseSchema = z7.object({
|
|
576
598
|
items: z7.array(ragCollectionSchema)
|
|
@@ -617,6 +639,48 @@ var ragDocumentSchema = z7.object({
|
|
|
617
639
|
pending_version: z7.number().int().min(1).nullable().optional()
|
|
618
640
|
});
|
|
619
641
|
var ragFilterSchema = jsonObject();
|
|
642
|
+
var ragDateValueSchema = z7.string().length(10).regex(/^[0-9]{4}-[0-9]{2}-[0-9]{2}$/).refine((value) => {
|
|
643
|
+
const year = Number(value.slice(0, 4));
|
|
644
|
+
const month = Number(value.slice(5, 7));
|
|
645
|
+
const day = Number(value.slice(8, 10));
|
|
646
|
+
const date = /* @__PURE__ */ new Date(0);
|
|
647
|
+
date.setUTCFullYear(year, month - 1, day);
|
|
648
|
+
return year >= 1 && year <= 9999 && date.getUTCFullYear() === year && date.getUTCMonth() === month - 1 && date.getUTCDate() === day;
|
|
649
|
+
}, "date must be a real YYYY-MM-DD day in years 0001\u20139999");
|
|
650
|
+
var ragRangeSchema = z7.discriminatedUnion("type", [
|
|
651
|
+
z7.strictObject({
|
|
652
|
+
field: ragRangeFieldSchema,
|
|
653
|
+
type: z7.literal("number"),
|
|
654
|
+
gt: z7.number().finite().optional(),
|
|
655
|
+
gte: z7.number().finite().optional(),
|
|
656
|
+
lt: z7.number().finite().optional(),
|
|
657
|
+
lte: z7.number().finite().optional()
|
|
658
|
+
}),
|
|
659
|
+
z7.strictObject({
|
|
660
|
+
field: ragRangeFieldSchema,
|
|
661
|
+
type: z7.literal("date"),
|
|
662
|
+
gt: ragDateValueSchema.optional(),
|
|
663
|
+
gte: ragDateValueSchema.optional(),
|
|
664
|
+
lt: ragDateValueSchema.optional(),
|
|
665
|
+
lte: ragDateValueSchema.optional()
|
|
666
|
+
})
|
|
667
|
+
]).superRefine((range, ctx) => {
|
|
668
|
+
const lower = range.gt ?? range.gte;
|
|
669
|
+
const upper = range.lt ?? range.lte;
|
|
670
|
+
if (lower === void 0 && upper === void 0) {
|
|
671
|
+
ctx.addIssue({ code: "custom", message: "range must have at least one bound" });
|
|
672
|
+
}
|
|
673
|
+
if (range.gt !== void 0 && range.gte !== void 0) {
|
|
674
|
+
ctx.addIssue({ code: "custom", message: "gt and gte cannot be combined", path: ["gte"] });
|
|
675
|
+
}
|
|
676
|
+
if (range.lt !== void 0 && range.lte !== void 0) {
|
|
677
|
+
ctx.addIssue({ code: "custom", message: "lt and lte cannot be combined", path: ["lte"] });
|
|
678
|
+
}
|
|
679
|
+
if (lower !== void 0 && upper !== void 0 && lower > upper) {
|
|
680
|
+
ctx.addIssue({ code: "custom", message: "lower bound must not exceed upper bound" });
|
|
681
|
+
}
|
|
682
|
+
});
|
|
683
|
+
var ragRangesSchema = z7.array(ragRangeSchema).max(10);
|
|
620
684
|
var listRagDocsQuerySchema = forwardPageQuerySchema.extend({
|
|
621
685
|
limit: limitParamSchema(STUDIO_RAG_DOCS_LIMIT_MAX).default(STUDIO_RAG_DOCS_LIMIT_DEFAULT),
|
|
622
686
|
// Filter is sent JSON-encoded inside a single query param so the UI can
|
|
@@ -641,6 +705,32 @@ var ragDocumentVersionListResponseSchema = z7.object({
|
|
|
641
705
|
items: z7.array(ragDocumentVersionSchema)
|
|
642
706
|
});
|
|
643
707
|
var ragDocumentVersionNumberSchema = z7.number().int().min(1).max(2147483647);
|
|
708
|
+
var updateRagDocumentValuesRequestSchema = z7.strictObject({
|
|
709
|
+
version: ragDocumentVersionNumberSchema.optional(),
|
|
710
|
+
values: jsonObject(),
|
|
711
|
+
add_fields: z7.array(ragExtractFieldSchema.omit({ removed: true })).refine(uniqueFieldNames, {
|
|
712
|
+
message: "Field names must be unique"
|
|
713
|
+
}).optional()
|
|
714
|
+
});
|
|
715
|
+
var updateRagDocumentValuesResponseSchema = z7.object({
|
|
716
|
+
id: z7.string(),
|
|
717
|
+
collection: z7.string(),
|
|
718
|
+
version: ragDocumentVersionNumberSchema,
|
|
719
|
+
metadata: jsonObject()
|
|
720
|
+
});
|
|
721
|
+
var extractRagDocumentRequestSchema = z7.strictObject({
|
|
722
|
+
version: ragDocumentVersionNumberSchema.optional()
|
|
723
|
+
});
|
|
724
|
+
var extractRagDocumentResponseSchema = z7.object({
|
|
725
|
+
proposal: z7.array(
|
|
726
|
+
ragExtractFieldSchema.omit({ removed: true }).extend({
|
|
727
|
+
value: z7.union([z7.number().finite(), z7.string()]),
|
|
728
|
+
new: z7.boolean()
|
|
729
|
+
})
|
|
730
|
+
),
|
|
731
|
+
failed: z7.boolean(),
|
|
732
|
+
truncated: z7.boolean()
|
|
733
|
+
});
|
|
644
734
|
var ragDocumentVersionParamSchema = z7.string().regex(/^\d+$/, { message: "version must be a positive integer" }).transform((value) => Number.parseInt(value, 10)).pipe(ragDocumentVersionNumberSchema);
|
|
645
735
|
var activateRagDocumentVersionResponseSchema = z7.object({
|
|
646
736
|
id: z7.string(),
|
|
@@ -655,7 +745,9 @@ var ragQueryRequestSchema = z7.object({
|
|
|
655
745
|
model: z7.string().min(1).optional(),
|
|
656
746
|
topK: z7.coerce.number().int().min(1).max(STUDIO_RAG_TOPK_MAX).default(STUDIO_RAG_TOPK_DEFAULT),
|
|
657
747
|
filter: ragFilterSchema.optional(),
|
|
658
|
-
|
|
748
|
+
ranges: ragRangesSchema.optional(),
|
|
749
|
+
preview: z7.object({ docId: z7.string().min(1), version: ragDocumentVersionNumberSchema }).optional(),
|
|
750
|
+
hybrid: z7.boolean().optional()
|
|
659
751
|
}).refine((value) => value.embedding !== void 0 || value.text !== void 0, {
|
|
660
752
|
message: "either `embedding` or `text` must be provided",
|
|
661
753
|
path: ["embedding"]
|
|
@@ -667,6 +759,11 @@ var ragQueryHitSchema = z7.object({
|
|
|
667
759
|
// query)` over `stackbone_rag_chunks`, so higher is closer. We surface the
|
|
668
760
|
// raw value; the UI formats it.
|
|
669
761
|
score: z7.number(),
|
|
762
|
+
// The reciprocal rank fusion score a hybrid search ordered by (STA3-57 D10).
|
|
763
|
+
// Absent when only the search by meaning ran, and from a box older than it.
|
|
764
|
+
mixed_score: z7.number().optional(),
|
|
765
|
+
dense_rank: z7.number().int().positive().optional(),
|
|
766
|
+
lexical_rank: z7.number().int().positive().optional(),
|
|
670
767
|
content: z7.string(),
|
|
671
768
|
metadata: jsonObject()
|
|
672
769
|
});
|
|
@@ -710,7 +807,12 @@ var createRagCollectionRequestSchema = z7.object({
|
|
|
710
807
|
name: ragCollectionNameParamSchema,
|
|
711
808
|
metadata: jsonObject().optional(),
|
|
712
809
|
embedding_provider: pickedModelProviderIdSchema.optional(),
|
|
713
|
-
embedding_model: z7.string().trim().min(1).optional()
|
|
810
|
+
embedding_model: z7.string().trim().min(1).optional(),
|
|
811
|
+
// Omitted, the box creates the collection with hybrid search off and `eu`.
|
|
812
|
+
hybrid_enabled: z7.boolean().optional(),
|
|
813
|
+
number_format: ragNumberFormatSchema.optional(),
|
|
814
|
+
extract_enabled: z7.boolean().optional(),
|
|
815
|
+
extract_fields: ragExtractFieldsRequestSchema.optional()
|
|
714
816
|
}).refine(
|
|
715
817
|
(value) => value.embedding_provider === void 0 === (value.embedding_model === void 0),
|
|
716
818
|
{
|
|
@@ -718,6 +820,22 @@ var createRagCollectionRequestSchema = z7.object({
|
|
|
718
820
|
path: ["embedding_model"]
|
|
719
821
|
}
|
|
720
822
|
);
|
|
823
|
+
var updateRagCollectionRequestSchema = z7.strictObject({
|
|
824
|
+
hybrid_enabled: z7.boolean().optional(),
|
|
825
|
+
number_format: ragNumberFormatSchema.optional(),
|
|
826
|
+
extract_enabled: z7.boolean().optional(),
|
|
827
|
+
extract_fields: ragExtractFieldsRequestSchema.optional()
|
|
828
|
+
}).refine((value) => Object.values(value).some((setting) => setting !== void 0), {
|
|
829
|
+
message: "Send at least one collection setting"
|
|
830
|
+
});
|
|
831
|
+
var updateRagCollectionResponseSchema = z7.object({
|
|
832
|
+
name: z7.string(),
|
|
833
|
+
hybrid_enabled: z7.boolean(),
|
|
834
|
+
number_format: ragNumberFormatSchema,
|
|
835
|
+
extract_enabled: z7.boolean().optional(),
|
|
836
|
+
extract_fields: z7.array(ragExtractFieldSchema).optional(),
|
|
837
|
+
unconverted: z7.number().int().nonnegative().optional()
|
|
838
|
+
});
|
|
721
839
|
var deleteRagCollectionResponseSchema = z7.object({
|
|
722
840
|
name: z7.string(),
|
|
723
841
|
deleted: z7.literal(true)
|
|
@@ -813,7 +931,8 @@ var ragSchema = z8.strictObject({
|
|
|
813
931
|
minScore: z8.number().min(0).max(1),
|
|
814
932
|
minWords: z8.int().min(0).max(20),
|
|
815
933
|
topK: z8.int().min(1).max(10),
|
|
816
|
-
timeoutMs: z8.int().min(200).max(1e4)
|
|
934
|
+
timeoutMs: z8.int().min(200).max(1e4),
|
|
935
|
+
previousExchanges: z8.int().min(0).max(5)
|
|
817
936
|
});
|
|
818
937
|
var effectiveVoiceSettingsObjectSchema = z8.strictObject({
|
|
819
938
|
/** BCP-47-ish tag the STT is hinted with and the TTS speaks (`es`, `en-US`), or `multi`. */
|
|
@@ -847,18 +966,16 @@ var effectiveVoiceSettingsObjectSchema = z8.strictObject({
|
|
|
847
966
|
* dispatch to the worker leaves it out (`callVoiceSettingsSchema`).
|
|
848
967
|
*/
|
|
849
968
|
recording: z8.boolean(),
|
|
969
|
+
/** The box runs the search before the model's turn, so the worker never reads it. */
|
|
970
|
+
rag: ragSchema,
|
|
850
971
|
/**
|
|
851
|
-
*
|
|
852
|
-
*
|
|
853
|
-
*
|
|
854
|
-
*
|
|
855
|
-
*
|
|
856
|
-
* one written. The box strips it off any other gateway. The SDK applies it when
|
|
857
|
-
* it builds the graph, so the worker never reads it.
|
|
972
|
+
* What the agent says when a guardrail stops the caller's sentence (FDE-66) or the
|
|
973
|
+
* agent's reply (STA3-59):
|
|
974
|
+
* a policy message written for a chat reader would leak the rule if a TTS read it.
|
|
975
|
+
* The box answers the turn itself, so the worker never reads it. Not nullable: a
|
|
976
|
+
* blocked turn always says something.
|
|
858
977
|
*/
|
|
859
|
-
|
|
860
|
-
/** The box runs the search before the model's turn, so the worker never reads it. */
|
|
861
|
-
rag: ragSchema
|
|
978
|
+
refusalPhrase: z8.string().trim().min(1).max(300)
|
|
862
979
|
});
|
|
863
980
|
var checkCrossFieldRules = (value, ctx) => {
|
|
864
981
|
const min = value.turn?.minEndpointingDelayMs;
|
|
@@ -888,8 +1005,8 @@ var checkCrossFieldRules = (value, ctx) => {
|
|
|
888
1005
|
var effectiveVoiceSettingsSchema = effectiveVoiceSettingsObjectSchema.superRefine(checkCrossFieldRules);
|
|
889
1006
|
var callVoiceSettingsObjectSchema = effectiveVoiceSettingsObjectSchema.omit({
|
|
890
1007
|
recording: true,
|
|
891
|
-
|
|
892
|
-
|
|
1008
|
+
rag: true,
|
|
1009
|
+
refusalPhrase: true
|
|
893
1010
|
});
|
|
894
1011
|
var callVoiceSettingsSchema = callVoiceSettingsObjectSchema.superRefine(checkCrossFieldRules);
|
|
895
1012
|
var VOICE_SETTINGS_LEAF_PATHS = [
|
|
@@ -913,12 +1030,13 @@ var VOICE_SETTINGS_LEAF_PATHS = [
|
|
|
913
1030
|
"phoneLine",
|
|
914
1031
|
"expressive",
|
|
915
1032
|
"recording",
|
|
916
|
-
"fastModelRouting",
|
|
917
1033
|
"rag.collection",
|
|
918
1034
|
"rag.minScore",
|
|
919
1035
|
"rag.minWords",
|
|
920
1036
|
"rag.topK",
|
|
921
|
-
"rag.timeoutMs"
|
|
1037
|
+
"rag.timeoutMs",
|
|
1038
|
+
"rag.previousExchanges",
|
|
1039
|
+
"refusalPhrase"
|
|
922
1040
|
];
|
|
923
1041
|
var voiceSettingOriginSchema = z8.enum(["studio", "code", "default"]);
|
|
924
1042
|
var effectiveVoiceSettingsViewSchema = z8.strictObject({
|
|
@@ -981,6 +1099,7 @@ var VOICE_TRACES_PATH = "/voice/v1/traces";
|
|
|
981
1099
|
var VOICE_WORKER_AGENT_NAME = "stackbone-voice";
|
|
982
1100
|
var VOICE_WORKER_AGENT_NAME_ENV = "STACKBONE_VOICE_AGENT_NAME";
|
|
983
1101
|
var VOICE_WORKER_PORT_ENV = "STACKBONE_VOICE_WORKER_PORT";
|
|
1102
|
+
var VOICE_TURN_METRICS_MESSAGE = "voice turn metrics";
|
|
984
1103
|
var VOICE_CALL_TOKEN_MAX = 2048;
|
|
985
1104
|
var voiceDispatchSchema = z8.strictObject({
|
|
986
1105
|
agent: deepAgentNameSchema,
|
|
@@ -1045,15 +1164,34 @@ var voiceCallTranscriptSchema = z8.array(
|
|
|
1045
1164
|
).max(1e3).refine((items) => items.reduce((size, item) => size + item.text.length, 0) <= 2e5, {
|
|
1046
1165
|
message: "A call transcript may contain at most 200000 characters."
|
|
1047
1166
|
});
|
|
1167
|
+
var VOICE_CALL_TURNS_MAX = 1e3;
|
|
1168
|
+
var AGUI_RUN_ID_MAX = 200;
|
|
1169
|
+
var VOICE_TURN_LATENCY_MAX_MS = 36e5;
|
|
1170
|
+
var turnLatencyMs = z8.number().int().min(0).max(VOICE_TURN_LATENCY_MAX_MS).optional();
|
|
1171
|
+
var voiceCallTurnMetricsSchema = z8.strictObject({
|
|
1172
|
+
/** The AG-UI `runId` of the box run that answered; absent on a reply the box did not run. */
|
|
1173
|
+
runId: z8.string().min(1).max(AGUI_RUN_ID_MAX).optional(),
|
|
1174
|
+
endOfTurnMs: turnLatencyMs,
|
|
1175
|
+
transcriptionMs: turnLatencyMs,
|
|
1176
|
+
llmTtftMs: turnLatencyMs,
|
|
1177
|
+
ttsTtfbMs: turnLatencyMs,
|
|
1178
|
+
e2eMs: turnLatencyMs
|
|
1179
|
+
});
|
|
1180
|
+
var voiceCallTurnsSchema = z8.array(voiceCallTurnMetricsSchema).max(VOICE_CALL_TURNS_MAX);
|
|
1048
1181
|
var voiceCallCompleteSchema = z8.strictObject({
|
|
1049
1182
|
transcript: voiceCallTranscriptSchema,
|
|
1050
|
-
outcome: z8.enum(["completed", "disconnected", "failed"])
|
|
1183
|
+
outcome: z8.enum(["completed", "disconnected", "failed"]),
|
|
1184
|
+
// The transcript, the memory and the `onEnd` start this body carries must not be
|
|
1185
|
+
// refused over a bad latency, so `turns` is any value here and read apart through
|
|
1186
|
+
// `readVoiceCallTurns`. Not `.catch()` on the array: this schema is a documented
|
|
1187
|
+
// route's body, and the OpenAPI generator cannot describe a catch (STA3-56).
|
|
1188
|
+
turns: z8.unknown().optional()
|
|
1051
1189
|
});
|
|
1052
1190
|
var voiceCallLifecycleResponseSchema = z8.strictObject({ ok: z8.literal(true) });
|
|
1053
1191
|
|
|
1054
1192
|
// libs/runtime/voice-worker/src/box-llm.ts
|
|
1055
1193
|
import { randomUUID } from "node:crypto";
|
|
1056
|
-
import { DEFAULT_API_CONNECT_OPTIONS, llm, log } from "@livekit/agents";
|
|
1194
|
+
import { DEFAULT_API_CONNECT_OPTIONS, llm as llm2, log } from "@livekit/agents";
|
|
1057
1195
|
|
|
1058
1196
|
// libs/shared/llm-wire/src/lib/sse/read-sse-data.ts
|
|
1059
1197
|
async function* readSseData(body) {
|
|
@@ -1098,8 +1236,9 @@ var BoxTurnError = class extends Error {
|
|
|
1098
1236
|
}
|
|
1099
1237
|
};
|
|
1100
1238
|
var boxEndpoint = (boxUrl, path) => `${boxUrl.replace(/\/+$/, "")}${path}`;
|
|
1239
|
+
var mintAguiRunId = () => `run-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
1101
1240
|
async function* aguiTurn(opts) {
|
|
1102
|
-
const runId =
|
|
1241
|
+
const runId = opts.runId ?? mintAguiRunId();
|
|
1103
1242
|
const heard = "toolResult" in opts ? void 0 : opts.heard;
|
|
1104
1243
|
const message = "toolResult" in opts ? {
|
|
1105
1244
|
id: `tool-${opts.toolResult.toolCallId}`,
|
|
@@ -1379,8 +1518,45 @@ var lastNewAssistantMessage = (chatCtx, seen) => {
|
|
|
1379
1518
|
return void 0;
|
|
1380
1519
|
};
|
|
1381
1520
|
|
|
1521
|
+
// libs/runtime/voice-worker/src/turn-metrics.ts
|
|
1522
|
+
import { llm, voice as voice2 } from "@livekit/agents";
|
|
1523
|
+
var AGUI_RUN_ID_EXTRA = "stackboneAguiRunId";
|
|
1524
|
+
var ms = (seconds) => {
|
|
1525
|
+
if (typeof seconds !== "number" || !Number.isFinite(seconds)) return void 0;
|
|
1526
|
+
const value = Math.round(seconds * 1e3);
|
|
1527
|
+
return value >= 0 && value <= VOICE_TURN_LATENCY_MAX_MS ? value : void 0;
|
|
1528
|
+
};
|
|
1529
|
+
var defined = (metrics) => Object.fromEntries(Object.entries(metrics).filter(([, value]) => value !== void 0));
|
|
1530
|
+
var watchTurnMetrics = (session, write, options = {}) => {
|
|
1531
|
+
let caller;
|
|
1532
|
+
let delivered;
|
|
1533
|
+
session.on(voice2.AgentSessionEventTypes.ConversationItemAdded, ({ item }) => {
|
|
1534
|
+
if (!(item instanceof llm.ChatMessage)) return;
|
|
1535
|
+
if (item.role === "user") {
|
|
1536
|
+
caller = item.metrics;
|
|
1537
|
+
delivered = options.endOfTurnMs?.();
|
|
1538
|
+
return;
|
|
1539
|
+
}
|
|
1540
|
+
if (item.role !== "assistant") return;
|
|
1541
|
+
const reply = item.metrics;
|
|
1542
|
+
const runId = item.extra[AGUI_RUN_ID_EXTRA];
|
|
1543
|
+
const e2eMs = ms(reply.e2eLatency);
|
|
1544
|
+
const metrics = defined({
|
|
1545
|
+
endOfTurnMs: delivered ?? ms(caller?.endOfTurnDelay),
|
|
1546
|
+
transcriptionMs: options.endOfTurnMs ? void 0 : ms(caller?.transcriptionDelay),
|
|
1547
|
+
llmTtftMs: ms(reply.llmNodeTtft),
|
|
1548
|
+
ttsTtfbMs: ms(reply.ttsNodeTtfb),
|
|
1549
|
+
e2eMs: e2eMs === void 0 || delivered === void 0 ? e2eMs : e2eMs + delivered
|
|
1550
|
+
});
|
|
1551
|
+
caller = void 0;
|
|
1552
|
+
delivered = void 0;
|
|
1553
|
+
if (Object.keys(metrics).length === 0) return;
|
|
1554
|
+
write(typeof runId === "string" ? { ...metrics, runId } : metrics);
|
|
1555
|
+
});
|
|
1556
|
+
};
|
|
1557
|
+
|
|
1382
1558
|
// libs/runtime/voice-worker/src/box-llm.ts
|
|
1383
|
-
var BoxLLM = class extends
|
|
1559
|
+
var BoxLLM = class extends llm2.LLM {
|
|
1384
1560
|
#opts;
|
|
1385
1561
|
#replies = new HeardTracker();
|
|
1386
1562
|
#pending = /* @__PURE__ */ new Set();
|
|
@@ -1477,7 +1653,7 @@ var lastUserText = (chatCtx) => {
|
|
|
1477
1653
|
}
|
|
1478
1654
|
return "";
|
|
1479
1655
|
};
|
|
1480
|
-
var BoxLLMStream = class extends
|
|
1656
|
+
var BoxLLMStream = class extends llm2.LLMStream {
|
|
1481
1657
|
#opts;
|
|
1482
1658
|
#replies;
|
|
1483
1659
|
#settle;
|
|
@@ -1507,6 +1683,7 @@ var BoxLLMStream = class extends llm.LLMStream {
|
|
|
1507
1683
|
...turn
|
|
1508
1684
|
} = this.#opts;
|
|
1509
1685
|
const id = randomUUID();
|
|
1686
|
+
const runId = mintAguiRunId();
|
|
1510
1687
|
const { signal } = this.abortController;
|
|
1511
1688
|
await pendingResume?.();
|
|
1512
1689
|
if (signal.aborted || isEnding?.()) return;
|
|
@@ -1531,7 +1708,8 @@ var BoxLLMStream = class extends llm.LLMStream {
|
|
|
1531
1708
|
tools: clientTools,
|
|
1532
1709
|
text: lastUserText(this.chatCtx),
|
|
1533
1710
|
...heard ? { heard } : {},
|
|
1534
|
-
signal: turnSignal
|
|
1711
|
+
signal: turnSignal,
|
|
1712
|
+
runId
|
|
1535
1713
|
})) {
|
|
1536
1714
|
restartIdle();
|
|
1537
1715
|
reply.observe(ev);
|
|
@@ -1542,7 +1720,7 @@ var BoxLLMStream = class extends llm.LLMStream {
|
|
|
1542
1720
|
delta: {
|
|
1543
1721
|
role: "assistant",
|
|
1544
1722
|
content: ev.text,
|
|
1545
|
-
extra: { [VOICE_RESPONSE_MARKER]: id }
|
|
1723
|
+
extra: { [VOICE_RESPONSE_MARKER]: id, [AGUI_RUN_ID_EXTRA]: runId }
|
|
1546
1724
|
}
|
|
1547
1725
|
});
|
|
1548
1726
|
break;
|
|
@@ -1589,6 +1767,49 @@ var BoxLLMStream = class extends llm.LLMStream {
|
|
|
1589
1767
|
}
|
|
1590
1768
|
};
|
|
1591
1769
|
|
|
1770
|
+
// libs/runtime/voice-worker/src/stt-delivery.ts
|
|
1771
|
+
import { stt } from "@livekit/agents";
|
|
1772
|
+
import { TransformStream } from "node:stream/web";
|
|
1773
|
+
var finalDeliveryMs = (event, audioSentSec) => {
|
|
1774
|
+
const alt = event.alternatives?.[0];
|
|
1775
|
+
if (alt === void 0 || !Number.isFinite(alt.endTime) || alt.endTime <= 0) return void 0;
|
|
1776
|
+
const segmentEndSec = alt.endTime - (alt.words?.[0]?.startTimeOffset ?? 0);
|
|
1777
|
+
const ms2 = Math.round((audioSentSec - segmentEndSec) * 1e3);
|
|
1778
|
+
return ms2 >= 0 ? ms2 : void 0;
|
|
1779
|
+
};
|
|
1780
|
+
var createSttDeliveryMeter = () => {
|
|
1781
|
+
let last;
|
|
1782
|
+
return {
|
|
1783
|
+
wrap: async (audio, inner) => {
|
|
1784
|
+
let audioSentSec = 0;
|
|
1785
|
+
async function* counted() {
|
|
1786
|
+
for await (const frame of audio) {
|
|
1787
|
+
audioSentSec += frame.samplesPerChannel / frame.sampleRate;
|
|
1788
|
+
yield frame;
|
|
1789
|
+
}
|
|
1790
|
+
}
|
|
1791
|
+
const events = await inner(counted());
|
|
1792
|
+
if (events === null) return null;
|
|
1793
|
+
return events.pipeThrough(
|
|
1794
|
+
new TransformStream({
|
|
1795
|
+
transform(event, controller) {
|
|
1796
|
+
if (typeof event !== "string" && event.type === stt.SpeechEventType.FINAL_TRANSCRIPT) {
|
|
1797
|
+
last = finalDeliveryMs(event, audioSentSec) ?? last;
|
|
1798
|
+
}
|
|
1799
|
+
controller.enqueue(event);
|
|
1800
|
+
}
|
|
1801
|
+
})
|
|
1802
|
+
);
|
|
1803
|
+
},
|
|
1804
|
+
take: () => {
|
|
1805
|
+
const taken = last;
|
|
1806
|
+
last = void 0;
|
|
1807
|
+
return taken;
|
|
1808
|
+
}
|
|
1809
|
+
};
|
|
1810
|
+
};
|
|
1811
|
+
var deliversItsOwnEndOfTurn = (settings) => settings.turn.mode === "stt" && settings.sttModel.startsWith("deepgram/flux");
|
|
1812
|
+
|
|
1592
1813
|
// libs/runtime/voice-worker/src/dispatch.ts
|
|
1593
1814
|
var readDispatch = (metadata) => {
|
|
1594
1815
|
if (!metadata) return null;
|
|
@@ -1665,17 +1886,17 @@ var createHangUp = ({
|
|
|
1665
1886
|
};
|
|
1666
1887
|
|
|
1667
1888
|
// libs/runtime/voice-worker/src/session-options.ts
|
|
1668
|
-
import { tts, voice as
|
|
1889
|
+
import { tts, voice as voice3 } from "@livekit/agents";
|
|
1669
1890
|
|
|
1670
1891
|
// libs/runtime/voice-worker/src/number-text.ts
|
|
1671
|
-
import { TransformStream } from "node:stream/web";
|
|
1892
|
+
import { TransformStream as TransformStream2 } from "node:stream/web";
|
|
1672
1893
|
var THOUSANDS_COMMA = /(\d),(?=\d{3}(?!\d))/g;
|
|
1673
1894
|
var TRAILING_NUMBER = /[\d.,]+$/;
|
|
1674
1895
|
var rewrite = (text) => text.replace(THOUSANDS_COMMA, "$1");
|
|
1675
1896
|
var dropThousandsComma = (input) => {
|
|
1676
1897
|
let pending = "";
|
|
1677
1898
|
return input.pipeThrough(
|
|
1678
|
-
new
|
|
1899
|
+
new TransformStream2({
|
|
1679
1900
|
transform(chunk, controller) {
|
|
1680
1901
|
const text = pending + chunk;
|
|
1681
1902
|
const tail = TRAILING_NUMBER.exec(text)?.[0] ?? "";
|
|
@@ -1692,7 +1913,7 @@ var dropThousandsComma = (input) => {
|
|
|
1692
1913
|
|
|
1693
1914
|
// libs/runtime/voice-worker/src/phone-voice.ts
|
|
1694
1915
|
import { AudioFrame as AudioFrame2 } from "@livekit/rtc-node";
|
|
1695
|
-
import { TransformStream as
|
|
1916
|
+
import { TransformStream as TransformStream3 } from "node:stream/web";
|
|
1696
1917
|
var LOW_CUT_HZ = 300;
|
|
1697
1918
|
var HIGH_CUT_HZ = 3400;
|
|
1698
1919
|
var Q = Math.SQRT1_2;
|
|
@@ -1728,7 +1949,7 @@ var phoneLine = (audio) => {
|
|
|
1728
1949
|
let chainRate = 0;
|
|
1729
1950
|
const norm = Math.tanh(DRIVE);
|
|
1730
1951
|
return audio.pipeThrough(
|
|
1731
|
-
new
|
|
1952
|
+
new TransformStream3({
|
|
1732
1953
|
transform(frame, controller) {
|
|
1733
1954
|
if (!chain || chainRate !== frame.sampleRate) {
|
|
1734
1955
|
chainRate = frame.sampleRate;
|
|
@@ -1781,7 +2002,16 @@ var markupContext = (expressive, speech) => {
|
|
|
1781
2002
|
});
|
|
1782
2003
|
return guide === void 0 ? [] : [{ description: VOICE_TTS_MARKUP_CONTEXT, value: guide }];
|
|
1783
2004
|
};
|
|
1784
|
-
var
|
|
2005
|
+
var VoiceCallAgent = class extends voice3.Agent {
|
|
2006
|
+
constructor() {
|
|
2007
|
+
super(...arguments);
|
|
2008
|
+
this.sttDelivery = createSttDeliveryMeter();
|
|
2009
|
+
}
|
|
2010
|
+
async sttNode(audio, modelSettings) {
|
|
2011
|
+
return this.sttDelivery.wrap(audio, (counted) => super.sttNode(counted, modelSettings));
|
|
2012
|
+
}
|
|
2013
|
+
};
|
|
2014
|
+
var PhoneLineAgent = class extends VoiceCallAgent {
|
|
1785
2015
|
async ttsNode(text, modelSettings) {
|
|
1786
2016
|
const audio = await super.ttsNode(text, modelSettings);
|
|
1787
2017
|
return audio && phoneLine(audio);
|
|
@@ -1835,7 +2065,7 @@ var voiceSessionOptions = (settings, pieces) => {
|
|
|
1835
2065
|
}
|
|
1836
2066
|
},
|
|
1837
2067
|
// The box holds the prompt: LiveKit's own chat context stays prompt-less.
|
|
1838
|
-
agent: settings.phoneLine ? new PhoneLineAgent({ instructions: "" }) : new
|
|
2068
|
+
agent: settings.phoneLine ? new PhoneLineAgent({ instructions: "" }) : new VoiceCallAgent({ instructions: "" }),
|
|
1839
2069
|
boxContext: markupContext(settings.expressive, speech)
|
|
1840
2070
|
};
|
|
1841
2071
|
};
|
|
@@ -1934,9 +2164,9 @@ var toolCallCue = () => {
|
|
|
1934
2164
|
};
|
|
1935
2165
|
};
|
|
1936
2166
|
var messageOf = (err) => err instanceof Error ? err.message : String(err);
|
|
1937
|
-
var callerNoiseCancellation = () => {
|
|
2167
|
+
var callerNoiseCancellation = (caller) => {
|
|
1938
2168
|
try {
|
|
1939
|
-
return BackgroundVoiceCancellation();
|
|
2169
|
+
return caller.kind === ParticipantKind.SIP ? TelephonyBackgroundVoiceCancellation() : BackgroundVoiceCancellation();
|
|
1940
2170
|
} catch (err) {
|
|
1941
2171
|
log4().warn({ error: messageOf(err) }, "voice worker: no noise cancellation on this call");
|
|
1942
2172
|
return void 0;
|
|
@@ -2070,6 +2300,7 @@ var runVoiceJob = async (ctx, deps) => {
|
|
|
2070
2300
|
onRunFinished
|
|
2071
2301
|
});
|
|
2072
2302
|
const session = deps.createSession({ ...plan.session, llm: brain });
|
|
2303
|
+
const turns = [];
|
|
2073
2304
|
let completion;
|
|
2074
2305
|
const finishCall = (outcome = "disconnected") => {
|
|
2075
2306
|
if (completion) return completion;
|
|
@@ -2078,14 +2309,20 @@ var runVoiceJob = async (ctx, deps) => {
|
|
|
2078
2309
|
await hangUp.pendingResume();
|
|
2079
2310
|
await session.close();
|
|
2080
2311
|
await brain.settled();
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2312
|
+
const transcript = brain.heardTranscript(session.history);
|
|
2313
|
+
const complete = (body) => voiceCallRequest(wire, "complete", voiceCallCompleteSchema.parse(body));
|
|
2314
|
+
try {
|
|
2315
|
+
await complete({ transcript, outcome, ...turns.length > 0 ? { turns } : {} });
|
|
2316
|
+
} catch (error) {
|
|
2317
|
+
if (!(turns.length > 0 && error instanceof BoxTurnError && error.status === 400)) {
|
|
2318
|
+
throw error;
|
|
2319
|
+
}
|
|
2320
|
+
log4().warn(
|
|
2321
|
+
{ agent: dispatch.agent, sessionKey: dispatch.sessionKey, status: error.status },
|
|
2322
|
+
"voice worker: the box refused the turn metrics, completing without them"
|
|
2323
|
+
);
|
|
2324
|
+
await complete({ transcript, outcome });
|
|
2325
|
+
}
|
|
2089
2326
|
}).catch((error) => {
|
|
2090
2327
|
log4().warn(
|
|
2091
2328
|
{ agent: dispatch.agent, sessionKey: dispatch.sessionKey, reason: "completion_failed" },
|
|
@@ -2096,11 +2333,22 @@ var runVoiceJob = async (ctx, deps) => {
|
|
|
2096
2333
|
return completion;
|
|
2097
2334
|
};
|
|
2098
2335
|
ctx.addShutdownCallback(() => finishCall());
|
|
2099
|
-
session.on(
|
|
2336
|
+
session.on(voice4.AgentSessionEventTypes.Close, () => {
|
|
2100
2337
|
if (completion || ending) return;
|
|
2101
2338
|
void finishCall().catch(() => void 0).finally(() => ctx.shutdown("caller left"));
|
|
2102
2339
|
});
|
|
2103
|
-
|
|
2340
|
+
watchTurnMetrics(
|
|
2341
|
+
session,
|
|
2342
|
+
(metrics) => {
|
|
2343
|
+
if (turns.length < VOICE_CALL_TURNS_MAX) turns.push(metrics);
|
|
2344
|
+
log4().info(
|
|
2345
|
+
{ agent: dispatch.agent, sessionKey: dispatch.sessionKey, room: ctx.room.name, ...metrics },
|
|
2346
|
+
VOICE_TURN_METRICS_MESSAGE
|
|
2347
|
+
);
|
|
2348
|
+
},
|
|
2349
|
+
deliversItsOwnEndOfTurn(settings) ? { endOfTurnMs: () => plan.agent.sttDelivery.take() } : {}
|
|
2350
|
+
);
|
|
2351
|
+
const noiseCancellation = callerNoiseCancellation(participant);
|
|
2104
2352
|
await session.start({
|
|
2105
2353
|
agent: plan.agent,
|
|
2106
2354
|
room: ctx.room,
|
|
@@ -2114,7 +2362,7 @@ var runVoiceJob = async (ctx, deps) => {
|
|
|
2114
2362
|
|
|
2115
2363
|
// libs/runtime/voice-worker/src/main.ts
|
|
2116
2364
|
var liveDeps = () => ({
|
|
2117
|
-
createSession: (options) => new
|
|
2365
|
+
createSession: (options) => new voice5.AgentSession(options),
|
|
2118
2366
|
createStt: (options) => new inference.STT(options),
|
|
2119
2367
|
createTts: (options) => new inference.TTS(options),
|
|
2120
2368
|
createTurnDetector: (options) => new inference.TurnDetector(options),
|