@vellumai/assistant 0.12.2-staging.4 → 0.12.2-staging.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +7 -7
- package/openapi.yaml +151 -53
- package/package.json +5 -4
- package/scripts/bundled-plugin-packages.ts +154 -0
- package/scripts/generate-bundled-plugin-packages.ts +29 -0
- package/scripts/test.ts +15 -13
- package/src/__tests__/managed-store.test.ts +121 -0
- package/src/__tests__/scaffold-managed-skill-tool.test.ts +88 -0
- package/src/calls/__tests__/voice-control-protocol.test.ts +24 -2
- package/src/calls/__tests__/voice-session-bridge.test.ts +22 -2
- package/src/calls/voice-control-protocol.ts +13 -4
- package/src/calls/voice-session-bridge.ts +27 -6
- package/src/cli/commands/__tests__/plugins.test.ts +66 -0
- package/src/cli/commands/plugins.ts +50 -18
- package/src/cli/lib/__tests__/install-from-github.test.ts +67 -0
- package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +169 -0
- package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +57 -0
- package/src/cli/lib/__tests__/plugin-catalog-platform.test.ts +14 -0
- package/src/cli/lib/__tests__/plugin-catalog-resolve.test.ts +27 -1
- package/src/cli/lib/__tests__/plugin-details.test.ts +9 -2
- package/src/cli/lib/__tests__/plugin-marketplace.test.ts +20 -0
- package/src/cli/lib/__tests__/plugins-install-offline.test.ts +43 -0
- package/src/cli/lib/__tests__/search-plugins.test.ts +31 -0
- package/src/cli/lib/bundled-plugin-packages.json +4 -0
- package/src/cli/lib/bundled-plugin-packages.ts +87 -0
- package/src/cli/lib/diff-plugin.ts +1 -1
- package/src/cli/lib/inspect-plugin.ts +80 -12
- package/src/cli/lib/install-from-github.ts +133 -76
- package/src/cli/lib/plugin-catalog-cache.ts +22 -3
- package/src/cli/lib/plugin-catalog-local.ts +13 -3
- package/src/cli/lib/plugin-catalog-platform.ts +6 -1
- package/src/cli/lib/plugin-catalog-resolve.ts +20 -0
- package/src/cli/lib/plugin-details.ts +12 -0
- package/src/cli/lib/plugin-marketplace.ts +121 -21
- package/src/cli/lib/plugin-pin-history.ts +5 -2
- package/src/cli/lib/search-plugins.ts +58 -16
- package/src/cli/lib/upgrade-plugin.ts +48 -3
- package/src/config/bundled-skills/skill-management/SKILL.md +1 -1
- package/src/config/bundled-skills/skill-management/TOOLS.json +7 -7
- package/src/daemon/conversation-tool-setup.ts +4 -22
- package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +374 -0
- package/src/live-voice/__tests__/live-voice-vad.test.ts +658 -3
- package/src/live-voice/__tests__/protocol.test.ts +48 -0
- package/src/live-voice/__tests__/session-controls.test.ts +43 -0
- package/src/live-voice/live-voice-session.ts +784 -64
- package/src/live-voice/protocol.ts +20 -0
- package/src/live-voice/session-controls.ts +61 -3
- package/src/monitoring/plugin-auto-update.ts +6 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +15 -0
- package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +1 -1
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +102 -0
- package/src/runtime/routes/plugins-routes.ts +69 -49
- package/src/skills/managed-store.ts +153 -31
- package/src/tools/skills/find-similar-skills.test.ts +214 -2
- package/src/tools/skills/find-similar-skills.ts +62 -9
- package/src/tools/skills/resolve-execute-invocation.ts +24 -0
- package/src/tools/skills/scaffold-managed.ts +16 -13
|
@@ -23,7 +23,10 @@ import {
|
|
|
23
23
|
SPOKEN_REPLY_PLAIN_TEXT_RULE,
|
|
24
24
|
} from "../calls/spoken-reply-rules.js";
|
|
25
25
|
import { sanitizeForTts } from "../calls/tts-text-sanitizer.js";
|
|
26
|
-
import {
|
|
26
|
+
import {
|
|
27
|
+
createControlMarkerHoldback,
|
|
28
|
+
TASK_STOP_MARKER,
|
|
29
|
+
} from "../calls/voice-control-protocol.js";
|
|
27
30
|
import {
|
|
28
31
|
createFrontDoorLegCoordinator,
|
|
29
32
|
type FrontDoorLegCoordinator,
|
|
@@ -97,7 +100,8 @@ import {
|
|
|
97
100
|
liveVoiceSilenceReason,
|
|
98
101
|
liveVoiceStartScreen,
|
|
99
102
|
} from "../telemetry/live-voice-funnel.js";
|
|
100
|
-
import { getToolOwner } from "../tools/registry.js";
|
|
103
|
+
import { getTool, getToolOwner } from "../tools/registry.js";
|
|
104
|
+
import { resolveSkillExecuteInvocation } from "../tools/skills/resolve-execute-invocation.js";
|
|
101
105
|
import {
|
|
102
106
|
createReasoningTagFilter,
|
|
103
107
|
type ReasoningTagFilter,
|
|
@@ -160,6 +164,11 @@ import {
|
|
|
160
164
|
} from "./protocol.js";
|
|
161
165
|
import {
|
|
162
166
|
type ClientSessionControlRequest,
|
|
167
|
+
isLookSessionControl,
|
|
168
|
+
LOOK_FOLLOW_UP_CONTENT,
|
|
169
|
+
LOOK_FRAME_REASON,
|
|
170
|
+
lookFollowUpNote,
|
|
171
|
+
type LookSessionControl,
|
|
163
172
|
progressConfigForCadence,
|
|
164
173
|
requestedSessionControl,
|
|
165
174
|
sessionControlTeaching,
|
|
@@ -265,11 +274,26 @@ const CONTINUATION_ANNOUNCE_SILENCE_MS = 1_500;
|
|
|
265
274
|
// reply's audio landing between two checks. Beyond the cap the announcement
|
|
266
275
|
// falls back to the stash rather than chasing a call that keeps talking.
|
|
267
276
|
const CONTINUATION_ANNOUNCE_MAX_DRAIN_REARMS = 3;
|
|
277
|
+
const FOREGROUND_TASK_RESUME_SILENCE_MS = 1_500;
|
|
278
|
+
const FOREGROUND_TASK_MAX_SUSPENDED_MS = 120_000;
|
|
279
|
+
const FOREGROUND_TASK_MAX_INTERVENING_TURNS = 3;
|
|
268
280
|
// `content` of an announcement turn. The answer never rides here — it goes in
|
|
269
281
|
// the model-facing control prompt (buildLiveDeliveryNote), so the only thing
|
|
270
282
|
// persisted on the user side is this marker, and it persists hidden.
|
|
271
283
|
export const CONTINUATION_DELIVERY_CONTENT =
|
|
272
284
|
"(background work finished — deliver it now)";
|
|
285
|
+
// How long a look control waits for the fresh frame the client takes for it.
|
|
286
|
+
// Long enough for a camera that has to open and warm up before its first keep;
|
|
287
|
+
// past it the look is dropped rather than answered into a later, unrelated
|
|
288
|
+
// silence.
|
|
289
|
+
const LOOK_FRAME_WAIT_MS = 10_000;
|
|
290
|
+
// How long after the look was asked for its answer may still start. Past it
|
|
291
|
+
// the floor has been taken for so long (a user talking through noise that never
|
|
292
|
+
// became a turn, say) that the look no longer answers what is on screen.
|
|
293
|
+
const LOOK_ANSWER_DEADLINE_MS = 15_000;
|
|
294
|
+
// The shortest wait before checking the floor again, for a blocker with no
|
|
295
|
+
// known end time (the look's own turn clearing, an utterance in capture).
|
|
296
|
+
const LOOK_FOLLOW_UP_REARM_MS = 250;
|
|
273
297
|
|
|
274
298
|
export type LiveVoiceStreamingTranscriberResolver = (
|
|
275
299
|
options: ResolveStreamingTranscriberOptions,
|
|
@@ -445,6 +469,12 @@ export interface LiveVoiceSessionOptions {
|
|
|
445
469
|
* `CONTINUATION_ANNOUNCE_SILENCE_MS`.
|
|
446
470
|
*/
|
|
447
471
|
continuationAnnounceSilenceMs?: number;
|
|
472
|
+
/** Overrides the silence before a suspended foreground task resumes (test hook). */
|
|
473
|
+
foregroundTaskResumeSilenceMs?: number;
|
|
474
|
+
/** Overrides the maximum time a foreground task may remain suspended (test hook). */
|
|
475
|
+
foregroundTaskMaxSuspendedMs?: number;
|
|
476
|
+
/** Overrides the committed user-turn limit while suspended (test hook). */
|
|
477
|
+
foregroundTaskMaxInterveningTurns?: number;
|
|
448
478
|
}
|
|
449
479
|
|
|
450
480
|
type LiveVoiceUtterancePhase =
|
|
@@ -697,10 +727,21 @@ interface ActiveAssistantTurn {
|
|
|
697
727
|
// no user utterance behind it — `content` is CONTINUATION_DELIVERY_CONTENT and
|
|
698
728
|
// the answer rides the control prompt (buildLiveDeliveryNote).
|
|
699
729
|
continuationDelivery: ContinuationDelivery | null;
|
|
730
|
+
// Set only on the turn that answers a look: which look it answers. The turn
|
|
731
|
+
// has no user utterance behind it; the instruction rides the control prompt
|
|
732
|
+
// (lookFollowUpNote).
|
|
733
|
+
lookFollowUp: LookSessionControl | null;
|
|
700
734
|
// The turn's content is an internal instruction rather than user speech (the
|
|
701
735
|
// greeting that opens a session, say). The row still persists and the model
|
|
702
736
|
// still sees it; `hiddenSyntheticPrompt` keeps it out of the transcript.
|
|
703
737
|
hiddenPrompt: boolean;
|
|
738
|
+
// The suspended foreground task this turn may continue. Set only after the user
|
|
739
|
+
// turn commits or on the synthetic resume turn; speculative discards never
|
|
740
|
+
// mutate the session's task state.
|
|
741
|
+
foregroundTaskEpoch: number | null;
|
|
742
|
+
// Snapshot retained while a tool-capable leg owns suspended work. If that
|
|
743
|
+
// leg fails asynchronously, the task returns to the same suspension budget.
|
|
744
|
+
foregroundTaskSuspensionSnapshot: SuspendedForegroundTask | null;
|
|
704
745
|
// Set when a barge-in handed the interrupted work to a background subagent:
|
|
705
746
|
// that request's transcript, so the model can tell the user the work is
|
|
706
747
|
// still running instead of appearing to have dropped it.
|
|
@@ -805,7 +846,7 @@ const LIVE_VOICE_SETUP_FLOW_TEACHING =
|
|
|
805
846
|
// off answering, rather than a fresh follow-up. Reaches the model only; it is
|
|
806
847
|
// not a user message and never renders as a transcript bubble.
|
|
807
848
|
function buildInterruptionMergeNote(interruptedRequest: string): string {
|
|
808
|
-
return `The user interrupted your previous, unfinished reply. Their earlier request was: "${interruptedRequest}". Treat their current message as a continuation of that request and address both together
|
|
849
|
+
return `The user interrupted your previous, unfinished reply. Their earlier request was: "${interruptedRequest}". Treat their current message as a continuation of that request and address both together. If they corrected or redirected the unfinished work, continue executing it with that correction instead of merely acknowledging it. If they clearly want the unfinished task abandoned, use the task-stop control.`;
|
|
809
850
|
}
|
|
810
851
|
|
|
811
852
|
// System-level guidance appended to the NEXT turn's control prompt after a
|
|
@@ -838,6 +879,14 @@ function describeInterruptedRequest(request: string): string {
|
|
|
838
879
|
: "their earlier request";
|
|
839
880
|
}
|
|
840
881
|
|
|
882
|
+
// A look control waiting on its fresh frame: which look, when it was asked
|
|
883
|
+
// for, and the wait's bound.
|
|
884
|
+
interface PendingLook {
|
|
885
|
+
action: LookSessionControl;
|
|
886
|
+
armedAtMs: number;
|
|
887
|
+
timer: ReturnType<typeof setTimeout>;
|
|
888
|
+
}
|
|
889
|
+
|
|
841
890
|
// A finished background continuation waiting to be delivered: the request it
|
|
842
891
|
// took over and the answer it produced.
|
|
843
892
|
interface ContinuationDelivery {
|
|
@@ -845,6 +894,26 @@ interface ContinuationDelivery {
|
|
|
845
894
|
answer: string;
|
|
846
895
|
}
|
|
847
896
|
|
|
897
|
+
interface OwnedForegroundTask {
|
|
898
|
+
phase: "owned";
|
|
899
|
+
epoch: number;
|
|
900
|
+
request: string;
|
|
901
|
+
ownerToken: symbol;
|
|
902
|
+
hostToolStarted: boolean;
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
interface SuspendedForegroundTask {
|
|
906
|
+
phase: "suspended";
|
|
907
|
+
epoch: number;
|
|
908
|
+
request: string;
|
|
909
|
+
suspendedAtMs: number;
|
|
910
|
+
interveningTurns: number;
|
|
911
|
+
resumePending: boolean;
|
|
912
|
+
hostToolStarted: boolean;
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
type ForegroundTaskState = OwnedForegroundTask | SuspendedForegroundTask;
|
|
916
|
+
|
|
848
917
|
// Appended to an announcement turn's control prompt: the turn the session
|
|
849
918
|
// starts on its own when a background continuation finishes while the call is
|
|
850
919
|
// live and nobody is speaking. There is no user utterance behind it, so this
|
|
@@ -856,6 +925,12 @@ function buildLiveDeliveryNote(request: string, answer: string): string {
|
|
|
856
925
|
return `The user interrupted you earlier, and in the background you finished ${what}. The call is still live and nobody is speaking, so tell them briefly that it is done and give them the result. Do not re-run any tool calls; the work is already complete, and do not repeat it verbatim if it no longer fits. What you produced was:\n\n${answer}`;
|
|
857
926
|
}
|
|
858
927
|
|
|
928
|
+
function buildForegroundTaskResumePrompt(request: string): string {
|
|
929
|
+
const task =
|
|
930
|
+
request.length > 0 ? JSON.stringify(request) : "the unfinished task";
|
|
931
|
+
return `Continue the unfinished task that began with ${task}. Follow the user's latest corrections and all newer conversation context. If the conversation has clearly moved on or the user no longer wants the task, do not continue it and output only ${TASK_STOP_MARKER}, with no spoken words before the marker.`;
|
|
932
|
+
}
|
|
933
|
+
|
|
859
934
|
// Assemble a leg's model-facing control prompt: the base live-voice rules, the
|
|
860
935
|
// screen-reveal and setup-flow teaching (both withheld from the front-door leg,
|
|
861
936
|
// see LIVE_VOICE_SCREEN_REVEAL_TEACHING), plus any pending barge-in merge
|
|
@@ -866,13 +941,20 @@ function buildVoiceControlPrompt(
|
|
|
866
941
|
turn: ActiveAssistantTurn,
|
|
867
942
|
leg: { frontDoor?: boolean },
|
|
868
943
|
sessionControls: readonly LiveVoiceSessionControl[],
|
|
944
|
+
client: { lookFrames: boolean },
|
|
869
945
|
): string {
|
|
870
946
|
let prompt =
|
|
871
947
|
LIVE_VOICE_CONTROL_PROMPT_BASE +
|
|
872
948
|
(leg.frontDoor === true
|
|
873
949
|
? ""
|
|
874
950
|
: LIVE_VOICE_SCREEN_REVEAL_TEACHING + LIVE_VOICE_SETUP_FLOW_TEACHING) +
|
|
875
|
-
sessionControlTeaching(sessionControls, leg
|
|
951
|
+
sessionControlTeaching(sessionControls, leg, {
|
|
952
|
+
...client,
|
|
953
|
+
unfinishedTaskPending:
|
|
954
|
+
turn.foregroundTaskEpoch !== null ||
|
|
955
|
+
turn.interruptedRequest !== null ||
|
|
956
|
+
turn.handedOffRequest !== null,
|
|
957
|
+
});
|
|
876
958
|
if (turn.language !== undefined) {
|
|
877
959
|
prompt = `${prompt}\n\nThe caller has been speaking the language with code "${turn.language}" this turn. Reply in that language unless they clearly switch to another.`;
|
|
878
960
|
}
|
|
@@ -885,6 +967,9 @@ function buildVoiceControlPrompt(
|
|
|
885
967
|
if (turn.handedOffRequest) {
|
|
886
968
|
prompt = `${prompt}\n\n${buildHandoffAnnouncementNote(turn.handedOffRequest)}`;
|
|
887
969
|
}
|
|
970
|
+
if (turn.lookFollowUp !== null) {
|
|
971
|
+
prompt = `${prompt}\n\n${lookFollowUpNote(turn.lookFollowUp)}`;
|
|
972
|
+
}
|
|
888
973
|
if (turn.continuationDelivery) {
|
|
889
974
|
prompt = `${prompt}\n\n${buildLiveDeliveryNote(
|
|
890
975
|
turn.continuationDelivery.request,
|
|
@@ -1037,6 +1122,20 @@ function foregroundToolContendsWithContinuation(
|
|
|
1037
1122
|
return isRefusedInReadOnlyPass(toolName, ownerKind);
|
|
1038
1123
|
}
|
|
1039
1124
|
|
|
1125
|
+
function hostInteractiveToolName(
|
|
1126
|
+
toolName: string,
|
|
1127
|
+
input?: Record<string, unknown>,
|
|
1128
|
+
allowedToolNames?: ReadonlySet<string>,
|
|
1129
|
+
): string | null {
|
|
1130
|
+
const effectiveToolName =
|
|
1131
|
+
toolName === "skill_execute"
|
|
1132
|
+
? resolveSkillExecuteInvocation(input ?? {}, allowedToolNames).name
|
|
1133
|
+
: toolName;
|
|
1134
|
+
return getTool(effectiveToolName)?.executionTarget === "host"
|
|
1135
|
+
? effectiveToolName
|
|
1136
|
+
: null;
|
|
1137
|
+
}
|
|
1138
|
+
|
|
1040
1139
|
// Upper bound on how long a barge-in waits for the interrupted turn's teardown
|
|
1041
1140
|
// to settle before giving up on the continuation. The teardown settles once the
|
|
1042
1141
|
// aborted turn's agent loop reaches its `finally`, which can wait out BOTH the
|
|
@@ -1135,6 +1234,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1135
1234
|
// The session controls the client declared it can carry out; the only ones
|
|
1136
1235
|
// the model is taught and the only ones a reply's marker can trigger.
|
|
1137
1236
|
private readonly sessionControls: readonly LiveVoiceSessionControl[];
|
|
1237
|
+
// The client declared `lookFrames`: it sends a fresh frame for every look
|
|
1238
|
+
// control it carries out, so the session answers a look on its own turn and
|
|
1239
|
+
// teaches the model that the reply asking for one is only the acknowledgement.
|
|
1240
|
+
private readonly lookFrames: boolean;
|
|
1138
1241
|
// How often progress updates are spoken, as the user last asked out loud.
|
|
1139
1242
|
// Session-scoped: it applies from the next turn to the end of the call.
|
|
1140
1243
|
private progressCadence: "fewer" | "normal" = "normal";
|
|
@@ -1222,6 +1325,23 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1222
1325
|
// answer.
|
|
1223
1326
|
private pendingAnnouncement: ContinuationDelivery | null = null;
|
|
1224
1327
|
private announcementTimer: ReturnType<typeof setTimeout> | null = null;
|
|
1328
|
+
// Host-backed work stays on the parent conversation across barge-ins. An
|
|
1329
|
+
// owned task belongs to the tool-capable turn doing the work; a suspended
|
|
1330
|
+
// task is waiting for either a merged escalation or a synthetic resume.
|
|
1331
|
+
private foregroundTaskState: ForegroundTaskState | null = null;
|
|
1332
|
+
private foregroundTaskEpoch = 0;
|
|
1333
|
+
private foregroundTaskResumeTimer: ReturnType<typeof setTimeout> | null =
|
|
1334
|
+
null;
|
|
1335
|
+
private readonly foregroundTaskResumeSilenceMs: number;
|
|
1336
|
+
private readonly foregroundTaskMaxSuspendedMs: number;
|
|
1337
|
+
private readonly foregroundTaskMaxInterveningTurns: number;
|
|
1338
|
+
// A look control sent to a client that declared `lookFrames`, waiting for
|
|
1339
|
+
// the fresh frame the client takes for it. The session answers the look on a
|
|
1340
|
+
// turn of its own once that frame is in the conversation (see
|
|
1341
|
+
// answerLookWhenFloorIsFree). Cleared when the frame lands, when the wait
|
|
1342
|
+
// runs out, and when the session closes.
|
|
1343
|
+
private pendingLook: PendingLook | null = null;
|
|
1344
|
+
private lookFollowUpTimer: ReturnType<typeof setTimeout> | null = null;
|
|
1225
1345
|
// Set when a continuation actually spawns: the request it took over, so the
|
|
1226
1346
|
// NEXT turn can tell the user the work is still running. Consumed by that
|
|
1227
1347
|
// turn; cleared when the continuation finishes (by then the result note
|
|
@@ -1236,6 +1356,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1236
1356
|
// the distance from it to a keep arriving is the client leg of the frame the
|
|
1237
1357
|
// onset asked for, measured from the daemon's own clock.
|
|
1238
1358
|
private lastSpeechStartedAtMs: number | null = null;
|
|
1359
|
+
// How many assistant turns have launched, so a look can tell whether one has
|
|
1360
|
+
// started since its frame landed. A count rather than a time: two launches
|
|
1361
|
+
// and a frame can share a millisecond.
|
|
1362
|
+
private turnsLaunched = 0;
|
|
1239
1363
|
private readonly maxPendingAudioBytes: number;
|
|
1240
1364
|
// Set on VAD speech onset; consumed when the first speech chunk is routed
|
|
1241
1365
|
// to an utterance so the metric lands on the right turn.
|
|
@@ -1357,6 +1481,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1357
1481
|
defaultDetachTeardownSettleTimeoutMs();
|
|
1358
1482
|
this.continuationAnnounceSilenceMs =
|
|
1359
1483
|
options.continuationAnnounceSilenceMs ?? CONTINUATION_ANNOUNCE_SILENCE_MS;
|
|
1484
|
+
this.foregroundTaskResumeSilenceMs =
|
|
1485
|
+
options.foregroundTaskResumeSilenceMs ??
|
|
1486
|
+
FOREGROUND_TASK_RESUME_SILENCE_MS;
|
|
1487
|
+
this.foregroundTaskMaxSuspendedMs =
|
|
1488
|
+
options.foregroundTaskMaxSuspendedMs ?? FOREGROUND_TASK_MAX_SUSPENDED_MS;
|
|
1489
|
+
this.foregroundTaskMaxInterveningTurns =
|
|
1490
|
+
options.foregroundTaskMaxInterveningTurns ??
|
|
1491
|
+
FOREGROUND_TASK_MAX_INTERVENING_TURNS;
|
|
1360
1492
|
this.emitMetrics = options.emitMetrics ?? false;
|
|
1361
1493
|
this.createTurnId = options.createTurnId ?? randomUUID;
|
|
1362
1494
|
this.conversationId =
|
|
@@ -1413,6 +1545,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1413
1545
|
SERVER_VAD_PENDING_AUDIO_MAX_SECONDS;
|
|
1414
1546
|
this.textInput = context.startFrame.textInput === true;
|
|
1415
1547
|
this.sessionControls = context.startFrame.sessionControls ?? [];
|
|
1548
|
+
this.lookFrames = context.startFrame.lookFrames === true;
|
|
1416
1549
|
}
|
|
1417
1550
|
|
|
1418
1551
|
get finalTranscriptText(): string {
|
|
@@ -1661,6 +1794,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1661
1794
|
},
|
|
1662
1795
|
"Sight frame timing",
|
|
1663
1796
|
);
|
|
1797
|
+
// Only once the row is in: the turn that answers the look reads the
|
|
1798
|
+
// conversation, so the frame has to be there before that turn starts.
|
|
1799
|
+
if (result.ok && frame.timing?.reason === LOOK_FRAME_REASON) {
|
|
1800
|
+
this.lookFrameLanded();
|
|
1801
|
+
}
|
|
1664
1802
|
if (!result.ok && !this.isClosed) {
|
|
1665
1803
|
void this.sendFrame({
|
|
1666
1804
|
type: "error",
|
|
@@ -1847,6 +1985,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1847
1985
|
|
|
1848
1986
|
const shouldEmitSessionEndMetrics = this.state !== "failed";
|
|
1849
1987
|
this.state = "closed";
|
|
1988
|
+
this.clearForegroundTask("session_closed");
|
|
1850
1989
|
// Retire the island before the teardown below starts awaiting things. A
|
|
1851
1990
|
// close can take a while (a pending continuation is delivered first), and
|
|
1852
1991
|
// an activity left asserting "Speaking…" through it is exactly the stale
|
|
@@ -1866,6 +2005,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1866
2005
|
// cleared: the announcement is dead either way.
|
|
1867
2006
|
await this.deliverPendingContinuationToConversation();
|
|
1868
2007
|
this.clearContinuationAnnouncement();
|
|
2008
|
+
this.clearPendingLook();
|
|
1869
2009
|
this.stopSessionTranscriber();
|
|
1870
2010
|
// Detached continuations outlive the call. A deliberate `interrupt()`
|
|
1871
2011
|
// aborts them; ending the session leaves them running. With no next voice
|
|
@@ -2784,6 +2924,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2784
2924
|
// do not collide with it in the collector. turn_cancelled flushes
|
|
2785
2925
|
// client playback, so the drain estimate resets with it.
|
|
2786
2926
|
this.assistantPlaybackTailUntilMs = 0;
|
|
2927
|
+
const keepForegroundTaskOnParent =
|
|
2928
|
+
!turn.assistantCompleted && this.suspendOwnedForegroundTask(turn);
|
|
2787
2929
|
// Carry the interrupted request into the next turn so it merges with the
|
|
2788
2930
|
// barge-in utterance rather than being answered as a fresh follow-up.
|
|
2789
2931
|
const interruptedRequest = turn.utterance.finalTranscriptSegments
|
|
@@ -2864,6 +3006,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2864
3006
|
this.publishActivity(turn, "");
|
|
2865
3007
|
await this.sendFrame({ type: "turn_cancelled", turnId: turn.turnId });
|
|
2866
3008
|
await this.cancelAssistantTurn("barge_in");
|
|
3009
|
+
if (keepForegroundTaskOnParent) {
|
|
3010
|
+
log.info(
|
|
3011
|
+
{ turnId: turn.turnId },
|
|
3012
|
+
"Voice duplex continuation skipped for parent-owned foreground task",
|
|
3013
|
+
);
|
|
3014
|
+
return;
|
|
3015
|
+
}
|
|
2867
3016
|
// Keep the interrupted turn's work alive on a background subagent; the
|
|
2868
3017
|
// detach waits for its teardown to settle the partial into history before
|
|
2869
3018
|
// forking.
|
|
@@ -2909,16 +3058,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2909
3058
|
// already finished, so bargeIn returns it to the stash instead.
|
|
2910
3059
|
turn.continuationDelivery !== null
|
|
2911
3060
|
? "announcement_turn"
|
|
2912
|
-
: //
|
|
2913
|
-
//
|
|
2914
|
-
|
|
2915
|
-
|
|
2916
|
-
|
|
2917
|
-
|
|
2918
|
-
//
|
|
2919
|
-
|
|
2920
|
-
? "
|
|
2921
|
-
:
|
|
3061
|
+
: // Nor over the answer to a look: there is no request behind it
|
|
3062
|
+
// either, and the user talking over it is them moving on.
|
|
3063
|
+
turn.lookFollowUp !== null
|
|
3064
|
+
? "look_follow_up"
|
|
3065
|
+
: // The model already finished generating (barge-in during TTS playback
|
|
3066
|
+
// of a complete reply): there is nothing to continue, so a
|
|
3067
|
+
// continuation would just re-do a finished answer.
|
|
3068
|
+
turn.assistantCompleted
|
|
3069
|
+
? "assistant_already_completed"
|
|
3070
|
+
: // A stop (interrupt/close) or a superseding invalidation landed
|
|
3071
|
+
// during the barge-in teardown: honor it.
|
|
3072
|
+
this.detachStopGeneration !== stopGeneration
|
|
3073
|
+
? "invalidated_during_barge_teardown"
|
|
3074
|
+
: null;
|
|
2922
3075
|
if (skipReason !== null) {
|
|
2923
3076
|
log.info(
|
|
2924
3077
|
{ turnId: turn.turnId, skipReason },
|
|
@@ -3137,6 +3290,350 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3137
3290
|
}
|
|
3138
3291
|
}
|
|
3139
3292
|
|
|
3293
|
+
private foregroundTaskRequestForTurn(turn: ActiveAssistantTurn): string {
|
|
3294
|
+
const currentRequest = turn.utterance.finalTranscriptSegments
|
|
3295
|
+
.join(" ")
|
|
3296
|
+
.trim();
|
|
3297
|
+
return currentRequest || turn.interruptedRequest || "";
|
|
3298
|
+
}
|
|
3299
|
+
|
|
3300
|
+
private createForegroundTaskOwnership(
|
|
3301
|
+
turn: ActiveAssistantTurn,
|
|
3302
|
+
hostToolStarted: boolean,
|
|
3303
|
+
): OwnedForegroundTask {
|
|
3304
|
+
const task: OwnedForegroundTask = {
|
|
3305
|
+
phase: "owned",
|
|
3306
|
+
epoch: ++this.foregroundTaskEpoch,
|
|
3307
|
+
request: this.foregroundTaskRequestForTurn(turn),
|
|
3308
|
+
ownerToken: turn.token,
|
|
3309
|
+
hostToolStarted,
|
|
3310
|
+
};
|
|
3311
|
+
turn.foregroundTaskEpoch = task.epoch;
|
|
3312
|
+
this.foregroundTaskState = task;
|
|
3313
|
+
return task;
|
|
3314
|
+
}
|
|
3315
|
+
|
|
3316
|
+
private noteEscalatedForegroundTask(turn: ActiveAssistantTurn): void {
|
|
3317
|
+
if (
|
|
3318
|
+
this.activeAssistantTurn?.token !== turn.token ||
|
|
3319
|
+
turn.discardRequested ||
|
|
3320
|
+
turn.abortController.signal.aborted
|
|
3321
|
+
) {
|
|
3322
|
+
return;
|
|
3323
|
+
}
|
|
3324
|
+
if (
|
|
3325
|
+
this.claimSuspendedForegroundTask(turn) ||
|
|
3326
|
+
this.foregroundTaskState !== null
|
|
3327
|
+
) {
|
|
3328
|
+
return;
|
|
3329
|
+
}
|
|
3330
|
+
|
|
3331
|
+
const task = this.createForegroundTaskOwnership(turn, false);
|
|
3332
|
+
log.info(
|
|
3333
|
+
{ turnId: turn.turnId, epoch: task.epoch },
|
|
3334
|
+
"Escalated live voice turn provisionally owns a foreground task",
|
|
3335
|
+
);
|
|
3336
|
+
}
|
|
3337
|
+
|
|
3338
|
+
private noteHostToolStarted(
|
|
3339
|
+
turn: ActiveAssistantTurn,
|
|
3340
|
+
effectiveToolName: string,
|
|
3341
|
+
): void {
|
|
3342
|
+
const current = this.foregroundTaskState;
|
|
3343
|
+
if (current?.phase === "owned" && current.ownerToken === turn.token) {
|
|
3344
|
+
if (!current.hostToolStarted) {
|
|
3345
|
+
this.foregroundTaskState = { ...current, hostToolStarted: true };
|
|
3346
|
+
log.info(
|
|
3347
|
+
{ turnId: turn.turnId, epoch: current.epoch, effectiveToolName },
|
|
3348
|
+
"Foreground task confirmed by host tool use",
|
|
3349
|
+
);
|
|
3350
|
+
}
|
|
3351
|
+
return;
|
|
3352
|
+
}
|
|
3353
|
+
if (this.claimSuspendedForegroundTask(turn, true)) {
|
|
3354
|
+
return;
|
|
3355
|
+
}
|
|
3356
|
+
|
|
3357
|
+
const task = this.createForegroundTaskOwnership(turn, true);
|
|
3358
|
+
log.info(
|
|
3359
|
+
{ turnId: turn.turnId, epoch: task.epoch, effectiveToolName },
|
|
3360
|
+
"Live voice turn started a foreground task",
|
|
3361
|
+
);
|
|
3362
|
+
}
|
|
3363
|
+
|
|
3364
|
+
private claimSuspendedForegroundTask(
|
|
3365
|
+
turn: ActiveAssistantTurn,
|
|
3366
|
+
hostToolStarted = false,
|
|
3367
|
+
): boolean {
|
|
3368
|
+
const current = this.foregroundTaskState;
|
|
3369
|
+
if (
|
|
3370
|
+
current?.phase !== "suspended" ||
|
|
3371
|
+
turn.foregroundTaskEpoch !== current.epoch ||
|
|
3372
|
+
this.activeAssistantTurn?.token !== turn.token ||
|
|
3373
|
+
turn.discardRequested ||
|
|
3374
|
+
turn.abortController.signal.aborted
|
|
3375
|
+
) {
|
|
3376
|
+
return false;
|
|
3377
|
+
}
|
|
3378
|
+
this.foregroundTaskState = {
|
|
3379
|
+
phase: "owned",
|
|
3380
|
+
epoch: current.epoch,
|
|
3381
|
+
request: current.request,
|
|
3382
|
+
ownerToken: turn.token,
|
|
3383
|
+
hostToolStarted: current.hostToolStarted || hostToolStarted,
|
|
3384
|
+
};
|
|
3385
|
+
this.clearForegroundTaskResumeTimer();
|
|
3386
|
+
log.info(
|
|
3387
|
+
{
|
|
3388
|
+
turnId: turn.turnId,
|
|
3389
|
+
epoch: current.epoch,
|
|
3390
|
+
hostToolStarted: current.hostToolStarted || hostToolStarted,
|
|
3391
|
+
},
|
|
3392
|
+
"Suspended foreground task claimed by tool-capable turn",
|
|
3393
|
+
);
|
|
3394
|
+
return true;
|
|
3395
|
+
}
|
|
3396
|
+
|
|
3397
|
+
private suspendOwnedForegroundTask(turn: ActiveAssistantTurn): boolean {
|
|
3398
|
+
const current = this.foregroundTaskState;
|
|
3399
|
+
if (current?.phase === "owned" && current.ownerToken === turn.token) {
|
|
3400
|
+
this.foregroundTaskState = {
|
|
3401
|
+
phase: "suspended",
|
|
3402
|
+
epoch: current.epoch,
|
|
3403
|
+
request: current.request,
|
|
3404
|
+
suspendedAtMs: Date.now(),
|
|
3405
|
+
interveningTurns: 0,
|
|
3406
|
+
resumePending: false,
|
|
3407
|
+
hostToolStarted: current.hostToolStarted,
|
|
3408
|
+
};
|
|
3409
|
+
this.clearForegroundTaskResumeTimer();
|
|
3410
|
+
log.info(
|
|
3411
|
+
{ turnId: turn.turnId, epoch: current.epoch },
|
|
3412
|
+
"Foreground task suspended by voice barge-in",
|
|
3413
|
+
);
|
|
3414
|
+
return true;
|
|
3415
|
+
}
|
|
3416
|
+
return current !== null;
|
|
3417
|
+
}
|
|
3418
|
+
|
|
3419
|
+
private clearForegroundTask(reason: string): void {
|
|
3420
|
+
const current = this.foregroundTaskState;
|
|
3421
|
+
this.foregroundTaskEpoch += 1;
|
|
3422
|
+
this.foregroundTaskState = null;
|
|
3423
|
+
this.clearForegroundTaskResumeTimer();
|
|
3424
|
+
if (current !== null) {
|
|
3425
|
+
log.info(
|
|
3426
|
+
{
|
|
3427
|
+
epoch: current.epoch,
|
|
3428
|
+
phase: current.phase,
|
|
3429
|
+
hostToolStarted: current.hostToolStarted,
|
|
3430
|
+
reason,
|
|
3431
|
+
},
|
|
3432
|
+
"Live voice foreground task cleared",
|
|
3433
|
+
);
|
|
3434
|
+
}
|
|
3435
|
+
}
|
|
3436
|
+
|
|
3437
|
+
private stopOutstandingInterruptedWork(reason: string): void {
|
|
3438
|
+
this.clearForegroundTask(reason);
|
|
3439
|
+
this.abortDetachedRuns({ reason });
|
|
3440
|
+
}
|
|
3441
|
+
|
|
3442
|
+
private foregroundTaskSuspensionExpired(
|
|
3443
|
+
task: SuspendedForegroundTask,
|
|
3444
|
+
): "turn_limit" | "time_limit" | null {
|
|
3445
|
+
if (task.interveningTurns > this.foregroundTaskMaxInterveningTurns) {
|
|
3446
|
+
return "turn_limit";
|
|
3447
|
+
}
|
|
3448
|
+
if (Date.now() - task.suspendedAtMs >= this.foregroundTaskMaxSuspendedMs) {
|
|
3449
|
+
return "time_limit";
|
|
3450
|
+
}
|
|
3451
|
+
return null;
|
|
3452
|
+
}
|
|
3453
|
+
|
|
3454
|
+
private queueForegroundTaskResume(turn: ActiveAssistantTurn): void {
|
|
3455
|
+
const current = this.foregroundTaskState;
|
|
3456
|
+
if (
|
|
3457
|
+
current?.phase !== "suspended" ||
|
|
3458
|
+
turn.foregroundTaskEpoch !== current.epoch ||
|
|
3459
|
+
turn.hiddenPrompt ||
|
|
3460
|
+
turn.discardRequested ||
|
|
3461
|
+
turn.speculativePending
|
|
3462
|
+
) {
|
|
3463
|
+
return;
|
|
3464
|
+
}
|
|
3465
|
+
const next: SuspendedForegroundTask = {
|
|
3466
|
+
...current,
|
|
3467
|
+
interveningTurns: current.interveningTurns + 1,
|
|
3468
|
+
resumePending: true,
|
|
3469
|
+
};
|
|
3470
|
+
const expired = this.foregroundTaskSuspensionExpired(next);
|
|
3471
|
+
if (expired !== null) {
|
|
3472
|
+
this.clearForegroundTask(`suspended_${expired}`);
|
|
3473
|
+
return;
|
|
3474
|
+
}
|
|
3475
|
+
this.foregroundTaskState = next;
|
|
3476
|
+
if (this.activeAssistantTurn === null) {
|
|
3477
|
+
this.scheduleForegroundTaskResume();
|
|
3478
|
+
}
|
|
3479
|
+
}
|
|
3480
|
+
|
|
3481
|
+
private markForegroundTaskResumePending(epoch: number): boolean {
|
|
3482
|
+
const current = this.foregroundTaskState;
|
|
3483
|
+
if (current?.phase !== "suspended" || current.epoch !== epoch) {
|
|
3484
|
+
return false;
|
|
3485
|
+
}
|
|
3486
|
+
this.foregroundTaskState = { ...current, resumePending: true };
|
|
3487
|
+
return true;
|
|
3488
|
+
}
|
|
3489
|
+
|
|
3490
|
+
private rearmForegroundTaskAfterLegFailure(
|
|
3491
|
+
turn: ActiveAssistantTurn,
|
|
3492
|
+
): boolean {
|
|
3493
|
+
const epoch = turn.foregroundTaskEpoch;
|
|
3494
|
+
if (epoch === null) {
|
|
3495
|
+
return false;
|
|
3496
|
+
}
|
|
3497
|
+
const current = this.foregroundTaskState;
|
|
3498
|
+
if (current?.phase === "suspended" && current.epoch === epoch) {
|
|
3499
|
+
return this.markForegroundTaskResumePending(epoch);
|
|
3500
|
+
}
|
|
3501
|
+
const snapshot = turn.foregroundTaskSuspensionSnapshot;
|
|
3502
|
+
if (
|
|
3503
|
+
current?.phase !== "owned" ||
|
|
3504
|
+
current.epoch !== epoch ||
|
|
3505
|
+
current.ownerToken !== turn.token ||
|
|
3506
|
+
(snapshot !== null && snapshot.epoch !== epoch)
|
|
3507
|
+
) {
|
|
3508
|
+
return false;
|
|
3509
|
+
}
|
|
3510
|
+
const next: SuspendedForegroundTask = {
|
|
3511
|
+
...(snapshot ?? {
|
|
3512
|
+
phase: "suspended",
|
|
3513
|
+
epoch,
|
|
3514
|
+
request: current.request,
|
|
3515
|
+
suspendedAtMs: Date.now(),
|
|
3516
|
+
interveningTurns: 0,
|
|
3517
|
+
hostToolStarted: current.hostToolStarted,
|
|
3518
|
+
}),
|
|
3519
|
+
resumePending: true,
|
|
3520
|
+
};
|
|
3521
|
+
const expired = this.foregroundTaskSuspensionExpired(next);
|
|
3522
|
+
if (expired !== null) {
|
|
3523
|
+
this.clearForegroundTask(`suspended_${expired}`);
|
|
3524
|
+
return false;
|
|
3525
|
+
}
|
|
3526
|
+
this.foregroundTaskState = next;
|
|
3527
|
+
log.info(
|
|
3528
|
+
{ turnId: turn.turnId, epoch },
|
|
3529
|
+
"Foreground task re-armed after tool-capable leg failure",
|
|
3530
|
+
);
|
|
3531
|
+
return true;
|
|
3532
|
+
}
|
|
3533
|
+
|
|
3534
|
+
private clearForegroundTaskResumeTimer(): void {
|
|
3535
|
+
if (this.foregroundTaskResumeTimer !== null) {
|
|
3536
|
+
clearTimeout(this.foregroundTaskResumeTimer);
|
|
3537
|
+
this.foregroundTaskResumeTimer = null;
|
|
3538
|
+
}
|
|
3539
|
+
}
|
|
3540
|
+
|
|
3541
|
+
private scheduleFloorCheck(
|
|
3542
|
+
currentTimer: ReturnType<typeof setTimeout> | null,
|
|
3543
|
+
silenceMs: number,
|
|
3544
|
+
check: (blockedBy: string | null) => void,
|
|
3545
|
+
): ReturnType<typeof setTimeout> {
|
|
3546
|
+
if (currentTimer !== null) {
|
|
3547
|
+
clearTimeout(currentTimer);
|
|
3548
|
+
}
|
|
3549
|
+
const drainMs = Math.max(0, this.assistantPlaybackTailUntilMs - Date.now());
|
|
3550
|
+
return setTimeout(
|
|
3551
|
+
() => check(this.sessionTurnFloorBlocker()),
|
|
3552
|
+
drainMs + silenceMs,
|
|
3553
|
+
);
|
|
3554
|
+
}
|
|
3555
|
+
|
|
3556
|
+
private scheduleForegroundTaskResume(): void {
|
|
3557
|
+
const task = this.foregroundTaskState;
|
|
3558
|
+
if (
|
|
3559
|
+
task?.phase !== "suspended" ||
|
|
3560
|
+
!task.resumePending ||
|
|
3561
|
+
this.isClosed ||
|
|
3562
|
+
this.state === "failed"
|
|
3563
|
+
) {
|
|
3564
|
+
this.clearForegroundTaskResumeTimer();
|
|
3565
|
+
return;
|
|
3566
|
+
}
|
|
3567
|
+
const expired = this.foregroundTaskSuspensionExpired(task);
|
|
3568
|
+
if (expired !== null) {
|
|
3569
|
+
this.clearForegroundTask(`suspended_${expired}`);
|
|
3570
|
+
return;
|
|
3571
|
+
}
|
|
3572
|
+
const epoch = task.epoch;
|
|
3573
|
+
this.foregroundTaskResumeTimer = this.scheduleFloorCheck(
|
|
3574
|
+
this.foregroundTaskResumeTimer,
|
|
3575
|
+
this.foregroundTaskResumeSilenceMs,
|
|
3576
|
+
(blockedBy) => {
|
|
3577
|
+
this.foregroundTaskResumeTimer = null;
|
|
3578
|
+
const current = this.foregroundTaskState;
|
|
3579
|
+
if (
|
|
3580
|
+
current?.phase !== "suspended" ||
|
|
3581
|
+
current.epoch !== epoch ||
|
|
3582
|
+
!current.resumePending
|
|
3583
|
+
) {
|
|
3584
|
+
return;
|
|
3585
|
+
}
|
|
3586
|
+
const currentExpired = this.foregroundTaskSuspensionExpired(current);
|
|
3587
|
+
if (currentExpired !== null) {
|
|
3588
|
+
this.clearForegroundTask(`suspended_${currentExpired}`);
|
|
3589
|
+
return;
|
|
3590
|
+
}
|
|
3591
|
+
if (blockedBy === null) {
|
|
3592
|
+
void this.resumeForegroundTask(current).catch((err: unknown) => {
|
|
3593
|
+
log.warn({ err, epoch }, "Foreground task resume failed");
|
|
3594
|
+
});
|
|
3595
|
+
return;
|
|
3596
|
+
}
|
|
3597
|
+
if (blockedBy === "session_unavailable") {
|
|
3598
|
+
this.clearForegroundTask("session_unavailable");
|
|
3599
|
+
return;
|
|
3600
|
+
}
|
|
3601
|
+
if (blockedBy !== "turn_active") {
|
|
3602
|
+
this.scheduleForegroundTaskResume();
|
|
3603
|
+
}
|
|
3604
|
+
},
|
|
3605
|
+
);
|
|
3606
|
+
}
|
|
3607
|
+
|
|
3608
|
+
private async resumeForegroundTask(
|
|
3609
|
+
task: SuspendedForegroundTask,
|
|
3610
|
+
): Promise<void> {
|
|
3611
|
+
const current = this.foregroundTaskState;
|
|
3612
|
+
if (
|
|
3613
|
+
current?.phase !== "suspended" ||
|
|
3614
|
+
current.epoch !== task.epoch ||
|
|
3615
|
+
!current.resumePending
|
|
3616
|
+
) {
|
|
3617
|
+
return;
|
|
3618
|
+
}
|
|
3619
|
+
this.foregroundTaskState = { ...current, resumePending: false };
|
|
3620
|
+
const started = await this.launchAssistantTurn(
|
|
3621
|
+
createSyntheticUtterance(),
|
|
3622
|
+
buildForegroundTaskResumePrompt(current.request),
|
|
3623
|
+
{
|
|
3624
|
+
hiddenPrompt: true,
|
|
3625
|
+
initialLeg: "escalated",
|
|
3626
|
+
foregroundTaskEpoch: current.epoch,
|
|
3627
|
+
},
|
|
3628
|
+
);
|
|
3629
|
+
if (started) {
|
|
3630
|
+
return;
|
|
3631
|
+
}
|
|
3632
|
+
if (this.markForegroundTaskResumePending(current.epoch)) {
|
|
3633
|
+
this.scheduleForegroundTaskResume();
|
|
3634
|
+
}
|
|
3635
|
+
}
|
|
3636
|
+
|
|
3140
3637
|
// Abort every background continuation this session started and drop its
|
|
3141
3638
|
// handle. Called on a hard stop (client interrupt / session close), when a
|
|
3142
3639
|
// newer barge-in supersedes the detached runs, and on a foreground-wins
|
|
@@ -3263,50 +3760,53 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3263
3760
|
retry = false,
|
|
3264
3761
|
drainRearms = 0,
|
|
3265
3762
|
): void {
|
|
3266
|
-
|
|
3267
|
-
|
|
3268
|
-
|
|
3269
|
-
|
|
3270
|
-
|
|
3271
|
-
|
|
3272
|
-
|
|
3273
|
-
|
|
3274
|
-
|
|
3275
|
-
|
|
3276
|
-
|
|
3277
|
-
|
|
3278
|
-
|
|
3279
|
-
|
|
3280
|
-
|
|
3281
|
-
|
|
3282
|
-
|
|
3283
|
-
|
|
3284
|
-
|
|
3285
|
-
|
|
3286
|
-
|
|
3287
|
-
|
|
3288
|
-
|
|
3289
|
-
|
|
3290
|
-
|
|
3291
|
-
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3295
|
-
|
|
3296
|
-
|
|
3297
|
-
|
|
3298
|
-
|
|
3299
|
-
|
|
3300
|
-
|
|
3301
|
-
|
|
3302
|
-
|
|
3303
|
-
|
|
3304
|
-
|
|
3305
|
-
|
|
3306
|
-
|
|
3307
|
-
|
|
3308
|
-
|
|
3309
|
-
|
|
3763
|
+
this.announcementTimer = this.scheduleFloorCheck(
|
|
3764
|
+
this.announcementTimer,
|
|
3765
|
+
this.continuationAnnounceSilenceMs,
|
|
3766
|
+
() => {
|
|
3767
|
+
this.announcementTimer = null;
|
|
3768
|
+
const blockedBy = this.continuationAnnouncementBlocker();
|
|
3769
|
+
if (blockedBy === null) {
|
|
3770
|
+
void this.announceContinuation().catch((err: unknown) => {
|
|
3771
|
+
log.warn({ err }, "Voice duplex continuation announcement failed");
|
|
3772
|
+
});
|
|
3773
|
+
return;
|
|
3774
|
+
}
|
|
3775
|
+
log.debug(
|
|
3776
|
+
{
|
|
3777
|
+
conversationId: this.conversationId,
|
|
3778
|
+
blockedBy,
|
|
3779
|
+
retry,
|
|
3780
|
+
drainRearms,
|
|
3781
|
+
},
|
|
3782
|
+
"Voice duplex continuation announcement deferred",
|
|
3783
|
+
);
|
|
3784
|
+
// A tail that grew while the timer ran (the reply was still streaming
|
|
3785
|
+
// TTS when it was armed) is a wait of known length rather than a busy
|
|
3786
|
+
// call, so it re-arms against the new deadline without spending the
|
|
3787
|
+
// single retry the busy-call blockers get. Otherwise a long reply's
|
|
3788
|
+
// playback alone outlasts the budget and the announcement is dropped
|
|
3789
|
+
// even though the call goes idle right after. The tail only extends
|
|
3790
|
+
// while a turn is active, which blocks earlier, so the re-arms converge;
|
|
3791
|
+
// the cap bounds the interleaving where a fresh turn's audio lands
|
|
3792
|
+
// between two checks.
|
|
3793
|
+
if (
|
|
3794
|
+
blockedBy === "playback_draining" &&
|
|
3795
|
+
drainRearms < CONTINUATION_ANNOUNCE_MAX_DRAIN_REARMS
|
|
3796
|
+
) {
|
|
3797
|
+
this.scheduleContinuationAnnouncement(retry, drainRearms + 1);
|
|
3798
|
+
return;
|
|
3799
|
+
}
|
|
3800
|
+
if (blockedBy === "turn_active") {
|
|
3801
|
+
return;
|
|
3802
|
+
}
|
|
3803
|
+
if (retry || this.pendingAnnouncement === null) {
|
|
3804
|
+
this.pendingAnnouncement = null;
|
|
3805
|
+
return;
|
|
3806
|
+
}
|
|
3807
|
+
this.scheduleContinuationAnnouncement(true, drainRearms);
|
|
3808
|
+
},
|
|
3809
|
+
);
|
|
3310
3810
|
}
|
|
3311
3811
|
|
|
3312
3812
|
/**
|
|
@@ -3496,6 +3996,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3496
3996
|
) {
|
|
3497
3997
|
this.scheduleContinuationAnnouncement();
|
|
3498
3998
|
}
|
|
3999
|
+
if (
|
|
4000
|
+
this.foregroundTaskState?.phase === "suspended" &&
|
|
4001
|
+
this.foregroundTaskState.resumePending &&
|
|
4002
|
+
!this.isClosed &&
|
|
4003
|
+
this.state !== "failed"
|
|
4004
|
+
) {
|
|
4005
|
+
this.scheduleForegroundTaskResume();
|
|
4006
|
+
}
|
|
3499
4007
|
}
|
|
3500
4008
|
|
|
3501
4009
|
// Why the session must not speak up right now, or null when the call is
|
|
@@ -3565,6 +4073,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3565
4073
|
// own it leaves manual mode blind to a user who is talking right now and
|
|
3566
4074
|
// has no text yet — hence the captured-audio flag, which manual ingress
|
|
3567
4075
|
// sets from the first chunk.
|
|
4076
|
+
//
|
|
4077
|
+
// A partial counts only when it has words. Deepgram Flux sends interim
|
|
4078
|
+
// updates through silence too, each an empty partial, and one of those
|
|
4079
|
+
// would otherwise hold the floor until the user next spoke.
|
|
3568
4080
|
if (
|
|
3569
4081
|
utterance !== null &&
|
|
3570
4082
|
!utterance.completed &&
|
|
@@ -3572,13 +4084,129 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3572
4084
|
utterance.assistantTurnStarted ||
|
|
3573
4085
|
(utterance.manualAudioCaptured && opts?.ignoreManualCapture !== true) ||
|
|
3574
4086
|
utterance.finalTranscriptSegments.length > 0 ||
|
|
3575
|
-
utterance.latestPartialText
|
|
4087
|
+
(utterance.latestPartialText?.trim() ?? "").length > 0)
|
|
3576
4088
|
) {
|
|
3577
4089
|
return "utterance_in_flight";
|
|
3578
4090
|
}
|
|
3579
4091
|
return null;
|
|
3580
4092
|
}
|
|
3581
4093
|
|
|
4094
|
+
/**
|
|
4095
|
+
* Wait for the fresh frame a look control asks the client for.
|
|
4096
|
+
*
|
|
4097
|
+
* Only for a client that declared `lookFrames`: any other sends no such
|
|
4098
|
+
* frame, and its look keeps the old shape, where the user's next words are
|
|
4099
|
+
* what the frame gets answered on. A newer look replaces an older one still
|
|
4100
|
+
* waiting, and the wait is bounded so a frame that never comes (a camera
|
|
4101
|
+
* that would not open, a share the desktop refused) cannot turn up minutes
|
|
4102
|
+
* later as a reply to nothing.
|
|
4103
|
+
*/
|
|
4104
|
+
private awaitLookFrame(action: LookSessionControl): void {
|
|
4105
|
+
if (!this.lookFrames) {
|
|
4106
|
+
return;
|
|
4107
|
+
}
|
|
4108
|
+
this.clearPendingLook();
|
|
4109
|
+
const timer = setTimeout(() => {
|
|
4110
|
+
if (this.pendingLook?.timer !== timer) {
|
|
4111
|
+
return;
|
|
4112
|
+
}
|
|
4113
|
+
this.pendingLook = null;
|
|
4114
|
+
log.info(
|
|
4115
|
+
{ conversationId: this.conversationId, action },
|
|
4116
|
+
"Live voice look dropped: no frame arrived",
|
|
4117
|
+
);
|
|
4118
|
+
}, LOOK_FRAME_WAIT_MS);
|
|
4119
|
+
this.pendingLook = { action, armedAtMs: Date.now(), timer };
|
|
4120
|
+
}
|
|
4121
|
+
|
|
4122
|
+
private clearPendingLook(): void {
|
|
4123
|
+
if (this.pendingLook !== null) {
|
|
4124
|
+
clearTimeout(this.pendingLook.timer);
|
|
4125
|
+
this.pendingLook = null;
|
|
4126
|
+
}
|
|
4127
|
+
if (this.lookFollowUpTimer !== null) {
|
|
4128
|
+
clearTimeout(this.lookFollowUpTimer);
|
|
4129
|
+
this.lookFollowUpTimer = null;
|
|
4130
|
+
}
|
|
4131
|
+
}
|
|
4132
|
+
|
|
4133
|
+
/**
|
|
4134
|
+
* The frame a look asked for is in the conversation: answer the look.
|
|
4135
|
+
*
|
|
4136
|
+
* A look frame with no look waiting (the wait ran out, or the session never
|
|
4137
|
+
* armed one) is only a frame, like any other keep.
|
|
4138
|
+
*/
|
|
4139
|
+
private lookFrameLanded(): void {
|
|
4140
|
+
const pending = this.pendingLook;
|
|
4141
|
+
if (pending === null || this.isClosed) {
|
|
4142
|
+
return;
|
|
4143
|
+
}
|
|
4144
|
+
clearTimeout(pending.timer);
|
|
4145
|
+
this.pendingLook = null;
|
|
4146
|
+
this.answerLookWhenFloorIsFree(pending, this.turnsLaunched, 0);
|
|
4147
|
+
}
|
|
4148
|
+
|
|
4149
|
+
/**
|
|
4150
|
+
* Start the turn that answers a look, once nothing else holds the floor.
|
|
4151
|
+
*
|
|
4152
|
+
* The acknowledgement ("taking a look") can still be playing when the frame
|
|
4153
|
+
* lands, and the look's own turn can still be clearing, so both are waited
|
|
4154
|
+
* out, as is the user mid-utterance. A turn that starts once the frame is in
|
|
4155
|
+
* the conversation is not: it reads the frame, so answering the look as well
|
|
4156
|
+
* would answer it twice. A turn that started before the frame landed did not
|
|
4157
|
+
* see it, so the look is still answered once that turn is done.
|
|
4158
|
+
*
|
|
4159
|
+
* `turnsAtFrame` is how many turns had launched when the frame landed.
|
|
4160
|
+
*/
|
|
4161
|
+
private answerLookWhenFloorIsFree(
|
|
4162
|
+
look: PendingLook,
|
|
4163
|
+
turnsAtFrame: number,
|
|
4164
|
+
rearms: number,
|
|
4165
|
+
): void {
|
|
4166
|
+
if (this.lookFollowUpTimer !== null) {
|
|
4167
|
+
clearTimeout(this.lookFollowUpTimer);
|
|
4168
|
+
this.lookFollowUpTimer = null;
|
|
4169
|
+
}
|
|
4170
|
+
const { action, armedAtMs } = look;
|
|
4171
|
+
// A turn launched since the frame landed read it already. Speech that
|
|
4172
|
+
// never became a turn (a cough, noise that transcribed to nothing) is only
|
|
4173
|
+
// waited out.
|
|
4174
|
+
const blockedBy =
|
|
4175
|
+
this.turnsLaunched > turnsAtFrame
|
|
4176
|
+
? "turn_since_look"
|
|
4177
|
+
: this.sessionTurnFloorBlocker();
|
|
4178
|
+
if (blockedBy === null) {
|
|
4179
|
+
void this.launchAssistantTurn(
|
|
4180
|
+
createSyntheticUtterance(),
|
|
4181
|
+
LOOK_FOLLOW_UP_CONTENT,
|
|
4182
|
+
{ lookFollowUp: action, hiddenPrompt: true },
|
|
4183
|
+
).catch((err: unknown) => {
|
|
4184
|
+
log.warn(
|
|
4185
|
+
{ err, conversationId: this.conversationId, action },
|
|
4186
|
+
"Live voice look follow-up failed to start",
|
|
4187
|
+
);
|
|
4188
|
+
});
|
|
4189
|
+
return;
|
|
4190
|
+
}
|
|
4191
|
+
const waitable =
|
|
4192
|
+
blockedBy !== "turn_since_look" && blockedBy !== "session_unavailable";
|
|
4193
|
+
if (!waitable || Date.now() - armedAtMs >= LOOK_ANSWER_DEADLINE_MS) {
|
|
4194
|
+
log.info(
|
|
4195
|
+
{ conversationId: this.conversationId, action, blockedBy, rearms },
|
|
4196
|
+
"Live voice look follow-up skipped",
|
|
4197
|
+
);
|
|
4198
|
+
return;
|
|
4199
|
+
}
|
|
4200
|
+
const drainMs = Math.max(0, this.assistantPlaybackTailUntilMs - Date.now());
|
|
4201
|
+
this.lookFollowUpTimer = setTimeout(
|
|
4202
|
+
() => {
|
|
4203
|
+
this.lookFollowUpTimer = null;
|
|
4204
|
+
this.answerLookWhenFloorIsFree(look, turnsAtFrame, rearms + 1);
|
|
4205
|
+
},
|
|
4206
|
+
Math.max(drainMs, LOOK_FOLLOW_UP_REARM_MS),
|
|
4207
|
+
);
|
|
4208
|
+
}
|
|
4209
|
+
|
|
3582
4210
|
// Speak the finished continuation's result on a turn the session starts
|
|
3583
4211
|
// itself. The turn rides a synthetic cycle rather than the armed VAD one —
|
|
3584
4212
|
// `currentUtterance` belongs to the capture loop, and overwriting it would
|
|
@@ -4768,8 +5396,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4768
5396
|
// the next turn is now stale (the interrupted utterance may be discarded
|
|
4769
5397
|
// without ever reaching finalizePendingUtterance).
|
|
4770
5398
|
this.pendingInterruptedRequest = null;
|
|
5399
|
+
this.clearForegroundTask("client_interrupt");
|
|
4771
5400
|
// ...and it hard-stops any detached background continuations.
|
|
4772
5401
|
this.abortDetachedRuns({ reason: "client_interrupt" });
|
|
5402
|
+
// ...and a look still waiting to be answered: the user stopped the call
|
|
5403
|
+
// talking, and a reply starting on its own a moment later is not a stop.
|
|
5404
|
+
this.clearPendingLook();
|
|
4773
5405
|
const utterance = this.currentUtterance;
|
|
4774
5406
|
this.stopSessionTranscriber();
|
|
4775
5407
|
if (utterance) {
|
|
@@ -4991,6 +5623,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4991
5623
|
// Set on an announcement turn: the finished continuation this turn exists
|
|
4992
5624
|
// to deliver. Its answer goes in the control prompt, not in `content`.
|
|
4993
5625
|
continuationDelivery?: ContinuationDelivery | null;
|
|
5626
|
+
// Set on the turn that answers a look: which look. Its instruction goes
|
|
5627
|
+
// in the control prompt, not in `content`.
|
|
5628
|
+
lookFollowUp?: LookSessionControl;
|
|
4994
5629
|
// Unified front-door: dispatch without releasing the utterance. The
|
|
4995
5630
|
// thinking frame and floor-holding timers are deferred until the leg's
|
|
4996
5631
|
// leading verdict commits the turn (see commitSpeculativeTurn); a hold
|
|
@@ -5000,6 +5635,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5000
5635
|
// the row persists and drives the turn but never renders: no echo, and
|
|
5001
5636
|
// `/messages` filters it after a reload.
|
|
5002
5637
|
hiddenPrompt?: boolean;
|
|
5638
|
+
// Synthetic host-task resumes start on the tool-capable profile without
|
|
5639
|
+
// passing through the front door.
|
|
5640
|
+
initialLeg?: VoiceRoutingLeg;
|
|
5641
|
+
// The suspended foreground task this synthetic turn resumes.
|
|
5642
|
+
foregroundTaskEpoch?: number;
|
|
5003
5643
|
},
|
|
5004
5644
|
): Promise<boolean> {
|
|
5005
5645
|
utterance.assistantTurnStarted = true;
|
|
@@ -5015,6 +5655,15 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5015
5655
|
this.startMetricsTurnIfNeeded(utterance, turnId);
|
|
5016
5656
|
this.markAssistantDispatch(utterance, turnId);
|
|
5017
5657
|
const abortController = new AbortController();
|
|
5658
|
+
const suspendedForegroundTask =
|
|
5659
|
+
this.foregroundTaskState?.phase === "suspended"
|
|
5660
|
+
? this.foregroundTaskState
|
|
5661
|
+
: null;
|
|
5662
|
+
const foregroundTaskEpoch =
|
|
5663
|
+
opts?.foregroundTaskEpoch ??
|
|
5664
|
+
(opts?.hiddenPrompt !== true
|
|
5665
|
+
? (suspendedForegroundTask?.epoch ?? null)
|
|
5666
|
+
: null);
|
|
5018
5667
|
const activeTurn: ActiveAssistantTurn = {
|
|
5019
5668
|
token,
|
|
5020
5669
|
turnId,
|
|
@@ -5085,7 +5734,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5085
5734
|
consumedAnnouncement: pending?.announcement ?? null,
|
|
5086
5735
|
pendingContextStopGeneration: this.detachStopGeneration,
|
|
5087
5736
|
continuationDelivery: opts?.continuationDelivery ?? null,
|
|
5737
|
+
lookFollowUp: opts?.lookFollowUp ?? null,
|
|
5088
5738
|
hiddenPrompt: opts?.hiddenPrompt === true,
|
|
5739
|
+
foregroundTaskEpoch,
|
|
5740
|
+
foregroundTaskSuspensionSnapshot:
|
|
5741
|
+
suspendedForegroundTask?.epoch === foregroundTaskEpoch
|
|
5742
|
+
? { ...suspendedForegroundTask }
|
|
5743
|
+
: null,
|
|
5089
5744
|
deltaEpoch: 0,
|
|
5090
5745
|
frontDoor: null,
|
|
5091
5746
|
ttsBuffer: "",
|
|
@@ -5100,6 +5755,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5100
5755
|
assistantAudioMimeType: "audio/pcm",
|
|
5101
5756
|
};
|
|
5102
5757
|
this.activeAssistantTurn = activeTurn;
|
|
5758
|
+
this.turnsLaunched += 1;
|
|
5103
5759
|
|
|
5104
5760
|
// A speculative turn defers the thinking frame and both floor-holding
|
|
5105
5761
|
// timers to commitSpeculativeTurn: until the verdict arrives, the pause
|
|
@@ -5149,13 +5805,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5149
5805
|
activeTurn.progress.arm();
|
|
5150
5806
|
}
|
|
5151
5807
|
|
|
5152
|
-
|
|
5153
|
-
// call site pins it) and may hand off to a quality leg on the escalate
|
|
5154
|
-
// verdict.
|
|
5808
|
+
const initialLeg = opts?.initialLeg ?? "front-door";
|
|
5155
5809
|
const started = await this.startAssistantLeg(activeTurn, {
|
|
5156
5810
|
content,
|
|
5157
|
-
routingLeg:
|
|
5158
|
-
|
|
5811
|
+
routingLeg: initialLeg,
|
|
5812
|
+
...(initialLeg === "front-door"
|
|
5813
|
+
? { frontDoor: true }
|
|
5814
|
+
: { directEscalated: true }),
|
|
5159
5815
|
// Only this leg: an escalated leg persists its own continuation row, and
|
|
5160
5816
|
// the frame belongs to the row carrying the user's words.
|
|
5161
5817
|
...(activeTurn.turnAttachmentId
|
|
@@ -5200,6 +5856,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5200
5856
|
overrideProfile?: string;
|
|
5201
5857
|
routingLeg?: VoiceRoutingLeg;
|
|
5202
5858
|
frontDoor?: boolean;
|
|
5859
|
+
directEscalated?: boolean;
|
|
5203
5860
|
spokenEscalationBridge?: string;
|
|
5204
5861
|
attachments?: readonly string[];
|
|
5205
5862
|
},
|
|
@@ -5280,6 +5937,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5280
5937
|
speakBridge: (bridge) =>
|
|
5281
5938
|
this.speakEscalationBridge(activeTurn, bridge),
|
|
5282
5939
|
startEscalatedLeg: (escalated) => {
|
|
5940
|
+
this.noteEscalatedForegroundTask(activeTurn);
|
|
5283
5941
|
void this.startAssistantLeg(activeTurn, escalated);
|
|
5284
5942
|
},
|
|
5285
5943
|
},
|
|
@@ -5342,6 +6000,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5342
6000
|
: {}),
|
|
5343
6001
|
},
|
|
5344
6002
|
this.sessionControls,
|
|
6003
|
+
{ lookFrames: this.lookFrames },
|
|
5345
6004
|
),
|
|
5346
6005
|
onApprovalPending: (requestId) => {
|
|
5347
6006
|
this.revealRoomForPendingApproval(activeTurn, requestId);
|
|
@@ -5367,6 +6026,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5367
6026
|
...(leg.spokenEscalationBridge != null
|
|
5368
6027
|
? { spokenEscalationBridge: leg.spokenEscalationBridge }
|
|
5369
6028
|
: {}),
|
|
6029
|
+
...(leg.directEscalated === true ? { directEscalated: true } : {}),
|
|
5370
6030
|
// A speculative front-door leg's decision rule includes the hold
|
|
5371
6031
|
// branch so its leading tokens can be the hold verdict — but only
|
|
5372
6032
|
// on the utterance's FIRST dispatch. Extension replays (the
|
|
@@ -5383,6 +6043,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5383
6043
|
if (!this.isForwardingAssistantText(token)) {
|
|
5384
6044
|
return;
|
|
5385
6045
|
}
|
|
6046
|
+
if (!leg.frontDoor) {
|
|
6047
|
+
this.claimSuspendedForegroundTask(activeTurn);
|
|
6048
|
+
}
|
|
5386
6049
|
if (coordinator !== null) {
|
|
5387
6050
|
// Verdict-first: the leg's leading tokens decide the turn's
|
|
5388
6051
|
// fate, and the coordinator acts on them. The bridge hands
|
|
@@ -5418,6 +6081,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5418
6081
|
) {
|
|
5419
6082
|
return;
|
|
5420
6083
|
}
|
|
6084
|
+
if (!leg.frontDoor) {
|
|
6085
|
+
this.claimSuspendedForegroundTask(current);
|
|
6086
|
+
}
|
|
5421
6087
|
// A speculative leg that finished without a single delta (empty
|
|
5422
6088
|
// output, provider hiccup) carries no verdict — fail open to a
|
|
5423
6089
|
// committed turn so the utterance releases and finalizes like a
|
|
@@ -5463,10 +6129,18 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5463
6129
|
{ turnId, cadence: request.cadence },
|
|
5464
6130
|
"Live voice progress cadence changed",
|
|
5465
6131
|
);
|
|
6132
|
+
} else if (request?.action === "task_stop") {
|
|
6133
|
+
this.stopOutstandingInterruptedWork("spoken_task_stop");
|
|
5466
6134
|
} else {
|
|
6135
|
+
if (request?.action === "end") {
|
|
6136
|
+
this.clearForegroundTask("call_end_requested");
|
|
6137
|
+
}
|
|
5467
6138
|
current.sessionControlRequested = request;
|
|
5468
6139
|
}
|
|
5469
6140
|
}
|
|
6141
|
+
if (leg.frontDoor) {
|
|
6142
|
+
this.queueForegroundTaskResume(current);
|
|
6143
|
+
}
|
|
5470
6144
|
current.assistantCompleted = true;
|
|
5471
6145
|
if (msg.type === "generation_cancelled") {
|
|
5472
6146
|
void this.finalizeAssistantTurn(
|
|
@@ -5501,6 +6175,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5501
6175
|
if (current?.token !== token) {
|
|
5502
6176
|
return;
|
|
5503
6177
|
}
|
|
6178
|
+
const effectiveHostTool = hostInteractiveToolName(
|
|
6179
|
+
toolName,
|
|
6180
|
+
detail?.input,
|
|
6181
|
+
detail?.allowedToolNames,
|
|
6182
|
+
);
|
|
6183
|
+
if (effectiveHostTool !== null) {
|
|
6184
|
+
this.noteHostToolStarted(current, effectiveHostTool);
|
|
6185
|
+
}
|
|
5504
6186
|
// Foreground wins the workspace: the continuation runs with full
|
|
5505
6187
|
// subagent abilities (it can write files, run commands), so the
|
|
5506
6188
|
// moment a live turn starts a consequential tool the two could
|
|
@@ -5587,7 +6269,15 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5587
6269
|
if (currentTurn?.token !== token) {
|
|
5588
6270
|
return;
|
|
5589
6271
|
}
|
|
6272
|
+
const foregroundTaskResumeRearmed =
|
|
6273
|
+
!leg.frontDoor &&
|
|
6274
|
+
!currentTurn.discardRequested &&
|
|
6275
|
+
!currentTurn.abortController.signal.aborted &&
|
|
6276
|
+
this.rearmForegroundTaskAfterLegFailure(currentTurn);
|
|
5590
6277
|
await this.finalizeAssistantTurn(currentTurn, "cancelled", "error");
|
|
6278
|
+
if (foregroundTaskResumeRearmed) {
|
|
6279
|
+
this.scheduleForegroundTaskResume();
|
|
6280
|
+
}
|
|
5591
6281
|
})();
|
|
5592
6282
|
},
|
|
5593
6283
|
});
|
|
@@ -5623,6 +6313,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5623
6313
|
this.clearActiveAssistantTurn(token);
|
|
5624
6314
|
return true;
|
|
5625
6315
|
}
|
|
6316
|
+
if (!leg.frontDoor) {
|
|
6317
|
+
this.claimSuspendedForegroundTask(current);
|
|
6318
|
+
}
|
|
5626
6319
|
// The front-door leg may have handed off before its handle resolved;
|
|
5627
6320
|
// abort it rather than exposing it as the turn's live handle.
|
|
5628
6321
|
if (coordinator?.handedOff === true) {
|
|
@@ -5637,8 +6330,17 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5637
6330
|
return false;
|
|
5638
6331
|
}
|
|
5639
6332
|
|
|
6333
|
+
const foregroundTaskResumeRearmed =
|
|
6334
|
+
!leg.frontDoor &&
|
|
6335
|
+
leg.directEscalated !== true &&
|
|
6336
|
+
!activeTurn.discardRequested &&
|
|
6337
|
+
!activeTurn.abortController.signal.aborted &&
|
|
6338
|
+
this.rearmForegroundTaskAfterLegFailure(activeTurn);
|
|
5640
6339
|
this.clearFillerTimers(activeTurn);
|
|
5641
6340
|
this.clearActiveAssistantTurn(token);
|
|
6341
|
+
if (foregroundTaskResumeRearmed) {
|
|
6342
|
+
this.scheduleForegroundTaskResume();
|
|
6343
|
+
}
|
|
5642
6344
|
await this.sendFrame({
|
|
5643
6345
|
type: "error",
|
|
5644
6346
|
code: LiveVoiceProtocolErrorCode.InvalidField,
|
|
@@ -5897,6 +6599,16 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5897
6599
|
if (sessionControl.action === "end") {
|
|
5898
6600
|
currentTurn.minimizeRequested = false;
|
|
5899
6601
|
}
|
|
6602
|
+
// Armed before the send, so a client quick enough to answer with
|
|
6603
|
+
// its frame before this await resolves still finds the look waiting.
|
|
6604
|
+
// Never from the turn that answers a look: that turn is the answer,
|
|
6605
|
+
// and a marker it emits anyway must not chain another.
|
|
6606
|
+
if (
|
|
6607
|
+
isLookSessionControl(sessionControl.action) &&
|
|
6608
|
+
currentTurn.lookFollowUp === null
|
|
6609
|
+
) {
|
|
6610
|
+
this.awaitLookFrame(sessionControl.action);
|
|
6611
|
+
}
|
|
5900
6612
|
await this.sendFrame(
|
|
5901
6613
|
{
|
|
5902
6614
|
type: "session_control",
|
|
@@ -6436,6 +7148,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6436
7148
|
|
|
6437
7149
|
turn.finalized = true;
|
|
6438
7150
|
this.clearFillerTimers(turn);
|
|
7151
|
+
if (
|
|
7152
|
+
this.foregroundTaskState?.phase === "owned" &&
|
|
7153
|
+
this.foregroundTaskState.ownerToken === turn.token
|
|
7154
|
+
) {
|
|
7155
|
+
this.clearForegroundTask(
|
|
7156
|
+
status === "completed" ? "owner_completed" : `owner_${reason}`,
|
|
7157
|
+
);
|
|
7158
|
+
}
|
|
6439
7159
|
turn.utterance.completed = true;
|
|
6440
7160
|
await this.archiveBufferedAudio({
|
|
6441
7161
|
turnId: turn.turnId,
|