@vellumai/assistant 0.12.2-staging.4 → 0.12.2-staging.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/Dockerfile +7 -7
  2. package/openapi.yaml +151 -53
  3. package/package.json +5 -4
  4. package/scripts/bundled-plugin-packages.ts +154 -0
  5. package/scripts/generate-bundled-plugin-packages.ts +29 -0
  6. package/scripts/test.ts +15 -13
  7. package/src/__tests__/managed-store.test.ts +121 -0
  8. package/src/__tests__/scaffold-managed-skill-tool.test.ts +88 -0
  9. package/src/calls/__tests__/voice-control-protocol.test.ts +24 -2
  10. package/src/calls/__tests__/voice-session-bridge.test.ts +22 -2
  11. package/src/calls/voice-control-protocol.ts +13 -4
  12. package/src/calls/voice-session-bridge.ts +27 -6
  13. package/src/cli/commands/__tests__/plugins.test.ts +66 -0
  14. package/src/cli/commands/plugins.ts +50 -18
  15. package/src/cli/lib/__tests__/install-from-github.test.ts +67 -0
  16. package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +169 -0
  17. package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +57 -0
  18. package/src/cli/lib/__tests__/plugin-catalog-platform.test.ts +14 -0
  19. package/src/cli/lib/__tests__/plugin-catalog-resolve.test.ts +27 -1
  20. package/src/cli/lib/__tests__/plugin-details.test.ts +9 -2
  21. package/src/cli/lib/__tests__/plugin-marketplace.test.ts +20 -0
  22. package/src/cli/lib/__tests__/plugins-install-offline.test.ts +43 -0
  23. package/src/cli/lib/__tests__/search-plugins.test.ts +31 -0
  24. package/src/cli/lib/bundled-plugin-packages.json +4 -0
  25. package/src/cli/lib/bundled-plugin-packages.ts +87 -0
  26. package/src/cli/lib/diff-plugin.ts +1 -1
  27. package/src/cli/lib/inspect-plugin.ts +80 -12
  28. package/src/cli/lib/install-from-github.ts +133 -76
  29. package/src/cli/lib/plugin-catalog-cache.ts +22 -3
  30. package/src/cli/lib/plugin-catalog-local.ts +13 -3
  31. package/src/cli/lib/plugin-catalog-platform.ts +6 -1
  32. package/src/cli/lib/plugin-catalog-resolve.ts +20 -0
  33. package/src/cli/lib/plugin-details.ts +12 -0
  34. package/src/cli/lib/plugin-marketplace.ts +121 -21
  35. package/src/cli/lib/plugin-pin-history.ts +5 -2
  36. package/src/cli/lib/search-plugins.ts +58 -16
  37. package/src/cli/lib/upgrade-plugin.ts +48 -3
  38. package/src/config/bundled-skills/skill-management/SKILL.md +1 -1
  39. package/src/config/bundled-skills/skill-management/TOOLS.json +7 -7
  40. package/src/daemon/conversation-tool-setup.ts +4 -22
  41. package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +374 -0
  42. package/src/live-voice/__tests__/live-voice-vad.test.ts +658 -3
  43. package/src/live-voice/__tests__/protocol.test.ts +48 -0
  44. package/src/live-voice/__tests__/session-controls.test.ts +43 -0
  45. package/src/live-voice/live-voice-session.ts +784 -64
  46. package/src/live-voice/protocol.ts +20 -0
  47. package/src/live-voice/session-controls.ts +61 -3
  48. package/src/monitoring/plugin-auto-update.ts +6 -0
  49. package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +15 -0
  50. package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +1 -1
  51. package/src/runtime/routes/__tests__/plugins-routes.test.ts +102 -0
  52. package/src/runtime/routes/plugins-routes.ts +69 -49
  53. package/src/skills/managed-store.ts +153 -31
  54. package/src/tools/skills/find-similar-skills.test.ts +214 -2
  55. package/src/tools/skills/find-similar-skills.ts +62 -9
  56. package/src/tools/skills/resolve-execute-invocation.ts +24 -0
  57. package/src/tools/skills/scaffold-managed.ts +16 -13
@@ -23,7 +23,10 @@ import {
23
23
  SPOKEN_REPLY_PLAIN_TEXT_RULE,
24
24
  } from "../calls/spoken-reply-rules.js";
25
25
  import { sanitizeForTts } from "../calls/tts-text-sanitizer.js";
26
- import { createControlMarkerHoldback } from "../calls/voice-control-protocol.js";
26
+ import {
27
+ createControlMarkerHoldback,
28
+ TASK_STOP_MARKER,
29
+ } from "../calls/voice-control-protocol.js";
27
30
  import {
28
31
  createFrontDoorLegCoordinator,
29
32
  type FrontDoorLegCoordinator,
@@ -97,7 +100,8 @@ import {
97
100
  liveVoiceSilenceReason,
98
101
  liveVoiceStartScreen,
99
102
  } from "../telemetry/live-voice-funnel.js";
100
- import { getToolOwner } from "../tools/registry.js";
103
+ import { getTool, getToolOwner } from "../tools/registry.js";
104
+ import { resolveSkillExecuteInvocation } from "../tools/skills/resolve-execute-invocation.js";
101
105
  import {
102
106
  createReasoningTagFilter,
103
107
  type ReasoningTagFilter,
@@ -160,6 +164,11 @@ import {
160
164
  } from "./protocol.js";
161
165
  import {
162
166
  type ClientSessionControlRequest,
167
+ isLookSessionControl,
168
+ LOOK_FOLLOW_UP_CONTENT,
169
+ LOOK_FRAME_REASON,
170
+ lookFollowUpNote,
171
+ type LookSessionControl,
163
172
  progressConfigForCadence,
164
173
  requestedSessionControl,
165
174
  sessionControlTeaching,
@@ -265,11 +274,26 @@ const CONTINUATION_ANNOUNCE_SILENCE_MS = 1_500;
265
274
  // reply's audio landing between two checks. Beyond the cap the announcement
266
275
  // falls back to the stash rather than chasing a call that keeps talking.
267
276
  const CONTINUATION_ANNOUNCE_MAX_DRAIN_REARMS = 3;
277
+ const FOREGROUND_TASK_RESUME_SILENCE_MS = 1_500;
278
+ const FOREGROUND_TASK_MAX_SUSPENDED_MS = 120_000;
279
+ const FOREGROUND_TASK_MAX_INTERVENING_TURNS = 3;
268
280
  // `content` of an announcement turn. The answer never rides here — it goes in
269
281
  // the model-facing control prompt (buildLiveDeliveryNote), so the only thing
270
282
  // persisted on the user side is this marker, and it persists hidden.
271
283
  export const CONTINUATION_DELIVERY_CONTENT =
272
284
  "(background work finished — deliver it now)";
285
+ // How long a look control waits for the fresh frame the client takes for it.
286
+ // Long enough for a camera that has to open and warm up before its first keep;
287
+ // past it the look is dropped rather than answered into a later, unrelated
288
+ // silence.
289
+ const LOOK_FRAME_WAIT_MS = 10_000;
290
+ // How long after the look was asked for its answer may still start. Past it
291
+ // the floor has been taken for so long (a user talking through noise that never
292
+ // became a turn, say) that the look no longer answers what is on screen.
293
+ const LOOK_ANSWER_DEADLINE_MS = 15_000;
294
+ // The shortest wait before checking the floor again, for a blocker with no
295
+ // known end time (the look's own turn clearing, an utterance in capture).
296
+ const LOOK_FOLLOW_UP_REARM_MS = 250;
273
297
 
274
298
  export type LiveVoiceStreamingTranscriberResolver = (
275
299
  options: ResolveStreamingTranscriberOptions,
@@ -445,6 +469,12 @@ export interface LiveVoiceSessionOptions {
445
469
  * `CONTINUATION_ANNOUNCE_SILENCE_MS`.
446
470
  */
447
471
  continuationAnnounceSilenceMs?: number;
472
+ /** Overrides the silence before a suspended foreground task resumes (test hook). */
473
+ foregroundTaskResumeSilenceMs?: number;
474
+ /** Overrides the maximum time a foreground task may remain suspended (test hook). */
475
+ foregroundTaskMaxSuspendedMs?: number;
476
+ /** Overrides the committed user-turn limit while suspended (test hook). */
477
+ foregroundTaskMaxInterveningTurns?: number;
448
478
  }
449
479
 
450
480
  type LiveVoiceUtterancePhase =
@@ -697,10 +727,21 @@ interface ActiveAssistantTurn {
697
727
  // no user utterance behind it — `content` is CONTINUATION_DELIVERY_CONTENT and
698
728
  // the answer rides the control prompt (buildLiveDeliveryNote).
699
729
  continuationDelivery: ContinuationDelivery | null;
730
+ // Set only on the turn that answers a look: which look it answers. The turn
731
+ // has no user utterance behind it; the instruction rides the control prompt
732
+ // (lookFollowUpNote).
733
+ lookFollowUp: LookSessionControl | null;
700
734
  // The turn's content is an internal instruction rather than user speech (the
701
735
  // greeting that opens a session, say). The row still persists and the model
702
736
  // still sees it; `hiddenSyntheticPrompt` keeps it out of the transcript.
703
737
  hiddenPrompt: boolean;
738
+ // The suspended foreground task this turn may continue. Set only after the user
739
+ // turn commits or on the synthetic resume turn; speculative discards never
740
+ // mutate the session's task state.
741
+ foregroundTaskEpoch: number | null;
742
+ // Snapshot retained while a tool-capable leg owns suspended work. If that
743
+ // leg fails asynchronously, the task returns to the same suspension budget.
744
+ foregroundTaskSuspensionSnapshot: SuspendedForegroundTask | null;
704
745
  // Set when a barge-in handed the interrupted work to a background subagent:
705
746
  // that request's transcript, so the model can tell the user the work is
706
747
  // still running instead of appearing to have dropped it.
@@ -805,7 +846,7 @@ const LIVE_VOICE_SETUP_FLOW_TEACHING =
805
846
  // off answering, rather than a fresh follow-up. Reaches the model only; it is
806
847
  // not a user message and never renders as a transcript bubble.
807
848
  function buildInterruptionMergeNote(interruptedRequest: string): string {
808
- return `The user interrupted your previous, unfinished reply. Their earlier request was: "${interruptedRequest}". Treat their current message as a continuation of that request and address both together, or stay silent if they only want you to stop.`;
849
+ return `The user interrupted your previous, unfinished reply. Their earlier request was: "${interruptedRequest}". Treat their current message as a continuation of that request and address both together. If they corrected or redirected the unfinished work, continue executing it with that correction instead of merely acknowledging it. If they clearly want the unfinished task abandoned, use the task-stop control.`;
809
850
  }
810
851
 
811
852
  // System-level guidance appended to the NEXT turn's control prompt after a
@@ -838,6 +879,14 @@ function describeInterruptedRequest(request: string): string {
838
879
  : "their earlier request";
839
880
  }
840
881
 
882
+ // A look control waiting on its fresh frame: which look, when it was asked
883
+ // for, and the wait's bound.
884
+ interface PendingLook {
885
+ action: LookSessionControl;
886
+ armedAtMs: number;
887
+ timer: ReturnType<typeof setTimeout>;
888
+ }
889
+
841
890
  // A finished background continuation waiting to be delivered: the request it
842
891
  // took over and the answer it produced.
843
892
  interface ContinuationDelivery {
@@ -845,6 +894,26 @@ interface ContinuationDelivery {
845
894
  answer: string;
846
895
  }
847
896
 
897
+ interface OwnedForegroundTask {
898
+ phase: "owned";
899
+ epoch: number;
900
+ request: string;
901
+ ownerToken: symbol;
902
+ hostToolStarted: boolean;
903
+ }
904
+
905
+ interface SuspendedForegroundTask {
906
+ phase: "suspended";
907
+ epoch: number;
908
+ request: string;
909
+ suspendedAtMs: number;
910
+ interveningTurns: number;
911
+ resumePending: boolean;
912
+ hostToolStarted: boolean;
913
+ }
914
+
915
+ type ForegroundTaskState = OwnedForegroundTask | SuspendedForegroundTask;
916
+
848
917
  // Appended to an announcement turn's control prompt: the turn the session
849
918
  // starts on its own when a background continuation finishes while the call is
850
919
  // live and nobody is speaking. There is no user utterance behind it, so this
@@ -856,6 +925,12 @@ function buildLiveDeliveryNote(request: string, answer: string): string {
856
925
  return `The user interrupted you earlier, and in the background you finished ${what}. The call is still live and nobody is speaking, so tell them briefly that it is done and give them the result. Do not re-run any tool calls; the work is already complete, and do not repeat it verbatim if it no longer fits. What you produced was:\n\n${answer}`;
857
926
  }
858
927
 
928
+ function buildForegroundTaskResumePrompt(request: string): string {
929
+ const task =
930
+ request.length > 0 ? JSON.stringify(request) : "the unfinished task";
931
+ return `Continue the unfinished task that began with ${task}. Follow the user's latest corrections and all newer conversation context. If the conversation has clearly moved on or the user no longer wants the task, do not continue it and output only ${TASK_STOP_MARKER}, with no spoken words before the marker.`;
932
+ }
933
+
859
934
  // Assemble a leg's model-facing control prompt: the base live-voice rules, the
860
935
  // screen-reveal and setup-flow teaching (both withheld from the front-door leg,
861
936
  // see LIVE_VOICE_SCREEN_REVEAL_TEACHING), plus any pending barge-in merge
@@ -866,13 +941,20 @@ function buildVoiceControlPrompt(
866
941
  turn: ActiveAssistantTurn,
867
942
  leg: { frontDoor?: boolean },
868
943
  sessionControls: readonly LiveVoiceSessionControl[],
944
+ client: { lookFrames: boolean },
869
945
  ): string {
870
946
  let prompt =
871
947
  LIVE_VOICE_CONTROL_PROMPT_BASE +
872
948
  (leg.frontDoor === true
873
949
  ? ""
874
950
  : LIVE_VOICE_SCREEN_REVEAL_TEACHING + LIVE_VOICE_SETUP_FLOW_TEACHING) +
875
- sessionControlTeaching(sessionControls, leg);
951
+ sessionControlTeaching(sessionControls, leg, {
952
+ ...client,
953
+ unfinishedTaskPending:
954
+ turn.foregroundTaskEpoch !== null ||
955
+ turn.interruptedRequest !== null ||
956
+ turn.handedOffRequest !== null,
957
+ });
876
958
  if (turn.language !== undefined) {
877
959
  prompt = `${prompt}\n\nThe caller has been speaking the language with code "${turn.language}" this turn. Reply in that language unless they clearly switch to another.`;
878
960
  }
@@ -885,6 +967,9 @@ function buildVoiceControlPrompt(
885
967
  if (turn.handedOffRequest) {
886
968
  prompt = `${prompt}\n\n${buildHandoffAnnouncementNote(turn.handedOffRequest)}`;
887
969
  }
970
+ if (turn.lookFollowUp !== null) {
971
+ prompt = `${prompt}\n\n${lookFollowUpNote(turn.lookFollowUp)}`;
972
+ }
888
973
  if (turn.continuationDelivery) {
889
974
  prompt = `${prompt}\n\n${buildLiveDeliveryNote(
890
975
  turn.continuationDelivery.request,
@@ -1037,6 +1122,20 @@ function foregroundToolContendsWithContinuation(
1037
1122
  return isRefusedInReadOnlyPass(toolName, ownerKind);
1038
1123
  }
1039
1124
 
1125
+ function hostInteractiveToolName(
1126
+ toolName: string,
1127
+ input?: Record<string, unknown>,
1128
+ allowedToolNames?: ReadonlySet<string>,
1129
+ ): string | null {
1130
+ const effectiveToolName =
1131
+ toolName === "skill_execute"
1132
+ ? resolveSkillExecuteInvocation(input ?? {}, allowedToolNames).name
1133
+ : toolName;
1134
+ return getTool(effectiveToolName)?.executionTarget === "host"
1135
+ ? effectiveToolName
1136
+ : null;
1137
+ }
1138
+
1040
1139
  // Upper bound on how long a barge-in waits for the interrupted turn's teardown
1041
1140
  // to settle before giving up on the continuation. The teardown settles once the
1042
1141
  // aborted turn's agent loop reaches its `finally`, which can wait out BOTH the
@@ -1135,6 +1234,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1135
1234
  // The session controls the client declared it can carry out; the only ones
1136
1235
  // the model is taught and the only ones a reply's marker can trigger.
1137
1236
  private readonly sessionControls: readonly LiveVoiceSessionControl[];
1237
+ // The client declared `lookFrames`: it sends a fresh frame for every look
1238
+ // control it carries out, so the session answers a look on its own turn and
1239
+ // teaches the model that the reply asking for one is only the acknowledgement.
1240
+ private readonly lookFrames: boolean;
1138
1241
  // How often progress updates are spoken, as the user last asked out loud.
1139
1242
  // Session-scoped: it applies from the next turn to the end of the call.
1140
1243
  private progressCadence: "fewer" | "normal" = "normal";
@@ -1222,6 +1325,23 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1222
1325
  // answer.
1223
1326
  private pendingAnnouncement: ContinuationDelivery | null = null;
1224
1327
  private announcementTimer: ReturnType<typeof setTimeout> | null = null;
1328
+ // Host-backed work stays on the parent conversation across barge-ins. An
1329
+ // owned task belongs to the tool-capable turn doing the work; a suspended
1330
+ // task is waiting for either a merged escalation or a synthetic resume.
1331
+ private foregroundTaskState: ForegroundTaskState | null = null;
1332
+ private foregroundTaskEpoch = 0;
1333
+ private foregroundTaskResumeTimer: ReturnType<typeof setTimeout> | null =
1334
+ null;
1335
+ private readonly foregroundTaskResumeSilenceMs: number;
1336
+ private readonly foregroundTaskMaxSuspendedMs: number;
1337
+ private readonly foregroundTaskMaxInterveningTurns: number;
1338
+ // A look control sent to a client that declared `lookFrames`, waiting for
1339
+ // the fresh frame the client takes for it. The session answers the look on a
1340
+ // turn of its own once that frame is in the conversation (see
1341
+ // answerLookWhenFloorIsFree). Cleared when the frame lands, when the wait
1342
+ // runs out, and when the session closes.
1343
+ private pendingLook: PendingLook | null = null;
1344
+ private lookFollowUpTimer: ReturnType<typeof setTimeout> | null = null;
1225
1345
  // Set when a continuation actually spawns: the request it took over, so the
1226
1346
  // NEXT turn can tell the user the work is still running. Consumed by that
1227
1347
  // turn; cleared when the continuation finishes (by then the result note
@@ -1236,6 +1356,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1236
1356
  // the distance from it to a keep arriving is the client leg of the frame the
1237
1357
  // onset asked for, measured from the daemon's own clock.
1238
1358
  private lastSpeechStartedAtMs: number | null = null;
1359
+ // How many assistant turns have launched, so a look can tell whether one has
1360
+ // started since its frame landed. A count rather than a time: two launches
1361
+ // and a frame can share a millisecond.
1362
+ private turnsLaunched = 0;
1239
1363
  private readonly maxPendingAudioBytes: number;
1240
1364
  // Set on VAD speech onset; consumed when the first speech chunk is routed
1241
1365
  // to an utterance so the metric lands on the right turn.
@@ -1357,6 +1481,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1357
1481
  defaultDetachTeardownSettleTimeoutMs();
1358
1482
  this.continuationAnnounceSilenceMs =
1359
1483
  options.continuationAnnounceSilenceMs ?? CONTINUATION_ANNOUNCE_SILENCE_MS;
1484
+ this.foregroundTaskResumeSilenceMs =
1485
+ options.foregroundTaskResumeSilenceMs ??
1486
+ FOREGROUND_TASK_RESUME_SILENCE_MS;
1487
+ this.foregroundTaskMaxSuspendedMs =
1488
+ options.foregroundTaskMaxSuspendedMs ?? FOREGROUND_TASK_MAX_SUSPENDED_MS;
1489
+ this.foregroundTaskMaxInterveningTurns =
1490
+ options.foregroundTaskMaxInterveningTurns ??
1491
+ FOREGROUND_TASK_MAX_INTERVENING_TURNS;
1360
1492
  this.emitMetrics = options.emitMetrics ?? false;
1361
1493
  this.createTurnId = options.createTurnId ?? randomUUID;
1362
1494
  this.conversationId =
@@ -1413,6 +1545,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1413
1545
  SERVER_VAD_PENDING_AUDIO_MAX_SECONDS;
1414
1546
  this.textInput = context.startFrame.textInput === true;
1415
1547
  this.sessionControls = context.startFrame.sessionControls ?? [];
1548
+ this.lookFrames = context.startFrame.lookFrames === true;
1416
1549
  }
1417
1550
 
1418
1551
  get finalTranscriptText(): string {
@@ -1661,6 +1794,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1661
1794
  },
1662
1795
  "Sight frame timing",
1663
1796
  );
1797
+ // Only once the row is in: the turn that answers the look reads the
1798
+ // conversation, so the frame has to be there before that turn starts.
1799
+ if (result.ok && frame.timing?.reason === LOOK_FRAME_REASON) {
1800
+ this.lookFrameLanded();
1801
+ }
1664
1802
  if (!result.ok && !this.isClosed) {
1665
1803
  void this.sendFrame({
1666
1804
  type: "error",
@@ -1847,6 +1985,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1847
1985
 
1848
1986
  const shouldEmitSessionEndMetrics = this.state !== "failed";
1849
1987
  this.state = "closed";
1988
+ this.clearForegroundTask("session_closed");
1850
1989
  // Retire the island before the teardown below starts awaiting things. A
1851
1990
  // close can take a while (a pending continuation is delivered first), and
1852
1991
  // an activity left asserting "Speaking…" through it is exactly the stale
@@ -1866,6 +2005,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1866
2005
  // cleared: the announcement is dead either way.
1867
2006
  await this.deliverPendingContinuationToConversation();
1868
2007
  this.clearContinuationAnnouncement();
2008
+ this.clearPendingLook();
1869
2009
  this.stopSessionTranscriber();
1870
2010
  // Detached continuations outlive the call. A deliberate `interrupt()`
1871
2011
  // aborts them; ending the session leaves them running. With no next voice
@@ -2784,6 +2924,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2784
2924
  // do not collide with it in the collector. turn_cancelled flushes
2785
2925
  // client playback, so the drain estimate resets with it.
2786
2926
  this.assistantPlaybackTailUntilMs = 0;
2927
+ const keepForegroundTaskOnParent =
2928
+ !turn.assistantCompleted && this.suspendOwnedForegroundTask(turn);
2787
2929
  // Carry the interrupted request into the next turn so it merges with the
2788
2930
  // barge-in utterance rather than being answered as a fresh follow-up.
2789
2931
  const interruptedRequest = turn.utterance.finalTranscriptSegments
@@ -2864,6 +3006,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2864
3006
  this.publishActivity(turn, "");
2865
3007
  await this.sendFrame({ type: "turn_cancelled", turnId: turn.turnId });
2866
3008
  await this.cancelAssistantTurn("barge_in");
3009
+ if (keepForegroundTaskOnParent) {
3010
+ log.info(
3011
+ { turnId: turn.turnId },
3012
+ "Voice duplex continuation skipped for parent-owned foreground task",
3013
+ );
3014
+ return;
3015
+ }
2867
3016
  // Keep the interrupted turn's work alive on a background subagent; the
2868
3017
  // detach waits for its teardown to settle the partial into history before
2869
3018
  // forking.
@@ -2909,16 +3058,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2909
3058
  // already finished, so bargeIn returns it to the stash instead.
2910
3059
  turn.continuationDelivery !== null
2911
3060
  ? "announcement_turn"
2912
- : // The model already finished generating (barge-in during TTS playback
2913
- // of a complete reply): there is nothing to continue, so a
2914
- // continuation would just re-do a finished answer.
2915
- turn.assistantCompleted
2916
- ? "assistant_already_completed"
2917
- : // A stop (interrupt/close) or a superseding invalidation landed
2918
- // during the barge-in teardown: honor it.
2919
- this.detachStopGeneration !== stopGeneration
2920
- ? "invalidated_during_barge_teardown"
2921
- : null;
3061
+ : // Nor over the answer to a look: there is no request behind it
3062
+ // either, and the user talking over it is them moving on.
3063
+ turn.lookFollowUp !== null
3064
+ ? "look_follow_up"
3065
+ : // The model already finished generating (barge-in during TTS playback
3066
+ // of a complete reply): there is nothing to continue, so a
3067
+ // continuation would just re-do a finished answer.
3068
+ turn.assistantCompleted
3069
+ ? "assistant_already_completed"
3070
+ : // A stop (interrupt/close) or a superseding invalidation landed
3071
+ // during the barge-in teardown: honor it.
3072
+ this.detachStopGeneration !== stopGeneration
3073
+ ? "invalidated_during_barge_teardown"
3074
+ : null;
2922
3075
  if (skipReason !== null) {
2923
3076
  log.info(
2924
3077
  { turnId: turn.turnId, skipReason },
@@ -3137,6 +3290,350 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3137
3290
  }
3138
3291
  }
3139
3292
 
3293
+ private foregroundTaskRequestForTurn(turn: ActiveAssistantTurn): string {
3294
+ const currentRequest = turn.utterance.finalTranscriptSegments
3295
+ .join(" ")
3296
+ .trim();
3297
+ return currentRequest || turn.interruptedRequest || "";
3298
+ }
3299
+
3300
+ private createForegroundTaskOwnership(
3301
+ turn: ActiveAssistantTurn,
3302
+ hostToolStarted: boolean,
3303
+ ): OwnedForegroundTask {
3304
+ const task: OwnedForegroundTask = {
3305
+ phase: "owned",
3306
+ epoch: ++this.foregroundTaskEpoch,
3307
+ request: this.foregroundTaskRequestForTurn(turn),
3308
+ ownerToken: turn.token,
3309
+ hostToolStarted,
3310
+ };
3311
+ turn.foregroundTaskEpoch = task.epoch;
3312
+ this.foregroundTaskState = task;
3313
+ return task;
3314
+ }
3315
+
3316
+ private noteEscalatedForegroundTask(turn: ActiveAssistantTurn): void {
3317
+ if (
3318
+ this.activeAssistantTurn?.token !== turn.token ||
3319
+ turn.discardRequested ||
3320
+ turn.abortController.signal.aborted
3321
+ ) {
3322
+ return;
3323
+ }
3324
+ if (
3325
+ this.claimSuspendedForegroundTask(turn) ||
3326
+ this.foregroundTaskState !== null
3327
+ ) {
3328
+ return;
3329
+ }
3330
+
3331
+ const task = this.createForegroundTaskOwnership(turn, false);
3332
+ log.info(
3333
+ { turnId: turn.turnId, epoch: task.epoch },
3334
+ "Escalated live voice turn provisionally owns a foreground task",
3335
+ );
3336
+ }
3337
+
3338
+ private noteHostToolStarted(
3339
+ turn: ActiveAssistantTurn,
3340
+ effectiveToolName: string,
3341
+ ): void {
3342
+ const current = this.foregroundTaskState;
3343
+ if (current?.phase === "owned" && current.ownerToken === turn.token) {
3344
+ if (!current.hostToolStarted) {
3345
+ this.foregroundTaskState = { ...current, hostToolStarted: true };
3346
+ log.info(
3347
+ { turnId: turn.turnId, epoch: current.epoch, effectiveToolName },
3348
+ "Foreground task confirmed by host tool use",
3349
+ );
3350
+ }
3351
+ return;
3352
+ }
3353
+ if (this.claimSuspendedForegroundTask(turn, true)) {
3354
+ return;
3355
+ }
3356
+
3357
+ const task = this.createForegroundTaskOwnership(turn, true);
3358
+ log.info(
3359
+ { turnId: turn.turnId, epoch: task.epoch, effectiveToolName },
3360
+ "Live voice turn started a foreground task",
3361
+ );
3362
+ }
3363
+
3364
+ private claimSuspendedForegroundTask(
3365
+ turn: ActiveAssistantTurn,
3366
+ hostToolStarted = false,
3367
+ ): boolean {
3368
+ const current = this.foregroundTaskState;
3369
+ if (
3370
+ current?.phase !== "suspended" ||
3371
+ turn.foregroundTaskEpoch !== current.epoch ||
3372
+ this.activeAssistantTurn?.token !== turn.token ||
3373
+ turn.discardRequested ||
3374
+ turn.abortController.signal.aborted
3375
+ ) {
3376
+ return false;
3377
+ }
3378
+ this.foregroundTaskState = {
3379
+ phase: "owned",
3380
+ epoch: current.epoch,
3381
+ request: current.request,
3382
+ ownerToken: turn.token,
3383
+ hostToolStarted: current.hostToolStarted || hostToolStarted,
3384
+ };
3385
+ this.clearForegroundTaskResumeTimer();
3386
+ log.info(
3387
+ {
3388
+ turnId: turn.turnId,
3389
+ epoch: current.epoch,
3390
+ hostToolStarted: current.hostToolStarted || hostToolStarted,
3391
+ },
3392
+ "Suspended foreground task claimed by tool-capable turn",
3393
+ );
3394
+ return true;
3395
+ }
3396
+
3397
+ private suspendOwnedForegroundTask(turn: ActiveAssistantTurn): boolean {
3398
+ const current = this.foregroundTaskState;
3399
+ if (current?.phase === "owned" && current.ownerToken === turn.token) {
3400
+ this.foregroundTaskState = {
3401
+ phase: "suspended",
3402
+ epoch: current.epoch,
3403
+ request: current.request,
3404
+ suspendedAtMs: Date.now(),
3405
+ interveningTurns: 0,
3406
+ resumePending: false,
3407
+ hostToolStarted: current.hostToolStarted,
3408
+ };
3409
+ this.clearForegroundTaskResumeTimer();
3410
+ log.info(
3411
+ { turnId: turn.turnId, epoch: current.epoch },
3412
+ "Foreground task suspended by voice barge-in",
3413
+ );
3414
+ return true;
3415
+ }
3416
+ return current !== null;
3417
+ }
3418
+
3419
+ private clearForegroundTask(reason: string): void {
3420
+ const current = this.foregroundTaskState;
3421
+ this.foregroundTaskEpoch += 1;
3422
+ this.foregroundTaskState = null;
3423
+ this.clearForegroundTaskResumeTimer();
3424
+ if (current !== null) {
3425
+ log.info(
3426
+ {
3427
+ epoch: current.epoch,
3428
+ phase: current.phase,
3429
+ hostToolStarted: current.hostToolStarted,
3430
+ reason,
3431
+ },
3432
+ "Live voice foreground task cleared",
3433
+ );
3434
+ }
3435
+ }
3436
+
3437
+ private stopOutstandingInterruptedWork(reason: string): void {
3438
+ this.clearForegroundTask(reason);
3439
+ this.abortDetachedRuns({ reason });
3440
+ }
3441
+
3442
+ private foregroundTaskSuspensionExpired(
3443
+ task: SuspendedForegroundTask,
3444
+ ): "turn_limit" | "time_limit" | null {
3445
+ if (task.interveningTurns > this.foregroundTaskMaxInterveningTurns) {
3446
+ return "turn_limit";
3447
+ }
3448
+ if (Date.now() - task.suspendedAtMs >= this.foregroundTaskMaxSuspendedMs) {
3449
+ return "time_limit";
3450
+ }
3451
+ return null;
3452
+ }
3453
+
3454
+ private queueForegroundTaskResume(turn: ActiveAssistantTurn): void {
3455
+ const current = this.foregroundTaskState;
3456
+ if (
3457
+ current?.phase !== "suspended" ||
3458
+ turn.foregroundTaskEpoch !== current.epoch ||
3459
+ turn.hiddenPrompt ||
3460
+ turn.discardRequested ||
3461
+ turn.speculativePending
3462
+ ) {
3463
+ return;
3464
+ }
3465
+ const next: SuspendedForegroundTask = {
3466
+ ...current,
3467
+ interveningTurns: current.interveningTurns + 1,
3468
+ resumePending: true,
3469
+ };
3470
+ const expired = this.foregroundTaskSuspensionExpired(next);
3471
+ if (expired !== null) {
3472
+ this.clearForegroundTask(`suspended_${expired}`);
3473
+ return;
3474
+ }
3475
+ this.foregroundTaskState = next;
3476
+ if (this.activeAssistantTurn === null) {
3477
+ this.scheduleForegroundTaskResume();
3478
+ }
3479
+ }
3480
+
3481
+ private markForegroundTaskResumePending(epoch: number): boolean {
3482
+ const current = this.foregroundTaskState;
3483
+ if (current?.phase !== "suspended" || current.epoch !== epoch) {
3484
+ return false;
3485
+ }
3486
+ this.foregroundTaskState = { ...current, resumePending: true };
3487
+ return true;
3488
+ }
3489
+
3490
+ private rearmForegroundTaskAfterLegFailure(
3491
+ turn: ActiveAssistantTurn,
3492
+ ): boolean {
3493
+ const epoch = turn.foregroundTaskEpoch;
3494
+ if (epoch === null) {
3495
+ return false;
3496
+ }
3497
+ const current = this.foregroundTaskState;
3498
+ if (current?.phase === "suspended" && current.epoch === epoch) {
3499
+ return this.markForegroundTaskResumePending(epoch);
3500
+ }
3501
+ const snapshot = turn.foregroundTaskSuspensionSnapshot;
3502
+ if (
3503
+ current?.phase !== "owned" ||
3504
+ current.epoch !== epoch ||
3505
+ current.ownerToken !== turn.token ||
3506
+ (snapshot !== null && snapshot.epoch !== epoch)
3507
+ ) {
3508
+ return false;
3509
+ }
3510
+ const next: SuspendedForegroundTask = {
3511
+ ...(snapshot ?? {
3512
+ phase: "suspended",
3513
+ epoch,
3514
+ request: current.request,
3515
+ suspendedAtMs: Date.now(),
3516
+ interveningTurns: 0,
3517
+ hostToolStarted: current.hostToolStarted,
3518
+ }),
3519
+ resumePending: true,
3520
+ };
3521
+ const expired = this.foregroundTaskSuspensionExpired(next);
3522
+ if (expired !== null) {
3523
+ this.clearForegroundTask(`suspended_${expired}`);
3524
+ return false;
3525
+ }
3526
+ this.foregroundTaskState = next;
3527
+ log.info(
3528
+ { turnId: turn.turnId, epoch },
3529
+ "Foreground task re-armed after tool-capable leg failure",
3530
+ );
3531
+ return true;
3532
+ }
3533
+
3534
+ private clearForegroundTaskResumeTimer(): void {
3535
+ if (this.foregroundTaskResumeTimer !== null) {
3536
+ clearTimeout(this.foregroundTaskResumeTimer);
3537
+ this.foregroundTaskResumeTimer = null;
3538
+ }
3539
+ }
3540
+
3541
+ private scheduleFloorCheck(
3542
+ currentTimer: ReturnType<typeof setTimeout> | null,
3543
+ silenceMs: number,
3544
+ check: (blockedBy: string | null) => void,
3545
+ ): ReturnType<typeof setTimeout> {
3546
+ if (currentTimer !== null) {
3547
+ clearTimeout(currentTimer);
3548
+ }
3549
+ const drainMs = Math.max(0, this.assistantPlaybackTailUntilMs - Date.now());
3550
+ return setTimeout(
3551
+ () => check(this.sessionTurnFloorBlocker()),
3552
+ drainMs + silenceMs,
3553
+ );
3554
+ }
3555
+
3556
+ private scheduleForegroundTaskResume(): void {
3557
+ const task = this.foregroundTaskState;
3558
+ if (
3559
+ task?.phase !== "suspended" ||
3560
+ !task.resumePending ||
3561
+ this.isClosed ||
3562
+ this.state === "failed"
3563
+ ) {
3564
+ this.clearForegroundTaskResumeTimer();
3565
+ return;
3566
+ }
3567
+ const expired = this.foregroundTaskSuspensionExpired(task);
3568
+ if (expired !== null) {
3569
+ this.clearForegroundTask(`suspended_${expired}`);
3570
+ return;
3571
+ }
3572
+ const epoch = task.epoch;
3573
+ this.foregroundTaskResumeTimer = this.scheduleFloorCheck(
3574
+ this.foregroundTaskResumeTimer,
3575
+ this.foregroundTaskResumeSilenceMs,
3576
+ (blockedBy) => {
3577
+ this.foregroundTaskResumeTimer = null;
3578
+ const current = this.foregroundTaskState;
3579
+ if (
3580
+ current?.phase !== "suspended" ||
3581
+ current.epoch !== epoch ||
3582
+ !current.resumePending
3583
+ ) {
3584
+ return;
3585
+ }
3586
+ const currentExpired = this.foregroundTaskSuspensionExpired(current);
3587
+ if (currentExpired !== null) {
3588
+ this.clearForegroundTask(`suspended_${currentExpired}`);
3589
+ return;
3590
+ }
3591
+ if (blockedBy === null) {
3592
+ void this.resumeForegroundTask(current).catch((err: unknown) => {
3593
+ log.warn({ err, epoch }, "Foreground task resume failed");
3594
+ });
3595
+ return;
3596
+ }
3597
+ if (blockedBy === "session_unavailable") {
3598
+ this.clearForegroundTask("session_unavailable");
3599
+ return;
3600
+ }
3601
+ if (blockedBy !== "turn_active") {
3602
+ this.scheduleForegroundTaskResume();
3603
+ }
3604
+ },
3605
+ );
3606
+ }
3607
+
3608
+ private async resumeForegroundTask(
3609
+ task: SuspendedForegroundTask,
3610
+ ): Promise<void> {
3611
+ const current = this.foregroundTaskState;
3612
+ if (
3613
+ current?.phase !== "suspended" ||
3614
+ current.epoch !== task.epoch ||
3615
+ !current.resumePending
3616
+ ) {
3617
+ return;
3618
+ }
3619
+ this.foregroundTaskState = { ...current, resumePending: false };
3620
+ const started = await this.launchAssistantTurn(
3621
+ createSyntheticUtterance(),
3622
+ buildForegroundTaskResumePrompt(current.request),
3623
+ {
3624
+ hiddenPrompt: true,
3625
+ initialLeg: "escalated",
3626
+ foregroundTaskEpoch: current.epoch,
3627
+ },
3628
+ );
3629
+ if (started) {
3630
+ return;
3631
+ }
3632
+ if (this.markForegroundTaskResumePending(current.epoch)) {
3633
+ this.scheduleForegroundTaskResume();
3634
+ }
3635
+ }
3636
+
3140
3637
  // Abort every background continuation this session started and drop its
3141
3638
  // handle. Called on a hard stop (client interrupt / session close), when a
3142
3639
  // newer barge-in supersedes the detached runs, and on a foreground-wins
@@ -3263,50 +3760,53 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3263
3760
  retry = false,
3264
3761
  drainRearms = 0,
3265
3762
  ): void {
3266
- if (this.announcementTimer) {
3267
- clearTimeout(this.announcementTimer);
3268
- }
3269
- // Queued client-side audio is the previous reply still being audible, so
3270
- // the silence window only begins once it has drained.
3271
- const drainMs = Math.max(0, this.assistantPlaybackTailUntilMs - Date.now());
3272
- this.announcementTimer = setTimeout(() => {
3273
- this.announcementTimer = null;
3274
- const blockedBy = this.continuationAnnouncementBlocker();
3275
- if (blockedBy === null) {
3276
- void this.announceContinuation().catch((err: unknown) => {
3277
- log.warn({ err }, "Voice duplex continuation announcement failed");
3278
- });
3279
- return;
3280
- }
3281
- log.debug(
3282
- { conversationId: this.conversationId, blockedBy, retry, drainRearms },
3283
- "Voice duplex continuation announcement deferred",
3284
- );
3285
- // A tail that grew while the timer ran (the reply was still streaming
3286
- // TTS when it was armed) is a wait of known length rather than a busy
3287
- // call, so it re-arms against the new deadline without spending the
3288
- // single retry the busy-call blockers get — otherwise a long reply's
3289
- // playback alone outlasts the budget and the announcement is dropped
3290
- // even though the call goes idle right after. The tail only extends
3291
- // while a turn is active, which blocks earlier, so the re-arms converge;
3292
- // the cap bounds the interleaving where a fresh turn's audio lands
3293
- // between two checks.
3294
- if (
3295
- blockedBy === "playback_draining" &&
3296
- drainRearms < CONTINUATION_ANNOUNCE_MAX_DRAIN_REARMS
3297
- ) {
3298
- this.scheduleContinuationAnnouncement(retry, drainRearms + 1);
3299
- return;
3300
- }
3301
- if (blockedBy === "turn_active") {
3302
- return;
3303
- }
3304
- if (retry || this.pendingAnnouncement === null) {
3305
- this.pendingAnnouncement = null;
3306
- return;
3307
- }
3308
- this.scheduleContinuationAnnouncement(true, drainRearms);
3309
- }, drainMs + this.continuationAnnounceSilenceMs);
3763
+ this.announcementTimer = this.scheduleFloorCheck(
3764
+ this.announcementTimer,
3765
+ this.continuationAnnounceSilenceMs,
3766
+ () => {
3767
+ this.announcementTimer = null;
3768
+ const blockedBy = this.continuationAnnouncementBlocker();
3769
+ if (blockedBy === null) {
3770
+ void this.announceContinuation().catch((err: unknown) => {
3771
+ log.warn({ err }, "Voice duplex continuation announcement failed");
3772
+ });
3773
+ return;
3774
+ }
3775
+ log.debug(
3776
+ {
3777
+ conversationId: this.conversationId,
3778
+ blockedBy,
3779
+ retry,
3780
+ drainRearms,
3781
+ },
3782
+ "Voice duplex continuation announcement deferred",
3783
+ );
3784
+ // A tail that grew while the timer ran (the reply was still streaming
3785
+ // TTS when it was armed) is a wait of known length rather than a busy
3786
+ // call, so it re-arms against the new deadline without spending the
3787
+ // single retry the busy-call blockers get. Otherwise a long reply's
3788
+ // playback alone outlasts the budget and the announcement is dropped
3789
+ // even though the call goes idle right after. The tail only extends
3790
+ // while a turn is active, which blocks earlier, so the re-arms converge;
3791
+ // the cap bounds the interleaving where a fresh turn's audio lands
3792
+ // between two checks.
3793
+ if (
3794
+ blockedBy === "playback_draining" &&
3795
+ drainRearms < CONTINUATION_ANNOUNCE_MAX_DRAIN_REARMS
3796
+ ) {
3797
+ this.scheduleContinuationAnnouncement(retry, drainRearms + 1);
3798
+ return;
3799
+ }
3800
+ if (blockedBy === "turn_active") {
3801
+ return;
3802
+ }
3803
+ if (retry || this.pendingAnnouncement === null) {
3804
+ this.pendingAnnouncement = null;
3805
+ return;
3806
+ }
3807
+ this.scheduleContinuationAnnouncement(true, drainRearms);
3808
+ },
3809
+ );
3310
3810
  }
3311
3811
 
3312
3812
  /**
@@ -3496,6 +3996,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3496
3996
  ) {
3497
3997
  this.scheduleContinuationAnnouncement();
3498
3998
  }
3999
+ if (
4000
+ this.foregroundTaskState?.phase === "suspended" &&
4001
+ this.foregroundTaskState.resumePending &&
4002
+ !this.isClosed &&
4003
+ this.state !== "failed"
4004
+ ) {
4005
+ this.scheduleForegroundTaskResume();
4006
+ }
3499
4007
  }
3500
4008
 
3501
4009
  // Why the session must not speak up right now, or null when the call is
@@ -3565,6 +4073,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3565
4073
  // own it leaves manual mode blind to a user who is talking right now and
3566
4074
  // has no text yet — hence the captured-audio flag, which manual ingress
3567
4075
  // sets from the first chunk.
4076
+ //
4077
+ // A partial counts only when it has words. Deepgram Flux sends interim
4078
+ // updates through silence too, each an empty partial, and one of those
4079
+ // would otherwise hold the floor until the user next spoke.
3568
4080
  if (
3569
4081
  utterance !== null &&
3570
4082
  !utterance.completed &&
@@ -3572,13 +4084,129 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3572
4084
  utterance.assistantTurnStarted ||
3573
4085
  (utterance.manualAudioCaptured && opts?.ignoreManualCapture !== true) ||
3574
4086
  utterance.finalTranscriptSegments.length > 0 ||
3575
- utterance.latestPartialText !== null)
4087
+ (utterance.latestPartialText?.trim() ?? "").length > 0)
3576
4088
  ) {
3577
4089
  return "utterance_in_flight";
3578
4090
  }
3579
4091
  return null;
3580
4092
  }
3581
4093
 
4094
+ /**
4095
+ * Wait for the fresh frame a look control asks the client for.
4096
+ *
4097
+ * Only for a client that declared `lookFrames`: any other sends no such
4098
+ * frame, and its look keeps the old shape, where the user's next words are
4099
+ * what the frame gets answered on. A newer look replaces an older one still
4100
+ * waiting, and the wait is bounded so a frame that never comes (a camera
4101
+ * that would not open, a share the desktop refused) cannot turn up minutes
4102
+ * later as a reply to nothing.
4103
+ */
4104
+ private awaitLookFrame(action: LookSessionControl): void {
4105
+ if (!this.lookFrames) {
4106
+ return;
4107
+ }
4108
+ this.clearPendingLook();
4109
+ const timer = setTimeout(() => {
4110
+ if (this.pendingLook?.timer !== timer) {
4111
+ return;
4112
+ }
4113
+ this.pendingLook = null;
4114
+ log.info(
4115
+ { conversationId: this.conversationId, action },
4116
+ "Live voice look dropped: no frame arrived",
4117
+ );
4118
+ }, LOOK_FRAME_WAIT_MS);
4119
+ this.pendingLook = { action, armedAtMs: Date.now(), timer };
4120
+ }
4121
+
4122
+ private clearPendingLook(): void {
4123
+ if (this.pendingLook !== null) {
4124
+ clearTimeout(this.pendingLook.timer);
4125
+ this.pendingLook = null;
4126
+ }
4127
+ if (this.lookFollowUpTimer !== null) {
4128
+ clearTimeout(this.lookFollowUpTimer);
4129
+ this.lookFollowUpTimer = null;
4130
+ }
4131
+ }
4132
+
4133
+ /**
4134
+ * The frame a look asked for is in the conversation: answer the look.
4135
+ *
4136
+ * A look frame with no look waiting (the wait ran out, or the session never
4137
+ * armed one) is only a frame, like any other keep.
4138
+ */
4139
+ private lookFrameLanded(): void {
4140
+ const pending = this.pendingLook;
4141
+ if (pending === null || this.isClosed) {
4142
+ return;
4143
+ }
4144
+ clearTimeout(pending.timer);
4145
+ this.pendingLook = null;
4146
+ this.answerLookWhenFloorIsFree(pending, this.turnsLaunched, 0);
4147
+ }
4148
+
4149
+ /**
4150
+ * Start the turn that answers a look, once nothing else holds the floor.
4151
+ *
4152
+ * The acknowledgement ("taking a look") can still be playing when the frame
4153
+ * lands, and the look's own turn can still be clearing, so both are waited
4154
+ * out, as is the user mid-utterance. A turn that starts once the frame is in
4155
+ * the conversation is not: it reads the frame, so answering the look as well
4156
+ * would answer it twice. A turn that started before the frame landed did not
4157
+ * see it, so the look is still answered once that turn is done.
4158
+ *
4159
+ * `turnsAtFrame` is how many turns had launched when the frame landed.
4160
+ */
4161
+ private answerLookWhenFloorIsFree(
4162
+ look: PendingLook,
4163
+ turnsAtFrame: number,
4164
+ rearms: number,
4165
+ ): void {
4166
+ if (this.lookFollowUpTimer !== null) {
4167
+ clearTimeout(this.lookFollowUpTimer);
4168
+ this.lookFollowUpTimer = null;
4169
+ }
4170
+ const { action, armedAtMs } = look;
4171
+ // A turn launched since the frame landed read it already. Speech that
4172
+ // never became a turn (a cough, noise that transcribed to nothing) is only
4173
+ // waited out.
4174
+ const blockedBy =
4175
+ this.turnsLaunched > turnsAtFrame
4176
+ ? "turn_since_look"
4177
+ : this.sessionTurnFloorBlocker();
4178
+ if (blockedBy === null) {
4179
+ void this.launchAssistantTurn(
4180
+ createSyntheticUtterance(),
4181
+ LOOK_FOLLOW_UP_CONTENT,
4182
+ { lookFollowUp: action, hiddenPrompt: true },
4183
+ ).catch((err: unknown) => {
4184
+ log.warn(
4185
+ { err, conversationId: this.conversationId, action },
4186
+ "Live voice look follow-up failed to start",
4187
+ );
4188
+ });
4189
+ return;
4190
+ }
4191
+ const waitable =
4192
+ blockedBy !== "turn_since_look" && blockedBy !== "session_unavailable";
4193
+ if (!waitable || Date.now() - armedAtMs >= LOOK_ANSWER_DEADLINE_MS) {
4194
+ log.info(
4195
+ { conversationId: this.conversationId, action, blockedBy, rearms },
4196
+ "Live voice look follow-up skipped",
4197
+ );
4198
+ return;
4199
+ }
4200
+ const drainMs = Math.max(0, this.assistantPlaybackTailUntilMs - Date.now());
4201
+ this.lookFollowUpTimer = setTimeout(
4202
+ () => {
4203
+ this.lookFollowUpTimer = null;
4204
+ this.answerLookWhenFloorIsFree(look, turnsAtFrame, rearms + 1);
4205
+ },
4206
+ Math.max(drainMs, LOOK_FOLLOW_UP_REARM_MS),
4207
+ );
4208
+ }
4209
+
3582
4210
  // Speak the finished continuation's result on a turn the session starts
3583
4211
  // itself. The turn rides a synthetic cycle rather than the armed VAD one —
3584
4212
  // `currentUtterance` belongs to the capture loop, and overwriting it would
@@ -4768,8 +5396,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4768
5396
  // the next turn is now stale (the interrupted utterance may be discarded
4769
5397
  // without ever reaching finalizePendingUtterance).
4770
5398
  this.pendingInterruptedRequest = null;
5399
+ this.clearForegroundTask("client_interrupt");
4771
5400
  // ...and it hard-stops any detached background continuations.
4772
5401
  this.abortDetachedRuns({ reason: "client_interrupt" });
5402
+ // ...and a look still waiting to be answered: the user stopped the call
5403
+ // talking, and a reply starting on its own a moment later is not a stop.
5404
+ this.clearPendingLook();
4773
5405
  const utterance = this.currentUtterance;
4774
5406
  this.stopSessionTranscriber();
4775
5407
  if (utterance) {
@@ -4991,6 +5623,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4991
5623
  // Set on an announcement turn: the finished continuation this turn exists
4992
5624
  // to deliver. Its answer goes in the control prompt, not in `content`.
4993
5625
  continuationDelivery?: ContinuationDelivery | null;
5626
+ // Set on the turn that answers a look: which look. Its instruction goes
5627
+ // in the control prompt, not in `content`.
5628
+ lookFollowUp?: LookSessionControl;
4994
5629
  // Unified front-door: dispatch without releasing the utterance. The
4995
5630
  // thinking frame and floor-holding timers are deferred until the leg's
4996
5631
  // leading verdict commits the turn (see commitSpeculativeTurn); a hold
@@ -5000,6 +5635,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5000
5635
  // the row persists and drives the turn but never renders: no echo, and
5001
5636
  // `/messages` filters it after a reload.
5002
5637
  hiddenPrompt?: boolean;
5638
+ // Synthetic host-task resumes start on the tool-capable profile without
5639
+ // passing through the front door.
5640
+ initialLeg?: VoiceRoutingLeg;
5641
+ // The suspended foreground task this synthetic turn resumes.
5642
+ foregroundTaskEpoch?: number;
5003
5643
  },
5004
5644
  ): Promise<boolean> {
5005
5645
  utterance.assistantTurnStarted = true;
@@ -5015,6 +5655,15 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5015
5655
  this.startMetricsTurnIfNeeded(utterance, turnId);
5016
5656
  this.markAssistantDispatch(utterance, turnId);
5017
5657
  const abortController = new AbortController();
5658
+ const suspendedForegroundTask =
5659
+ this.foregroundTaskState?.phase === "suspended"
5660
+ ? this.foregroundTaskState
5661
+ : null;
5662
+ const foregroundTaskEpoch =
5663
+ opts?.foregroundTaskEpoch ??
5664
+ (opts?.hiddenPrompt !== true
5665
+ ? (suspendedForegroundTask?.epoch ?? null)
5666
+ : null);
5018
5667
  const activeTurn: ActiveAssistantTurn = {
5019
5668
  token,
5020
5669
  turnId,
@@ -5085,7 +5734,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5085
5734
  consumedAnnouncement: pending?.announcement ?? null,
5086
5735
  pendingContextStopGeneration: this.detachStopGeneration,
5087
5736
  continuationDelivery: opts?.continuationDelivery ?? null,
5737
+ lookFollowUp: opts?.lookFollowUp ?? null,
5088
5738
  hiddenPrompt: opts?.hiddenPrompt === true,
5739
+ foregroundTaskEpoch,
5740
+ foregroundTaskSuspensionSnapshot:
5741
+ suspendedForegroundTask?.epoch === foregroundTaskEpoch
5742
+ ? { ...suspendedForegroundTask }
5743
+ : null,
5089
5744
  deltaEpoch: 0,
5090
5745
  frontDoor: null,
5091
5746
  ttsBuffer: "",
@@ -5100,6 +5755,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5100
5755
  assistantAudioMimeType: "audio/pcm",
5101
5756
  };
5102
5757
  this.activeAssistantTurn = activeTurn;
5758
+ this.turnsLaunched += 1;
5103
5759
 
5104
5760
  // A speculative turn defers the thinking frame and both floor-holding
5105
5761
  // timers to commitSpeculativeTurn: until the verdict arrives, the pause
@@ -5149,13 +5805,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5149
5805
  activeTurn.progress.arm();
5150
5806
  }
5151
5807
 
5152
- // Front-door leg: a fast model fronts every turn (the `voiceFrontDoor`
5153
- // call site pins it) and may hand off to a quality leg on the escalate
5154
- // verdict.
5808
+ const initialLeg = opts?.initialLeg ?? "front-door";
5155
5809
  const started = await this.startAssistantLeg(activeTurn, {
5156
5810
  content,
5157
- routingLeg: "front-door",
5158
- frontDoor: true,
5811
+ routingLeg: initialLeg,
5812
+ ...(initialLeg === "front-door"
5813
+ ? { frontDoor: true }
5814
+ : { directEscalated: true }),
5159
5815
  // Only this leg: an escalated leg persists its own continuation row, and
5160
5816
  // the frame belongs to the row carrying the user's words.
5161
5817
  ...(activeTurn.turnAttachmentId
@@ -5200,6 +5856,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5200
5856
  overrideProfile?: string;
5201
5857
  routingLeg?: VoiceRoutingLeg;
5202
5858
  frontDoor?: boolean;
5859
+ directEscalated?: boolean;
5203
5860
  spokenEscalationBridge?: string;
5204
5861
  attachments?: readonly string[];
5205
5862
  },
@@ -5280,6 +5937,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5280
5937
  speakBridge: (bridge) =>
5281
5938
  this.speakEscalationBridge(activeTurn, bridge),
5282
5939
  startEscalatedLeg: (escalated) => {
5940
+ this.noteEscalatedForegroundTask(activeTurn);
5283
5941
  void this.startAssistantLeg(activeTurn, escalated);
5284
5942
  },
5285
5943
  },
@@ -5342,6 +6000,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5342
6000
  : {}),
5343
6001
  },
5344
6002
  this.sessionControls,
6003
+ { lookFrames: this.lookFrames },
5345
6004
  ),
5346
6005
  onApprovalPending: (requestId) => {
5347
6006
  this.revealRoomForPendingApproval(activeTurn, requestId);
@@ -5367,6 +6026,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5367
6026
  ...(leg.spokenEscalationBridge != null
5368
6027
  ? { spokenEscalationBridge: leg.spokenEscalationBridge }
5369
6028
  : {}),
6029
+ ...(leg.directEscalated === true ? { directEscalated: true } : {}),
5370
6030
  // A speculative front-door leg's decision rule includes the hold
5371
6031
  // branch so its leading tokens can be the hold verdict — but only
5372
6032
  // on the utterance's FIRST dispatch. Extension replays (the
@@ -5383,6 +6043,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5383
6043
  if (!this.isForwardingAssistantText(token)) {
5384
6044
  return;
5385
6045
  }
6046
+ if (!leg.frontDoor) {
6047
+ this.claimSuspendedForegroundTask(activeTurn);
6048
+ }
5386
6049
  if (coordinator !== null) {
5387
6050
  // Verdict-first: the leg's leading tokens decide the turn's
5388
6051
  // fate, and the coordinator acts on them. The bridge hands
@@ -5418,6 +6081,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5418
6081
  ) {
5419
6082
  return;
5420
6083
  }
6084
+ if (!leg.frontDoor) {
6085
+ this.claimSuspendedForegroundTask(current);
6086
+ }
5421
6087
  // A speculative leg that finished without a single delta (empty
5422
6088
  // output, provider hiccup) carries no verdict — fail open to a
5423
6089
  // committed turn so the utterance releases and finalizes like a
@@ -5463,10 +6129,18 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5463
6129
  { turnId, cadence: request.cadence },
5464
6130
  "Live voice progress cadence changed",
5465
6131
  );
6132
+ } else if (request?.action === "task_stop") {
6133
+ this.stopOutstandingInterruptedWork("spoken_task_stop");
5466
6134
  } else {
6135
+ if (request?.action === "end") {
6136
+ this.clearForegroundTask("call_end_requested");
6137
+ }
5467
6138
  current.sessionControlRequested = request;
5468
6139
  }
5469
6140
  }
6141
+ if (leg.frontDoor) {
6142
+ this.queueForegroundTaskResume(current);
6143
+ }
5470
6144
  current.assistantCompleted = true;
5471
6145
  if (msg.type === "generation_cancelled") {
5472
6146
  void this.finalizeAssistantTurn(
@@ -5501,6 +6175,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5501
6175
  if (current?.token !== token) {
5502
6176
  return;
5503
6177
  }
6178
+ const effectiveHostTool = hostInteractiveToolName(
6179
+ toolName,
6180
+ detail?.input,
6181
+ detail?.allowedToolNames,
6182
+ );
6183
+ if (effectiveHostTool !== null) {
6184
+ this.noteHostToolStarted(current, effectiveHostTool);
6185
+ }
5504
6186
  // Foreground wins the workspace: the continuation runs with full
5505
6187
  // subagent abilities (it can write files, run commands), so the
5506
6188
  // moment a live turn starts a consequential tool the two could
@@ -5587,7 +6269,15 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5587
6269
  if (currentTurn?.token !== token) {
5588
6270
  return;
5589
6271
  }
6272
+ const foregroundTaskResumeRearmed =
6273
+ !leg.frontDoor &&
6274
+ !currentTurn.discardRequested &&
6275
+ !currentTurn.abortController.signal.aborted &&
6276
+ this.rearmForegroundTaskAfterLegFailure(currentTurn);
5590
6277
  await this.finalizeAssistantTurn(currentTurn, "cancelled", "error");
6278
+ if (foregroundTaskResumeRearmed) {
6279
+ this.scheduleForegroundTaskResume();
6280
+ }
5591
6281
  })();
5592
6282
  },
5593
6283
  });
@@ -5623,6 +6313,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5623
6313
  this.clearActiveAssistantTurn(token);
5624
6314
  return true;
5625
6315
  }
6316
+ if (!leg.frontDoor) {
6317
+ this.claimSuspendedForegroundTask(current);
6318
+ }
5626
6319
  // The front-door leg may have handed off before its handle resolved;
5627
6320
  // abort it rather than exposing it as the turn's live handle.
5628
6321
  if (coordinator?.handedOff === true) {
@@ -5637,8 +6330,17 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5637
6330
  return false;
5638
6331
  }
5639
6332
 
6333
+ const foregroundTaskResumeRearmed =
6334
+ !leg.frontDoor &&
6335
+ leg.directEscalated !== true &&
6336
+ !activeTurn.discardRequested &&
6337
+ !activeTurn.abortController.signal.aborted &&
6338
+ this.rearmForegroundTaskAfterLegFailure(activeTurn);
5640
6339
  this.clearFillerTimers(activeTurn);
5641
6340
  this.clearActiveAssistantTurn(token);
6341
+ if (foregroundTaskResumeRearmed) {
6342
+ this.scheduleForegroundTaskResume();
6343
+ }
5642
6344
  await this.sendFrame({
5643
6345
  type: "error",
5644
6346
  code: LiveVoiceProtocolErrorCode.InvalidField,
@@ -5897,6 +6599,16 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5897
6599
  if (sessionControl.action === "end") {
5898
6600
  currentTurn.minimizeRequested = false;
5899
6601
  }
6602
+ // Armed before the send, so a client quick enough to answer with
6603
+ // its frame before this await resolves still finds the look waiting.
6604
+ // Never from the turn that answers a look: that turn is the answer,
6605
+ // and a marker it emits anyway must not chain another.
6606
+ if (
6607
+ isLookSessionControl(sessionControl.action) &&
6608
+ currentTurn.lookFollowUp === null
6609
+ ) {
6610
+ this.awaitLookFrame(sessionControl.action);
6611
+ }
5900
6612
  await this.sendFrame(
5901
6613
  {
5902
6614
  type: "session_control",
@@ -6436,6 +7148,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6436
7148
 
6437
7149
  turn.finalized = true;
6438
7150
  this.clearFillerTimers(turn);
7151
+ if (
7152
+ this.foregroundTaskState?.phase === "owned" &&
7153
+ this.foregroundTaskState.ownerToken === turn.token
7154
+ ) {
7155
+ this.clearForegroundTask(
7156
+ status === "completed" ? "owner_completed" : `owner_${reason}`,
7157
+ );
7158
+ }
6439
7159
  turn.utterance.completed = true;
6440
7160
  await this.archiveBufferedAudio({
6441
7161
  turnId: turn.turnId,