@vellumai/assistant 0.12.2-staging.5 → 0.12.2-staging.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/Dockerfile +7 -7
  2. package/openapi.yaml +151 -53
  3. package/package.json +5 -4
  4. package/scripts/bundled-plugin-packages.ts +154 -0
  5. package/scripts/generate-bundled-plugin-packages.ts +29 -0
  6. package/scripts/test.ts +15 -13
  7. package/src/__tests__/managed-store.test.ts +121 -0
  8. package/src/__tests__/scaffold-managed-skill-tool.test.ts +88 -0
  9. package/src/calls/__tests__/voice-control-protocol.test.ts +24 -2
  10. package/src/calls/__tests__/voice-session-bridge.test.ts +22 -2
  11. package/src/calls/voice-control-protocol.ts +13 -4
  12. package/src/calls/voice-session-bridge.ts +27 -6
  13. package/src/cli/commands/__tests__/plugins.test.ts +66 -0
  14. package/src/cli/commands/plugins.ts +50 -18
  15. package/src/cli/lib/__tests__/install-from-github.test.ts +67 -0
  16. package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +169 -0
  17. package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +57 -0
  18. package/src/cli/lib/__tests__/plugin-catalog-platform.test.ts +14 -0
  19. package/src/cli/lib/__tests__/plugin-catalog-resolve.test.ts +27 -1
  20. package/src/cli/lib/__tests__/plugin-details.test.ts +9 -2
  21. package/src/cli/lib/__tests__/plugin-marketplace.test.ts +20 -0
  22. package/src/cli/lib/__tests__/plugins-install-offline.test.ts +43 -0
  23. package/src/cli/lib/__tests__/search-plugins.test.ts +31 -0
  24. package/src/cli/lib/bundled-plugin-packages.json +4 -0
  25. package/src/cli/lib/bundled-plugin-packages.ts +87 -0
  26. package/src/cli/lib/diff-plugin.ts +1 -1
  27. package/src/cli/lib/inspect-plugin.ts +80 -12
  28. package/src/cli/lib/install-from-github.ts +133 -76
  29. package/src/cli/lib/plugin-catalog-cache.ts +22 -3
  30. package/src/cli/lib/plugin-catalog-local.ts +13 -3
  31. package/src/cli/lib/plugin-catalog-platform.ts +6 -1
  32. package/src/cli/lib/plugin-catalog-resolve.ts +20 -0
  33. package/src/cli/lib/plugin-details.ts +12 -0
  34. package/src/cli/lib/plugin-marketplace.ts +121 -21
  35. package/src/cli/lib/plugin-pin-history.ts +5 -2
  36. package/src/cli/lib/search-plugins.ts +58 -16
  37. package/src/cli/lib/upgrade-plugin.ts +48 -3
  38. package/src/config/bundled-skills/skill-management/SKILL.md +1 -1
  39. package/src/config/bundled-skills/skill-management/TOOLS.json +6 -6
  40. package/src/daemon/conversation-tool-setup.ts +4 -22
  41. package/src/live-voice/__tests__/live-voice-vad.test.ts +624 -3
  42. package/src/live-voice/__tests__/session-controls.test.ts +18 -0
  43. package/src/live-voice/live-voice-session.ts +575 -53
  44. package/src/live-voice/session-controls.ts +7 -3
  45. package/src/monitoring/plugin-auto-update.ts +6 -0
  46. package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +5 -0
  47. package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +1 -1
  48. package/src/runtime/routes/__tests__/plugins-routes.test.ts +102 -0
  49. package/src/runtime/routes/plugins-routes.ts +69 -49
  50. package/src/skills/managed-store.ts +98 -32
  51. package/src/tools/skills/find-similar-skills.test.ts +3 -0
  52. package/src/tools/skills/resolve-execute-invocation.ts +24 -0
  53. package/src/tools/skills/scaffold-managed.ts +16 -13
@@ -23,7 +23,10 @@ import {
23
23
  SPOKEN_REPLY_PLAIN_TEXT_RULE,
24
24
  } from "../calls/spoken-reply-rules.js";
25
25
  import { sanitizeForTts } from "../calls/tts-text-sanitizer.js";
26
- import { createControlMarkerHoldback } from "../calls/voice-control-protocol.js";
26
+ import {
27
+ createControlMarkerHoldback,
28
+ TASK_STOP_MARKER,
29
+ } from "../calls/voice-control-protocol.js";
27
30
  import {
28
31
  createFrontDoorLegCoordinator,
29
32
  type FrontDoorLegCoordinator,
@@ -97,7 +100,8 @@ import {
97
100
  liveVoiceSilenceReason,
98
101
  liveVoiceStartScreen,
99
102
  } from "../telemetry/live-voice-funnel.js";
100
- import { getToolOwner } from "../tools/registry.js";
103
+ import { getTool, getToolOwner } from "../tools/registry.js";
104
+ import { resolveSkillExecuteInvocation } from "../tools/skills/resolve-execute-invocation.js";
101
105
  import {
102
106
  createReasoningTagFilter,
103
107
  type ReasoningTagFilter,
@@ -270,6 +274,9 @@ const CONTINUATION_ANNOUNCE_SILENCE_MS = 1_500;
270
274
  // reply's audio landing between two checks. Beyond the cap the announcement
271
275
  // falls back to the stash rather than chasing a call that keeps talking.
272
276
  const CONTINUATION_ANNOUNCE_MAX_DRAIN_REARMS = 3;
277
+ const FOREGROUND_TASK_RESUME_SILENCE_MS = 1_500;
278
+ const FOREGROUND_TASK_MAX_SUSPENDED_MS = 120_000;
279
+ const FOREGROUND_TASK_MAX_INTERVENING_TURNS = 3;
273
280
  // `content` of an announcement turn. The answer never rides here — it goes in
274
281
  // the model-facing control prompt (buildLiveDeliveryNote), so the only thing
275
282
  // persisted on the user side is this marker, and it persists hidden.
@@ -462,6 +469,12 @@ export interface LiveVoiceSessionOptions {
462
469
  * `CONTINUATION_ANNOUNCE_SILENCE_MS`.
463
470
  */
464
471
  continuationAnnounceSilenceMs?: number;
472
+ /** Overrides the silence before a suspended foreground task resumes (test hook). */
473
+ foregroundTaskResumeSilenceMs?: number;
474
+ /** Overrides the maximum time a foreground task may remain suspended (test hook). */
475
+ foregroundTaskMaxSuspendedMs?: number;
476
+ /** Overrides the committed user-turn limit while suspended (test hook). */
477
+ foregroundTaskMaxInterveningTurns?: number;
465
478
  }
466
479
 
467
480
  type LiveVoiceUtterancePhase =
@@ -722,6 +735,13 @@ interface ActiveAssistantTurn {
722
735
  // greeting that opens a session, say). The row still persists and the model
723
736
  // still sees it; `hiddenSyntheticPrompt` keeps it out of the transcript.
724
737
  hiddenPrompt: boolean;
738
+ // The suspended foreground task this turn may continue. Set only after the user
739
+ // turn commits or on the synthetic resume turn; speculative discards never
740
+ // mutate the session's task state.
741
+ foregroundTaskEpoch: number | null;
742
+ // Snapshot retained while a tool-capable leg owns suspended work. If that
743
+ // leg fails asynchronously, the task returns to the same suspension budget.
744
+ foregroundTaskSuspensionSnapshot: SuspendedForegroundTask | null;
725
745
  // Set when a barge-in handed the interrupted work to a background subagent:
726
746
  // that request's transcript, so the model can tell the user the work is
727
747
  // still running instead of appearing to have dropped it.
@@ -826,7 +846,7 @@ const LIVE_VOICE_SETUP_FLOW_TEACHING =
826
846
  // off answering, rather than a fresh follow-up. Reaches the model only; it is
827
847
  // not a user message and never renders as a transcript bubble.
828
848
  function buildInterruptionMergeNote(interruptedRequest: string): string {
829
- return `The user interrupted your previous, unfinished reply. Their earlier request was: "${interruptedRequest}". Treat their current message as a continuation of that request and address both together, or stay silent if they only want you to stop.`;
849
+ return `The user interrupted your previous, unfinished reply. Their earlier request was: "${interruptedRequest}". Treat their current message as a continuation of that request and address both together. If they corrected or redirected the unfinished work, continue executing it with that correction instead of merely acknowledging it. If they clearly want the unfinished task abandoned, use the task-stop control.`;
830
850
  }
831
851
 
832
852
  // System-level guidance appended to the NEXT turn's control prompt after a
@@ -874,6 +894,26 @@ interface ContinuationDelivery {
874
894
  answer: string;
875
895
  }
876
896
 
897
+ interface OwnedForegroundTask {
898
+ phase: "owned";
899
+ epoch: number;
900
+ request: string;
901
+ ownerToken: symbol;
902
+ hostToolStarted: boolean;
903
+ }
904
+
905
+ interface SuspendedForegroundTask {
906
+ phase: "suspended";
907
+ epoch: number;
908
+ request: string;
909
+ suspendedAtMs: number;
910
+ interveningTurns: number;
911
+ resumePending: boolean;
912
+ hostToolStarted: boolean;
913
+ }
914
+
915
+ type ForegroundTaskState = OwnedForegroundTask | SuspendedForegroundTask;
916
+
877
917
  // Appended to an announcement turn's control prompt: the turn the session
878
918
  // starts on its own when a background continuation finishes while the call is
879
919
  // live and nobody is speaking. There is no user utterance behind it, so this
@@ -885,6 +925,12 @@ function buildLiveDeliveryNote(request: string, answer: string): string {
885
925
  return `The user interrupted you earlier, and in the background you finished ${what}. The call is still live and nobody is speaking, so tell them briefly that it is done and give them the result. Do not re-run any tool calls; the work is already complete, and do not repeat it verbatim if it no longer fits. What you produced was:\n\n${answer}`;
886
926
  }
887
927
 
928
+ function buildForegroundTaskResumePrompt(request: string): string {
929
+ const task =
930
+ request.length > 0 ? JSON.stringify(request) : "the unfinished task";
931
+ return `Continue the unfinished task that began with ${task}. Follow the user's latest corrections and all newer conversation context. If the conversation has clearly moved on or the user no longer wants the task, do not continue it and output only ${TASK_STOP_MARKER}, with no spoken words before the marker.`;
932
+ }
933
+
888
934
  // Assemble a leg's model-facing control prompt: the base live-voice rules, the
889
935
  // screen-reveal and setup-flow teaching (both withheld from the front-door leg,
890
936
  // see LIVE_VOICE_SCREEN_REVEAL_TEACHING), plus any pending barge-in merge
@@ -902,7 +948,13 @@ function buildVoiceControlPrompt(
902
948
  (leg.frontDoor === true
903
949
  ? ""
904
950
  : LIVE_VOICE_SCREEN_REVEAL_TEACHING + LIVE_VOICE_SETUP_FLOW_TEACHING) +
905
- sessionControlTeaching(sessionControls, leg, client);
951
+ sessionControlTeaching(sessionControls, leg, {
952
+ ...client,
953
+ unfinishedTaskPending:
954
+ turn.foregroundTaskEpoch !== null ||
955
+ turn.interruptedRequest !== null ||
956
+ turn.handedOffRequest !== null,
957
+ });
906
958
  if (turn.language !== undefined) {
907
959
  prompt = `${prompt}\n\nThe caller has been speaking the language with code "${turn.language}" this turn. Reply in that language unless they clearly switch to another.`;
908
960
  }
@@ -1070,6 +1122,20 @@ function foregroundToolContendsWithContinuation(
1070
1122
  return isRefusedInReadOnlyPass(toolName, ownerKind);
1071
1123
  }
1072
1124
 
1125
+ function hostInteractiveToolName(
1126
+ toolName: string,
1127
+ input?: Record<string, unknown>,
1128
+ allowedToolNames?: ReadonlySet<string>,
1129
+ ): string | null {
1130
+ const effectiveToolName =
1131
+ toolName === "skill_execute"
1132
+ ? resolveSkillExecuteInvocation(input ?? {}, allowedToolNames).name
1133
+ : toolName;
1134
+ return getTool(effectiveToolName)?.executionTarget === "host"
1135
+ ? effectiveToolName
1136
+ : null;
1137
+ }
1138
+
1073
1139
  // Upper bound on how long a barge-in waits for the interrupted turn's teardown
1074
1140
  // to settle before giving up on the continuation. The teardown settles once the
1075
1141
  // aborted turn's agent loop reaches its `finally`, which can wait out BOTH the
@@ -1259,6 +1325,16 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1259
1325
  // answer.
1260
1326
  private pendingAnnouncement: ContinuationDelivery | null = null;
1261
1327
  private announcementTimer: ReturnType<typeof setTimeout> | null = null;
1328
+ // Host-backed work stays on the parent conversation across barge-ins. An
1329
+ // owned task belongs to the tool-capable turn doing the work; a suspended
1330
+ // task is waiting for either a merged escalation or a synthetic resume.
1331
+ private foregroundTaskState: ForegroundTaskState | null = null;
1332
+ private foregroundTaskEpoch = 0;
1333
+ private foregroundTaskResumeTimer: ReturnType<typeof setTimeout> | null =
1334
+ null;
1335
+ private readonly foregroundTaskResumeSilenceMs: number;
1336
+ private readonly foregroundTaskMaxSuspendedMs: number;
1337
+ private readonly foregroundTaskMaxInterveningTurns: number;
1262
1338
  // A look control sent to a client that declared `lookFrames`, waiting for
1263
1339
  // the fresh frame the client takes for it. The session answers the look on a
1264
1340
  // turn of its own once that frame is in the conversation (see
@@ -1405,6 +1481,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1405
1481
  defaultDetachTeardownSettleTimeoutMs();
1406
1482
  this.continuationAnnounceSilenceMs =
1407
1483
  options.continuationAnnounceSilenceMs ?? CONTINUATION_ANNOUNCE_SILENCE_MS;
1484
+ this.foregroundTaskResumeSilenceMs =
1485
+ options.foregroundTaskResumeSilenceMs ??
1486
+ FOREGROUND_TASK_RESUME_SILENCE_MS;
1487
+ this.foregroundTaskMaxSuspendedMs =
1488
+ options.foregroundTaskMaxSuspendedMs ?? FOREGROUND_TASK_MAX_SUSPENDED_MS;
1489
+ this.foregroundTaskMaxInterveningTurns =
1490
+ options.foregroundTaskMaxInterveningTurns ??
1491
+ FOREGROUND_TASK_MAX_INTERVENING_TURNS;
1408
1492
  this.emitMetrics = options.emitMetrics ?? false;
1409
1493
  this.createTurnId = options.createTurnId ?? randomUUID;
1410
1494
  this.conversationId =
@@ -1901,6 +1985,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1901
1985
 
1902
1986
  const shouldEmitSessionEndMetrics = this.state !== "failed";
1903
1987
  this.state = "closed";
1988
+ this.clearForegroundTask("session_closed");
1904
1989
  // Retire the island before the teardown below starts awaiting things. A
1905
1990
  // close can take a while (a pending continuation is delivered first), and
1906
1991
  // an activity left asserting "Speaking…" through it is exactly the stale
@@ -2839,6 +2924,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2839
2924
  // do not collide with it in the collector. turn_cancelled flushes
2840
2925
  // client playback, so the drain estimate resets with it.
2841
2926
  this.assistantPlaybackTailUntilMs = 0;
2927
+ const keepForegroundTaskOnParent =
2928
+ !turn.assistantCompleted && this.suspendOwnedForegroundTask(turn);
2842
2929
  // Carry the interrupted request into the next turn so it merges with the
2843
2930
  // barge-in utterance rather than being answered as a fresh follow-up.
2844
2931
  const interruptedRequest = turn.utterance.finalTranscriptSegments
@@ -2919,6 +3006,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2919
3006
  this.publishActivity(turn, "");
2920
3007
  await this.sendFrame({ type: "turn_cancelled", turnId: turn.turnId });
2921
3008
  await this.cancelAssistantTurn("barge_in");
3009
+ if (keepForegroundTaskOnParent) {
3010
+ log.info(
3011
+ { turnId: turn.turnId },
3012
+ "Voice duplex continuation skipped for parent-owned foreground task",
3013
+ );
3014
+ return;
3015
+ }
2922
3016
  // Keep the interrupted turn's work alive on a background subagent; the
2923
3017
  // detach waits for its teardown to settle the partial into history before
2924
3018
  // forking.
@@ -3196,6 +3290,350 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3196
3290
  }
3197
3291
  }
3198
3292
 
3293
+ private foregroundTaskRequestForTurn(turn: ActiveAssistantTurn): string {
3294
+ const currentRequest = turn.utterance.finalTranscriptSegments
3295
+ .join(" ")
3296
+ .trim();
3297
+ return currentRequest || turn.interruptedRequest || "";
3298
+ }
3299
+
3300
+ private createForegroundTaskOwnership(
3301
+ turn: ActiveAssistantTurn,
3302
+ hostToolStarted: boolean,
3303
+ ): OwnedForegroundTask {
3304
+ const task: OwnedForegroundTask = {
3305
+ phase: "owned",
3306
+ epoch: ++this.foregroundTaskEpoch,
3307
+ request: this.foregroundTaskRequestForTurn(turn),
3308
+ ownerToken: turn.token,
3309
+ hostToolStarted,
3310
+ };
3311
+ turn.foregroundTaskEpoch = task.epoch;
3312
+ this.foregroundTaskState = task;
3313
+ return task;
3314
+ }
3315
+
3316
+ private noteEscalatedForegroundTask(turn: ActiveAssistantTurn): void {
3317
+ if (
3318
+ this.activeAssistantTurn?.token !== turn.token ||
3319
+ turn.discardRequested ||
3320
+ turn.abortController.signal.aborted
3321
+ ) {
3322
+ return;
3323
+ }
3324
+ if (
3325
+ this.claimSuspendedForegroundTask(turn) ||
3326
+ this.foregroundTaskState !== null
3327
+ ) {
3328
+ return;
3329
+ }
3330
+
3331
+ const task = this.createForegroundTaskOwnership(turn, false);
3332
+ log.info(
3333
+ { turnId: turn.turnId, epoch: task.epoch },
3334
+ "Escalated live voice turn provisionally owns a foreground task",
3335
+ );
3336
+ }
3337
+
3338
+ private noteHostToolStarted(
3339
+ turn: ActiveAssistantTurn,
3340
+ effectiveToolName: string,
3341
+ ): void {
3342
+ const current = this.foregroundTaskState;
3343
+ if (current?.phase === "owned" && current.ownerToken === turn.token) {
3344
+ if (!current.hostToolStarted) {
3345
+ this.foregroundTaskState = { ...current, hostToolStarted: true };
3346
+ log.info(
3347
+ { turnId: turn.turnId, epoch: current.epoch, effectiveToolName },
3348
+ "Foreground task confirmed by host tool use",
3349
+ );
3350
+ }
3351
+ return;
3352
+ }
3353
+ if (this.claimSuspendedForegroundTask(turn, true)) {
3354
+ return;
3355
+ }
3356
+
3357
+ const task = this.createForegroundTaskOwnership(turn, true);
3358
+ log.info(
3359
+ { turnId: turn.turnId, epoch: task.epoch, effectiveToolName },
3360
+ "Live voice turn started a foreground task",
3361
+ );
3362
+ }
3363
+
3364
+ private claimSuspendedForegroundTask(
3365
+ turn: ActiveAssistantTurn,
3366
+ hostToolStarted = false,
3367
+ ): boolean {
3368
+ const current = this.foregroundTaskState;
3369
+ if (
3370
+ current?.phase !== "suspended" ||
3371
+ turn.foregroundTaskEpoch !== current.epoch ||
3372
+ this.activeAssistantTurn?.token !== turn.token ||
3373
+ turn.discardRequested ||
3374
+ turn.abortController.signal.aborted
3375
+ ) {
3376
+ return false;
3377
+ }
3378
+ this.foregroundTaskState = {
3379
+ phase: "owned",
3380
+ epoch: current.epoch,
3381
+ request: current.request,
3382
+ ownerToken: turn.token,
3383
+ hostToolStarted: current.hostToolStarted || hostToolStarted,
3384
+ };
3385
+ this.clearForegroundTaskResumeTimer();
3386
+ log.info(
3387
+ {
3388
+ turnId: turn.turnId,
3389
+ epoch: current.epoch,
3390
+ hostToolStarted: current.hostToolStarted || hostToolStarted,
3391
+ },
3392
+ "Suspended foreground task claimed by tool-capable turn",
3393
+ );
3394
+ return true;
3395
+ }
3396
+
3397
+ private suspendOwnedForegroundTask(turn: ActiveAssistantTurn): boolean {
3398
+ const current = this.foregroundTaskState;
3399
+ if (current?.phase === "owned" && current.ownerToken === turn.token) {
3400
+ this.foregroundTaskState = {
3401
+ phase: "suspended",
3402
+ epoch: current.epoch,
3403
+ request: current.request,
3404
+ suspendedAtMs: Date.now(),
3405
+ interveningTurns: 0,
3406
+ resumePending: false,
3407
+ hostToolStarted: current.hostToolStarted,
3408
+ };
3409
+ this.clearForegroundTaskResumeTimer();
3410
+ log.info(
3411
+ { turnId: turn.turnId, epoch: current.epoch },
3412
+ "Foreground task suspended by voice barge-in",
3413
+ );
3414
+ return true;
3415
+ }
3416
+ return current !== null;
3417
+ }
3418
+
3419
+ private clearForegroundTask(reason: string): void {
3420
+ const current = this.foregroundTaskState;
3421
+ this.foregroundTaskEpoch += 1;
3422
+ this.foregroundTaskState = null;
3423
+ this.clearForegroundTaskResumeTimer();
3424
+ if (current !== null) {
3425
+ log.info(
3426
+ {
3427
+ epoch: current.epoch,
3428
+ phase: current.phase,
3429
+ hostToolStarted: current.hostToolStarted,
3430
+ reason,
3431
+ },
3432
+ "Live voice foreground task cleared",
3433
+ );
3434
+ }
3435
+ }
3436
+
3437
+ private stopOutstandingInterruptedWork(reason: string): void {
3438
+ this.clearForegroundTask(reason);
3439
+ this.abortDetachedRuns({ reason });
3440
+ }
3441
+
3442
+ private foregroundTaskSuspensionExpired(
3443
+ task: SuspendedForegroundTask,
3444
+ ): "turn_limit" | "time_limit" | null {
3445
+ if (task.interveningTurns > this.foregroundTaskMaxInterveningTurns) {
3446
+ return "turn_limit";
3447
+ }
3448
+ if (Date.now() - task.suspendedAtMs >= this.foregroundTaskMaxSuspendedMs) {
3449
+ return "time_limit";
3450
+ }
3451
+ return null;
3452
+ }
3453
+
3454
+ private queueForegroundTaskResume(turn: ActiveAssistantTurn): void {
3455
+ const current = this.foregroundTaskState;
3456
+ if (
3457
+ current?.phase !== "suspended" ||
3458
+ turn.foregroundTaskEpoch !== current.epoch ||
3459
+ turn.hiddenPrompt ||
3460
+ turn.discardRequested ||
3461
+ turn.speculativePending
3462
+ ) {
3463
+ return;
3464
+ }
3465
+ const next: SuspendedForegroundTask = {
3466
+ ...current,
3467
+ interveningTurns: current.interveningTurns + 1,
3468
+ resumePending: true,
3469
+ };
3470
+ const expired = this.foregroundTaskSuspensionExpired(next);
3471
+ if (expired !== null) {
3472
+ this.clearForegroundTask(`suspended_${expired}`);
3473
+ return;
3474
+ }
3475
+ this.foregroundTaskState = next;
3476
+ if (this.activeAssistantTurn === null) {
3477
+ this.scheduleForegroundTaskResume();
3478
+ }
3479
+ }
3480
+
3481
+ private markForegroundTaskResumePending(epoch: number): boolean {
3482
+ const current = this.foregroundTaskState;
3483
+ if (current?.phase !== "suspended" || current.epoch !== epoch) {
3484
+ return false;
3485
+ }
3486
+ this.foregroundTaskState = { ...current, resumePending: true };
3487
+ return true;
3488
+ }
3489
+
3490
+ private rearmForegroundTaskAfterLegFailure(
3491
+ turn: ActiveAssistantTurn,
3492
+ ): boolean {
3493
+ const epoch = turn.foregroundTaskEpoch;
3494
+ if (epoch === null) {
3495
+ return false;
3496
+ }
3497
+ const current = this.foregroundTaskState;
3498
+ if (current?.phase === "suspended" && current.epoch === epoch) {
3499
+ return this.markForegroundTaskResumePending(epoch);
3500
+ }
3501
+ const snapshot = turn.foregroundTaskSuspensionSnapshot;
3502
+ if (
3503
+ current?.phase !== "owned" ||
3504
+ current.epoch !== epoch ||
3505
+ current.ownerToken !== turn.token ||
3506
+ (snapshot !== null && snapshot.epoch !== epoch)
3507
+ ) {
3508
+ return false;
3509
+ }
3510
+ const next: SuspendedForegroundTask = {
3511
+ ...(snapshot ?? {
3512
+ phase: "suspended",
3513
+ epoch,
3514
+ request: current.request,
3515
+ suspendedAtMs: Date.now(),
3516
+ interveningTurns: 0,
3517
+ hostToolStarted: current.hostToolStarted,
3518
+ }),
3519
+ resumePending: true,
3520
+ };
3521
+ const expired = this.foregroundTaskSuspensionExpired(next);
3522
+ if (expired !== null) {
3523
+ this.clearForegroundTask(`suspended_${expired}`);
3524
+ return false;
3525
+ }
3526
+ this.foregroundTaskState = next;
3527
+ log.info(
3528
+ { turnId: turn.turnId, epoch },
3529
+ "Foreground task re-armed after tool-capable leg failure",
3530
+ );
3531
+ return true;
3532
+ }
3533
+
3534
+ private clearForegroundTaskResumeTimer(): void {
3535
+ if (this.foregroundTaskResumeTimer !== null) {
3536
+ clearTimeout(this.foregroundTaskResumeTimer);
3537
+ this.foregroundTaskResumeTimer = null;
3538
+ }
3539
+ }
3540
+
3541
+ private scheduleFloorCheck(
3542
+ currentTimer: ReturnType<typeof setTimeout> | null,
3543
+ silenceMs: number,
3544
+ check: (blockedBy: string | null) => void,
3545
+ ): ReturnType<typeof setTimeout> {
3546
+ if (currentTimer !== null) {
3547
+ clearTimeout(currentTimer);
3548
+ }
3549
+ const drainMs = Math.max(0, this.assistantPlaybackTailUntilMs - Date.now());
3550
+ return setTimeout(
3551
+ () => check(this.sessionTurnFloorBlocker()),
3552
+ drainMs + silenceMs,
3553
+ );
3554
+ }
3555
+
3556
+ private scheduleForegroundTaskResume(): void {
3557
+ const task = this.foregroundTaskState;
3558
+ if (
3559
+ task?.phase !== "suspended" ||
3560
+ !task.resumePending ||
3561
+ this.isClosed ||
3562
+ this.state === "failed"
3563
+ ) {
3564
+ this.clearForegroundTaskResumeTimer();
3565
+ return;
3566
+ }
3567
+ const expired = this.foregroundTaskSuspensionExpired(task);
3568
+ if (expired !== null) {
3569
+ this.clearForegroundTask(`suspended_${expired}`);
3570
+ return;
3571
+ }
3572
+ const epoch = task.epoch;
3573
+ this.foregroundTaskResumeTimer = this.scheduleFloorCheck(
3574
+ this.foregroundTaskResumeTimer,
3575
+ this.foregroundTaskResumeSilenceMs,
3576
+ (blockedBy) => {
3577
+ this.foregroundTaskResumeTimer = null;
3578
+ const current = this.foregroundTaskState;
3579
+ if (
3580
+ current?.phase !== "suspended" ||
3581
+ current.epoch !== epoch ||
3582
+ !current.resumePending
3583
+ ) {
3584
+ return;
3585
+ }
3586
+ const currentExpired = this.foregroundTaskSuspensionExpired(current);
3587
+ if (currentExpired !== null) {
3588
+ this.clearForegroundTask(`suspended_${currentExpired}`);
3589
+ return;
3590
+ }
3591
+ if (blockedBy === null) {
3592
+ void this.resumeForegroundTask(current).catch((err: unknown) => {
3593
+ log.warn({ err, epoch }, "Foreground task resume failed");
3594
+ });
3595
+ return;
3596
+ }
3597
+ if (blockedBy === "session_unavailable") {
3598
+ this.clearForegroundTask("session_unavailable");
3599
+ return;
3600
+ }
3601
+ if (blockedBy !== "turn_active") {
3602
+ this.scheduleForegroundTaskResume();
3603
+ }
3604
+ },
3605
+ );
3606
+ }
3607
+
3608
+ private async resumeForegroundTask(
3609
+ task: SuspendedForegroundTask,
3610
+ ): Promise<void> {
3611
+ const current = this.foregroundTaskState;
3612
+ if (
3613
+ current?.phase !== "suspended" ||
3614
+ current.epoch !== task.epoch ||
3615
+ !current.resumePending
3616
+ ) {
3617
+ return;
3618
+ }
3619
+ this.foregroundTaskState = { ...current, resumePending: false };
3620
+ const started = await this.launchAssistantTurn(
3621
+ createSyntheticUtterance(),
3622
+ buildForegroundTaskResumePrompt(current.request),
3623
+ {
3624
+ hiddenPrompt: true,
3625
+ initialLeg: "escalated",
3626
+ foregroundTaskEpoch: current.epoch,
3627
+ },
3628
+ );
3629
+ if (started) {
3630
+ return;
3631
+ }
3632
+ if (this.markForegroundTaskResumePending(current.epoch)) {
3633
+ this.scheduleForegroundTaskResume();
3634
+ }
3635
+ }
3636
+
3199
3637
  // Abort every background continuation this session started and drop its
3200
3638
  // handle. Called on a hard stop (client interrupt / session close), when a
3201
3639
  // newer barge-in supersedes the detached runs, and on a foreground-wins
@@ -3322,50 +3760,53 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3322
3760
  retry = false,
3323
3761
  drainRearms = 0,
3324
3762
  ): void {
3325
- if (this.announcementTimer) {
3326
- clearTimeout(this.announcementTimer);
3327
- }
3328
- // Queued client-side audio is the previous reply still being audible, so
3329
- // the silence window only begins once it has drained.
3330
- const drainMs = Math.max(0, this.assistantPlaybackTailUntilMs - Date.now());
3331
- this.announcementTimer = setTimeout(() => {
3332
- this.announcementTimer = null;
3333
- const blockedBy = this.continuationAnnouncementBlocker();
3334
- if (blockedBy === null) {
3335
- void this.announceContinuation().catch((err: unknown) => {
3336
- log.warn({ err }, "Voice duplex continuation announcement failed");
3337
- });
3338
- return;
3339
- }
3340
- log.debug(
3341
- { conversationId: this.conversationId, blockedBy, retry, drainRearms },
3342
- "Voice duplex continuation announcement deferred",
3343
- );
3344
- // A tail that grew while the timer ran (the reply was still streaming
3345
- // TTS when it was armed) is a wait of known length rather than a busy
3346
- // call, so it re-arms against the new deadline without spending the
3347
- // single retry the busy-call blockers get — otherwise a long reply's
3348
- // playback alone outlasts the budget and the announcement is dropped
3349
- // even though the call goes idle right after. The tail only extends
3350
- // while a turn is active, which blocks earlier, so the re-arms converge;
3351
- // the cap bounds the interleaving where a fresh turn's audio lands
3352
- // between two checks.
3353
- if (
3354
- blockedBy === "playback_draining" &&
3355
- drainRearms < CONTINUATION_ANNOUNCE_MAX_DRAIN_REARMS
3356
- ) {
3357
- this.scheduleContinuationAnnouncement(retry, drainRearms + 1);
3358
- return;
3359
- }
3360
- if (blockedBy === "turn_active") {
3361
- return;
3362
- }
3363
- if (retry || this.pendingAnnouncement === null) {
3364
- this.pendingAnnouncement = null;
3365
- return;
3366
- }
3367
- this.scheduleContinuationAnnouncement(true, drainRearms);
3368
- }, drainMs + this.continuationAnnounceSilenceMs);
3763
+ this.announcementTimer = this.scheduleFloorCheck(
3764
+ this.announcementTimer,
3765
+ this.continuationAnnounceSilenceMs,
3766
+ () => {
3767
+ this.announcementTimer = null;
3768
+ const blockedBy = this.continuationAnnouncementBlocker();
3769
+ if (blockedBy === null) {
3770
+ void this.announceContinuation().catch((err: unknown) => {
3771
+ log.warn({ err }, "Voice duplex continuation announcement failed");
3772
+ });
3773
+ return;
3774
+ }
3775
+ log.debug(
3776
+ {
3777
+ conversationId: this.conversationId,
3778
+ blockedBy,
3779
+ retry,
3780
+ drainRearms,
3781
+ },
3782
+ "Voice duplex continuation announcement deferred",
3783
+ );
3784
+ // A tail that grew while the timer ran (the reply was still streaming
3785
+ // TTS when it was armed) is a wait of known length rather than a busy
3786
+ // call, so it re-arms against the new deadline without spending the
3787
+ // single retry the busy-call blockers get. Otherwise a long reply's
3788
+ // playback alone outlasts the budget and the announcement is dropped
3789
+ // even though the call goes idle right after. The tail only extends
3790
+ // while a turn is active, which blocks earlier, so the re-arms converge;
3791
+ // the cap bounds the interleaving where a fresh turn's audio lands
3792
+ // between two checks.
3793
+ if (
3794
+ blockedBy === "playback_draining" &&
3795
+ drainRearms < CONTINUATION_ANNOUNCE_MAX_DRAIN_REARMS
3796
+ ) {
3797
+ this.scheduleContinuationAnnouncement(retry, drainRearms + 1);
3798
+ return;
3799
+ }
3800
+ if (blockedBy === "turn_active") {
3801
+ return;
3802
+ }
3803
+ if (retry || this.pendingAnnouncement === null) {
3804
+ this.pendingAnnouncement = null;
3805
+ return;
3806
+ }
3807
+ this.scheduleContinuationAnnouncement(true, drainRearms);
3808
+ },
3809
+ );
3369
3810
  }
3370
3811
 
3371
3812
  /**
@@ -3555,6 +3996,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3555
3996
  ) {
3556
3997
  this.scheduleContinuationAnnouncement();
3557
3998
  }
3999
+ if (
4000
+ this.foregroundTaskState?.phase === "suspended" &&
4001
+ this.foregroundTaskState.resumePending &&
4002
+ !this.isClosed &&
4003
+ this.state !== "failed"
4004
+ ) {
4005
+ this.scheduleForegroundTaskResume();
4006
+ }
3558
4007
  }
3559
4008
 
3560
4009
  // Why the session must not speak up right now, or null when the call is
@@ -4947,6 +5396,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4947
5396
  // the next turn is now stale (the interrupted utterance may be discarded
4948
5397
  // without ever reaching finalizePendingUtterance).
4949
5398
  this.pendingInterruptedRequest = null;
5399
+ this.clearForegroundTask("client_interrupt");
4950
5400
  // ...and it hard-stops any detached background continuations.
4951
5401
  this.abortDetachedRuns({ reason: "client_interrupt" });
4952
5402
  // ...and a look still waiting to be answered: the user stopped the call
@@ -5185,6 +5635,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5185
5635
  // the row persists and drives the turn but never renders: no echo, and
5186
5636
  // `/messages` filters it after a reload.
5187
5637
  hiddenPrompt?: boolean;
5638
+ // Synthetic host-task resumes start on the tool-capable profile without
5639
+ // passing through the front door.
5640
+ initialLeg?: VoiceRoutingLeg;
5641
+ // The suspended foreground task this synthetic turn resumes.
5642
+ foregroundTaskEpoch?: number;
5188
5643
  },
5189
5644
  ): Promise<boolean> {
5190
5645
  utterance.assistantTurnStarted = true;
@@ -5200,6 +5655,15 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5200
5655
  this.startMetricsTurnIfNeeded(utterance, turnId);
5201
5656
  this.markAssistantDispatch(utterance, turnId);
5202
5657
  const abortController = new AbortController();
5658
+ const suspendedForegroundTask =
5659
+ this.foregroundTaskState?.phase === "suspended"
5660
+ ? this.foregroundTaskState
5661
+ : null;
5662
+ const foregroundTaskEpoch =
5663
+ opts?.foregroundTaskEpoch ??
5664
+ (opts?.hiddenPrompt !== true
5665
+ ? (suspendedForegroundTask?.epoch ?? null)
5666
+ : null);
5203
5667
  const activeTurn: ActiveAssistantTurn = {
5204
5668
  token,
5205
5669
  turnId,
@@ -5272,6 +5736,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5272
5736
  continuationDelivery: opts?.continuationDelivery ?? null,
5273
5737
  lookFollowUp: opts?.lookFollowUp ?? null,
5274
5738
  hiddenPrompt: opts?.hiddenPrompt === true,
5739
+ foregroundTaskEpoch,
5740
+ foregroundTaskSuspensionSnapshot:
5741
+ suspendedForegroundTask?.epoch === foregroundTaskEpoch
5742
+ ? { ...suspendedForegroundTask }
5743
+ : null,
5275
5744
  deltaEpoch: 0,
5276
5745
  frontDoor: null,
5277
5746
  ttsBuffer: "",
@@ -5336,13 +5805,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5336
5805
  activeTurn.progress.arm();
5337
5806
  }
5338
5807
 
5339
- // Front-door leg: a fast model fronts every turn (the `voiceFrontDoor`
5340
- // call site pins it) and may hand off to a quality leg on the escalate
5341
- // verdict.
5808
+ const initialLeg = opts?.initialLeg ?? "front-door";
5342
5809
  const started = await this.startAssistantLeg(activeTurn, {
5343
5810
  content,
5344
- routingLeg: "front-door",
5345
- frontDoor: true,
5811
+ routingLeg: initialLeg,
5812
+ ...(initialLeg === "front-door"
5813
+ ? { frontDoor: true }
5814
+ : { directEscalated: true }),
5346
5815
  // Only this leg: an escalated leg persists its own continuation row, and
5347
5816
  // the frame belongs to the row carrying the user's words.
5348
5817
  ...(activeTurn.turnAttachmentId
@@ -5387,6 +5856,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5387
5856
  overrideProfile?: string;
5388
5857
  routingLeg?: VoiceRoutingLeg;
5389
5858
  frontDoor?: boolean;
5859
+ directEscalated?: boolean;
5390
5860
  spokenEscalationBridge?: string;
5391
5861
  attachments?: readonly string[];
5392
5862
  },
@@ -5467,6 +5937,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5467
5937
  speakBridge: (bridge) =>
5468
5938
  this.speakEscalationBridge(activeTurn, bridge),
5469
5939
  startEscalatedLeg: (escalated) => {
5940
+ this.noteEscalatedForegroundTask(activeTurn);
5470
5941
  void this.startAssistantLeg(activeTurn, escalated);
5471
5942
  },
5472
5943
  },
@@ -5555,6 +6026,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5555
6026
  ...(leg.spokenEscalationBridge != null
5556
6027
  ? { spokenEscalationBridge: leg.spokenEscalationBridge }
5557
6028
  : {}),
6029
+ ...(leg.directEscalated === true ? { directEscalated: true } : {}),
5558
6030
  // A speculative front-door leg's decision rule includes the hold
5559
6031
  // branch so its leading tokens can be the hold verdict — but only
5560
6032
  // on the utterance's FIRST dispatch. Extension replays (the
@@ -5571,6 +6043,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5571
6043
  if (!this.isForwardingAssistantText(token)) {
5572
6044
  return;
5573
6045
  }
6046
+ if (!leg.frontDoor) {
6047
+ this.claimSuspendedForegroundTask(activeTurn);
6048
+ }
5574
6049
  if (coordinator !== null) {
5575
6050
  // Verdict-first: the leg's leading tokens decide the turn's
5576
6051
  // fate, and the coordinator acts on them. The bridge hands
@@ -5606,6 +6081,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5606
6081
  ) {
5607
6082
  return;
5608
6083
  }
6084
+ if (!leg.frontDoor) {
6085
+ this.claimSuspendedForegroundTask(current);
6086
+ }
5609
6087
  // A speculative leg that finished without a single delta (empty
5610
6088
  // output, provider hiccup) carries no verdict — fail open to a
5611
6089
  // committed turn so the utterance releases and finalizes like a
@@ -5651,10 +6129,18 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5651
6129
  { turnId, cadence: request.cadence },
5652
6130
  "Live voice progress cadence changed",
5653
6131
  );
6132
+ } else if (request?.action === "task_stop") {
6133
+ this.stopOutstandingInterruptedWork("spoken_task_stop");
5654
6134
  } else {
6135
+ if (request?.action === "end") {
6136
+ this.clearForegroundTask("call_end_requested");
6137
+ }
5655
6138
  current.sessionControlRequested = request;
5656
6139
  }
5657
6140
  }
6141
+ if (leg.frontDoor) {
6142
+ this.queueForegroundTaskResume(current);
6143
+ }
5658
6144
  current.assistantCompleted = true;
5659
6145
  if (msg.type === "generation_cancelled") {
5660
6146
  void this.finalizeAssistantTurn(
@@ -5689,6 +6175,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5689
6175
  if (current?.token !== token) {
5690
6176
  return;
5691
6177
  }
6178
+ const effectiveHostTool = hostInteractiveToolName(
6179
+ toolName,
6180
+ detail?.input,
6181
+ detail?.allowedToolNames,
6182
+ );
6183
+ if (effectiveHostTool !== null) {
6184
+ this.noteHostToolStarted(current, effectiveHostTool);
6185
+ }
5692
6186
  // Foreground wins the workspace: the continuation runs with full
5693
6187
  // subagent abilities (it can write files, run commands), so the
5694
6188
  // moment a live turn starts a consequential tool the two could
@@ -5775,7 +6269,15 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5775
6269
  if (currentTurn?.token !== token) {
5776
6270
  return;
5777
6271
  }
6272
+ const foregroundTaskResumeRearmed =
6273
+ !leg.frontDoor &&
6274
+ !currentTurn.discardRequested &&
6275
+ !currentTurn.abortController.signal.aborted &&
6276
+ this.rearmForegroundTaskAfterLegFailure(currentTurn);
5778
6277
  await this.finalizeAssistantTurn(currentTurn, "cancelled", "error");
6278
+ if (foregroundTaskResumeRearmed) {
6279
+ this.scheduleForegroundTaskResume();
6280
+ }
5779
6281
  })();
5780
6282
  },
5781
6283
  });
@@ -5811,6 +6313,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5811
6313
  this.clearActiveAssistantTurn(token);
5812
6314
  return true;
5813
6315
  }
6316
+ if (!leg.frontDoor) {
6317
+ this.claimSuspendedForegroundTask(current);
6318
+ }
5814
6319
  // The front-door leg may have handed off before its handle resolved;
5815
6320
  // abort it rather than exposing it as the turn's live handle.
5816
6321
  if (coordinator?.handedOff === true) {
@@ -5825,8 +6330,17 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5825
6330
  return false;
5826
6331
  }
5827
6332
 
6333
+ const foregroundTaskResumeRearmed =
6334
+ !leg.frontDoor &&
6335
+ leg.directEscalated !== true &&
6336
+ !activeTurn.discardRequested &&
6337
+ !activeTurn.abortController.signal.aborted &&
6338
+ this.rearmForegroundTaskAfterLegFailure(activeTurn);
5828
6339
  this.clearFillerTimers(activeTurn);
5829
6340
  this.clearActiveAssistantTurn(token);
6341
+ if (foregroundTaskResumeRearmed) {
6342
+ this.scheduleForegroundTaskResume();
6343
+ }
5830
6344
  await this.sendFrame({
5831
6345
  type: "error",
5832
6346
  code: LiveVoiceProtocolErrorCode.InvalidField,
@@ -6634,6 +7148,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6634
7148
 
6635
7149
  turn.finalized = true;
6636
7150
  this.clearFillerTimers(turn);
7151
+ if (
7152
+ this.foregroundTaskState?.phase === "owned" &&
7153
+ this.foregroundTaskState.ownerToken === turn.token
7154
+ ) {
7155
+ this.clearForegroundTask(
7156
+ status === "completed" ? "owner_completed" : `owner_${reason}`,
7157
+ );
7158
+ }
6637
7159
  turn.utterance.completed = true;
6638
7160
  await this.archiveBufferedAudio({
6639
7161
  turnId: turn.turnId,