@vellumai/assistant 0.12.2-staging.4 → 0.12.2-staging.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/Dockerfile +7 -7
  2. package/openapi.yaml +151 -53
  3. package/package.json +5 -4
  4. package/scripts/bundled-plugin-packages.ts +154 -0
  5. package/scripts/generate-bundled-plugin-packages.ts +29 -0
  6. package/scripts/test.ts +15 -13
  7. package/src/__tests__/managed-store.test.ts +121 -0
  8. package/src/__tests__/scaffold-managed-skill-tool.test.ts +88 -0
  9. package/src/calls/__tests__/voice-control-protocol.test.ts +24 -2
  10. package/src/calls/__tests__/voice-session-bridge.test.ts +22 -2
  11. package/src/calls/voice-control-protocol.ts +13 -4
  12. package/src/calls/voice-session-bridge.ts +27 -6
  13. package/src/cli/commands/__tests__/plugins.test.ts +66 -0
  14. package/src/cli/commands/plugins.ts +50 -18
  15. package/src/cli/lib/__tests__/install-from-github.test.ts +67 -0
  16. package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +169 -0
  17. package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +57 -0
  18. package/src/cli/lib/__tests__/plugin-catalog-platform.test.ts +14 -0
  19. package/src/cli/lib/__tests__/plugin-catalog-resolve.test.ts +27 -1
  20. package/src/cli/lib/__tests__/plugin-details.test.ts +9 -2
  21. package/src/cli/lib/__tests__/plugin-marketplace.test.ts +20 -0
  22. package/src/cli/lib/__tests__/plugins-install-offline.test.ts +43 -0
  23. package/src/cli/lib/__tests__/search-plugins.test.ts +31 -0
  24. package/src/cli/lib/bundled-plugin-packages.json +4 -0
  25. package/src/cli/lib/bundled-plugin-packages.ts +87 -0
  26. package/src/cli/lib/diff-plugin.ts +1 -1
  27. package/src/cli/lib/inspect-plugin.ts +80 -12
  28. package/src/cli/lib/install-from-github.ts +133 -76
  29. package/src/cli/lib/plugin-catalog-cache.ts +22 -3
  30. package/src/cli/lib/plugin-catalog-local.ts +13 -3
  31. package/src/cli/lib/plugin-catalog-platform.ts +6 -1
  32. package/src/cli/lib/plugin-catalog-resolve.ts +20 -0
  33. package/src/cli/lib/plugin-details.ts +12 -0
  34. package/src/cli/lib/plugin-marketplace.ts +121 -21
  35. package/src/cli/lib/plugin-pin-history.ts +5 -2
  36. package/src/cli/lib/search-plugins.ts +58 -16
  37. package/src/cli/lib/upgrade-plugin.ts +48 -3
  38. package/src/config/bundled-skills/skill-management/SKILL.md +1 -1
  39. package/src/config/bundled-skills/skill-management/TOOLS.json +7 -7
  40. package/src/daemon/conversation-tool-setup.ts +4 -22
  41. package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +374 -0
  42. package/src/live-voice/__tests__/live-voice-vad.test.ts +658 -3
  43. package/src/live-voice/__tests__/protocol.test.ts +48 -0
  44. package/src/live-voice/__tests__/session-controls.test.ts +43 -0
  45. package/src/live-voice/live-voice-session.ts +784 -64
  46. package/src/live-voice/protocol.ts +20 -0
  47. package/src/live-voice/session-controls.ts +61 -3
  48. package/src/monitoring/plugin-auto-update.ts +6 -0
  49. package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +15 -0
  50. package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +1 -1
  51. package/src/runtime/routes/__tests__/plugins-routes.test.ts +102 -0
  52. package/src/runtime/routes/plugins-routes.ts +69 -49
  53. package/src/skills/managed-store.ts +153 -31
  54. package/src/tools/skills/find-similar-skills.test.ts +214 -2
  55. package/src/tools/skills/find-similar-skills.ts +62 -9
  56. package/src/tools/skills/resolve-execute-invocation.ts +24 -0
  57. package/src/tools/skills/scaffold-managed.ts +16 -13
@@ -1,6 +1,6 @@
1
1
  import { mkdirSync, writeFileSync } from "node:fs";
2
2
  import { join } from "node:path";
3
- import { beforeAll, describe, expect, mock, test } from "bun:test";
3
+ import { afterAll, beforeAll, describe, expect, mock, test } from "bun:test";
4
4
 
5
5
  import type { TurnDetectorConfig } from "../../calls/media-turn-detector.js";
6
6
  import type {
@@ -17,7 +17,16 @@ import type {
17
17
  StreamingTranscriber,
18
18
  SttStreamServerEvent,
19
19
  } from "../../stt/types.js";
20
- import { __resetRegistryForTesting } from "../../tools/registry.js";
20
+ import {
21
+ computerUseKeyTool,
22
+ computerUseObserveTool,
23
+ } from "../../tools/computer-use/definitions.js";
24
+ import {
25
+ __resetRegistryForTesting,
26
+ registerSkillTools,
27
+ unregisterSkillTools,
28
+ } from "../../tools/registry.js";
29
+ import { finalizeTool } from "../../tools/tool-defaults.js";
21
30
  import { getWorkspaceSkillsDir } from "../../util/platform.js";
22
31
  import type { LiveVoiceContinuationLabeler } from "../continuation-label.js";
23
32
  import type { LiveVoiceAudioArchiveResult } from "../live-voice-archive.js";
@@ -202,6 +211,9 @@ function createHarness(options: {
202
211
  getTurnTeardown?: (conversationId: string) => Promise<void> | undefined;
203
212
  detachTeardownSettleTimeoutMs?: number;
204
213
  continuationAnnounceSilenceMs?: number;
214
+ foregroundTaskResumeSilenceMs?: number;
215
+ foregroundTaskMaxSuspendedMs?: number;
216
+ foregroundTaskMaxInterveningTurns?: number;
205
217
  }) {
206
218
  const sequencer = createLiveVoiceServerFrameSequencer();
207
219
  const frames: LiveVoiceServerFrame[] = [];
@@ -290,6 +302,18 @@ function createHarness(options: {
290
302
  continuationAnnounceSilenceMs: options.continuationAnnounceSilenceMs,
291
303
  }
292
304
  : {}),
305
+ ...(options.foregroundTaskResumeSilenceMs !== undefined
306
+ ? { foregroundTaskResumeSilenceMs: options.foregroundTaskResumeSilenceMs }
307
+ : {}),
308
+ ...(options.foregroundTaskMaxSuspendedMs !== undefined
309
+ ? { foregroundTaskMaxSuspendedMs: options.foregroundTaskMaxSuspendedMs }
310
+ : {}),
311
+ ...(options.foregroundTaskMaxInterveningTurns !== undefined
312
+ ? {
313
+ foregroundTaskMaxInterveningTurns:
314
+ options.foregroundTaskMaxInterveningTurns,
315
+ }
316
+ : {}),
293
317
  };
294
318
  const session = options.viaFactory
295
319
  ? createLiveVoiceSession(context, {
@@ -310,6 +334,26 @@ function countType(frames: LiveVoiceServerFrame[], type: string): number {
310
334
  return frames.filter((frame) => frame.type === type).length;
311
335
  }
312
336
 
337
+ function foregroundTaskStateOf(session: LiveVoiceSession): {
338
+ phase: string;
339
+ request?: string;
340
+ interveningTurns?: number;
341
+ resumePending?: boolean;
342
+ hostToolStarted?: boolean;
343
+ } | null {
344
+ return (
345
+ session as unknown as {
346
+ foregroundTaskState: {
347
+ phase: string;
348
+ request?: string;
349
+ interveningTurns?: number;
350
+ resumePending?: boolean;
351
+ hostToolStarted?: boolean;
352
+ } | null;
353
+ }
354
+ ).foregroundTaskState;
355
+ }
356
+
313
357
  function makeTtsChunk(text: string): LiveVoiceTtsAudioChunk {
314
358
  return {
315
359
  type: "tts_audio",
@@ -463,11 +507,88 @@ async function startForegroundWinsScenario(): Promise<{
463
507
  };
464
508
  }
465
509
 
510
+ async function startForegroundTaskBargeInScenario(options?: {
511
+ foregroundTaskMaxInterveningTurns?: number;
512
+ foregroundTaskMaxSuspendedMs?: number;
513
+ foregroundTaskResumeSilenceMs?: number;
514
+ skillExecuteInput?: Record<string, unknown>;
515
+ skillExecuteAllowedToolNames?: ReadonlySet<string>;
516
+ startHostTool?: boolean;
517
+ }): Promise<{
518
+ calls: VoiceTurnOptions[];
519
+ frames: LiveVoiceServerFrame[];
520
+ session: LiveVoiceSession;
521
+ spawnBackgroundContinuation: ReturnType<typeof mock>;
522
+ }> {
523
+ const calls: VoiceTurnOptions[] = [];
524
+ const startVoiceTurn = mock(async (turn: VoiceTurnOptions) => {
525
+ calls.push(turn);
526
+ return {
527
+ turnId: `bridge-turn-${calls.length}`,
528
+ abort: mock(),
529
+ discard: mock(async () => {}),
530
+ };
531
+ });
532
+ const spawnBackgroundContinuation = mock(async () => "");
533
+ const { frames, session } = createHarness({
534
+ finals: ["change the title", "what title is there now"],
535
+ startVoiceTurn,
536
+ streamTtsAudio: mock(async (tts: LiveVoiceTtsOptions) => {
537
+ tts.onAudioChunk(makeTtsChunk("assistant audio"));
538
+ return makeTtsResult("assistant audio");
539
+ }),
540
+ spawnBackgroundContinuation,
541
+ foregroundTaskResumeSilenceMs: options?.foregroundTaskResumeSilenceMs ?? 20,
542
+ ...(options?.foregroundTaskMaxInterveningTurns !== undefined
543
+ ? {
544
+ foregroundTaskMaxInterveningTurns:
545
+ options.foregroundTaskMaxInterveningTurns,
546
+ }
547
+ : {}),
548
+ ...(options?.foregroundTaskMaxSuspendedMs !== undefined
549
+ ? { foregroundTaskMaxSuspendedMs: options.foregroundTaskMaxSuspendedMs }
550
+ : {}),
551
+ });
552
+
553
+ await session.start();
554
+ await session.handleBinaryAudio(LOUD_CHUNK);
555
+ await waitFor(() => calls.length === 1);
556
+
557
+ calls[0]?.callbacks?.assistant_text_delta?.(makeTextDelta("[1] One moment."));
558
+ calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
559
+ await waitFor(() => calls.length === 2);
560
+
561
+ if (options?.startHostTool !== false) {
562
+ calls[1]?.callbacks?.tool_use_start?.("skill_execute", {
563
+ toolUseId: "tool-1",
564
+ input: options?.skillExecuteInput ?? {
565
+ tool: "computer_use_observe",
566
+ input: {},
567
+ },
568
+ ...(options?.skillExecuteAllowedToolNames !== undefined
569
+ ? { allowedToolNames: options.skillExecuteAllowedToolNames }
570
+ : {}),
571
+ });
572
+ }
573
+ await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
574
+ await waitFor(() => calls.length === 3);
575
+
576
+ return { calls, frames, session, spawnBackgroundContinuation };
577
+ }
578
+
466
579
  describe("LiveVoiceSession server VAD", () => {
467
580
  // The foreground-wins classification consults the tool registry's owner map
468
581
  // (a tool is only provably read-only when it is the trusted built-in);
469
582
  // register the core baseline so built-in names resolve like in the daemon.
470
- beforeAll(() => __resetRegistryForTesting());
583
+ beforeAll(() => {
584
+ __resetRegistryForTesting();
585
+ registerSkillTools("computer-use-test", [
586
+ finalizeTool(computerUseKeyTool),
587
+ finalizeTool(computerUseObserveTool),
588
+ finalizeTool({ ...computerUseObserveTool, name: "read_file" }),
589
+ ]);
590
+ });
591
+ afterAll(() => unregisterSkillTools("computer-use-test"));
471
592
 
472
593
  test("ready echoes turnDetection server_vad", async () => {
473
594
  const { frames, session } = createHarness({});
@@ -580,6 +701,506 @@ describe("LiveVoiceSession server VAD", () => {
580
701
  expect(countType(frames, "utterance_end")).toBe(2);
581
702
  });
582
703
 
704
+ test("an escalated task interrupted before its first tool stays on the parent", async () => {
705
+ const { calls, session, spawnBackgroundContinuation } =
706
+ await startForegroundTaskBargeInScenario({ startHostTool: false });
707
+
708
+ expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
709
+ expect(foregroundTaskStateOf(session)).toMatchObject({
710
+ phase: "suspended",
711
+ hostToolStarted: false,
712
+ });
713
+
714
+ const frontDoorAnswer = calls[2];
715
+ frontDoorAnswer?.callbacks?.assistant_text_delta?.(
716
+ makeTextDelta("The title is Project Theta."),
717
+ );
718
+ frontDoorAnswer?.callbacks?.message_complete?.(makeMessageComplete());
719
+ await waitFor(() => calls.length === 4);
720
+
721
+ const resume = calls[3];
722
+ expect(resume).toMatchObject({
723
+ routingLeg: "escalated",
724
+ directEscalated: true,
725
+ hiddenSyntheticPrompt: true,
726
+ });
727
+ expect(resume?.content).toContain("change the title");
728
+ resume?.callbacks?.tool_use_start?.("skill_execute", {
729
+ toolUseId: "tool-2",
730
+ input: { tool: "computer_use_observe", input: {} },
731
+ });
732
+ expect(foregroundTaskStateOf(session)).toMatchObject({
733
+ phase: "owned",
734
+ hostToolStarted: true,
735
+ });
736
+
737
+ resume?.callbacks?.assistant_text_delta?.(makeTextDelta("Done."));
738
+ resume?.callbacks?.message_complete?.(makeMessageComplete());
739
+ await waitFor(() => foregroundTaskStateOf(session) === null);
740
+ });
741
+
742
+ test("a new barged-in task becomes the resume anchor", async () => {
743
+ const calls: VoiceTurnOptions[] = [];
744
+ const startVoiceTurn = mock(async (turn: VoiceTurnOptions) => {
745
+ calls.push(turn);
746
+ return {
747
+ turnId: `bridge-turn-${calls.length}`,
748
+ abort: mock(),
749
+ discard: mock(async () => {}),
750
+ };
751
+ });
752
+ let releaseFirstCompletionTts: (() => void) | undefined;
753
+ const { session } = createHarness({
754
+ finals: ["change the title", "add three bullets", "what is the title"],
755
+ startVoiceTurn,
756
+ streamTtsAudio: mock(async (tts: LiveVoiceTtsOptions) => {
757
+ tts.onAudioChunk(makeTtsChunk("assistant audio"));
758
+ if (tts.text.includes("First task done")) {
759
+ await new Promise<void>((resolve) => {
760
+ releaseFirstCompletionTts = resolve;
761
+ });
762
+ }
763
+ return makeTtsResult("assistant audio");
764
+ }),
765
+ foregroundTaskResumeSilenceMs: 20,
766
+ });
767
+
768
+ await session.start();
769
+ await session.handleBinaryAudio(LOUD_CHUNK);
770
+ await waitFor(() => calls.length === 1);
771
+ calls[0]?.callbacks?.assistant_text_delta?.(
772
+ makeTextDelta("[1] One moment."),
773
+ );
774
+ calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
775
+ await waitFor(() => calls.length === 2);
776
+ calls[1]?.callbacks?.tool_use_start?.("skill_execute", {
777
+ toolUseId: "tool-1",
778
+ input: { tool: "computer_use_observe", input: {} },
779
+ });
780
+ calls[1]?.callbacks?.assistant_text_delta?.(
781
+ makeTextDelta("First task done."),
782
+ );
783
+ calls[1]?.callbacks?.message_complete?.(makeMessageComplete());
784
+ await waitFor(() => releaseFirstCompletionTts !== undefined);
785
+
786
+ await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
787
+ await waitFor(() => calls.length === 3);
788
+ await waitFor(() => foregroundTaskStateOf(session) === null);
789
+ releaseFirstCompletionTts?.();
790
+ calls[2]?.callbacks?.assistant_text_delta?.(
791
+ makeTextDelta("[1] One moment."),
792
+ );
793
+ calls[2]?.callbacks?.message_complete?.(makeMessageComplete());
794
+ await waitFor(() => calls.length === 4);
795
+ expect(foregroundTaskStateOf(session)).toMatchObject({
796
+ phase: "owned",
797
+ request: "add three bullets",
798
+ hostToolStarted: false,
799
+ });
800
+
801
+ await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
802
+ await waitFor(() => calls.length === 5);
803
+ calls[4]?.callbacks?.assistant_text_delta?.(
804
+ makeTextDelta("The title is Project Alpha."),
805
+ );
806
+ calls[4]?.callbacks?.message_complete?.(makeMessageComplete());
807
+ await waitFor(() => calls.length === 6);
808
+
809
+ const resume = calls[5];
810
+ expect(resume).toMatchObject({
811
+ routingLeg: "escalated",
812
+ directEscalated: true,
813
+ hiddenSyntheticPrompt: true,
814
+ });
815
+ expect(resume?.content).toContain("add three bullets");
816
+ expect(resume?.content).not.toContain("change the title");
817
+ resume?.callbacks?.assistant_text_delta?.(makeTextDelta("[TASK:STOP]"));
818
+ resume?.callbacks?.message_complete?.(makeMessageComplete());
819
+ await waitFor(() => foregroundTaskStateOf(session) === null);
820
+ });
821
+
822
+ test("an escalated task retries when its first tool-capable leg fails", async () => {
823
+ const calls: VoiceTurnOptions[] = [];
824
+ const startVoiceTurn = mock(async (turn: VoiceTurnOptions) => {
825
+ calls.push(turn);
826
+ return {
827
+ turnId: `bridge-turn-${calls.length}`,
828
+ abort: mock(),
829
+ discard: mock(async () => {}),
830
+ };
831
+ });
832
+ const { session } = createHarness({
833
+ finals: ["change the title"],
834
+ startVoiceTurn,
835
+ streamTtsAudio: mock(async (tts: LiveVoiceTtsOptions) => {
836
+ tts.onAudioChunk(makeTtsChunk("assistant audio"));
837
+ return makeTtsResult("assistant audio");
838
+ }),
839
+ foregroundTaskResumeSilenceMs: 20,
840
+ });
841
+
842
+ await session.start();
843
+ await session.handleBinaryAudio(LOUD_CHUNK);
844
+ await waitFor(() => calls.length === 1);
845
+ calls[0]?.callbacks?.assistant_text_delta?.(
846
+ makeTextDelta("[1] One moment."),
847
+ );
848
+ calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
849
+ await waitFor(() => calls.length === 2);
850
+ expect(foregroundTaskStateOf(session)).toMatchObject({
851
+ phase: "owned",
852
+ hostToolStarted: false,
853
+ });
854
+
855
+ calls[1]?.onError?.("strong leg failed");
856
+ await waitFor(() => calls.length === 3);
857
+ expect(calls[2]).toMatchObject({
858
+ routingLeg: "escalated",
859
+ directEscalated: true,
860
+ hiddenSyntheticPrompt: true,
861
+ });
862
+ calls[2]?.callbacks?.assistant_text_delta?.(makeTextDelta("[TASK:STOP]"));
863
+ calls[2]?.callbacks?.message_complete?.(makeMessageComplete());
864
+ await waitFor(() => foregroundTaskStateOf(session) === null);
865
+ });
866
+
867
+ test("a barged-in skill host tool stays on the parent and resumes after the front door answers", async () => {
868
+ const { calls, frames, session, spawnBackgroundContinuation } =
869
+ await startForegroundTaskBargeInScenario();
870
+ expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
871
+
872
+ const frontDoorAnswer = calls[2];
873
+ expect(frontDoorAnswer?.routingLeg).toBe("front-door");
874
+ expect(frontDoorAnswer?.voiceControlPrompt).toContain("[TASK:STOP]");
875
+ expect(frontDoorAnswer?.voiceControlPrompt).toContain("change the title");
876
+
877
+ // A direct answer cannot accidentally end the task. Once its speech has
878
+ // drained, the task resumes on the strong, tool-capable leg.
879
+ frontDoorAnswer?.callbacks?.assistant_text_delta?.(
880
+ makeTextDelta("The title is Project Alpha."),
881
+ );
882
+ frontDoorAnswer?.callbacks?.message_complete?.(makeMessageComplete());
883
+ await waitFor(() => calls.length === 4);
884
+
885
+ const resume = calls[3];
886
+ expect(resume).toMatchObject({
887
+ routingLeg: "escalated",
888
+ directEscalated: true,
889
+ hiddenSyntheticPrompt: true,
890
+ });
891
+ expect(resume?.content).toContain("change the title");
892
+ expect(resume?.content).toContain("[TASK:STOP]");
893
+ expect(
894
+ frames.filter((frame) => frame.type === "thinking").length,
895
+ ).toBeGreaterThanOrEqual(3);
896
+
897
+ // The strong leg can decline a stale resume silently and clear ownership.
898
+ const spokenBeforeStop = countType(frames, "assistant_text_delta");
899
+ resume?.callbacks?.assistant_text_delta?.(makeTextDelta("[TASK:STOP]"));
900
+ resume?.callbacks?.message_complete?.(makeMessageComplete());
901
+ await flushAsyncCallbacks();
902
+ expect(countType(frames, "assistant_text_delta")).toBe(spokenBeforeStop);
903
+ expect(foregroundTaskStateOf(session)).toBeNull();
904
+ });
905
+
906
+ test.each([
907
+ [
908
+ "a provider-wrapped inner tool",
909
+ {
910
+ _raw: JSON.stringify({
911
+ tool: "computer_use_observe",
912
+ input: {},
913
+ }),
914
+ },
915
+ ],
916
+ [
917
+ "an aliased inner tool",
918
+ {
919
+ tool: "computer_use_press_key",
920
+ input: { key: "Enter", reasoning: "Continue the task" },
921
+ },
922
+ ],
923
+ ])("recognizes host ownership for %s", async (_label, skillExecuteInput) => {
924
+ const { session, spawnBackgroundContinuation } =
925
+ await startForegroundTaskBargeInScenario({ skillExecuteInput });
926
+
927
+ expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
928
+ expect(foregroundTaskStateOf(session)).toMatchObject({
929
+ phase: "suspended",
930
+ });
931
+ await session.close("client_end");
932
+ });
933
+
934
+ test("preserves an allowed host tool whose name is also an alias", async () => {
935
+ const { session, spawnBackgroundContinuation } =
936
+ await startForegroundTaskBargeInScenario({
937
+ skillExecuteInput: { tool: "read_file", input: {} },
938
+ skillExecuteAllowedToolNames: new Set(["read_file"]),
939
+ });
940
+
941
+ expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
942
+ expect(foregroundTaskStateOf(session)).toMatchObject({
943
+ phase: "suspended",
944
+ });
945
+ await session.close("client_end");
946
+ });
947
+
948
+ test("retries suspended host work when the handed-off leg fails to start", async () => {
949
+ const calls: VoiceTurnOptions[] = [];
950
+ const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
951
+ calls.push(options);
952
+ if (calls.length === 4) {
953
+ throw new Error("strong leg unavailable");
954
+ }
955
+ return {
956
+ turnId: `bridge-turn-${calls.length}`,
957
+ abort: mock(),
958
+ discard: mock(async () => {}),
959
+ };
960
+ });
961
+ const { session } = createHarness({
962
+ finals: ["change the title", "make it shorter"],
963
+ startVoiceTurn,
964
+ streamTtsAudio: mock(async (tts: LiveVoiceTtsOptions) => {
965
+ tts.onAudioChunk(makeTtsChunk("assistant audio"));
966
+ return makeTtsResult("assistant audio");
967
+ }),
968
+ foregroundTaskResumeSilenceMs: 20,
969
+ });
970
+
971
+ await session.start();
972
+ await session.handleBinaryAudio(LOUD_CHUNK);
973
+ await waitFor(() => calls.length === 1);
974
+ calls[0]?.callbacks?.assistant_text_delta?.(
975
+ makeTextDelta("[1] One moment."),
976
+ );
977
+ calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
978
+ await waitFor(() => calls.length === 2);
979
+ calls[1]?.callbacks?.tool_use_start?.("skill_execute", {
980
+ toolUseId: "tool-1",
981
+ input: { tool: "computer_use_observe", input: {} },
982
+ });
983
+
984
+ await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
985
+ await waitFor(() => calls.length === 3);
986
+ calls[2]?.callbacks?.assistant_text_delta?.(
987
+ makeTextDelta("[1] I will continue."),
988
+ );
989
+ calls[2]?.callbacks?.message_complete?.(makeMessageComplete());
990
+ await waitFor(() => calls.length === 5);
991
+
992
+ expect(calls[3]).toMatchObject({ routingLeg: "escalated" });
993
+ expect(calls[4]).toMatchObject({
994
+ routingLeg: "escalated",
995
+ directEscalated: true,
996
+ hiddenSyntheticPrompt: true,
997
+ });
998
+ calls[4]?.callbacks?.assistant_text_delta?.(makeTextDelta("[TASK:STOP]"));
999
+ calls[4]?.callbacks?.message_complete?.(makeMessageComplete());
1000
+ await waitFor(() => foregroundTaskStateOf(session) === null);
1001
+ });
1002
+
1003
+ test("retries suspended host work when the handed-off leg fails asynchronously", async () => {
1004
+ const { calls, session } = await startForegroundTaskBargeInScenario();
1005
+
1006
+ calls[2]?.callbacks?.assistant_text_delta?.(
1007
+ makeTextDelta("[1] I will continue."),
1008
+ );
1009
+ calls[2]?.callbacks?.message_complete?.(makeMessageComplete());
1010
+ await waitFor(() => calls.length === 4);
1011
+ await waitFor(() => foregroundTaskStateOf(session)?.phase === "owned");
1012
+
1013
+ calls[3]?.onError?.("strong leg failed");
1014
+ await waitFor(() => calls.length === 5);
1015
+
1016
+ expect(calls[4]).toMatchObject({
1017
+ routingLeg: "escalated",
1018
+ directEscalated: true,
1019
+ hiddenSyntheticPrompt: true,
1020
+ });
1021
+ calls[4]?.callbacks?.assistant_text_delta?.(makeTextDelta("[TASK:STOP]"));
1022
+ calls[4]?.callbacks?.message_complete?.(makeMessageComplete());
1023
+ await waitFor(() => foregroundTaskStateOf(session) === null);
1024
+ });
1025
+
1026
+ test("a front-door task-stop answer abandons the suspended foreground task", async () => {
1027
+ const { calls, frames, session } =
1028
+ await startForegroundTaskBargeInScenario();
1029
+
1030
+ const frontDoorAnswer = calls[2];
1031
+ frontDoorAnswer?.callbacks?.assistant_text_delta?.(
1032
+ makeTextDelta("Okay, stopping. [TASK:STOP]"),
1033
+ );
1034
+ frontDoorAnswer?.callbacks?.message_complete?.(makeMessageComplete());
1035
+ await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
1036
+ await new Promise((resolve) => setTimeout(resolve, 80));
1037
+
1038
+ expect(calls).toHaveLength(3);
1039
+ expect(foregroundTaskStateOf(session)).toBeNull();
1040
+ expect(
1041
+ frames.filter((frame) => frame.type === "assistant_text_delta").at(-1),
1042
+ ).toMatchObject({ text: "Okay, stopping. " });
1043
+ });
1044
+
1045
+ test("barging into completed host-task speech does not revive the task", async () => {
1046
+ const calls: VoiceTurnOptions[] = [];
1047
+ const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
1048
+ calls.push(options);
1049
+ return { turnId: `bridge-turn-${calls.length}`, abort: mock() };
1050
+ });
1051
+ let releaseDoneTts: (() => void) | undefined;
1052
+ const streamTtsAudio = mock(async (options: LiveVoiceTtsOptions) => {
1053
+ options.onAudioChunk(makeTtsChunk("assistant audio"));
1054
+ if (options.text.includes("Done")) {
1055
+ await new Promise<void>((resolve) => {
1056
+ releaseDoneTts = resolve;
1057
+ });
1058
+ }
1059
+ return makeTtsResult("assistant audio");
1060
+ });
1061
+ const { session } = createHarness({
1062
+ finals: ["change the title", "one more thing"],
1063
+ startVoiceTurn,
1064
+ streamTtsAudio,
1065
+ foregroundTaskResumeSilenceMs: 20,
1066
+ });
1067
+
1068
+ await session.start();
1069
+ await session.handleBinaryAudio(LOUD_CHUNK);
1070
+ await waitFor(() => calls.length === 1);
1071
+ calls[0]?.callbacks?.assistant_text_delta?.(
1072
+ makeTextDelta("[1] One moment."),
1073
+ );
1074
+ calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
1075
+ await waitFor(() => calls.length === 2);
1076
+ calls[1]?.callbacks?.tool_use_start?.("skill_execute", {
1077
+ toolUseId: "tool-1",
1078
+ input: { tool: "computer_use_observe", input: {} },
1079
+ });
1080
+ calls[1]?.callbacks?.assistant_text_delta?.(makeTextDelta("Done."));
1081
+ calls[1]?.callbacks?.message_complete?.(makeMessageComplete());
1082
+ await waitFor(() => releaseDoneTts !== undefined);
1083
+
1084
+ await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
1085
+ await waitFor(() => calls.length === 3);
1086
+ await waitFor(() => foregroundTaskStateOf(session) === null);
1087
+ calls[2]?.callbacks?.assistant_text_delta?.(makeTextDelta("Sure."));
1088
+ calls[2]?.callbacks?.message_complete?.(makeMessageComplete());
1089
+ releaseDoneTts?.();
1090
+ await new Promise((resolve) => setTimeout(resolve, 80));
1091
+
1092
+ expect(calls).toHaveLength(3);
1093
+ });
1094
+
1095
+ test("too many intervening answers abandon a suspended foreground task", async () => {
1096
+ const { calls, session } = await startForegroundTaskBargeInScenario({
1097
+ foregroundTaskMaxInterveningTurns: 1,
1098
+ foregroundTaskResumeSilenceMs: 60_000,
1099
+ });
1100
+
1101
+ calls[2]?.callbacks?.assistant_text_delta?.(makeTextDelta("First answer."));
1102
+ calls[2]?.callbacks?.message_complete?.(makeMessageComplete());
1103
+ await waitFor(() => foregroundTaskStateOf(session)?.interveningTurns === 1);
1104
+
1105
+ await session.handleBinaryAudio(LOUD_CHUNK);
1106
+ await waitFor(() => calls.length === 4);
1107
+ calls[3]?.callbacks?.assistant_text_delta?.(
1108
+ makeTextDelta("Second answer."),
1109
+ );
1110
+ calls[3]?.callbacks?.message_complete?.(makeMessageComplete());
1111
+ await waitFor(() => foregroundTaskStateOf(session) === null);
1112
+
1113
+ expect(calls).toHaveLength(4);
1114
+ await session.close("client_end");
1115
+ });
1116
+
1117
+ test("an old suspended foreground task is not revived", async () => {
1118
+ const { calls, session } = await startForegroundTaskBargeInScenario({
1119
+ foregroundTaskMaxSuspendedMs: 1,
1120
+ foregroundTaskResumeSilenceMs: 20,
1121
+ });
1122
+ await new Promise((resolve) => setTimeout(resolve, 5));
1123
+
1124
+ calls[2]?.callbacks?.assistant_text_delta?.(makeTextDelta("Answer."));
1125
+ calls[2]?.callbacks?.message_complete?.(makeMessageComplete());
1126
+ await waitFor(() => foregroundTaskStateOf(session) === null);
1127
+ await new Promise((resolve) => setTimeout(resolve, 60));
1128
+
1129
+ expect(calls).toHaveLength(3);
1130
+ });
1131
+
1132
+ test.each(["hold_verdict", "speech_resumed"] as const)(
1133
+ "a discarded speculative front-door turn leaves host ownership suspended: %s",
1134
+ async (discardReason) => {
1135
+ const calls: VoiceTurnOptions[] = [];
1136
+ const discards: Array<ReturnType<typeof mock>> = [];
1137
+ const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
1138
+ calls.push(options);
1139
+ const discard = mock(async () => {});
1140
+ discards.push(discard);
1141
+ if (
1142
+ discardReason === "hold_verdict" &&
1143
+ options.unifiedVerdict === true
1144
+ ) {
1145
+ setTimeout(() => {
1146
+ options.callbacks?.assistant_text_delta?.(makeTextDelta("[0]"));
1147
+ options.callbacks?.message_complete?.(makeMessageComplete());
1148
+ }, 0);
1149
+ }
1150
+ return {
1151
+ turnId: `bridge-turn-${calls.length}`,
1152
+ abort: mock(),
1153
+ discard,
1154
+ };
1155
+ });
1156
+ const spawnBackgroundContinuation = mock(async () => "");
1157
+ const { session, transcribers } = createHarness({
1158
+ finals: ["change the title", "actually just change it"],
1159
+ startVoiceTurn,
1160
+ streamTtsAudio: mock(async (tts: LiveVoiceTtsOptions) => {
1161
+ tts.onAudioChunk(makeTtsChunk("assistant audio"));
1162
+ return makeTtsResult("assistant audio");
1163
+ }),
1164
+ spawnBackgroundContinuation,
1165
+ holdStopEventsFor: [1],
1166
+ turnDetectorConfig: { silenceThresholdMs: 120 },
1167
+ foregroundTaskResumeSilenceMs: 60_000,
1168
+ });
1169
+
1170
+ await session.start();
1171
+ await session.handleBinaryAudio(LOUD_CHUNK);
1172
+ await waitFor(() => calls.length === 1);
1173
+ calls[0]?.callbacks?.assistant_text_delta?.(
1174
+ makeTextDelta("[1] One moment."),
1175
+ );
1176
+ calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
1177
+ await waitFor(() => calls.length === 2);
1178
+ calls[1]?.callbacks?.tool_use_start?.("skill_execute", {
1179
+ toolUseId: "tool-1",
1180
+ input: { tool: "computer_use_observe", input: {} },
1181
+ });
1182
+
1183
+ await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
1184
+ await waitFor(() => transcribers.length === 2);
1185
+ transcribers[1]?.emit({ type: "partial", text: "actually" });
1186
+ await waitFor(() => calls.length === 3);
1187
+ expect(calls[2]?.unifiedVerdict).toBe(true);
1188
+
1189
+ if (discardReason === "speech_resumed") {
1190
+ await session.handleBinaryAudio(LOUD_CHUNK);
1191
+ }
1192
+ await waitFor(() => (discards[2]?.mock.calls.length ?? 0) === 1);
1193
+
1194
+ expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
1195
+ expect(foregroundTaskStateOf(session)).toMatchObject({
1196
+ phase: "suspended",
1197
+ interveningTurns: 0,
1198
+ resumePending: false,
1199
+ });
1200
+ await session.close("client_end");
1201
+ },
1202
+ );
1203
+
583
1204
  test("speech while the first tts_audio send is stuck in the queue does not cancel; barge-in works once it lands", async () => {
584
1205
  let callbacks: VoiceTurnCallbacks | undefined;
585
1206
  const abort = mock();
@@ -1873,6 +2494,40 @@ describe("LiveVoiceSession server VAD", () => {
1873
2494
  expect(announcement?.voiceControlPrompt).toContain("first question");
1874
2495
  });
1875
2496
 
2497
+ // Deepgram Flux sends interim updates through silence, each an empty
2498
+ // partial. A call nobody is talking on is still idle.
2499
+ test("an empty partial on an idle call does not hold the announcement back", async () => {
2500
+ const continuation = makeControlledContinuation();
2501
+ const { startVoiceTurn, calls } = makeResurfaceTurnStarter();
2502
+ const { frames, session, transcribers } = createHarness({
2503
+ finals: ["first question", ""],
2504
+ startVoiceTurn,
2505
+ streamTtsAudio: makeImmediateTts(),
2506
+ spawnBackgroundContinuation: continuation.spawnBackgroundContinuation,
2507
+ continuationAnnounceSilenceMs: 20,
2508
+ });
2509
+
2510
+ await session.start();
2511
+ await session.handleBinaryAudio(LOUD_CHUNK);
2512
+ await waitFor(() => frames.some((frame) => frame.type === "thinking"));
2513
+ await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
2514
+ await waitFor(
2515
+ () => continuation.spawnBackgroundContinuation.mock.calls.length === 1,
2516
+ );
2517
+ await waitFor(() =>
2518
+ frames.some((frame) => frame.type === "utterance_discarded"),
2519
+ );
2520
+ await waitFor(() => transcribers.some((t) => !t.stopped));
2521
+ for (const transcriber of transcribers) {
2522
+ if (!transcriber.stopped) {
2523
+ transcriber.emit({ type: "partial", text: "" });
2524
+ }
2525
+ }
2526
+
2527
+ continuation.finish("THE_RESULT");
2528
+ await waitFor(() => announcementOf(calls) !== undefined);
2529
+ });
2530
+
1876
2531
  test("an announcement persists hidden and is never delivered twice", async () => {
1877
2532
  const continuation = makeControlledContinuation();
1878
2533
  const { startVoiceTurn, calls } = makeResurfaceTurnStarter();