@vellumai/assistant 0.12.2-staging.4 → 0.12.2-staging.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +7 -7
- package/openapi.yaml +151 -53
- package/package.json +5 -4
- package/scripts/bundled-plugin-packages.ts +154 -0
- package/scripts/generate-bundled-plugin-packages.ts +29 -0
- package/scripts/test.ts +15 -13
- package/src/__tests__/managed-store.test.ts +121 -0
- package/src/__tests__/scaffold-managed-skill-tool.test.ts +88 -0
- package/src/calls/__tests__/voice-control-protocol.test.ts +24 -2
- package/src/calls/__tests__/voice-session-bridge.test.ts +22 -2
- package/src/calls/voice-control-protocol.ts +13 -4
- package/src/calls/voice-session-bridge.ts +27 -6
- package/src/cli/commands/__tests__/plugins.test.ts +66 -0
- package/src/cli/commands/plugins.ts +50 -18
- package/src/cli/lib/__tests__/install-from-github.test.ts +67 -0
- package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +169 -0
- package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +57 -0
- package/src/cli/lib/__tests__/plugin-catalog-platform.test.ts +14 -0
- package/src/cli/lib/__tests__/plugin-catalog-resolve.test.ts +27 -1
- package/src/cli/lib/__tests__/plugin-details.test.ts +9 -2
- package/src/cli/lib/__tests__/plugin-marketplace.test.ts +20 -0
- package/src/cli/lib/__tests__/plugins-install-offline.test.ts +43 -0
- package/src/cli/lib/__tests__/search-plugins.test.ts +31 -0
- package/src/cli/lib/bundled-plugin-packages.json +4 -0
- package/src/cli/lib/bundled-plugin-packages.ts +87 -0
- package/src/cli/lib/diff-plugin.ts +1 -1
- package/src/cli/lib/inspect-plugin.ts +80 -12
- package/src/cli/lib/install-from-github.ts +133 -76
- package/src/cli/lib/plugin-catalog-cache.ts +22 -3
- package/src/cli/lib/plugin-catalog-local.ts +13 -3
- package/src/cli/lib/plugin-catalog-platform.ts +6 -1
- package/src/cli/lib/plugin-catalog-resolve.ts +20 -0
- package/src/cli/lib/plugin-details.ts +12 -0
- package/src/cli/lib/plugin-marketplace.ts +121 -21
- package/src/cli/lib/plugin-pin-history.ts +5 -2
- package/src/cli/lib/search-plugins.ts +58 -16
- package/src/cli/lib/upgrade-plugin.ts +48 -3
- package/src/config/bundled-skills/skill-management/SKILL.md +1 -1
- package/src/config/bundled-skills/skill-management/TOOLS.json +7 -7
- package/src/daemon/conversation-tool-setup.ts +4 -22
- package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +374 -0
- package/src/live-voice/__tests__/live-voice-vad.test.ts +658 -3
- package/src/live-voice/__tests__/protocol.test.ts +48 -0
- package/src/live-voice/__tests__/session-controls.test.ts +43 -0
- package/src/live-voice/live-voice-session.ts +784 -64
- package/src/live-voice/protocol.ts +20 -0
- package/src/live-voice/session-controls.ts +61 -3
- package/src/monitoring/plugin-auto-update.ts +6 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +15 -0
- package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +1 -1
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +102 -0
- package/src/runtime/routes/plugins-routes.ts +69 -49
- package/src/skills/managed-store.ts +153 -31
- package/src/tools/skills/find-similar-skills.test.ts +214 -2
- package/src/tools/skills/find-similar-skills.ts +62 -9
- package/src/tools/skills/resolve-execute-invocation.ts +24 -0
- package/src/tools/skills/scaffold-managed.ts +16 -13
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { mkdirSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
|
-
import { beforeAll, describe, expect, mock, test } from "bun:test";
|
|
3
|
+
import { afterAll, beforeAll, describe, expect, mock, test } from "bun:test";
|
|
4
4
|
|
|
5
5
|
import type { TurnDetectorConfig } from "../../calls/media-turn-detector.js";
|
|
6
6
|
import type {
|
|
@@ -17,7 +17,16 @@ import type {
|
|
|
17
17
|
StreamingTranscriber,
|
|
18
18
|
SttStreamServerEvent,
|
|
19
19
|
} from "../../stt/types.js";
|
|
20
|
-
import {
|
|
20
|
+
import {
|
|
21
|
+
computerUseKeyTool,
|
|
22
|
+
computerUseObserveTool,
|
|
23
|
+
} from "../../tools/computer-use/definitions.js";
|
|
24
|
+
import {
|
|
25
|
+
__resetRegistryForTesting,
|
|
26
|
+
registerSkillTools,
|
|
27
|
+
unregisterSkillTools,
|
|
28
|
+
} from "../../tools/registry.js";
|
|
29
|
+
import { finalizeTool } from "../../tools/tool-defaults.js";
|
|
21
30
|
import { getWorkspaceSkillsDir } from "../../util/platform.js";
|
|
22
31
|
import type { LiveVoiceContinuationLabeler } from "../continuation-label.js";
|
|
23
32
|
import type { LiveVoiceAudioArchiveResult } from "../live-voice-archive.js";
|
|
@@ -202,6 +211,9 @@ function createHarness(options: {
|
|
|
202
211
|
getTurnTeardown?: (conversationId: string) => Promise<void> | undefined;
|
|
203
212
|
detachTeardownSettleTimeoutMs?: number;
|
|
204
213
|
continuationAnnounceSilenceMs?: number;
|
|
214
|
+
foregroundTaskResumeSilenceMs?: number;
|
|
215
|
+
foregroundTaskMaxSuspendedMs?: number;
|
|
216
|
+
foregroundTaskMaxInterveningTurns?: number;
|
|
205
217
|
}) {
|
|
206
218
|
const sequencer = createLiveVoiceServerFrameSequencer();
|
|
207
219
|
const frames: LiveVoiceServerFrame[] = [];
|
|
@@ -290,6 +302,18 @@ function createHarness(options: {
|
|
|
290
302
|
continuationAnnounceSilenceMs: options.continuationAnnounceSilenceMs,
|
|
291
303
|
}
|
|
292
304
|
: {}),
|
|
305
|
+
...(options.foregroundTaskResumeSilenceMs !== undefined
|
|
306
|
+
? { foregroundTaskResumeSilenceMs: options.foregroundTaskResumeSilenceMs }
|
|
307
|
+
: {}),
|
|
308
|
+
...(options.foregroundTaskMaxSuspendedMs !== undefined
|
|
309
|
+
? { foregroundTaskMaxSuspendedMs: options.foregroundTaskMaxSuspendedMs }
|
|
310
|
+
: {}),
|
|
311
|
+
...(options.foregroundTaskMaxInterveningTurns !== undefined
|
|
312
|
+
? {
|
|
313
|
+
foregroundTaskMaxInterveningTurns:
|
|
314
|
+
options.foregroundTaskMaxInterveningTurns,
|
|
315
|
+
}
|
|
316
|
+
: {}),
|
|
293
317
|
};
|
|
294
318
|
const session = options.viaFactory
|
|
295
319
|
? createLiveVoiceSession(context, {
|
|
@@ -310,6 +334,26 @@ function countType(frames: LiveVoiceServerFrame[], type: string): number {
|
|
|
310
334
|
return frames.filter((frame) => frame.type === type).length;
|
|
311
335
|
}
|
|
312
336
|
|
|
337
|
+
function foregroundTaskStateOf(session: LiveVoiceSession): {
|
|
338
|
+
phase: string;
|
|
339
|
+
request?: string;
|
|
340
|
+
interveningTurns?: number;
|
|
341
|
+
resumePending?: boolean;
|
|
342
|
+
hostToolStarted?: boolean;
|
|
343
|
+
} | null {
|
|
344
|
+
return (
|
|
345
|
+
session as unknown as {
|
|
346
|
+
foregroundTaskState: {
|
|
347
|
+
phase: string;
|
|
348
|
+
request?: string;
|
|
349
|
+
interveningTurns?: number;
|
|
350
|
+
resumePending?: boolean;
|
|
351
|
+
hostToolStarted?: boolean;
|
|
352
|
+
} | null;
|
|
353
|
+
}
|
|
354
|
+
).foregroundTaskState;
|
|
355
|
+
}
|
|
356
|
+
|
|
313
357
|
function makeTtsChunk(text: string): LiveVoiceTtsAudioChunk {
|
|
314
358
|
return {
|
|
315
359
|
type: "tts_audio",
|
|
@@ -463,11 +507,88 @@ async function startForegroundWinsScenario(): Promise<{
|
|
|
463
507
|
};
|
|
464
508
|
}
|
|
465
509
|
|
|
510
|
+
async function startForegroundTaskBargeInScenario(options?: {
|
|
511
|
+
foregroundTaskMaxInterveningTurns?: number;
|
|
512
|
+
foregroundTaskMaxSuspendedMs?: number;
|
|
513
|
+
foregroundTaskResumeSilenceMs?: number;
|
|
514
|
+
skillExecuteInput?: Record<string, unknown>;
|
|
515
|
+
skillExecuteAllowedToolNames?: ReadonlySet<string>;
|
|
516
|
+
startHostTool?: boolean;
|
|
517
|
+
}): Promise<{
|
|
518
|
+
calls: VoiceTurnOptions[];
|
|
519
|
+
frames: LiveVoiceServerFrame[];
|
|
520
|
+
session: LiveVoiceSession;
|
|
521
|
+
spawnBackgroundContinuation: ReturnType<typeof mock>;
|
|
522
|
+
}> {
|
|
523
|
+
const calls: VoiceTurnOptions[] = [];
|
|
524
|
+
const startVoiceTurn = mock(async (turn: VoiceTurnOptions) => {
|
|
525
|
+
calls.push(turn);
|
|
526
|
+
return {
|
|
527
|
+
turnId: `bridge-turn-${calls.length}`,
|
|
528
|
+
abort: mock(),
|
|
529
|
+
discard: mock(async () => {}),
|
|
530
|
+
};
|
|
531
|
+
});
|
|
532
|
+
const spawnBackgroundContinuation = mock(async () => "");
|
|
533
|
+
const { frames, session } = createHarness({
|
|
534
|
+
finals: ["change the title", "what title is there now"],
|
|
535
|
+
startVoiceTurn,
|
|
536
|
+
streamTtsAudio: mock(async (tts: LiveVoiceTtsOptions) => {
|
|
537
|
+
tts.onAudioChunk(makeTtsChunk("assistant audio"));
|
|
538
|
+
return makeTtsResult("assistant audio");
|
|
539
|
+
}),
|
|
540
|
+
spawnBackgroundContinuation,
|
|
541
|
+
foregroundTaskResumeSilenceMs: options?.foregroundTaskResumeSilenceMs ?? 20,
|
|
542
|
+
...(options?.foregroundTaskMaxInterveningTurns !== undefined
|
|
543
|
+
? {
|
|
544
|
+
foregroundTaskMaxInterveningTurns:
|
|
545
|
+
options.foregroundTaskMaxInterveningTurns,
|
|
546
|
+
}
|
|
547
|
+
: {}),
|
|
548
|
+
...(options?.foregroundTaskMaxSuspendedMs !== undefined
|
|
549
|
+
? { foregroundTaskMaxSuspendedMs: options.foregroundTaskMaxSuspendedMs }
|
|
550
|
+
: {}),
|
|
551
|
+
});
|
|
552
|
+
|
|
553
|
+
await session.start();
|
|
554
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
555
|
+
await waitFor(() => calls.length === 1);
|
|
556
|
+
|
|
557
|
+
calls[0]?.callbacks?.assistant_text_delta?.(makeTextDelta("[1] One moment."));
|
|
558
|
+
calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
559
|
+
await waitFor(() => calls.length === 2);
|
|
560
|
+
|
|
561
|
+
if (options?.startHostTool !== false) {
|
|
562
|
+
calls[1]?.callbacks?.tool_use_start?.("skill_execute", {
|
|
563
|
+
toolUseId: "tool-1",
|
|
564
|
+
input: options?.skillExecuteInput ?? {
|
|
565
|
+
tool: "computer_use_observe",
|
|
566
|
+
input: {},
|
|
567
|
+
},
|
|
568
|
+
...(options?.skillExecuteAllowedToolNames !== undefined
|
|
569
|
+
? { allowedToolNames: options.skillExecuteAllowedToolNames }
|
|
570
|
+
: {}),
|
|
571
|
+
});
|
|
572
|
+
}
|
|
573
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
574
|
+
await waitFor(() => calls.length === 3);
|
|
575
|
+
|
|
576
|
+
return { calls, frames, session, spawnBackgroundContinuation };
|
|
577
|
+
}
|
|
578
|
+
|
|
466
579
|
describe("LiveVoiceSession server VAD", () => {
|
|
467
580
|
// The foreground-wins classification consults the tool registry's owner map
|
|
468
581
|
// (a tool is only provably read-only when it is the trusted built-in);
|
|
469
582
|
// register the core baseline so built-in names resolve like in the daemon.
|
|
470
|
-
beforeAll(() =>
|
|
583
|
+
beforeAll(() => {
|
|
584
|
+
__resetRegistryForTesting();
|
|
585
|
+
registerSkillTools("computer-use-test", [
|
|
586
|
+
finalizeTool(computerUseKeyTool),
|
|
587
|
+
finalizeTool(computerUseObserveTool),
|
|
588
|
+
finalizeTool({ ...computerUseObserveTool, name: "read_file" }),
|
|
589
|
+
]);
|
|
590
|
+
});
|
|
591
|
+
afterAll(() => unregisterSkillTools("computer-use-test"));
|
|
471
592
|
|
|
472
593
|
test("ready echoes turnDetection server_vad", async () => {
|
|
473
594
|
const { frames, session } = createHarness({});
|
|
@@ -580,6 +701,506 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
580
701
|
expect(countType(frames, "utterance_end")).toBe(2);
|
|
581
702
|
});
|
|
582
703
|
|
|
704
|
+
test("an escalated task interrupted before its first tool stays on the parent", async () => {
|
|
705
|
+
const { calls, session, spawnBackgroundContinuation } =
|
|
706
|
+
await startForegroundTaskBargeInScenario({ startHostTool: false });
|
|
707
|
+
|
|
708
|
+
expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
|
|
709
|
+
expect(foregroundTaskStateOf(session)).toMatchObject({
|
|
710
|
+
phase: "suspended",
|
|
711
|
+
hostToolStarted: false,
|
|
712
|
+
});
|
|
713
|
+
|
|
714
|
+
const frontDoorAnswer = calls[2];
|
|
715
|
+
frontDoorAnswer?.callbacks?.assistant_text_delta?.(
|
|
716
|
+
makeTextDelta("The title is Project Theta."),
|
|
717
|
+
);
|
|
718
|
+
frontDoorAnswer?.callbacks?.message_complete?.(makeMessageComplete());
|
|
719
|
+
await waitFor(() => calls.length === 4);
|
|
720
|
+
|
|
721
|
+
const resume = calls[3];
|
|
722
|
+
expect(resume).toMatchObject({
|
|
723
|
+
routingLeg: "escalated",
|
|
724
|
+
directEscalated: true,
|
|
725
|
+
hiddenSyntheticPrompt: true,
|
|
726
|
+
});
|
|
727
|
+
expect(resume?.content).toContain("change the title");
|
|
728
|
+
resume?.callbacks?.tool_use_start?.("skill_execute", {
|
|
729
|
+
toolUseId: "tool-2",
|
|
730
|
+
input: { tool: "computer_use_observe", input: {} },
|
|
731
|
+
});
|
|
732
|
+
expect(foregroundTaskStateOf(session)).toMatchObject({
|
|
733
|
+
phase: "owned",
|
|
734
|
+
hostToolStarted: true,
|
|
735
|
+
});
|
|
736
|
+
|
|
737
|
+
resume?.callbacks?.assistant_text_delta?.(makeTextDelta("Done."));
|
|
738
|
+
resume?.callbacks?.message_complete?.(makeMessageComplete());
|
|
739
|
+
await waitFor(() => foregroundTaskStateOf(session) === null);
|
|
740
|
+
});
|
|
741
|
+
|
|
742
|
+
test("a new barged-in task becomes the resume anchor", async () => {
|
|
743
|
+
const calls: VoiceTurnOptions[] = [];
|
|
744
|
+
const startVoiceTurn = mock(async (turn: VoiceTurnOptions) => {
|
|
745
|
+
calls.push(turn);
|
|
746
|
+
return {
|
|
747
|
+
turnId: `bridge-turn-${calls.length}`,
|
|
748
|
+
abort: mock(),
|
|
749
|
+
discard: mock(async () => {}),
|
|
750
|
+
};
|
|
751
|
+
});
|
|
752
|
+
let releaseFirstCompletionTts: (() => void) | undefined;
|
|
753
|
+
const { session } = createHarness({
|
|
754
|
+
finals: ["change the title", "add three bullets", "what is the title"],
|
|
755
|
+
startVoiceTurn,
|
|
756
|
+
streamTtsAudio: mock(async (tts: LiveVoiceTtsOptions) => {
|
|
757
|
+
tts.onAudioChunk(makeTtsChunk("assistant audio"));
|
|
758
|
+
if (tts.text.includes("First task done")) {
|
|
759
|
+
await new Promise<void>((resolve) => {
|
|
760
|
+
releaseFirstCompletionTts = resolve;
|
|
761
|
+
});
|
|
762
|
+
}
|
|
763
|
+
return makeTtsResult("assistant audio");
|
|
764
|
+
}),
|
|
765
|
+
foregroundTaskResumeSilenceMs: 20,
|
|
766
|
+
});
|
|
767
|
+
|
|
768
|
+
await session.start();
|
|
769
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
770
|
+
await waitFor(() => calls.length === 1);
|
|
771
|
+
calls[0]?.callbacks?.assistant_text_delta?.(
|
|
772
|
+
makeTextDelta("[1] One moment."),
|
|
773
|
+
);
|
|
774
|
+
calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
775
|
+
await waitFor(() => calls.length === 2);
|
|
776
|
+
calls[1]?.callbacks?.tool_use_start?.("skill_execute", {
|
|
777
|
+
toolUseId: "tool-1",
|
|
778
|
+
input: { tool: "computer_use_observe", input: {} },
|
|
779
|
+
});
|
|
780
|
+
calls[1]?.callbacks?.assistant_text_delta?.(
|
|
781
|
+
makeTextDelta("First task done."),
|
|
782
|
+
);
|
|
783
|
+
calls[1]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
784
|
+
await waitFor(() => releaseFirstCompletionTts !== undefined);
|
|
785
|
+
|
|
786
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
787
|
+
await waitFor(() => calls.length === 3);
|
|
788
|
+
await waitFor(() => foregroundTaskStateOf(session) === null);
|
|
789
|
+
releaseFirstCompletionTts?.();
|
|
790
|
+
calls[2]?.callbacks?.assistant_text_delta?.(
|
|
791
|
+
makeTextDelta("[1] One moment."),
|
|
792
|
+
);
|
|
793
|
+
calls[2]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
794
|
+
await waitFor(() => calls.length === 4);
|
|
795
|
+
expect(foregroundTaskStateOf(session)).toMatchObject({
|
|
796
|
+
phase: "owned",
|
|
797
|
+
request: "add three bullets",
|
|
798
|
+
hostToolStarted: false,
|
|
799
|
+
});
|
|
800
|
+
|
|
801
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
802
|
+
await waitFor(() => calls.length === 5);
|
|
803
|
+
calls[4]?.callbacks?.assistant_text_delta?.(
|
|
804
|
+
makeTextDelta("The title is Project Alpha."),
|
|
805
|
+
);
|
|
806
|
+
calls[4]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
807
|
+
await waitFor(() => calls.length === 6);
|
|
808
|
+
|
|
809
|
+
const resume = calls[5];
|
|
810
|
+
expect(resume).toMatchObject({
|
|
811
|
+
routingLeg: "escalated",
|
|
812
|
+
directEscalated: true,
|
|
813
|
+
hiddenSyntheticPrompt: true,
|
|
814
|
+
});
|
|
815
|
+
expect(resume?.content).toContain("add three bullets");
|
|
816
|
+
expect(resume?.content).not.toContain("change the title");
|
|
817
|
+
resume?.callbacks?.assistant_text_delta?.(makeTextDelta("[TASK:STOP]"));
|
|
818
|
+
resume?.callbacks?.message_complete?.(makeMessageComplete());
|
|
819
|
+
await waitFor(() => foregroundTaskStateOf(session) === null);
|
|
820
|
+
});
|
|
821
|
+
|
|
822
|
+
test("an escalated task retries when its first tool-capable leg fails", async () => {
|
|
823
|
+
const calls: VoiceTurnOptions[] = [];
|
|
824
|
+
const startVoiceTurn = mock(async (turn: VoiceTurnOptions) => {
|
|
825
|
+
calls.push(turn);
|
|
826
|
+
return {
|
|
827
|
+
turnId: `bridge-turn-${calls.length}`,
|
|
828
|
+
abort: mock(),
|
|
829
|
+
discard: mock(async () => {}),
|
|
830
|
+
};
|
|
831
|
+
});
|
|
832
|
+
const { session } = createHarness({
|
|
833
|
+
finals: ["change the title"],
|
|
834
|
+
startVoiceTurn,
|
|
835
|
+
streamTtsAudio: mock(async (tts: LiveVoiceTtsOptions) => {
|
|
836
|
+
tts.onAudioChunk(makeTtsChunk("assistant audio"));
|
|
837
|
+
return makeTtsResult("assistant audio");
|
|
838
|
+
}),
|
|
839
|
+
foregroundTaskResumeSilenceMs: 20,
|
|
840
|
+
});
|
|
841
|
+
|
|
842
|
+
await session.start();
|
|
843
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
844
|
+
await waitFor(() => calls.length === 1);
|
|
845
|
+
calls[0]?.callbacks?.assistant_text_delta?.(
|
|
846
|
+
makeTextDelta("[1] One moment."),
|
|
847
|
+
);
|
|
848
|
+
calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
849
|
+
await waitFor(() => calls.length === 2);
|
|
850
|
+
expect(foregroundTaskStateOf(session)).toMatchObject({
|
|
851
|
+
phase: "owned",
|
|
852
|
+
hostToolStarted: false,
|
|
853
|
+
});
|
|
854
|
+
|
|
855
|
+
calls[1]?.onError?.("strong leg failed");
|
|
856
|
+
await waitFor(() => calls.length === 3);
|
|
857
|
+
expect(calls[2]).toMatchObject({
|
|
858
|
+
routingLeg: "escalated",
|
|
859
|
+
directEscalated: true,
|
|
860
|
+
hiddenSyntheticPrompt: true,
|
|
861
|
+
});
|
|
862
|
+
calls[2]?.callbacks?.assistant_text_delta?.(makeTextDelta("[TASK:STOP]"));
|
|
863
|
+
calls[2]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
864
|
+
await waitFor(() => foregroundTaskStateOf(session) === null);
|
|
865
|
+
});
|
|
866
|
+
|
|
867
|
+
test("a barged-in skill host tool stays on the parent and resumes after the front door answers", async () => {
|
|
868
|
+
const { calls, frames, session, spawnBackgroundContinuation } =
|
|
869
|
+
await startForegroundTaskBargeInScenario();
|
|
870
|
+
expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
|
|
871
|
+
|
|
872
|
+
const frontDoorAnswer = calls[2];
|
|
873
|
+
expect(frontDoorAnswer?.routingLeg).toBe("front-door");
|
|
874
|
+
expect(frontDoorAnswer?.voiceControlPrompt).toContain("[TASK:STOP]");
|
|
875
|
+
expect(frontDoorAnswer?.voiceControlPrompt).toContain("change the title");
|
|
876
|
+
|
|
877
|
+
// A direct answer cannot accidentally end the task. Once its speech has
|
|
878
|
+
// drained, the task resumes on the strong, tool-capable leg.
|
|
879
|
+
frontDoorAnswer?.callbacks?.assistant_text_delta?.(
|
|
880
|
+
makeTextDelta("The title is Project Alpha."),
|
|
881
|
+
);
|
|
882
|
+
frontDoorAnswer?.callbacks?.message_complete?.(makeMessageComplete());
|
|
883
|
+
await waitFor(() => calls.length === 4);
|
|
884
|
+
|
|
885
|
+
const resume = calls[3];
|
|
886
|
+
expect(resume).toMatchObject({
|
|
887
|
+
routingLeg: "escalated",
|
|
888
|
+
directEscalated: true,
|
|
889
|
+
hiddenSyntheticPrompt: true,
|
|
890
|
+
});
|
|
891
|
+
expect(resume?.content).toContain("change the title");
|
|
892
|
+
expect(resume?.content).toContain("[TASK:STOP]");
|
|
893
|
+
expect(
|
|
894
|
+
frames.filter((frame) => frame.type === "thinking").length,
|
|
895
|
+
).toBeGreaterThanOrEqual(3);
|
|
896
|
+
|
|
897
|
+
// The strong leg can decline a stale resume silently and clear ownership.
|
|
898
|
+
const spokenBeforeStop = countType(frames, "assistant_text_delta");
|
|
899
|
+
resume?.callbacks?.assistant_text_delta?.(makeTextDelta("[TASK:STOP]"));
|
|
900
|
+
resume?.callbacks?.message_complete?.(makeMessageComplete());
|
|
901
|
+
await flushAsyncCallbacks();
|
|
902
|
+
expect(countType(frames, "assistant_text_delta")).toBe(spokenBeforeStop);
|
|
903
|
+
expect(foregroundTaskStateOf(session)).toBeNull();
|
|
904
|
+
});
|
|
905
|
+
|
|
906
|
+
test.each([
|
|
907
|
+
[
|
|
908
|
+
"a provider-wrapped inner tool",
|
|
909
|
+
{
|
|
910
|
+
_raw: JSON.stringify({
|
|
911
|
+
tool: "computer_use_observe",
|
|
912
|
+
input: {},
|
|
913
|
+
}),
|
|
914
|
+
},
|
|
915
|
+
],
|
|
916
|
+
[
|
|
917
|
+
"an aliased inner tool",
|
|
918
|
+
{
|
|
919
|
+
tool: "computer_use_press_key",
|
|
920
|
+
input: { key: "Enter", reasoning: "Continue the task" },
|
|
921
|
+
},
|
|
922
|
+
],
|
|
923
|
+
])("recognizes host ownership for %s", async (_label, skillExecuteInput) => {
|
|
924
|
+
const { session, spawnBackgroundContinuation } =
|
|
925
|
+
await startForegroundTaskBargeInScenario({ skillExecuteInput });
|
|
926
|
+
|
|
927
|
+
expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
|
|
928
|
+
expect(foregroundTaskStateOf(session)).toMatchObject({
|
|
929
|
+
phase: "suspended",
|
|
930
|
+
});
|
|
931
|
+
await session.close("client_end");
|
|
932
|
+
});
|
|
933
|
+
|
|
934
|
+
test("preserves an allowed host tool whose name is also an alias", async () => {
|
|
935
|
+
const { session, spawnBackgroundContinuation } =
|
|
936
|
+
await startForegroundTaskBargeInScenario({
|
|
937
|
+
skillExecuteInput: { tool: "read_file", input: {} },
|
|
938
|
+
skillExecuteAllowedToolNames: new Set(["read_file"]),
|
|
939
|
+
});
|
|
940
|
+
|
|
941
|
+
expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
|
|
942
|
+
expect(foregroundTaskStateOf(session)).toMatchObject({
|
|
943
|
+
phase: "suspended",
|
|
944
|
+
});
|
|
945
|
+
await session.close("client_end");
|
|
946
|
+
});
|
|
947
|
+
|
|
948
|
+
test("retries suspended host work when the handed-off leg fails to start", async () => {
|
|
949
|
+
const calls: VoiceTurnOptions[] = [];
|
|
950
|
+
const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
|
|
951
|
+
calls.push(options);
|
|
952
|
+
if (calls.length === 4) {
|
|
953
|
+
throw new Error("strong leg unavailable");
|
|
954
|
+
}
|
|
955
|
+
return {
|
|
956
|
+
turnId: `bridge-turn-${calls.length}`,
|
|
957
|
+
abort: mock(),
|
|
958
|
+
discard: mock(async () => {}),
|
|
959
|
+
};
|
|
960
|
+
});
|
|
961
|
+
const { session } = createHarness({
|
|
962
|
+
finals: ["change the title", "make it shorter"],
|
|
963
|
+
startVoiceTurn,
|
|
964
|
+
streamTtsAudio: mock(async (tts: LiveVoiceTtsOptions) => {
|
|
965
|
+
tts.onAudioChunk(makeTtsChunk("assistant audio"));
|
|
966
|
+
return makeTtsResult("assistant audio");
|
|
967
|
+
}),
|
|
968
|
+
foregroundTaskResumeSilenceMs: 20,
|
|
969
|
+
});
|
|
970
|
+
|
|
971
|
+
await session.start();
|
|
972
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
973
|
+
await waitFor(() => calls.length === 1);
|
|
974
|
+
calls[0]?.callbacks?.assistant_text_delta?.(
|
|
975
|
+
makeTextDelta("[1] One moment."),
|
|
976
|
+
);
|
|
977
|
+
calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
978
|
+
await waitFor(() => calls.length === 2);
|
|
979
|
+
calls[1]?.callbacks?.tool_use_start?.("skill_execute", {
|
|
980
|
+
toolUseId: "tool-1",
|
|
981
|
+
input: { tool: "computer_use_observe", input: {} },
|
|
982
|
+
});
|
|
983
|
+
|
|
984
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
985
|
+
await waitFor(() => calls.length === 3);
|
|
986
|
+
calls[2]?.callbacks?.assistant_text_delta?.(
|
|
987
|
+
makeTextDelta("[1] I will continue."),
|
|
988
|
+
);
|
|
989
|
+
calls[2]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
990
|
+
await waitFor(() => calls.length === 5);
|
|
991
|
+
|
|
992
|
+
expect(calls[3]).toMatchObject({ routingLeg: "escalated" });
|
|
993
|
+
expect(calls[4]).toMatchObject({
|
|
994
|
+
routingLeg: "escalated",
|
|
995
|
+
directEscalated: true,
|
|
996
|
+
hiddenSyntheticPrompt: true,
|
|
997
|
+
});
|
|
998
|
+
calls[4]?.callbacks?.assistant_text_delta?.(makeTextDelta("[TASK:STOP]"));
|
|
999
|
+
calls[4]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1000
|
+
await waitFor(() => foregroundTaskStateOf(session) === null);
|
|
1001
|
+
});
|
|
1002
|
+
|
|
1003
|
+
test("retries suspended host work when the handed-off leg fails asynchronously", async () => {
|
|
1004
|
+
const { calls, session } = await startForegroundTaskBargeInScenario();
|
|
1005
|
+
|
|
1006
|
+
calls[2]?.callbacks?.assistant_text_delta?.(
|
|
1007
|
+
makeTextDelta("[1] I will continue."),
|
|
1008
|
+
);
|
|
1009
|
+
calls[2]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1010
|
+
await waitFor(() => calls.length === 4);
|
|
1011
|
+
await waitFor(() => foregroundTaskStateOf(session)?.phase === "owned");
|
|
1012
|
+
|
|
1013
|
+
calls[3]?.onError?.("strong leg failed");
|
|
1014
|
+
await waitFor(() => calls.length === 5);
|
|
1015
|
+
|
|
1016
|
+
expect(calls[4]).toMatchObject({
|
|
1017
|
+
routingLeg: "escalated",
|
|
1018
|
+
directEscalated: true,
|
|
1019
|
+
hiddenSyntheticPrompt: true,
|
|
1020
|
+
});
|
|
1021
|
+
calls[4]?.callbacks?.assistant_text_delta?.(makeTextDelta("[TASK:STOP]"));
|
|
1022
|
+
calls[4]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1023
|
+
await waitFor(() => foregroundTaskStateOf(session) === null);
|
|
1024
|
+
});
|
|
1025
|
+
|
|
1026
|
+
test("a front-door task-stop answer abandons the suspended foreground task", async () => {
|
|
1027
|
+
const { calls, frames, session } =
|
|
1028
|
+
await startForegroundTaskBargeInScenario();
|
|
1029
|
+
|
|
1030
|
+
const frontDoorAnswer = calls[2];
|
|
1031
|
+
frontDoorAnswer?.callbacks?.assistant_text_delta?.(
|
|
1032
|
+
makeTextDelta("Okay, stopping. [TASK:STOP]"),
|
|
1033
|
+
);
|
|
1034
|
+
frontDoorAnswer?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1035
|
+
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
1036
|
+
await new Promise((resolve) => setTimeout(resolve, 80));
|
|
1037
|
+
|
|
1038
|
+
expect(calls).toHaveLength(3);
|
|
1039
|
+
expect(foregroundTaskStateOf(session)).toBeNull();
|
|
1040
|
+
expect(
|
|
1041
|
+
frames.filter((frame) => frame.type === "assistant_text_delta").at(-1),
|
|
1042
|
+
).toMatchObject({ text: "Okay, stopping. " });
|
|
1043
|
+
});
|
|
1044
|
+
|
|
1045
|
+
test("barging into completed host-task speech does not revive the task", async () => {
|
|
1046
|
+
const calls: VoiceTurnOptions[] = [];
|
|
1047
|
+
const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
|
|
1048
|
+
calls.push(options);
|
|
1049
|
+
return { turnId: `bridge-turn-${calls.length}`, abort: mock() };
|
|
1050
|
+
});
|
|
1051
|
+
let releaseDoneTts: (() => void) | undefined;
|
|
1052
|
+
const streamTtsAudio = mock(async (options: LiveVoiceTtsOptions) => {
|
|
1053
|
+
options.onAudioChunk(makeTtsChunk("assistant audio"));
|
|
1054
|
+
if (options.text.includes("Done")) {
|
|
1055
|
+
await new Promise<void>((resolve) => {
|
|
1056
|
+
releaseDoneTts = resolve;
|
|
1057
|
+
});
|
|
1058
|
+
}
|
|
1059
|
+
return makeTtsResult("assistant audio");
|
|
1060
|
+
});
|
|
1061
|
+
const { session } = createHarness({
|
|
1062
|
+
finals: ["change the title", "one more thing"],
|
|
1063
|
+
startVoiceTurn,
|
|
1064
|
+
streamTtsAudio,
|
|
1065
|
+
foregroundTaskResumeSilenceMs: 20,
|
|
1066
|
+
});
|
|
1067
|
+
|
|
1068
|
+
await session.start();
|
|
1069
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1070
|
+
await waitFor(() => calls.length === 1);
|
|
1071
|
+
calls[0]?.callbacks?.assistant_text_delta?.(
|
|
1072
|
+
makeTextDelta("[1] One moment."),
|
|
1073
|
+
);
|
|
1074
|
+
calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1075
|
+
await waitFor(() => calls.length === 2);
|
|
1076
|
+
calls[1]?.callbacks?.tool_use_start?.("skill_execute", {
|
|
1077
|
+
toolUseId: "tool-1",
|
|
1078
|
+
input: { tool: "computer_use_observe", input: {} },
|
|
1079
|
+
});
|
|
1080
|
+
calls[1]?.callbacks?.assistant_text_delta?.(makeTextDelta("Done."));
|
|
1081
|
+
calls[1]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1082
|
+
await waitFor(() => releaseDoneTts !== undefined);
|
|
1083
|
+
|
|
1084
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
1085
|
+
await waitFor(() => calls.length === 3);
|
|
1086
|
+
await waitFor(() => foregroundTaskStateOf(session) === null);
|
|
1087
|
+
calls[2]?.callbacks?.assistant_text_delta?.(makeTextDelta("Sure."));
|
|
1088
|
+
calls[2]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1089
|
+
releaseDoneTts?.();
|
|
1090
|
+
await new Promise((resolve) => setTimeout(resolve, 80));
|
|
1091
|
+
|
|
1092
|
+
expect(calls).toHaveLength(3);
|
|
1093
|
+
});
|
|
1094
|
+
|
|
1095
|
+
test("too many intervening answers abandon a suspended foreground task", async () => {
|
|
1096
|
+
const { calls, session } = await startForegroundTaskBargeInScenario({
|
|
1097
|
+
foregroundTaskMaxInterveningTurns: 1,
|
|
1098
|
+
foregroundTaskResumeSilenceMs: 60_000,
|
|
1099
|
+
});
|
|
1100
|
+
|
|
1101
|
+
calls[2]?.callbacks?.assistant_text_delta?.(makeTextDelta("First answer."));
|
|
1102
|
+
calls[2]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1103
|
+
await waitFor(() => foregroundTaskStateOf(session)?.interveningTurns === 1);
|
|
1104
|
+
|
|
1105
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1106
|
+
await waitFor(() => calls.length === 4);
|
|
1107
|
+
calls[3]?.callbacks?.assistant_text_delta?.(
|
|
1108
|
+
makeTextDelta("Second answer."),
|
|
1109
|
+
);
|
|
1110
|
+
calls[3]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1111
|
+
await waitFor(() => foregroundTaskStateOf(session) === null);
|
|
1112
|
+
|
|
1113
|
+
expect(calls).toHaveLength(4);
|
|
1114
|
+
await session.close("client_end");
|
|
1115
|
+
});
|
|
1116
|
+
|
|
1117
|
+
test("an old suspended foreground task is not revived", async () => {
|
|
1118
|
+
const { calls, session } = await startForegroundTaskBargeInScenario({
|
|
1119
|
+
foregroundTaskMaxSuspendedMs: 1,
|
|
1120
|
+
foregroundTaskResumeSilenceMs: 20,
|
|
1121
|
+
});
|
|
1122
|
+
await new Promise((resolve) => setTimeout(resolve, 5));
|
|
1123
|
+
|
|
1124
|
+
calls[2]?.callbacks?.assistant_text_delta?.(makeTextDelta("Answer."));
|
|
1125
|
+
calls[2]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1126
|
+
await waitFor(() => foregroundTaskStateOf(session) === null);
|
|
1127
|
+
await new Promise((resolve) => setTimeout(resolve, 60));
|
|
1128
|
+
|
|
1129
|
+
expect(calls).toHaveLength(3);
|
|
1130
|
+
});
|
|
1131
|
+
|
|
1132
|
+
test.each(["hold_verdict", "speech_resumed"] as const)(
|
|
1133
|
+
"a discarded speculative front-door turn leaves host ownership suspended: %s",
|
|
1134
|
+
async (discardReason) => {
|
|
1135
|
+
const calls: VoiceTurnOptions[] = [];
|
|
1136
|
+
const discards: Array<ReturnType<typeof mock>> = [];
|
|
1137
|
+
const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
|
|
1138
|
+
calls.push(options);
|
|
1139
|
+
const discard = mock(async () => {});
|
|
1140
|
+
discards.push(discard);
|
|
1141
|
+
if (
|
|
1142
|
+
discardReason === "hold_verdict" &&
|
|
1143
|
+
options.unifiedVerdict === true
|
|
1144
|
+
) {
|
|
1145
|
+
setTimeout(() => {
|
|
1146
|
+
options.callbacks?.assistant_text_delta?.(makeTextDelta("[0]"));
|
|
1147
|
+
options.callbacks?.message_complete?.(makeMessageComplete());
|
|
1148
|
+
}, 0);
|
|
1149
|
+
}
|
|
1150
|
+
return {
|
|
1151
|
+
turnId: `bridge-turn-${calls.length}`,
|
|
1152
|
+
abort: mock(),
|
|
1153
|
+
discard,
|
|
1154
|
+
};
|
|
1155
|
+
});
|
|
1156
|
+
const spawnBackgroundContinuation = mock(async () => "");
|
|
1157
|
+
const { session, transcribers } = createHarness({
|
|
1158
|
+
finals: ["change the title", "actually just change it"],
|
|
1159
|
+
startVoiceTurn,
|
|
1160
|
+
streamTtsAudio: mock(async (tts: LiveVoiceTtsOptions) => {
|
|
1161
|
+
tts.onAudioChunk(makeTtsChunk("assistant audio"));
|
|
1162
|
+
return makeTtsResult("assistant audio");
|
|
1163
|
+
}),
|
|
1164
|
+
spawnBackgroundContinuation,
|
|
1165
|
+
holdStopEventsFor: [1],
|
|
1166
|
+
turnDetectorConfig: { silenceThresholdMs: 120 },
|
|
1167
|
+
foregroundTaskResumeSilenceMs: 60_000,
|
|
1168
|
+
});
|
|
1169
|
+
|
|
1170
|
+
await session.start();
|
|
1171
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1172
|
+
await waitFor(() => calls.length === 1);
|
|
1173
|
+
calls[0]?.callbacks?.assistant_text_delta?.(
|
|
1174
|
+
makeTextDelta("[1] One moment."),
|
|
1175
|
+
);
|
|
1176
|
+
calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1177
|
+
await waitFor(() => calls.length === 2);
|
|
1178
|
+
calls[1]?.callbacks?.tool_use_start?.("skill_execute", {
|
|
1179
|
+
toolUseId: "tool-1",
|
|
1180
|
+
input: { tool: "computer_use_observe", input: {} },
|
|
1181
|
+
});
|
|
1182
|
+
|
|
1183
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
1184
|
+
await waitFor(() => transcribers.length === 2);
|
|
1185
|
+
transcribers[1]?.emit({ type: "partial", text: "actually" });
|
|
1186
|
+
await waitFor(() => calls.length === 3);
|
|
1187
|
+
expect(calls[2]?.unifiedVerdict).toBe(true);
|
|
1188
|
+
|
|
1189
|
+
if (discardReason === "speech_resumed") {
|
|
1190
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1191
|
+
}
|
|
1192
|
+
await waitFor(() => (discards[2]?.mock.calls.length ?? 0) === 1);
|
|
1193
|
+
|
|
1194
|
+
expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
|
|
1195
|
+
expect(foregroundTaskStateOf(session)).toMatchObject({
|
|
1196
|
+
phase: "suspended",
|
|
1197
|
+
interveningTurns: 0,
|
|
1198
|
+
resumePending: false,
|
|
1199
|
+
});
|
|
1200
|
+
await session.close("client_end");
|
|
1201
|
+
},
|
|
1202
|
+
);
|
|
1203
|
+
|
|
583
1204
|
test("speech while the first tts_audio send is stuck in the queue does not cancel; barge-in works once it lands", async () => {
|
|
584
1205
|
let callbacks: VoiceTurnCallbacks | undefined;
|
|
585
1206
|
const abort = mock();
|
|
@@ -1873,6 +2494,40 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
1873
2494
|
expect(announcement?.voiceControlPrompt).toContain("first question");
|
|
1874
2495
|
});
|
|
1875
2496
|
|
|
2497
|
+
// Deepgram Flux sends interim updates through silence, each an empty
|
|
2498
|
+
// partial. A call nobody is talking on is still idle.
|
|
2499
|
+
test("an empty partial on an idle call does not hold the announcement back", async () => {
|
|
2500
|
+
const continuation = makeControlledContinuation();
|
|
2501
|
+
const { startVoiceTurn, calls } = makeResurfaceTurnStarter();
|
|
2502
|
+
const { frames, session, transcribers } = createHarness({
|
|
2503
|
+
finals: ["first question", ""],
|
|
2504
|
+
startVoiceTurn,
|
|
2505
|
+
streamTtsAudio: makeImmediateTts(),
|
|
2506
|
+
spawnBackgroundContinuation: continuation.spawnBackgroundContinuation,
|
|
2507
|
+
continuationAnnounceSilenceMs: 20,
|
|
2508
|
+
});
|
|
2509
|
+
|
|
2510
|
+
await session.start();
|
|
2511
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2512
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
2513
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
2514
|
+
await waitFor(
|
|
2515
|
+
() => continuation.spawnBackgroundContinuation.mock.calls.length === 1,
|
|
2516
|
+
);
|
|
2517
|
+
await waitFor(() =>
|
|
2518
|
+
frames.some((frame) => frame.type === "utterance_discarded"),
|
|
2519
|
+
);
|
|
2520
|
+
await waitFor(() => transcribers.some((t) => !t.stopped));
|
|
2521
|
+
for (const transcriber of transcribers) {
|
|
2522
|
+
if (!transcriber.stopped) {
|
|
2523
|
+
transcriber.emit({ type: "partial", text: "" });
|
|
2524
|
+
}
|
|
2525
|
+
}
|
|
2526
|
+
|
|
2527
|
+
continuation.finish("THE_RESULT");
|
|
2528
|
+
await waitFor(() => announcementOf(calls) !== undefined);
|
|
2529
|
+
});
|
|
2530
|
+
|
|
1876
2531
|
test("an announcement persists hidden and is never delivered twice", async () => {
|
|
1877
2532
|
const continuation = makeControlledContinuation();
|
|
1878
2533
|
const { startVoiceTurn, calls } = makeResurfaceTurnStarter();
|