@vellumai/assistant 0.12.2-dev.202609172115.0b9be8f → 0.12.2-dev.202609172313.5ad9c4b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/live-voice-task-outcomes.md +83 -0
- package/docs/voice-input-diagnostics.md +46 -0
- package/package.json +1 -1
- package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
- package/src/__tests__/subagent-manager-notify.test.ts +1 -2
- package/src/__tests__/subagent-notify-parent.test.ts +146 -39
- package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
- package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
- package/src/calls/__tests__/voice-session-bridge.test.ts +38 -0
- package/src/calls/voice-control-protocol.ts +6 -0
- package/src/calls/voice-session-bridge.ts +7 -0
- package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +326 -0
- package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
- package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
- package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
- package/src/live-voice/__tests__/live-voice-vad.test.ts +492 -1
- package/src/live-voice/__tests__/subagent-notifications.test.ts +48 -0
- package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
- package/src/live-voice/live-voice-manager.ts +10 -0
- package/src/live-voice/live-voice-session-manager.ts +15 -0
- package/src/live-voice/live-voice-session.ts +456 -49
- package/src/live-voice/live-voice-tts.ts +22 -4
- package/src/live-voice/subagent-notifications.ts +54 -0
- package/src/live-voice/voice-input-diagnostics.ts +117 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
- package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
- package/src/stt/types.ts +4 -0
- package/src/subagent/notify.ts +54 -34
- package/src/subagent/parent-notification.ts +6 -0
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# Background task outcomes in live voice
|
|
2
|
+
|
|
3
|
+
The conversation's active live-voice session owns delivery of subagent updates
|
|
4
|
+
while the call is open. `injectMessageIntoParent` offers updates carrying
|
|
5
|
+
`subagentNotification` metadata to the existing `LiveVoiceSessionManager` before
|
|
6
|
+
enqueuing a generic parent turn. The manager only routes to its session with the
|
|
7
|
+
matching conversation ID. A closing session holds incoming updates for its
|
|
8
|
+
teardown handoff. Other conversations and fully closed calls retain the ordinary
|
|
9
|
+
parent delivery path.
|
|
10
|
+
|
|
11
|
+
The session keeps pending updates per subagent. A newer update from the same
|
|
12
|
+
subagent replaces its pending update; independent subagents keep separate entries.
|
|
13
|
+
Success, failure, abort and mid-run updates all follow this route. Silent/advisor
|
|
14
|
+
instructions remain in the update so the assistant can use findings internally.
|
|
15
|
+
|
|
16
|
+
Announcements use the shared voice floor check: user speech, an active response,
|
|
17
|
+
and queued audio playback delay delivery. A hidden, tool-capable voice turn
|
|
18
|
+
processes the update with current conversation context and produces speech through
|
|
19
|
+
the normal TTS pipeline. The original notification metadata and cron attribution
|
|
20
|
+
travel with the turn. No synthetic user bubble is echoed. The assistant can read
|
|
21
|
+
additional output or resolve a blocker without starting a competing text turn.
|
|
22
|
+
|
|
23
|
+
The assistant decides whether an update adds useful information. It speaks the
|
|
24
|
+
new outcome before optional visual-summary tools, or outputs the internal
|
|
25
|
+
`[TASK_UPDATE:SILENT]` marker for routine progress or findings already heard.
|
|
26
|
+
The marker is stripped from speech, captions, and persisted assistant text; a
|
|
27
|
+
completed silent decision acknowledges the update without audio.
|
|
28
|
+
|
|
29
|
+
Spoken results require forwarded reply audio and remain pending through the
|
|
30
|
+
estimated playback tail. Progress narration alone, empty speech synthesis, and
|
|
31
|
+
failed speech synthesis leave them pending. A barge-in preserves the interrupted outcome without
|
|
32
|
+
creating another worker. A failed launch or an
|
|
33
|
+
explicit stop defers retry until the next user turn or task update. Hang-up returns
|
|
34
|
+
undelivered updates to the ordinary conversation path after pending voice starts
|
|
35
|
+
and the voice turn's teardown settle, since its abort clears the parent queue.
|
|
36
|
+
The handoff bypasses voice routing so a replacement call cannot reclaim it during
|
|
37
|
+
teardown. All parent deliveries enqueue even when idle and trigger the existing
|
|
38
|
+
queue drain after turn finalization, so simultaneous outcomes share its processing
|
|
39
|
+
lock and retry behavior.
|
|
40
|
+
|
|
41
|
+
PCM is framed into at most 100ms chunks, with the first available samples emitted
|
|
42
|
+
immediately. Socket emission runs roughly 500ms ahead of estimated playback,
|
|
43
|
+
keeping prefetched synthesis from flooding the relay queue. Pacing waits stay
|
|
44
|
+
outside the shared outbound-frame queue and abort with the turn, so cancellation
|
|
45
|
+
and control frames are not delayed. Assistant audio recordings contain only
|
|
46
|
+
frames successfully sent to the transport. Hands-free clients reconnect on retryable
|
|
47
|
+
relay closes, including legacy backpressure code 4013.
|
|
48
|
+
|
|
49
|
+
The pending queue is session-local; this does not add a durable notification ledger
|
|
50
|
+
or client playback acknowledgements. Task records and worker output retain their
|
|
51
|
+
existing persistence. Automatic announcements follow the current exchange rather
|
|
52
|
+
than merging task updates into an already-running response.
|
|
53
|
+
|
|
54
|
+
## macOS companion QA
|
|
55
|
+
|
|
56
|
+
Start a fresh conversation from the companion and use a new investigation topic
|
|
57
|
+
so an earlier report cannot satisfy the request. Ask explicitly for a background
|
|
58
|
+
agent. Use ordinary conversational pauses; timed silence is not required.
|
|
59
|
+
Continuous Flux input and the diagnostics described in
|
|
60
|
+
[voice input diagnostics](voice-input-diagnostics.md) apply to these calls.
|
|
61
|
+
|
|
62
|
+
1. **Ordinary completion.** Ask for a small background investigation and wait.
|
|
63
|
+
Confirm the task actually starts, the assistant acknowledges the request, and
|
|
64
|
+
useful findings are spoken without asking for them. A completion-only notice
|
|
65
|
+
or visual summary without an audible explanation is not sufficient.
|
|
66
|
+
2. **Keep talking.** Start another investigation, then ask an unrelated question.
|
|
67
|
+
Include a natural pause partway through the question. The question should stay
|
|
68
|
+
together, the work should survive, and its results should wait for the current
|
|
69
|
+
exchange to finish before being announced.
|
|
70
|
+
3. **Interrupt the announcement.** Speak while findings are being announced.
|
|
71
|
+
The assistant should yield, answer the interruption, and retain the unheard
|
|
72
|
+
result for appropriate delivery. Interrupting an announcement should not spawn
|
|
73
|
+
another copy of the investigation.
|
|
74
|
+
4. **Two tasks.** Request two independent investigations. Keep chatting while
|
|
75
|
+
they run, then leave an opening. Both useful outcomes should arrive without
|
|
76
|
+
overlapping audio, repetitive status announcements, or a disconnect.
|
|
77
|
+
5. **Hang up before delivery.** End the call with work still running or a result
|
|
78
|
+
waiting to be heard. Confirm the outcome reaches the conversation afterward.
|
|
79
|
+
|
|
80
|
+
Record approximate times for requests, interruptions, missing results, and any
|
|
81
|
+
disconnect. Export support logs after the call. Distinguish the request being
|
|
82
|
+
acknowledged, the worker starting, and the result being heard; they exercise
|
|
83
|
+
different parts of the flow.
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# Voice input diagnostics
|
|
2
|
+
|
|
3
|
+
Hands-free voice records signal measurements in the ordinary support log export. This includes calls started from the macOS companion. These diagnostics do not change capture constraints, speech thresholds, interruption timing, or task cancellation. They add no PCM recordings, transcript text, microphone labels, or device identifiers.
|
|
4
|
+
|
|
5
|
+
## Collect a reproduction
|
|
6
|
+
|
|
7
|
+
Use the same microphone, output device, volume, and companion surface as the failing call. Note whether output is through headphones or speakers.
|
|
8
|
+
|
|
9
|
+
1. Start a companion voice call. Stay quiet for about 20 seconds to capture the room baseline.
|
|
10
|
+
2. Ask for an explanation long enough to produce sustained speech. Stay quiet while it answers; note the time of any unexpected cutoff.
|
|
11
|
+
3. Ask it to start a background investigation. Stay quiet during startup; note missing acknowledgements or cancelled starts.
|
|
12
|
+
4. Deliberately interrupt an answer with a short spoken question. This provides a real interruption to compare against the unexplained cutoffs.
|
|
13
|
+
5. End the call and export logs through the existing feedback log export.
|
|
14
|
+
|
|
15
|
+
Keep the initial reproduction on the original audio setup. A second call using headphones, or using the built-in microphone instead of a headset, can isolate a route-dependent problem after the first capture.
|
|
16
|
+
|
|
17
|
+
## Client records
|
|
18
|
+
|
|
19
|
+
`[live-voice-input]` entries are serialized JSON in `electron-main-logs.txt`. Electron collects them from every renderer, so export does not depend on which window hosts the companion call. The same events appear as `voice_input` in the web diagnostics lifecycle ring.
|
|
20
|
+
|
|
21
|
+
- `capture_started` and `input_switched` report applied echo cancellation, noise suppression, gain control, channel count, and track/context/output sample rates. `null` means the browser did not report a setting. An explicit device selection reports whether it matched, without logging either device identifier.
|
|
22
|
+
- `capture_mute`, `capture_unmute`, `capture_ended`, and `capture_statechange` expose track and audio-context transitions. Listeners retire with the old stream on a device switch or stop.
|
|
23
|
+
- `session_ready` links the client `captureId` to the server `sessionId`. Capture can open before the server is ready, so earlier entries have a null session ID. `entry` identifies the launch surface when supplied.
|
|
24
|
+
- `speech_started`, `turn_cancelled`, and `utterance_discarded` report whether playback was active, held, or actually resumed after an unusable utterance.
|
|
25
|
+
|
|
26
|
+
## Server records
|
|
27
|
+
|
|
28
|
+
Search the assistant log for `Live voice input diagnostics` or `voice_input_`. All records include session, input-turn, assistant-turn, and speech-generation identifiers where available, plus the effective threshold, room floor, echo configuration, playback estimate, and foreground-task state.
|
|
29
|
+
|
|
30
|
+
- `voice_input_window` summarizes each approximately five-second arrival window: received audio duration, zero-signal duration, duration-weighted mean amplitude, minimum/maximum chunk mean amplitude, classified speech/silence/echo duration, and maximum arrival gap. Window emission requires incoming audio; `voice_input_closed` flushes a partial window.
|
|
31
|
+
- The same window includes `sttSubmittedChunks`, `sttSubmittedAudioMs`, and `sttMaxSubmissionGapMs`. These measure handoff to the transcription adapter, not provider receipt. A probe can buffer audio before submission, and the final chunk of an arrival window can be submitted in the next window. Compare sustained windows to identify input that arrives normally but stops reaching transcription.
|
|
32
|
+
- `voice_input_provider_turn_start` and `voice_input_provider_turn_end` record provider boundaries alongside local VAD state. End records include confidence, the provider's `trigger` (`model`, `timeout`, or `manual`), decoded audio position in seconds from stream start, and elapsed time since locally detected speech. Missing provider fields are null. These records describe received decisions, including ones ignored as stale or after fallback; correlate with the turn metrics to identify the boundary that committed the request.
|
|
33
|
+
- `voice_input_barge_in_armed`, `voice_input_barge_in_fired`, and `voice_input_barge_in_expired` distinguish a candidate from an accepted or expired interruption. Fired events include accumulated speech duration, guard resets, and a bounded trace of the preceding two seconds (at most 40 input chunks).
|
|
34
|
+
- Each trace entry captures the amplitude and gate/echo state before classification, and classified durations, remaining probe buffer, and correlation result after classification. The classifier can hold a probe and release multiple older chunks together, so a row's classified duration can differ from its received duration. A null correlation means no comparison ran, not a measured zero correlation.
|
|
35
|
+
- `voice_input_speech_routed` binds the speech generation to the input turn that actually receives its audio. At onset, the prior spent cycle may still be installed, so use this routing record for correlation rather than assuming the onset's `inputTurnId` is the new utterance.
|
|
36
|
+
- `voice_input_transcript` reports committed character count and provider, never words. `voice_input_utterance_discarded` reports the discard reason and remaining transcript/partial state.
|
|
37
|
+
|
|
38
|
+
Amplitude uses the classifier's PCM16 mean-absolute scale, not the client's normalized UI meter. Playback timing is the server's estimate, not proof that sound was heard. An empty transcript is evidence of no usable transcription; it does not establish that the person was silent or that the detector was wrong.
|
|
39
|
+
|
|
40
|
+
These are local diagnostic logs, not new analytics events or voice protocol fields. Existing log retention and export apply.
|
|
41
|
+
|
|
42
|
+
## Flux pause handling
|
|
43
|
+
|
|
44
|
+
When Flux owns turn detection, it receives microphone audio through idle periods and pauses, including quiet speech below the local VAD gate. Confirmed playback echo is replaced by silence of the same duration so it advances the provider's audio clock without transcribing the assistant's voice. Echo probes can briefly buffer that audio. Local VAD still owns barge-in; its thresholds and guard are independent of this continuous transcription feed. Idle audio is not accumulated into request recordings.
|
|
45
|
+
|
|
46
|
+
For a focused reproduction, say an unfinished question such as "Did they help invent", pause briefly, then complete it with "ChatGPT". Repeat with a shorter and a longer pause, plus a complete short question. Note when the assistant starts answering and export the logs. Check that STT submissions continue during the pause, then use the provider end event's time, confidence, and trigger to distinguish model endpointing from a timeout. Tests verify audio delivery and event handling; they do not establish how Flux will segment real speech.
|
package/package.json
CHANGED
|
@@ -22,8 +22,7 @@ mock.module("../daemon/conversation-registry.js", () => ({
|
|
|
22
22
|
});
|
|
23
23
|
return { queued: true };
|
|
24
24
|
},
|
|
25
|
-
|
|
26
|
-
runAgentLoop: async () => {},
|
|
25
|
+
kickDrainQueue: async () => {},
|
|
27
26
|
}),
|
|
28
27
|
}));
|
|
29
28
|
|
|
@@ -27,8 +27,7 @@ mock.module("../daemon/conversation-registry.js", () => ({
|
|
|
27
27
|
});
|
|
28
28
|
return { queued: true };
|
|
29
29
|
},
|
|
30
|
-
|
|
31
|
-
runAgentLoop: async () => {},
|
|
30
|
+
kickDrainQueue: async () => {},
|
|
32
31
|
}),
|
|
33
32
|
}));
|
|
34
33
|
|
|
@@ -41,18 +41,12 @@ const capturedMessages: string[] = [];
|
|
|
41
41
|
/** Parent conversation ids that a notification was routed to (findConversation). */
|
|
42
42
|
const capturedParentIds: string[] = [];
|
|
43
43
|
|
|
44
|
-
/**
|
|
45
|
-
* Cron attribution seen by each delivery path. The queued path hands it to
|
|
46
|
-
* `enqueueMessage` (the drain applies it later); the immediate path hands it
|
|
47
|
-
* straight to `runAgentLoop`.
|
|
48
|
-
*/
|
|
49
44
|
const capturedEnqueueCronRunIds: (string | null | undefined)[] = [];
|
|
50
|
-
const
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
*/
|
|
45
|
+
const capturedQueueOptions: {
|
|
46
|
+
queueWhenIdle: boolean;
|
|
47
|
+
metadata?: Record<string, unknown>;
|
|
48
|
+
}[] = [];
|
|
49
|
+
const drainedParents: string[] = [];
|
|
56
50
|
let parentAcceptsEnqueue = true;
|
|
57
51
|
|
|
58
52
|
// Live subagent conversations, keyed by conversationId. notifyParentFromChild
|
|
@@ -74,19 +68,17 @@ mock.module("../daemon/conversation-registry.js", () => ({
|
|
|
74
68
|
hasInFlightWork: () => false,
|
|
75
69
|
enqueueMessage: (options: {
|
|
76
70
|
content: string;
|
|
71
|
+
queueWhenIdle: boolean;
|
|
72
|
+
metadata?: Record<string, unknown>;
|
|
77
73
|
cronRunId?: string | null;
|
|
78
74
|
}) => {
|
|
79
75
|
capturedMessages.push(options.content);
|
|
76
|
+
capturedQueueOptions.push(options);
|
|
80
77
|
capturedEnqueueCronRunIds.push(options.cronRunId);
|
|
81
78
|
return { queued: parentAcceptsEnqueue };
|
|
82
79
|
},
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
_message: string,
|
|
86
|
-
_messageId: string,
|
|
87
|
-
options?: { cronRunId?: string | null },
|
|
88
|
-
) => {
|
|
89
|
-
capturedLoopCronRunIds.push(options?.cronRunId);
|
|
80
|
+
kickDrainQueue: async () => {
|
|
81
|
+
drainedParents.push(id);
|
|
90
82
|
},
|
|
91
83
|
};
|
|
92
84
|
},
|
|
@@ -111,11 +103,15 @@ mock.module("../runtime/assistant-event-hub.js", () => ({
|
|
|
111
103
|
|
|
112
104
|
import type { Conversation } from "../daemon/conversation.js";
|
|
113
105
|
import { isToolActiveForContext } from "../daemon/conversation-tool-setup.js";
|
|
106
|
+
import { beginTurnFinalization } from "../daemon/turn-finalization.js";
|
|
107
|
+
import { setLiveVoiceSessionManagerForTesting } from "../live-voice/live-voice-manager.js";
|
|
108
|
+
import { LiveVoiceSessionManager } from "../live-voice/live-voice-session-manager.js";
|
|
114
109
|
import type { SubagentRecord } from "../persistence/subagent-store.js";
|
|
115
110
|
import {
|
|
116
111
|
injectMessageIntoParent,
|
|
117
112
|
notifyParentFromChild,
|
|
118
113
|
} from "../subagent/notify.js";
|
|
114
|
+
import type { SubagentParentNotification } from "../subagent/parent-notification.js";
|
|
119
115
|
import {
|
|
120
116
|
executeSubagentNotifyParent,
|
|
121
117
|
notifyParentTool,
|
|
@@ -183,12 +179,108 @@ function lastCapturedMessage(): string {
|
|
|
183
179
|
function clearCaptured(): void {
|
|
184
180
|
capturedMessages.length = 0;
|
|
185
181
|
capturedEnqueueCronRunIds.length = 0;
|
|
186
|
-
|
|
182
|
+
capturedQueueOptions.length = 0;
|
|
183
|
+
drainedParents.length = 0;
|
|
187
184
|
parentAcceptsEnqueue = true;
|
|
188
185
|
}
|
|
189
186
|
|
|
190
187
|
// ── Tool definition ────────────────────────────────────────────────
|
|
191
188
|
|
|
189
|
+
describe("voice parent notification routing", () => {
|
|
190
|
+
test("only the matching active call claims task updates, without starting a generic parent turn", async () => {
|
|
191
|
+
clearCaptured();
|
|
192
|
+
const received: SubagentParentNotification[] = [];
|
|
193
|
+
let finishClose: (() => void) | undefined;
|
|
194
|
+
const manager = new LiveVoiceSessionManager({
|
|
195
|
+
createSession: (context) => ({
|
|
196
|
+
start: async () => {
|
|
197
|
+
await context.sendFrame({
|
|
198
|
+
type: "ready",
|
|
199
|
+
sessionId: context.sessionId,
|
|
200
|
+
conversationId: "parent-voice",
|
|
201
|
+
});
|
|
202
|
+
},
|
|
203
|
+
handleClientFrame: () => {},
|
|
204
|
+
handleBinaryAudio: () => {},
|
|
205
|
+
close: () =>
|
|
206
|
+
new Promise<void>((resolve) => {
|
|
207
|
+
finishClose = resolve;
|
|
208
|
+
}),
|
|
209
|
+
receiveSubagentNotification: (notification) => {
|
|
210
|
+
received.push(notification);
|
|
211
|
+
return true;
|
|
212
|
+
},
|
|
213
|
+
}),
|
|
214
|
+
});
|
|
215
|
+
setLiveVoiceSessionManagerForTesting(manager);
|
|
216
|
+
const metadata = {
|
|
217
|
+
subagentNotification: {
|
|
218
|
+
subagentId: "task-1",
|
|
219
|
+
status: "completed",
|
|
220
|
+
conversationId: "child-1",
|
|
221
|
+
},
|
|
222
|
+
};
|
|
223
|
+
try {
|
|
224
|
+
await manager.startSession(
|
|
225
|
+
{
|
|
226
|
+
type: "start",
|
|
227
|
+
audio: { mimeType: "audio/pcm", sampleRate: 24_000, channels: 1 },
|
|
228
|
+
},
|
|
229
|
+
{ sendFrame: () => {} },
|
|
230
|
+
);
|
|
231
|
+
injectMessageIntoParent("parent-voice", "Task completed", metadata, {
|
|
232
|
+
cronRunId: "run-123",
|
|
233
|
+
});
|
|
234
|
+
expect(received).toEqual([
|
|
235
|
+
{
|
|
236
|
+
taskId: "child-1",
|
|
237
|
+
message: "Task completed",
|
|
238
|
+
metadata,
|
|
239
|
+
cronRunId: "run-123",
|
|
240
|
+
},
|
|
241
|
+
]);
|
|
242
|
+
expect(capturedMessages).toEqual([]);
|
|
243
|
+
expect(drainedParents).toEqual([]);
|
|
244
|
+
|
|
245
|
+
injectMessageIntoParent("parent-other", "Other task completed", metadata);
|
|
246
|
+
injectMessageIntoParent("parent-voice", "Generic event");
|
|
247
|
+
injectMessageIntoParent("parent-voice", "Hang-up fallback", metadata, {
|
|
248
|
+
bypassLiveVoice: true,
|
|
249
|
+
cronRunId: "run-123",
|
|
250
|
+
});
|
|
251
|
+
expect(received).toHaveLength(1);
|
|
252
|
+
expect(capturedMessages).toEqual([
|
|
253
|
+
"Other task completed",
|
|
254
|
+
"Generic event",
|
|
255
|
+
"Hang-up fallback",
|
|
256
|
+
]);
|
|
257
|
+
expect(capturedEnqueueCronRunIds.at(-1)).toBe("run-123");
|
|
258
|
+
|
|
259
|
+
const closing = manager.endActiveSession("manager_shutdown");
|
|
260
|
+
injectMessageIntoParent(
|
|
261
|
+
"parent-voice",
|
|
262
|
+
"Finished during hang-up",
|
|
263
|
+
metadata,
|
|
264
|
+
);
|
|
265
|
+
expect(received.at(-1)?.message).toBe("Finished during hang-up");
|
|
266
|
+
expect(capturedMessages).not.toContain("Finished during hang-up");
|
|
267
|
+
finishClose?.();
|
|
268
|
+
await closing;
|
|
269
|
+
injectMessageIntoParent(
|
|
270
|
+
"parent-voice",
|
|
271
|
+
"Finished after hang-up",
|
|
272
|
+
metadata,
|
|
273
|
+
);
|
|
274
|
+
expect(capturedMessages.at(-1)).toBe("Finished after hang-up");
|
|
275
|
+
} finally {
|
|
276
|
+
finishClose?.();
|
|
277
|
+
await manager.endActiveSession("manager_shutdown");
|
|
278
|
+
setLiveVoiceSessionManagerForTesting(null);
|
|
279
|
+
clearCaptured();
|
|
280
|
+
}
|
|
281
|
+
});
|
|
282
|
+
});
|
|
283
|
+
|
|
192
284
|
describe("notify_parent tool definition", () => {
|
|
193
285
|
test("has correct core tool definition", () => {
|
|
194
286
|
const def = notifyParentTool;
|
|
@@ -443,31 +535,46 @@ describe("injectMessageIntoParent cron attribution", () => {
|
|
|
443
535
|
expect(capturedEnqueueCronRunIds).toEqual(["cron-run-7"]);
|
|
444
536
|
});
|
|
445
537
|
|
|
446
|
-
test("
|
|
538
|
+
test("queues simultaneous idle-parent deliveries until turn finalization", async () => {
|
|
447
539
|
clearCaptured();
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
540
|
+
const finalize = beginTurnFinalization("parent-handoff");
|
|
541
|
+
try {
|
|
542
|
+
injectMessageIntoParent("parent-handoff", "first result", undefined, {
|
|
543
|
+
bypassLiveVoice: true,
|
|
544
|
+
});
|
|
545
|
+
injectMessageIntoParent("parent-handoff", "second result", undefined, {
|
|
546
|
+
bypassLiveVoice: true,
|
|
547
|
+
});
|
|
548
|
+
expect(capturedMessages).toEqual(["first result", "second result"]);
|
|
549
|
+
expect(capturedQueueOptions).toEqual([
|
|
550
|
+
expect.objectContaining({
|
|
551
|
+
queueWhenIdle: true,
|
|
552
|
+
metadata: { scripted: true },
|
|
553
|
+
}),
|
|
554
|
+
expect.objectContaining({
|
|
555
|
+
queueWhenIdle: true,
|
|
556
|
+
metadata: { scripted: true },
|
|
557
|
+
}),
|
|
558
|
+
]);
|
|
559
|
+
expect(drainedParents).toEqual([]);
|
|
560
|
+
} finally {
|
|
561
|
+
finalize();
|
|
562
|
+
}
|
|
563
|
+
await Promise.resolve();
|
|
564
|
+
expect(drainedParents).toEqual(["parent-handoff", "parent-handoff"]);
|
|
458
565
|
});
|
|
459
566
|
|
|
460
|
-
test("an
|
|
567
|
+
test("does not start an unqueued turn when the parent queue rejects delivery", () => {
|
|
461
568
|
clearCaptured();
|
|
462
|
-
|
|
463
|
-
injectMessageIntoParent("parent-plain", "notification");
|
|
464
569
|
parentAcceptsEnqueue = false;
|
|
465
|
-
injectMessageIntoParent("parent-
|
|
466
|
-
|
|
570
|
+
injectMessageIntoParent("parent-full", "notification");
|
|
571
|
+
expect(drainedParents).toEqual([]);
|
|
572
|
+
});
|
|
467
573
|
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
expect(
|
|
574
|
+
test("an unscheduled notification carries no run id", () => {
|
|
575
|
+
clearCaptured();
|
|
576
|
+
injectMessageIntoParent("parent-plain", "notification");
|
|
577
|
+
expect(capturedEnqueueCronRunIds).toEqual([undefined]);
|
|
578
|
+
expect(drainedParents).toEqual(["parent-plain"]);
|
|
472
579
|
});
|
|
473
580
|
});
|
|
@@ -22,8 +22,7 @@ mock.module("../daemon/conversation-registry.js", () => ({
|
|
|
22
22
|
delivered.push(options.content);
|
|
23
23
|
return { queued: true };
|
|
24
24
|
},
|
|
25
|
-
|
|
26
|
-
runAgentLoop: async () => {},
|
|
25
|
+
kickDrainQueue: async () => {},
|
|
27
26
|
};
|
|
28
27
|
},
|
|
29
28
|
}));
|
|
@@ -36,8 +35,7 @@ mock.module("../daemon/conversation-store.js", () => ({
|
|
|
36
35
|
delivered.push(`rebuilt:${options.content}`);
|
|
37
36
|
return { queued: true };
|
|
38
37
|
},
|
|
39
|
-
|
|
40
|
-
runAgentLoop: async () => {},
|
|
38
|
+
kickDrainQueue: async () => {},
|
|
41
39
|
};
|
|
42
40
|
},
|
|
43
41
|
}));
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
type SessionControlRequest,
|
|
11
11
|
stripInternalSpeechMarkers,
|
|
12
12
|
TASK_STOP_MARKER,
|
|
13
|
+
TASK_UPDATE_SILENT_MARKER,
|
|
13
14
|
terminalControlMarkerLength,
|
|
14
15
|
} from "../voice-control-protocol.js";
|
|
15
16
|
|
|
@@ -227,3 +228,17 @@ describe("session control markers", () => {
|
|
|
227
228
|
expect(terminalControlMarkerLength("The array [-1] sorts")).toBe(0);
|
|
228
229
|
});
|
|
229
230
|
});
|
|
231
|
+
|
|
232
|
+
test("silent task-update marker is held across every split and removed from the transcript", () => {
|
|
233
|
+
for (let split = 1; split < TASK_UPDATE_SILENT_MARKER.length; split += 1) {
|
|
234
|
+
const emitted: string[] = [];
|
|
235
|
+
const flush = createControlMarkerHoldback((text) => emitted.push(text));
|
|
236
|
+
flush(TASK_UPDATE_SILENT_MARKER.slice(0, split));
|
|
237
|
+
flush(TASK_UPDATE_SILENT_MARKER, { force: true });
|
|
238
|
+
expect(emitted.join("")).toBe("");
|
|
239
|
+
}
|
|
240
|
+
expect(terminalControlMarkerLength(TASK_UPDATE_SILENT_MARKER)).toBe(
|
|
241
|
+
TASK_UPDATE_SILENT_MARKER.length,
|
|
242
|
+
);
|
|
243
|
+
expect(parseTerminalSessionControl(TASK_UPDATE_SILENT_MARKER)).toBeNull();
|
|
244
|
+
});
|
|
@@ -724,6 +724,44 @@ describe("startVoiceTurn camera-frame attachments", () => {
|
|
|
724
724
|
});
|
|
725
725
|
|
|
726
726
|
describe("startVoiceTurn hiddenSyntheticPrompt", () => {
|
|
727
|
+
test("task announcements retain metadata and cron attribution without a user echo", async () => {
|
|
728
|
+
const fake = makeFakeConversation({ processing: false });
|
|
729
|
+
fakeConversation = fake.conversation;
|
|
730
|
+
const loopOptions: unknown[] = [];
|
|
731
|
+
fake.conversation.runAgentLoop = async (...args: unknown[]) => {
|
|
732
|
+
loopOptions.push(args[2]);
|
|
733
|
+
};
|
|
734
|
+
const metadata = {
|
|
735
|
+
subagentNotification: {
|
|
736
|
+
subagentId: "task-1",
|
|
737
|
+
label: "Compare options",
|
|
738
|
+
status: "completed",
|
|
739
|
+
},
|
|
740
|
+
};
|
|
741
|
+
const echoes = await collectUserMessageEchoes(async () => {
|
|
742
|
+
await startVoiceTurn({
|
|
743
|
+
...makeTurnOptions(),
|
|
744
|
+
content: "Comparison completed",
|
|
745
|
+
hiddenSyntheticPrompt: true,
|
|
746
|
+
subagentNotification: {
|
|
747
|
+
taskId: "task-1",
|
|
748
|
+
message: "Comparison completed",
|
|
749
|
+
metadata,
|
|
750
|
+
cronRunId: "run-123",
|
|
751
|
+
},
|
|
752
|
+
});
|
|
753
|
+
});
|
|
754
|
+
expect(fake.lastPersistOpts()?.metadata).toMatchObject({
|
|
755
|
+
...metadata,
|
|
756
|
+
hidden: true,
|
|
757
|
+
scripted: true,
|
|
758
|
+
voiceSessionTurn: true,
|
|
759
|
+
});
|
|
760
|
+
expect(loopOptions).toEqual([
|
|
761
|
+
expect.objectContaining({ cronRunId: "run-123" }),
|
|
762
|
+
]);
|
|
763
|
+
expect(echoes).toEqual([]);
|
|
764
|
+
});
|
|
727
765
|
// A caller whose internal instruction is composed per call carries no
|
|
728
766
|
// sentinel for the content comparisons to recognize, so it declares itself.
|
|
729
767
|
const SYNTHETIC_CONTENT =
|
|
@@ -14,6 +14,7 @@ export const CALL_OPENING_ACK_MARKER = "[CALL_OPENING_ACK]";
|
|
|
14
14
|
export const CALL_VERIFICATION_COMPLETE_MARKER = "[CALL_VERIFICATION_COMPLETE]";
|
|
15
15
|
export const END_CALL_MARKER = "[END_CALL]";
|
|
16
16
|
export const TASK_STOP_MARKER = "[TASK:STOP]";
|
|
17
|
+
export const TASK_UPDATE_SILENT_MARKER = "[TASK_UPDATE:SILENT]";
|
|
17
18
|
|
|
18
19
|
/**
|
|
19
20
|
* Verdict tokens for the fast "front-door" model (triage-and-escalate voice
|
|
@@ -220,6 +221,7 @@ export function stripInternalSpeechMarkers(text: string): string {
|
|
|
220
221
|
.replace(CALL_OPENING_ACK_MARKER_REGEX, "")
|
|
221
222
|
.replace(END_CALL_MARKER_REGEX, "")
|
|
222
223
|
.replace(TASK_STOP_MARKER_REGEX, "")
|
|
224
|
+
.replaceAll(TASK_UPDATE_SILENT_MARKER, "")
|
|
223
225
|
.replace(HOLD_VERDICT_TOKEN_REGEX, "")
|
|
224
226
|
.replace(ESCALATE_VERDICT_TOKEN_REGEX, "")
|
|
225
227
|
.replace(MINIMIZE_ROOM_MARKER_REGEX, "")
|
|
@@ -249,6 +251,7 @@ const CONTROL_MARKER_STRINGS = [
|
|
|
249
251
|
"[CALL_OPENING_ACK]",
|
|
250
252
|
"[END_CALL]",
|
|
251
253
|
TASK_STOP_MARKER,
|
|
254
|
+
TASK_UPDATE_SILENT_MARKER,
|
|
252
255
|
"[0]",
|
|
253
256
|
"[1]",
|
|
254
257
|
"[-1]",
|
|
@@ -435,6 +438,9 @@ export function parseTerminalSessionControl(
|
|
|
435
438
|
*/
|
|
436
439
|
export function terminalControlMarkerLength(text: string): number {
|
|
437
440
|
const trimmed = text.trimEnd();
|
|
441
|
+
if (trimmed.endsWith(TASK_UPDATE_SILENT_MARKER)) {
|
|
442
|
+
return TASK_UPDATE_SILENT_MARKER.length;
|
|
443
|
+
}
|
|
438
444
|
if (trimmed.endsWith(MINIMIZE_ROOM_MARKER)) {
|
|
439
445
|
return MINIMIZE_ROOM_MARKER.length;
|
|
440
446
|
}
|
|
@@ -57,6 +57,7 @@ import { getCurrentSeq } from "../runtime/assistant-stream-state.js";
|
|
|
57
57
|
import { publishConversationMessagesChanged } from "../runtime/sync/resource-sync-events.js";
|
|
58
58
|
import { computeToolApprovalDigest } from "../security/tool-approval-digest.js";
|
|
59
59
|
import { sttCatalogKeyForRole } from "../stt/roles.js";
|
|
60
|
+
import type { SubagentParentNotification } from "../subagent/parent-notification.js";
|
|
60
61
|
import { getAllTools } from "../tools/registry.js";
|
|
61
62
|
import { sensitiveToolReach } from "../tools/tool-approval-handler.js";
|
|
62
63
|
import { createAbortReason } from "../util/abort-reasons.js";
|
|
@@ -389,6 +390,8 @@ export interface VoiceTurnCallbacks {
|
|
|
389
390
|
}
|
|
390
391
|
|
|
391
392
|
export interface VoiceTurnOptions {
|
|
393
|
+
/** Internal task update delivered through the call, with its original attribution. */
|
|
394
|
+
subagentNotification?: SubagentParentNotification;
|
|
392
395
|
/** The conversation ID for this voice call's session. */
|
|
393
396
|
conversationId: string;
|
|
394
397
|
/** Voice session ID for scoped grant matching. Defaults to callSessionId. */
|
|
@@ -1219,6 +1222,7 @@ export async function startVoiceTurn(
|
|
|
1219
1222
|
...(turnAttachments.length > 0 ? { attachments: turnAttachments } : {}),
|
|
1220
1223
|
requestId,
|
|
1221
1224
|
metadata: {
|
|
1225
|
+
...opts.subagentNotification?.metadata,
|
|
1222
1226
|
// Durable "this turn came from an open voice session" marker; see
|
|
1223
1227
|
// `isVoiceSessionUserMessage` for why the channel fields cannot carry it.
|
|
1224
1228
|
voiceSessionTurn: true,
|
|
@@ -2056,6 +2060,9 @@ export async function startVoiceTurn(
|
|
|
2056
2060
|
);
|
|
2057
2061
|
}
|
|
2058
2062
|
await conversation.runAgentLoop(persistedContent, messageId, {
|
|
2063
|
+
...(opts.subagentNotification?.cronRunId
|
|
2064
|
+
? { cronRunId: opts.subagentNotification.cronRunId }
|
|
2065
|
+
: {}),
|
|
2059
2066
|
onEvent: (msg: AssistantEvent) => {
|
|
2060
2067
|
if (msg.type === "assistant_turn_start") {
|
|
2061
2068
|
reservedAssistantRowId = msg.messageId;
|