@vibesdotdev/agent-execution 0.0.0-stage → 1.0.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +18 -0
- package/README.md +53 -2
- package/dist/code-session/agent-code-session.consumer.descriptor.d.ts +68 -0
- package/dist/code-session/agent-code-session.consumer.descriptor.d.ts.map +1 -0
- package/dist/code-session/agent-code-session.consumer.descriptor.js +31 -0
- package/dist/code-session/agent-code-session.consumer.descriptor.js.map +1 -0
- package/dist/code-session/agent-code-session.consumer.plugin.d.ts +18 -0
- package/dist/code-session/agent-code-session.consumer.plugin.d.ts.map +1 -0
- package/dist/code-session/agent-code-session.consumer.plugin.js +47 -0
- package/dist/code-session/agent-code-session.consumer.plugin.js.map +1 -0
- package/dist/code-session/agent-code-session.descriptor.d.ts +163 -0
- package/dist/code-session/agent-code-session.descriptor.d.ts.map +1 -0
- package/dist/code-session/agent-code-session.descriptor.js +129 -0
- package/dist/code-session/agent-code-session.descriptor.js.map +1 -0
- package/dist/code-session/agent-code-session.impl.consumer.d.ts +224 -0
- package/dist/code-session/agent-code-session.impl.consumer.d.ts.map +1 -0
- package/dist/code-session/agent-code-session.impl.consumer.js +2131 -0
- package/dist/code-session/agent-code-session.impl.consumer.js.map +1 -0
- package/dist/code-session/agent-code-session.kind.d.ts +15 -0
- package/dist/code-session/agent-code-session.kind.d.ts.map +1 -0
- package/dist/code-session/agent-code-session.kind.js +17 -0
- package/dist/code-session/agent-code-session.kind.js.map +1 -0
- package/dist/code-session/agent-code-session.plugin.d.ts +17 -0
- package/dist/code-session/agent-code-session.plugin.d.ts.map +1 -0
- package/dist/code-session/agent-code-session.plugin.js +28 -0
- package/dist/code-session/agent-code-session.plugin.js.map +1 -0
- package/dist/code-session/billing/gates.d.ts +52 -0
- package/dist/code-session/billing/gates.d.ts.map +1 -0
- package/dist/code-session/billing/gates.js +75 -0
- package/dist/code-session/billing/gates.js.map +1 -0
- package/dist/code-session/client/index.d.ts +2 -0
- package/dist/code-session/client/index.d.ts.map +1 -0
- package/dist/code-session/client/index.js +2 -0
- package/dist/code-session/client/index.js.map +1 -0
- package/dist/code-session/client/resolve.d.ts +20 -0
- package/dist/code-session/client/resolve.d.ts.map +1 -0
- package/dist/code-session/client/resolve.js +145 -0
- package/dist/code-session/client/resolve.js.map +1 -0
- package/dist/code-session/docs/agent-code-session.docs.descriptor.d.ts +4 -0
- package/dist/code-session/docs/agent-code-session.docs.descriptor.d.ts.map +1 -0
- package/dist/code-session/docs/agent-code-session.docs.descriptor.js +109 -0
- package/dist/code-session/docs/agent-code-session.docs.descriptor.js.map +1 -0
- package/dist/code-session/git-credential.d.ts +87 -0
- package/dist/code-session/git-credential.d.ts.map +1 -0
- package/dist/code-session/git-credential.js +107 -0
- package/dist/code-session/git-credential.js.map +1 -0
- package/dist/code-session/host-target.d.ts +76 -0
- package/dist/code-session/host-target.d.ts.map +1 -0
- package/dist/code-session/host-target.js +81 -0
- package/dist/code-session/host-target.js.map +1 -0
- package/dist/code-session/index.d.ts +20 -0
- package/dist/code-session/index.d.ts.map +1 -0
- package/dist/code-session/index.js +20 -0
- package/dist/code-session/index.js.map +1 -0
- package/dist/code-session/interfaces/code-session.d.ts +448 -0
- package/dist/code-session/interfaces/code-session.d.ts.map +1 -0
- package/dist/code-session/interfaces/code-session.js +373 -0
- package/dist/code-session/interfaces/code-session.js.map +1 -0
- package/dist/code-session/operation-lease.d.ts +48 -0
- package/dist/code-session/operation-lease.d.ts.map +1 -0
- package/dist/code-session/operation-lease.js +73 -0
- package/dist/code-session/operation-lease.js.map +1 -0
- package/dist/code-session/remote-setup-custody.d.ts +59 -0
- package/dist/code-session/remote-setup-custody.d.ts.map +1 -0
- package/dist/code-session/remote-setup-custody.js +78 -0
- package/dist/code-session/remote-setup-custody.js.map +1 -0
- package/dist/code-session/remote-workspace-fs.d.ts +119 -0
- package/dist/code-session/remote-workspace-fs.d.ts.map +1 -0
- package/dist/code-session/remote-workspace-fs.js +257 -0
- package/dist/code-session/remote-workspace-fs.js.map +1 -0
- package/dist/code-session/session-metadata.d.ts +40 -0
- package/dist/code-session/session-metadata.d.ts.map +1 -0
- package/dist/code-session/session-metadata.js +69 -0
- package/dist/code-session/session-metadata.js.map +1 -0
- package/dist/code-session-relay/agent-code-session-relay.descriptor.d.ts +78 -0
- package/dist/code-session-relay/agent-code-session-relay.descriptor.d.ts.map +1 -0
- package/dist/code-session-relay/agent-code-session-relay.descriptor.js +38 -0
- package/dist/code-session-relay/agent-code-session-relay.descriptor.js.map +1 -0
- package/dist/code-session-relay/agent-code-session-relay.impl.d.ts +120 -0
- package/dist/code-session-relay/agent-code-session-relay.impl.d.ts.map +1 -0
- package/dist/code-session-relay/agent-code-session-relay.impl.js +1392 -0
- package/dist/code-session-relay/agent-code-session-relay.impl.js.map +1 -0
- package/dist/code-session-relay/agent-code-session-relay.plugin.d.ts +14 -0
- package/dist/code-session-relay/agent-code-session-relay.plugin.d.ts.map +1 -0
- package/dist/code-session-relay/agent-code-session-relay.plugin.js +25 -0
- package/dist/code-session-relay/agent-code-session-relay.plugin.js.map +1 -0
- package/dist/code-session-relay/index.d.ts +13 -0
- package/dist/code-session-relay/index.d.ts.map +1 -0
- package/dist/code-session-relay/index.js +13 -0
- package/dist/code-session-relay/index.js.map +1 -0
- package/dist/code-session-relay/relay-dial.d.ts +72 -0
- package/dist/code-session-relay/relay-dial.d.ts.map +1 -0
- package/dist/code-session-relay/relay-dial.js +177 -0
- package/dist/code-session-relay/relay-dial.js.map +1 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +10 -0
- package/dist/index.js.map +1 -0
- package/dist/sandbox/agent-sandbox.consumer.descriptor.d.ts +35 -0
- package/dist/sandbox/agent-sandbox.consumer.descriptor.d.ts.map +1 -0
- package/dist/sandbox/agent-sandbox.consumer.descriptor.js +19 -0
- package/dist/sandbox/agent-sandbox.consumer.descriptor.js.map +1 -0
- package/dist/sandbox/agent-sandbox.consumer.plugin.d.ts +9 -0
- package/dist/sandbox/agent-sandbox.consumer.plugin.d.ts.map +1 -0
- package/dist/sandbox/agent-sandbox.consumer.plugin.js +33 -0
- package/dist/sandbox/agent-sandbox.consumer.plugin.js.map +1 -0
- package/dist/sandbox/agent-sandbox.descriptor.d.ts +76 -0
- package/dist/sandbox/agent-sandbox.descriptor.d.ts.map +1 -0
- package/dist/sandbox/agent-sandbox.descriptor.js +57 -0
- package/dist/sandbox/agent-sandbox.descriptor.js.map +1 -0
- package/dist/sandbox/agent-sandbox.impl.consumer.d.ts +15 -0
- package/dist/sandbox/agent-sandbox.impl.consumer.d.ts.map +1 -0
- package/dist/sandbox/agent-sandbox.impl.consumer.js +23 -0
- package/dist/sandbox/agent-sandbox.impl.consumer.js.map +1 -0
- package/dist/sandbox/agent-sandbox.kind.d.ts +14 -0
- package/dist/sandbox/agent-sandbox.kind.d.ts.map +1 -0
- package/dist/sandbox/agent-sandbox.kind.js +16 -0
- package/dist/sandbox/agent-sandbox.kind.js.map +1 -0
- package/dist/sandbox/agent-sandbox.plugin.d.ts +15 -0
- package/dist/sandbox/agent-sandbox.plugin.d.ts.map +1 -0
- package/dist/sandbox/agent-sandbox.plugin.js +25 -0
- package/dist/sandbox/agent-sandbox.plugin.js.map +1 -0
- package/dist/sandbox/client/resolve.d.ts +21 -0
- package/dist/sandbox/client/resolve.d.ts.map +1 -0
- package/dist/sandbox/client/resolve.js +103 -0
- package/dist/sandbox/client/resolve.js.map +1 -0
- package/dist/sandbox/docs/agent-sandbox.docs.descriptor.d.ts +4 -0
- package/dist/sandbox/docs/agent-sandbox.docs.descriptor.d.ts.map +1 -0
- package/dist/sandbox/docs/agent-sandbox.docs.descriptor.js +75 -0
- package/dist/sandbox/docs/agent-sandbox.docs.descriptor.js.map +1 -0
- package/dist/sandbox/index.d.ts +15 -0
- package/dist/sandbox/index.d.ts.map +1 -0
- package/dist/sandbox/index.js +14 -0
- package/dist/sandbox/index.js.map +1 -0
- package/dist/sandbox/interfaces/sandbox.d.ts +61 -0
- package/dist/sandbox/interfaces/sandbox.d.ts.map +1 -0
- package/dist/sandbox/interfaces/sandbox.js +54 -0
- package/dist/sandbox/interfaces/sandbox.js.map +1 -0
- package/dist/schemas/code-session-resolution.d.ts +20 -0
- package/dist/schemas/code-session-resolution.d.ts.map +1 -0
- package/dist/schemas/code-session-resolution.js +11 -0
- package/dist/schemas/code-session-resolution.js.map +1 -0
- package/dist/schemas/session-operation.d.ts +73 -0
- package/dist/schemas/session-operation.d.ts.map +1 -0
- package/dist/schemas/session-operation.js +47 -0
- package/dist/schemas/session-operation.js.map +1 -0
- package/dist/schemas/voice.d.ts +87 -0
- package/dist/schemas/voice.d.ts.map +1 -0
- package/dist/schemas/voice.js +70 -0
- package/dist/schemas/voice.js.map +1 -0
- package/dist/voice/agent-voice.consumer.descriptor.d.ts +43 -0
- package/dist/voice/agent-voice.consumer.descriptor.d.ts.map +1 -0
- package/dist/voice/agent-voice.consumer.descriptor.js +20 -0
- package/dist/voice/agent-voice.consumer.descriptor.js.map +1 -0
- package/dist/voice/agent-voice.consumer.plugin.d.ts +4 -0
- package/dist/voice/agent-voice.consumer.plugin.d.ts.map +1 -0
- package/dist/voice/agent-voice.consumer.plugin.js +13 -0
- package/dist/voice/agent-voice.consumer.plugin.js.map +1 -0
- package/dist/voice/agent-voice.descriptor.d.ts +82 -0
- package/dist/voice/agent-voice.descriptor.d.ts.map +1 -0
- package/dist/voice/agent-voice.descriptor.js +79 -0
- package/dist/voice/agent-voice.descriptor.js.map +1 -0
- package/dist/voice/agent-voice.impl.consumer.d.ts +35 -0
- package/dist/voice/agent-voice.impl.consumer.d.ts.map +1 -0
- package/dist/voice/agent-voice.impl.consumer.js +50 -0
- package/dist/voice/agent-voice.impl.consumer.js.map +1 -0
- package/dist/voice/agent-voice.kind.d.ts +67 -0
- package/dist/voice/agent-voice.kind.d.ts.map +1 -0
- package/dist/voice/agent-voice.kind.js +26 -0
- package/dist/voice/agent-voice.kind.js.map +1 -0
- package/dist/voice/agent-voice.plugin.d.ts +4 -0
- package/dist/voice/agent-voice.plugin.d.ts.map +1 -0
- package/dist/voice/agent-voice.plugin.js +18 -0
- package/dist/voice/agent-voice.plugin.js.map +1 -0
- package/dist/voice/client.d.ts +38 -0
- package/dist/voice/client.d.ts.map +1 -0
- package/dist/voice/client.js +29 -0
- package/dist/voice/client.js.map +1 -0
- package/dist/voice/index.d.ts +14 -0
- package/dist/voice/index.d.ts.map +1 -0
- package/dist/voice/index.js +14 -0
- package/dist/voice/index.js.map +1 -0
- package/dist/voice/pipeline.d.ts +66 -0
- package/dist/voice/pipeline.d.ts.map +1 -0
- package/dist/voice/pipeline.js +330 -0
- package/dist/voice/pipeline.js.map +1 -0
- package/dist/voice/session.d.ts +29 -0
- package/dist/voice/session.d.ts.map +1 -0
- package/dist/voice/session.js +78 -0
- package/dist/voice/session.js.map +1 -0
- package/package.json +279 -4
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Half-duplex voice pipeline — the engine behind the `agent/voice` kind.
|
|
3
|
+
*
|
|
4
|
+
* One *turn* = one user utterance → one assistant reply:
|
|
5
|
+
*
|
|
6
|
+
* audio (PCM/encoded) ──STT──▶ transcript ──agent loop──▶ reply text ──TTS──▶ audio
|
|
7
|
+
*
|
|
8
|
+
* The pipeline is deliberately **dependency-injected**: STT, the text turn
|
|
9
|
+
* (agent loop), and TTS are passed in as async functions. The host (the
|
|
10
|
+
* chat-web voice route) resolves those from the runtime's `ai/generation`
|
|
11
|
+
* providers + the agent loop, so this package stays workerd-safe — it never
|
|
12
|
+
* imports `@vibesdotdev/filesystem`, Playwright, or any Node-only generation
|
|
13
|
+
* executor. It only depends on the runtime + ai *schemas*.
|
|
14
|
+
*
|
|
15
|
+
* Server-side silence detection (`SilenceSegmenter`) lives here so both the
|
|
16
|
+
* WS/POST transport and any future full-duplex transport can reuse the same
|
|
17
|
+
* turn-boundary logic.
|
|
18
|
+
*/
|
|
19
|
+
import { getLogger } from '@vibesdotdev/logging';
|
|
20
|
+
import { CapabilityError, CaughtFailureSchema, isCapabilityError, valueFreeFailure } from '@vibesdotdev/kernel';
|
|
21
|
+
import { AgentVoiceDescriptorSchema } from "./agent-voice.descriptor.js";
|
|
22
|
+
import { VoiceBytesSchema, VoiceTurnAudioSchema, VoiceTranscriptSchema, VoiceReplySchema, VoiceSynthesisSchema, VoiceHistorySchema, VoiceAbortSignalSchema, VoiceSystemPromptSchema } from "../schemas/voice.js";
|
|
23
|
+
export { VoiceMetadataSchema, VoiceTurnAudioSchema, VoiceTranscriptSchema, VoiceReplySchema, VoiceSynthesisSchema, VoiceHistoryMessageSchema, VoiceHistorySchema, VoiceAbortSignalSchema, VoiceSystemPromptSchema, VoiceTurnTimingSchema, VoiceTurnResultSchema, VoicePipelineDepsSchema } from "../schemas/voice.js";
|
|
24
|
+
export function parseVoiceSystemPrompt(value) {
|
|
25
|
+
try {
|
|
26
|
+
return VoiceSystemPromptSchema.parse(value);
|
|
27
|
+
}
|
|
28
|
+
catch (caught) {
|
|
29
|
+
const failure = CaughtFailureSchema.parse(caught);
|
|
30
|
+
throw new CapabilityError('Voice session input is invalid.', 'INVALID_INPUT', {
|
|
31
|
+
reason: 'voice-turn-invalid',
|
|
32
|
+
retryable: false,
|
|
33
|
+
cause: valueFreeFailure(failure)
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
function voiceFailureDetails(stage, failure) {
|
|
38
|
+
return { stage, failure: valueFreeFailure(failure) };
|
|
39
|
+
}
|
|
40
|
+
function stageFailure(stage) {
|
|
41
|
+
return new CapabilityError('A voice pipeline stage failed.', 'UNAVAILABLE', {
|
|
42
|
+
reason: 'voice-stage-failed',
|
|
43
|
+
retryable: true,
|
|
44
|
+
details: { stage }
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
function stageOutputFailure(stage) {
|
|
48
|
+
return new CapabilityError('A voice pipeline stage returned an invalid result.', 'INTERNAL', {
|
|
49
|
+
reason: 'voice-stage-output-invalid',
|
|
50
|
+
retryable: false,
|
|
51
|
+
details: { stage }
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
function parseVoiceBytes(value) {
|
|
55
|
+
try {
|
|
56
|
+
return VoiceBytesSchema.parse(value);
|
|
57
|
+
}
|
|
58
|
+
catch (caught) {
|
|
59
|
+
const failure = CaughtFailureSchema.parse(caught);
|
|
60
|
+
const facts = valueFreeFailure(failure);
|
|
61
|
+
throw new CapabilityError('Voice audio bytes are invalid.', 'INVALID_INPUT', {
|
|
62
|
+
reason: 'voice-turn-invalid',
|
|
63
|
+
retryable: false,
|
|
64
|
+
details: { failureClass: facts.name }
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Run a single half-duplex turn under the descriptor's `turnTimeoutMs`, a hard
|
|
70
|
+
* ceiling over the whole STT→agent→TTS chain. The ceiling is the owning
|
|
71
|
+
* runtime's bounded wait (RC07): `runtime` is the runtime the host was handed,
|
|
72
|
+
* so disposing that runtime aborts the turn and
|
|
73
|
+
* no timer outlives it. Every stage receives the one combined signal (the
|
|
74
|
+
* caller's, the runtime's and the deadline). Exceeding the ceiling rejects
|
|
75
|
+
* `CapabilityError` `voice-turn-timeout`.
|
|
76
|
+
*/
|
|
77
|
+
export async function runVoiceTurn(runtime, descriptor, deps, audio, history, externalSignal) {
|
|
78
|
+
const log = getLogger('agent.voice.pipeline', runtime);
|
|
79
|
+
let admittedDescriptor;
|
|
80
|
+
let admittedAudio;
|
|
81
|
+
let admittedHistory;
|
|
82
|
+
let admittedSignal;
|
|
83
|
+
try {
|
|
84
|
+
admittedDescriptor = AgentVoiceDescriptorSchema.parse(descriptor);
|
|
85
|
+
admittedAudio = VoiceTurnAudioSchema.parse(audio);
|
|
86
|
+
admittedHistory = VoiceHistorySchema.parse(history);
|
|
87
|
+
admittedSignal = VoiceAbortSignalSchema.optional().parse(externalSignal);
|
|
88
|
+
}
|
|
89
|
+
catch (caught) {
|
|
90
|
+
const failure = CaughtFailureSchema.parse(caught);
|
|
91
|
+
log.warn('voice turn input is invalid', voiceFailureDetails('input', failure));
|
|
92
|
+
throw new CapabilityError('Voice turn input is invalid.', 'INVALID_INPUT', {
|
|
93
|
+
reason: 'voice-turn-invalid',
|
|
94
|
+
retryable: false
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
const start = runtime.lifecycle.now();
|
|
98
|
+
const waitOptions = {
|
|
99
|
+
timeoutMs: admittedDescriptor.turnTimeoutMs,
|
|
100
|
+
reason: 'voice-turn'
|
|
101
|
+
};
|
|
102
|
+
if (admittedSignal)
|
|
103
|
+
waitOptions.signal = admittedSignal;
|
|
104
|
+
try {
|
|
105
|
+
return await runtime.lifecycle.wait(async (signal) => {
|
|
106
|
+
const sttStart = runtime.lifecycle.now();
|
|
107
|
+
let transcriptCandidate;
|
|
108
|
+
try {
|
|
109
|
+
transcriptCandidate = await deps.transcribe(admittedAudio, signal);
|
|
110
|
+
}
|
|
111
|
+
catch (caught) {
|
|
112
|
+
const failure = CaughtFailureSchema.parse(caught);
|
|
113
|
+
if (isCapabilityError(failure))
|
|
114
|
+
throw failure;
|
|
115
|
+
log.warn('voice stage failed', voiceFailureDetails('stt', failure));
|
|
116
|
+
throw stageFailure('stt');
|
|
117
|
+
}
|
|
118
|
+
let transcript;
|
|
119
|
+
try {
|
|
120
|
+
transcript = VoiceTranscriptSchema.parse(transcriptCandidate);
|
|
121
|
+
}
|
|
122
|
+
catch (caught) {
|
|
123
|
+
const failure = CaughtFailureSchema.parse(caught);
|
|
124
|
+
log.error('voice stage returned an invalid result', voiceFailureDetails('stt', failure));
|
|
125
|
+
throw stageOutputFailure('stt');
|
|
126
|
+
}
|
|
127
|
+
const sttMs = runtime.lifecycle.now() - sttStart;
|
|
128
|
+
if (!transcript.text.trim()) {
|
|
129
|
+
throw new CapabilityError('Speech recognition produced no transcript.', 'INVALID_INPUT', {
|
|
130
|
+
reason: 'voice-empty-transcript',
|
|
131
|
+
retryable: false
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
const agentStart = runtime.lifecycle.now();
|
|
135
|
+
let replyCandidate;
|
|
136
|
+
try {
|
|
137
|
+
replyCandidate = await deps.runAgentTurn(transcript.text, admittedHistory, signal);
|
|
138
|
+
}
|
|
139
|
+
catch (caught) {
|
|
140
|
+
const failure = CaughtFailureSchema.parse(caught);
|
|
141
|
+
if (isCapabilityError(failure))
|
|
142
|
+
throw failure;
|
|
143
|
+
log.warn('voice stage failed', voiceFailureDetails('agent', failure));
|
|
144
|
+
throw stageFailure('agent');
|
|
145
|
+
}
|
|
146
|
+
let reply;
|
|
147
|
+
try {
|
|
148
|
+
reply = VoiceReplySchema.parse(replyCandidate);
|
|
149
|
+
}
|
|
150
|
+
catch (caught) {
|
|
151
|
+
const failure = CaughtFailureSchema.parse(caught);
|
|
152
|
+
log.error('voice stage returned an invalid result', voiceFailureDetails('agent', failure));
|
|
153
|
+
throw stageOutputFailure('agent');
|
|
154
|
+
}
|
|
155
|
+
const agentMs = runtime.lifecycle.now() - agentStart;
|
|
156
|
+
if (!reply.text.trim()) {
|
|
157
|
+
// An empty agent reply is a valid (if quiet) turn — skip TTS.
|
|
158
|
+
log.debug?.('voice turn: empty agent reply, skipping synthesis');
|
|
159
|
+
const quietResult = {
|
|
160
|
+
transcript: transcript.text,
|
|
161
|
+
replyText: '',
|
|
162
|
+
timing: { sttMs, agentMs, ttsMs: 0, totalMs: runtime.lifecycle.now() - start }
|
|
163
|
+
};
|
|
164
|
+
if (reply.metadata)
|
|
165
|
+
quietResult.metadata = reply.metadata;
|
|
166
|
+
return quietResult;
|
|
167
|
+
}
|
|
168
|
+
const ttsStart = runtime.lifecycle.now();
|
|
169
|
+
let synthesisCandidate;
|
|
170
|
+
try {
|
|
171
|
+
synthesisCandidate = await deps.synthesize(reply.text, signal);
|
|
172
|
+
}
|
|
173
|
+
catch (caught) {
|
|
174
|
+
const failure = CaughtFailureSchema.parse(caught);
|
|
175
|
+
if (isCapabilityError(failure))
|
|
176
|
+
throw failure;
|
|
177
|
+
log.warn('voice stage failed', voiceFailureDetails('tts', failure));
|
|
178
|
+
throw stageFailure('tts');
|
|
179
|
+
}
|
|
180
|
+
let synthesis;
|
|
181
|
+
try {
|
|
182
|
+
synthesis = VoiceSynthesisSchema.parse(synthesisCandidate);
|
|
183
|
+
}
|
|
184
|
+
catch (caught) {
|
|
185
|
+
const failure = CaughtFailureSchema.parse(caught);
|
|
186
|
+
log.error('voice stage returned an invalid result', voiceFailureDetails('tts', failure));
|
|
187
|
+
throw stageOutputFailure('tts');
|
|
188
|
+
}
|
|
189
|
+
const ttsMs = runtime.lifecycle.now() - ttsStart;
|
|
190
|
+
const result = {
|
|
191
|
+
transcript: transcript.text,
|
|
192
|
+
replyText: reply.text,
|
|
193
|
+
synthesis,
|
|
194
|
+
timing: { sttMs, agentMs, ttsMs, totalMs: runtime.lifecycle.now() - start }
|
|
195
|
+
};
|
|
196
|
+
if (reply.metadata)
|
|
197
|
+
result.metadata = reply.metadata;
|
|
198
|
+
return result;
|
|
199
|
+
}, waitOptions);
|
|
200
|
+
}
|
|
201
|
+
catch (caught) {
|
|
202
|
+
const failure = CaughtFailureSchema.parse(caught);
|
|
203
|
+
if (isCapabilityError(failure) && failure.code === 'TIMEOUT' && failure.reason === 'wait-timeout') {
|
|
204
|
+
throw new CapabilityError('The voice turn exceeded its deadline.', 'TIMEOUT', {
|
|
205
|
+
reason: 'voice-turn-timeout',
|
|
206
|
+
retryable: true
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
if (isCapabilityError(failure) && failure.code === 'CANCELLED' && failure.reason === 'wait-aborted') {
|
|
210
|
+
throw new CapabilityError('The voice turn was cancelled.', 'CANCELLED', {
|
|
211
|
+
reason: 'voice-turn-cancelled',
|
|
212
|
+
retryable: false
|
|
213
|
+
});
|
|
214
|
+
}
|
|
215
|
+
if (isCapabilityError(failure))
|
|
216
|
+
throw failure;
|
|
217
|
+
log.error('voice turn failed unexpectedly', voiceFailureDetails('input', failure));
|
|
218
|
+
throw new CapabilityError('The voice turn failed.', 'INTERNAL', {
|
|
219
|
+
reason: 'voice-turn-failed',
|
|
220
|
+
retryable: false
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* RMS amplitude of a 16-bit little-endian PCM frame, normalized to 0..1.
|
|
226
|
+
* Used by the silence segmenter. For non-PCM (encoded) frames the caller
|
|
227
|
+
* should not rely on this — silence detection is a PCM-stream concern.
|
|
228
|
+
*/
|
|
229
|
+
export function pcm16Rms(bytes) {
|
|
230
|
+
return pcm16RmsOf(parseVoiceBytes(bytes));
|
|
231
|
+
}
|
|
232
|
+
function pcm16RmsOf(bytes) {
|
|
233
|
+
if (bytes.length < 2)
|
|
234
|
+
return 0;
|
|
235
|
+
const samples = Math.floor(bytes.length / 2);
|
|
236
|
+
let sumSquares = 0;
|
|
237
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, samples * 2);
|
|
238
|
+
for (let i = 0; i < samples; i++) {
|
|
239
|
+
const s = view.getInt16(i * 2, true) / 32_768;
|
|
240
|
+
sumSquares += s * s;
|
|
241
|
+
}
|
|
242
|
+
return Math.sqrt(sumSquares / samples);
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* Server-side silence detection that segments an inbound PCM stream into turns.
|
|
246
|
+
*
|
|
247
|
+
* Feed frames as they arrive via {@link push}; when trailing silence exceeds
|
|
248
|
+
* `silenceTimeoutMs` after at least one voiced frame, {@link push} returns the
|
|
249
|
+
* concatenated utterance bytes (and resets). This is the half-duplex turn
|
|
250
|
+
* boundary: one completed utterance → one agent turn.
|
|
251
|
+
*/
|
|
252
|
+
export class SilenceSegmenter {
|
|
253
|
+
frames = [];
|
|
254
|
+
bytesBuffered = 0;
|
|
255
|
+
hasVoiced = false;
|
|
256
|
+
silenceAccumulatedMs = 0;
|
|
257
|
+
threshold;
|
|
258
|
+
silenceTimeoutMs;
|
|
259
|
+
bytesPerMs;
|
|
260
|
+
constructor(descriptor) {
|
|
261
|
+
let admitted;
|
|
262
|
+
try {
|
|
263
|
+
admitted = AgentVoiceDescriptorSchema.pick({
|
|
264
|
+
silenceThreshold: true,
|
|
265
|
+
silenceTimeoutMs: true,
|
|
266
|
+
sampleRate: true
|
|
267
|
+
}).parse(descriptor);
|
|
268
|
+
}
|
|
269
|
+
catch (caught) {
|
|
270
|
+
throw new CapabilityError('Voice segmenter input is invalid.', 'INVALID_INPUT', {
|
|
271
|
+
reason: 'voice-turn-invalid',
|
|
272
|
+
retryable: false,
|
|
273
|
+
cause: valueFreeFailure(CaughtFailureSchema.parse(caught))
|
|
274
|
+
});
|
|
275
|
+
}
|
|
276
|
+
this.threshold = admitted.silenceThreshold;
|
|
277
|
+
this.silenceTimeoutMs = admitted.silenceTimeoutMs;
|
|
278
|
+
// 16-bit mono PCM: 2 bytes per sample.
|
|
279
|
+
this.bytesPerMs = Math.max(1, (admitted.sampleRate * 2) / 1000);
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* Push a PCM frame. Returns the completed utterance bytes when trailing
|
|
283
|
+
* silence terminates a turn, otherwise `null`.
|
|
284
|
+
*/
|
|
285
|
+
push(frame) {
|
|
286
|
+
const admittedFrame = parseVoiceBytes(frame);
|
|
287
|
+
const rms = pcm16RmsOf(admittedFrame);
|
|
288
|
+
const frameMs = admittedFrame.length / this.bytesPerMs;
|
|
289
|
+
if (rms >= this.threshold) {
|
|
290
|
+
this.hasVoiced = true;
|
|
291
|
+
this.silenceAccumulatedMs = 0;
|
|
292
|
+
this.frames.push(admittedFrame);
|
|
293
|
+
this.bytesBuffered += admittedFrame.length;
|
|
294
|
+
return null;
|
|
295
|
+
}
|
|
296
|
+
// Silent frame. Keep buffering it (trailing silence is part of the turn
|
|
297
|
+
// until the segmenter fires) only once we've heard voice.
|
|
298
|
+
if (this.hasVoiced) {
|
|
299
|
+
this.frames.push(admittedFrame);
|
|
300
|
+
this.bytesBuffered += admittedFrame.length;
|
|
301
|
+
this.silenceAccumulatedMs += frameMs;
|
|
302
|
+
if (this.silenceAccumulatedMs >= this.silenceTimeoutMs) {
|
|
303
|
+
return this.flush();
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
return null;
|
|
307
|
+
}
|
|
308
|
+
/** Force-close the current utterance (e.g. transport closed). */
|
|
309
|
+
flush() {
|
|
310
|
+
if (!this.hasVoiced || this.bytesBuffered === 0) {
|
|
311
|
+
this.reset();
|
|
312
|
+
return null;
|
|
313
|
+
}
|
|
314
|
+
const out = new Uint8Array(this.bytesBuffered);
|
|
315
|
+
let offset = 0;
|
|
316
|
+
for (const f of this.frames) {
|
|
317
|
+
out.set(f, offset);
|
|
318
|
+
offset += f.length;
|
|
319
|
+
}
|
|
320
|
+
this.reset();
|
|
321
|
+
return out;
|
|
322
|
+
}
|
|
323
|
+
reset() {
|
|
324
|
+
this.frames.length = 0;
|
|
325
|
+
this.bytesBuffered = 0;
|
|
326
|
+
this.hasVoiced = false;
|
|
327
|
+
this.silenceAccumulatedMs = 0;
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
//# sourceMappingURL=pipeline.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/voice/pipeline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EACN,eAAe,EACf,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,EAIhB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACN,0BAA0B,EAE1B,MAAM,6BAA6B,CAAC;AAErC,OAAO,EACN,gBAAgB,EAChB,oBAAoB,EACpB,qBAAqB,EACrB,gBAAgB,EAChB,oBAAoB,EACpB,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,EAQvB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACN,mBAAmB,EACnB,oBAAoB,EACpB,qBAAqB,EACrB,gBAAgB,EAChB,oBAAoB,EACpB,yBAAyB,EACzB,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,EACvB,qBAAqB,EACrB,qBAAqB,EACrB,uBAAuB,EASvB,MAAM,qBAAqB,CAAC;AAE7B,MAAM,UAAU,sBAAsB,CAAS,KAAa;IAC3D,IAAI,CAAC;QACJ,OAAO,uBAAuB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC7C,CAAC;IAAC,OAAO,MAAM,EAAE,CAAC;QACjB,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAClD,MAAM,IAAI,eAAe,CAAC,iCAAiC,EAAE,eAAe,EAAE;YAC7E,MAAM,EAAE,oBAAoB;YAC5B,SAAS,EAAE,KAAK;YAChB,KAAK,EAAE,gBAAgB,CAAC,OAAO,CAAC;SAChC,CAAC,CAAC;IACJ,CAAC;AACF,CAAC;AAID,SAAS,mBAAmB,CAAC,KAA2B,EAAE,OAAsB;IAC/E,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC;AACtD,CAAC;AAED,SAAS,YAAY,CAAC,KAAiB;IACtC,OAAO,IAAI,eAAe,CAAC,gCAAgC,EAAE,aAAa,EAAE;QAC3E,MAAM,EAAE,oBAAoB;QAC5B,SAAS,EAAE,IAAI;QACf,OAAO,EAAE,EAAE,KAAK,EAAE;KAClB,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAiB;IAC5C,OAAO,IAAI,eAAe,CAAC,oDAAoD,EAAE,UAAU,EAAE;QAC5F,MAAM,EAAE,4BAA4B;QACpC,SAAS,EAAE,KAAK;QAChB,OAAO,EAAE,EAAE,KAAK,EAAE;KAClB,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACtC,IAAI,CAAC;QACJ,OAAO,gBAAgB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC;IAAC,OAAO,MAAM,EAAE,CAAC;QACjB,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAClD,MAAM,KAAK,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACxC,MAAM,IAAI,eAAe,CAAC,gCAAgC,EAAE,eAAe,EAAE;YAC5E,MAAM,EAAE,oBAAoB;YAC5B,SAAS,EAAE,KAAK;YAChB,OAAO,EAAE,EAAE,YAAY,EAAE,KAAK,CAAC,IAAI,EAAE;SACrC,CAAC,CAAC;IACJ,CAAC;AACF,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CACjC,OAAmB,EACnB,UAAgC,EAChC,IAAuB,EACvB,KAAqB,EACrB,OAA8B,EAC9B,cAA4B;IAE5B,MAAM,GAAG,GAAG,SAAS,CAAC,sBAAsB,EAAE,OAAO,CAAC,CAAC;IACvD,IAAI,kBAAwC,CAAC;IAC7C,IAAI,aAA6B,CAAC;IAClC,IAAI,eAAsC,CAAC;IAC3C,IAAI,cAAuC,CAAC;IAC5C,IAAI,CAAC;QACJ,kBAAkB,GAAG,0BAA0B,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAClE,aAAa,GAAG,oBAAoB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAClD,eAAe,GAAG,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACpD,cAAc,GAAG,sBAAsB,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IAC1E,CAAC;IAAC,OAAO,MAAM,EAAE,CAAC;QACjB,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAClD,GAAG,CAAC,IAAI,CAAC,6BAA6B,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QAC/E,MAAM,IAAI,eAAe,CAAC,8BAA8B,EAAE,eAAe,EAAE;YAC1E,MAAM,EAAE,oBAAoB;YAC5B,SAAS,EAAE,KAAK;SAChB,CAAC,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC;IACtC,MAAM,WAAW,GAAuB;QACvC,SAAS,EAAE,kBAAkB,CAAC,aAAa;QAC3C,MAAM,EAAE,YAAY;KACpB,CAAC;IACF,IAAI,cAAc;QAAE,WAAW,CAAC,MAAM,GAAG,cAAc,CAAC;IACxD,IAAI,CAAC;QACJ,OAAO,MAAM,OAAO,CAAC,SAAS,CAAC,IAAI,CAClC,KAAK,EAAE,MAAM,EAAE,EAAE;YAChB,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC;YACzC,IAAI,mBAA4B,CAAC;YACjC,IAAI,CAAC;gBACJ,mBAAmB,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YACpE,CAAC;YAAC,OAAO,MAAM,EAAE,CAAC;gBACjB,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAClD,IAAI,iBAAiB,CAAC,OAAO,CAAC;oBAAE,MAAM,OAAO,CAAC;gBAC9C,GAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,mBAAmB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC;gBACpE,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;YACD,IAAI,UAA2B,CAAC;YAChC,IAAI,CAAC;gBACJ,UAAU,GAAG,qBAAqB,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAC/D,CAAC;YAAC,OAAO,MAAM,EAAE,CAAC;gBACjB,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAClD,GAAG,CAAC,KAAK,CAAC,wCAAwC,EAAE,mBAAmB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC;gBACzF,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;YACD,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC;YACjD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC7B,MAAM,IAAI,eAAe,CAAC,4CAA4C,EAAE,eAAe,EAAE;oBACxF,MAAM,EAAE,wBAAwB;oBAChC,SAAS,EAAE,KAAK;iBAChB,CAAC,CAAC;YACJ,CAAC;YAED,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC;YAC3C,IAAI,cAAuB,CAAC;YAC5B,IAAI,CAAC;gBACJ,cAAc,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;YACpF,CAAC;YAAC,OAAO,MAAM,EAAE,CAAC;gBACjB,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAClD,IAAI,iBAAiB,CAAC,OAAO,CAAC;oBAAE,MAAM,OAAO,CAAC;gBAC9C,GAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;gBACtE,MAAM,YAAY,CAAC,OAAO,CAAC,CAAC;YAC7B,CAAC;YACD,IAAI,KAAiB,CAAC;YACtB,IAAI,CAAC;gBACJ,KAAK,GAAG,gBAAgB,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;YAChD,CAAC;YAAC,OAAO,MAAM,EAAE,CAAC;gBACjB,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAClD,GAAG,CAAC,KAAK,CAAC,wCAAwC,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;gBAC3F,MAAM,kBAAkB,CAAC,OAAO,CAAC,CAAC;YACnC,CAAC;YACD,MAAM,OAAO,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;YACrD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBACxB,8DAA8D;gBAC9D,GAAG,CAAC,KAAK,EAAE,CAAC,mDAAmD,CAAC,CAAC;gBACjE,MAAM,WAAW,GAAoB;oBACpC,UAAU,EAAE,UAAU,CAAC,IAAI;oBAC3B,SAAS,EAAE,EAAE;oBACb,MAAM,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE;iBAC9E,CAAC;gBACF,IAAI,KAAK,CAAC,QAAQ;oBAAE,WAAW,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;gBAC1D,OAAO,WAAW,CAAC;YACpB,CAAC;YAED,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC;YACzC,IAAI,kBAA2B,CAAC;YAChC,IAAI,CAAC;gBACJ,kBAAkB,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAChE,CAAC;YAAC,OAAO,MAAM,EAAE,CAAC;gBACjB,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAClD,IAAI,iBAAiB,CAAC,OAAO,CAAC;oBAAE,MAAM,OAAO,CAAC;gBAC9C,GAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,mBAAmB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC;gBACpE,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;YACD,IAAI,SAAyB,CAAC;YAC9B,IAAI,CAAC;gBACJ,SAAS,GAAG,oBAAoB,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YAC5D,CAAC;YAAC,OAAO,MAAM,EAAE,CAAC;gBACjB,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAClD,GAAG,CAAC,KAAK,CAAC,wCAAwC,EAAE,mBAAmB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC;gBACzF,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;YACD,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC;YAEjD,MAAM,MAAM,GAAoB;gBAC/B,UAAU,EAAE,UAAU,CAAC,IAAI;gBAC3B,SAAS,EAAE,KAAK,CAAC,IAAI;gBACrB,SAAS;gBACT,MAAM,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE;aAC3E,CAAC;YACF,IAAI,KAAK,CAAC,QAAQ;gBAAE,MAAM,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;YACrD,OAAO,MAAM,CAAC;QACf,CAAC,EACD,WAAW,CACX,CAAC;IACH,CAAC;IAAC,OAAO,MAAM,EAAE,CAAC;QACjB,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAClD,IAAI,iBAAiB,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;YACnG,MAAM,IAAI,eAAe,CAAC,uCAAuC,EAAE,SAAS,EAAE;gBAC7E,MAAM,EAAE,oBAAoB;gBAC5B,SAAS,EAAE,IAAI;aACf,CAAC,CAAC;QACJ,CAAC;QACD,IAAI,iBAAiB,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,IAAI,OAAO,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;YACrG,MAAM,IAAI,eAAe,CAAC,+BAA+B,EAAE,WAAW,EAAE;gBACvE,MAAM,EAAE,sBAAsB;gBAC9B,SAAS,EAAE,KAAK;aAChB,CAAC,CAAC;QACJ,CAAC;QACD,IAAI,iBAAiB,CAAC,OAAO,CAAC;YAAE,MAAM,OAAO,CAAC;QAC9C,GAAG,CAAC,KAAK,CAAC,gCAAgC,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QACnF,MAAM,IAAI,eAAe,CAAC,wBAAwB,EAAE,UAAU,EAAE;YAC/D,MAAM,EAAE,mBAAmB;YAC3B,SAAS,EAAE,KAAK;SAChB,CAAC,CAAC;IACJ,CAAC;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,QAAQ,CAAC,KAAiB;IACzC,OAAO,UAAU,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,UAAU,CAAC,KAAiB;IACpC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC7C,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;IACvE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAC,CAAC,GAAG,OAAO,EAAC,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,MAAM,CAAC;QAC9C,UAAU,IAAI,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,CAAC;AACxC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,OAAO,gBAAgB;IACX,MAAM,GAAiB,EAAE,CAAC;IACnC,aAAa,GAAG,CAAC,CAAC;IAClB,SAAS,GAAG,KAAK,CAAC;IAClB,oBAAoB,GAAG,CAAC,CAAC;IAChB,SAAS,CAAS;IAClB,gBAAgB,CAAS;IACzB,UAAU,CAAS;IAEpC,YAAY,UAA8F;QACzG,IAAI,QAGH,CAAC;QACF,IAAI,CAAC;YACJ,QAAQ,GAAG,0BAA0B,CAAC,IAAI,CAAC;gBAC1C,gBAAgB,EAAE,IAAI;gBACtB,gBAAgB,EAAE,IAAI;gBACtB,UAAU,EAAE,IAAI;aAChB,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACtB,CAAC;QAAC,OAAO,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,eAAe,CAAC,mCAAmC,EAAE,eAAe,EAAE;gBAC/E,MAAM,EAAE,oBAAoB;gBAC5B,SAAS,EAAE,KAAK;gBAChB,KAAK,EAAE,gBAAgB,CAAC,mBAAmB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;aAC1D,CAAC,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,gBAAgB,CAAC;QAC3C,IAAI,CAAC,gBAAgB,GAAG,QAAQ,CAAC,gBAAgB,CAAC;QAClD,uCAAuC;QACvC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IACjE,CAAC;IAED;;;OAGG;IACH,IAAI,CAAC,KAAiB;QACrB,MAAM,aAAa,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC7C,MAAM,GAAG,GAAG,UAAU,CAAC,aAAa,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC;QAEvD,IAAI,GAAG,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACtB,IAAI,CAAC,oBAAoB,GAAG,CAAC,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAChC,IAAI,CAAC,aAAa,IAAI,aAAa,CAAC,MAAM,CAAC;YAC3C,OAAO,IAAI,CAAC;QACb,CAAC;QAED,wEAAwE;QACxE,0DAA0D;QAC1D,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAChC,IAAI,CAAC,aAAa,IAAI,aAAa,CAAC,MAAM,CAAC;YAC3C,IAAI,CAAC,oBAAoB,IAAI,OAAO,CAAC;YACrC,IAAI,IAAI,CAAC,oBAAoB,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACxD,OAAO,IAAI,CAAC,KAAK,EAAE,CAAC;YACrB,CAAC;QACF,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC;IAED,iEAAiE;IACjE,KAAK;QACJ,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,aAAa,KAAK,CAAC,EAAE,CAAC;YACjD,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,OAAO,IAAI,CAAC;QACb,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC/C,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC7B,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YACnB,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC;QACpB,CAAC;QACD,IAAI,CAAC,KAAK,EAAE,CAAC;QACb,OAAO,GAAG,CAAC;IACZ,CAAC;IAEO,KAAK;QACZ,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,oBAAoB,GAAG,CAAC,CAAC;IAC/B,CAAC;CACD"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { type RuntimeRef } from '@vibesdotdev/kernel';
|
|
2
|
+
import { type AgentVoiceDescriptor } from './agent-voice.descriptor.ts';
|
|
3
|
+
import { type VoicePipelineDeps, type VoiceHistoryMessage, type VoiceTurnAudio, type VoiceTurnResult } from './pipeline.ts';
|
|
4
|
+
export declare class VoiceSession {
|
|
5
|
+
private readonly runtime;
|
|
6
|
+
private readonly descriptor;
|
|
7
|
+
private readonly deps;
|
|
8
|
+
private readonly onClose?;
|
|
9
|
+
private readonly segmenter;
|
|
10
|
+
private readonly history;
|
|
11
|
+
private closed;
|
|
12
|
+
constructor(runtime: RuntimeRef, descriptor: AgentVoiceDescriptor, deps: VoicePipelineDeps, systemPrompt?: string, onClose?: (() => void) | undefined);
|
|
13
|
+
/**
|
|
14
|
+
* Feed a PCM frame. Returns a completed turn result when trailing silence
|
|
15
|
+
* terminates the utterance, otherwise `null` (still listening).
|
|
16
|
+
*/
|
|
17
|
+
pushFrame(frame: Uint8Array, signal?: AbortSignal): Promise<VoiceTurnResult | null>;
|
|
18
|
+
/**
|
|
19
|
+
* Run one turn from an already-segmented utterance (pre-segmented
|
|
20
|
+
* transports / the POST turn endpoint). Updates history.
|
|
21
|
+
*/
|
|
22
|
+
submitUtterance(audio: VoiceTurnAudio, signal?: AbortSignal): Promise<VoiceTurnResult>;
|
|
23
|
+
/** Force-close the in-flight utterance (transport closed mid-turn). */
|
|
24
|
+
flush(signal?: AbortSignal): Promise<VoiceTurnResult> | null;
|
|
25
|
+
close(): void;
|
|
26
|
+
get turns(): readonly VoiceHistoryMessage[];
|
|
27
|
+
private assertOpen;
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=session.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/voice/session.ts"],"names":[],"mappings":"AACA,OAAO,EAAmB,KAAK,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACvE,OAAO,EACN,KAAK,oBAAoB,EACzB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAIN,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,MAAM,eAAe,CAAC;AAEvB,qBAAa,YAAY;IAMvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,IAAI;IAErB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;IAT1B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA6B;IACrD,OAAO,CAAC,MAAM,CAAS;gBAGL,OAAO,EAAE,UAAU,EACnB,UAAU,EAAE,oBAAoB,EAChC,IAAI,EAAE,iBAAiB,EACxC,YAAY,CAAC,EAAE,MAAM,EACJ,OAAO,CAAC,GAAE,MAAM,IAAI,aAAA;IAWtC;;;OAGG;IACG,SAAS,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IAOzF;;;OAGG;IACG,eAAe,CACpB,KAAK,EAAE,cAAc,EACrB,MAAM,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,eAAe,CAAC;IAe3B,uEAAuE;IACvE,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,eAAe,CAAC,GAAG,IAAI;IAM5D,KAAK,IAAI,IAAI;IAKb,IAAI,KAAK,IAAI,SAAS,mBAAmB,EAAE,CAE1C;IAED,OAAO,CAAC,UAAU;CAOlB"}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { getLogger } from '@vibesdotdev/logging';
|
|
2
|
+
import { CapabilityError } from '@vibesdotdev/kernel';
|
|
3
|
+
import {} from "./agent-voice.descriptor.js";
|
|
4
|
+
import { runVoiceTurn, SilenceSegmenter, parseVoiceSystemPrompt } from "./pipeline.js";
|
|
5
|
+
export class VoiceSession {
|
|
6
|
+
runtime;
|
|
7
|
+
descriptor;
|
|
8
|
+
deps;
|
|
9
|
+
onClose;
|
|
10
|
+
segmenter;
|
|
11
|
+
history = [];
|
|
12
|
+
closed = false;
|
|
13
|
+
constructor(runtime, descriptor, deps, systemPrompt, onClose) {
|
|
14
|
+
this.runtime = runtime;
|
|
15
|
+
this.descriptor = descriptor;
|
|
16
|
+
this.deps = deps;
|
|
17
|
+
this.onClose = onClose;
|
|
18
|
+
this.segmenter = new SilenceSegmenter(descriptor);
|
|
19
|
+
if (systemPrompt !== undefined) {
|
|
20
|
+
const admittedPrompt = parseVoiceSystemPrompt(systemPrompt);
|
|
21
|
+
if (admittedPrompt.trim()) {
|
|
22
|
+
this.history.push({ role: 'system', content: admittedPrompt });
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Feed a PCM frame. Returns a completed turn result when trailing silence
|
|
28
|
+
* terminates the utterance, otherwise `null` (still listening).
|
|
29
|
+
*/
|
|
30
|
+
async pushFrame(frame, signal) {
|
|
31
|
+
this.assertOpen();
|
|
32
|
+
const utterance = this.segmenter.push(frame);
|
|
33
|
+
if (!utterance)
|
|
34
|
+
return null;
|
|
35
|
+
return this.submitUtterance({ bytes: utterance }, signal);
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Run one turn from an already-segmented utterance (pre-segmented
|
|
39
|
+
* transports / the POST turn endpoint). Updates history.
|
|
40
|
+
*/
|
|
41
|
+
async submitUtterance(audio, signal) {
|
|
42
|
+
this.assertOpen();
|
|
43
|
+
const result = await runVoiceTurn(this.runtime, this.descriptor, this.deps, audio, this.history, signal);
|
|
44
|
+
this.history.push({ role: 'user', content: result.transcript });
|
|
45
|
+
if (result.replyText) {
|
|
46
|
+
this.history.push({ role: 'assistant', content: result.replyText });
|
|
47
|
+
}
|
|
48
|
+
getLogger('agent.voice.impl.consumer', this.runtime).debug?.('voice turn complete', {
|
|
49
|
+
transcriptChars: result.transcript.length,
|
|
50
|
+
replyChars: result.replyText.length,
|
|
51
|
+
timing: result.timing
|
|
52
|
+
});
|
|
53
|
+
return result;
|
|
54
|
+
}
|
|
55
|
+
/** Force-close the in-flight utterance (transport closed mid-turn). */
|
|
56
|
+
flush(signal) {
|
|
57
|
+
const utterance = this.segmenter.flush();
|
|
58
|
+
if (!utterance)
|
|
59
|
+
return null;
|
|
60
|
+
return this.submitUtterance({ bytes: utterance }, signal);
|
|
61
|
+
}
|
|
62
|
+
close() {
|
|
63
|
+
this.closed = true;
|
|
64
|
+
this.onClose?.();
|
|
65
|
+
}
|
|
66
|
+
get turns() {
|
|
67
|
+
return this.history;
|
|
68
|
+
}
|
|
69
|
+
assertOpen() {
|
|
70
|
+
if (!this.closed && !this.runtime.lifecycle.signal.aborted)
|
|
71
|
+
return;
|
|
72
|
+
throw new CapabilityError('The voice session is closed.', 'UNAVAILABLE', {
|
|
73
|
+
reason: 'voice-session-closed',
|
|
74
|
+
retryable: false
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
//# sourceMappingURL=session.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/voice/session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAmB,MAAM,qBAAqB,CAAC;AACvE,OAAO,EAEN,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACN,YAAY,EACZ,gBAAgB,EAChB,sBAAsB,EAKtB,MAAM,eAAe,CAAC;AAEvB,MAAM,OAAO,YAAY;IAMN;IACA;IACA;IAEA;IATD,SAAS,CAAmB;IAC5B,OAAO,GAA0B,EAAE,CAAC;IAC7C,MAAM,GAAG,KAAK,CAAC;IAEvB,YACkB,OAAmB,EACnB,UAAgC,EAChC,IAAuB,EACxC,YAAqB,EACJ,OAAoB;QAJpB,YAAO,GAAP,OAAO,CAAY;QACnB,eAAU,GAAV,UAAU,CAAsB;QAChC,SAAI,GAAJ,IAAI,CAAmB;QAEvB,YAAO,GAAP,OAAO,CAAa;QAErC,IAAI,CAAC,SAAS,GAAG,IAAI,gBAAgB,CAAC,UAAU,CAAC,CAAC;QAClD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAChC,MAAM,cAAc,GAAG,sBAAsB,CAAC,YAAY,CAAC,CAAC;YAC5D,IAAI,cAAc,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC3B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;YAChE,CAAC;QACF,CAAC;IACF,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,SAAS,CAAC,KAAiB,EAAE,MAAoB;QACtD,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAC5B,OAAO,IAAI,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,MAAM,CAAC,CAAC;IAC3D,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,eAAe,CACpB,KAAqB,EACrB,MAAoB;QAEpB,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACzG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;QAChE,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACtB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;QACrE,CAAC;QACD,SAAS,CAAC,2BAA2B,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,qBAAqB,EAAE;YACnF,eAAe,EAAE,MAAM,CAAC,UAAU,CAAC,MAAM;YACzC,UAAU,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM;YACnC,MAAM,EAAE,MAAM,CAAC,MAAM;SACrB,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IACf,CAAC;IAED,uEAAuE;IACvE,KAAK,CAAC,MAAoB;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QACzC,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAC5B,OAAO,IAAI,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,MAAM,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK;QACJ,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;IAClB,CAAC;IAED,IAAI,KAAK;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACrB,CAAC;IAEO,UAAU;QACjB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO;QACnE,MAAM,IAAI,eAAe,CAAC,8BAA8B,EAAE,aAAa,EAAE;YACxE,MAAM,EAAE,sBAAsB;YAC9B,SAAS,EAAE,KAAK;SAChB,CAAC,CAAC;IACJ,CAAC;CACD"}
|