agents 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -1
- package/dist/{agent-routing-CnkaHb-v.d.ts → agent-routing-DE5zmCQ8.d.ts} +1038 -394
- package/dist/agent-routing.d.ts +1 -1
- package/dist/agent-tool-types.d.ts +26 -26
- package/dist/{agent-tools-C0XJqWzB.d.ts → agent-tools-DtXMTDGM.d.ts} +7 -7
- package/dist/agent-tools.d.ts +21 -21
- package/dist/browser/ai.js +4 -1
- package/dist/browser/ai.js.map +1 -1
- package/dist/browser/tanstack-ai.js.map +1 -1
- package/dist/callable-decorator-DP__HhBA.d.ts +72 -0
- package/dist/callable-decorator.d.ts +18 -0
- package/dist/callable-decorator.js +71 -0
- package/dist/callable-decorator.js.map +1 -0
- package/dist/capability-BjSKYpzg.js.map +1 -1
- package/dist/capability-runner-Be_-PLR1.d.ts +459 -0
- package/dist/channel-Bnm4S7T2.d.ts +491 -0
- package/dist/channels/ai-sdk.d.ts +46 -0
- package/dist/channels/ai-sdk.js +120 -0
- package/dist/channels/ai-sdk.js.map +1 -0
- package/dist/channels/email.d.ts +95 -0
- package/dist/channels/email.js +323 -0
- package/dist/channels/email.js.map +1 -0
- package/dist/channels/index.d.ts +233 -0
- package/dist/channels/index.js +608 -0
- package/dist/channels/index.js.map +1 -0
- package/dist/channels/slack.d.ts +140 -0
- package/dist/channels/slack.js +614 -0
- package/dist/channels/slack.js.map +1 -0
- package/dist/channels/tanstack-ai.d.ts +39 -0
- package/dist/channels/tanstack-ai.js +17 -0
- package/dist/channels/tanstack-ai.js.map +1 -0
- package/dist/channels/telegram.d.ts +106 -0
- package/dist/channels/telegram.js +427 -0
- package/dist/channels/telegram.js.map +1 -0
- package/dist/channels/voice.d.ts +45 -0
- package/dist/channels/voice.js +122 -0
- package/dist/channels/voice.js.map +1 -0
- package/dist/chat/index.d.ts +2325 -2019
- package/dist/chat/index.js +888 -518
- package/dist/chat/index.js.map +1 -1
- package/dist/chat-sdk/index.d.ts +7 -7
- package/dist/chat-sdk/index.js +1 -1
- package/dist/client.d.ts +1 -1
- package/dist/context/index.d.ts +216 -0
- package/dist/context/index.js +454 -0
- package/dist/context/index.js.map +1 -0
- package/dist/{current-agent-CuMErtly.d.ts → current-agent-Da_C9a3b.d.ts} +90 -107
- package/dist/current-agent-DhoDkSnH.js.map +1 -1
- package/dist/{diagnostics-CaBjfz4J.js → diagnostics-BzvaX2UT.js} +5 -1
- package/dist/diagnostics-BzvaX2UT.js.map +1 -0
- package/dist/diagnostics-C4jcz3VK.js +360 -0
- package/dist/diagnostics-C4jcz3VK.js.map +1 -0
- package/dist/index-BB0kqhIz.d.ts +101 -0
- package/dist/index-BVVgDSdq.d.ts +1 -0
- package/dist/index-XDkuQ7zm.d.ts +89 -0
- package/dist/{index-DcSAZKsB.d.ts → index-YSKgfgg9.d.ts} +3 -1
- package/dist/index.d.ts +91 -82
- package/dist/index.js +3 -2
- package/dist/ingress-BfetZbMO.js +83 -0
- package/dist/ingress-BfetZbMO.js.map +1 -0
- package/dist/internal-CYlgHl1l.js +59 -0
- package/dist/internal-CYlgHl1l.js.map +1 -0
- package/dist/internal_context.d.ts +1 -1
- package/dist/lifecycle/index.d.ts +35 -17
- package/dist/lifecycle/index.js +1 -1
- package/dist/lifecycle-CMRGjZdw.js +1299 -0
- package/dist/lifecycle-CMRGjZdw.js.map +1 -0
- package/dist/mcp/client/index.d.ts +20 -20
- package/dist/mcp/index.d.ts +35 -35
- package/dist/mcp/index.js +1 -1
- package/dist/observability/index.d.ts +1 -1
- package/dist/observability/index.js +1 -1
- package/dist/react.d.ts +4 -4
- package/dist/{retries-CAvxtG9d.d.ts → retries-D9Ds-1lz.d.ts} +17 -6
- package/dist/retries.d.ts +8 -6
- package/dist/retries.js +13 -1
- package/dist/retries.js.map +1 -1
- package/dist/routing/index.d.ts +137 -0
- package/dist/routing/index.js +244 -0
- package/dist/routing/index.js.map +1 -0
- package/dist/sanitize-D9TujEK8.js +79 -0
- package/dist/sanitize-D9TujEK8.js.map +1 -0
- package/dist/scheduler-DD9NdYbF.js +665 -0
- package/dist/scheduler-DD9NdYbF.js.map +1 -0
- package/dist/{scheduler-DQoTGoAW.d.ts → scheduler-Dwh85ZGl.d.ts} +21 -22
- package/dist/schedules/index.d.ts +1 -1
- package/dist/schedules/index.js +1 -1
- package/dist/sentence-chunker-BAidJ4DA.d.ts +68 -0
- package/dist/serializable.d.ts +1 -1
- package/dist/sessions/index.d.ts +441 -0
- package/dist/sessions/index.js +2063 -0
- package/dist/sessions/index.js.map +1 -0
- package/dist/skills/index.d.ts +99 -0
- package/dist/skills/index.js +254 -5
- package/dist/skills/index.js.map +1 -1
- package/dist/{src-5W6JNKVb.js → src-DlSHshb2.js} +1460 -1110
- package/dist/src-DlSHshb2.js.map +1 -0
- package/dist/streams/index.d.ts +120 -0
- package/dist/streams/index.js +107 -0
- package/dist/streams/index.js.map +1 -0
- package/dist/streams-D6tJ0NN9.d.ts +370 -0
- package/dist/streams-DZKgAj9b.js +709 -0
- package/dist/streams-DZKgAj9b.js.map +1 -0
- package/dist/sub-routing.d.ts +12 -12
- package/dist/surface-bZZJqBka.js +17 -0
- package/dist/surface-bZZJqBka.js.map +1 -0
- package/dist/tasks/index.d.ts +64 -0
- package/dist/tasks/index.js +2 -0
- package/dist/tasks-BRJ5zgya.d.ts +517 -0
- package/dist/tasks-ylZgBjhj.js +1656 -0
- package/dist/tasks-ylZgBjhj.js.map +1 -0
- package/dist/text-segment-joiner-BtAFQSA_.js +57 -0
- package/dist/text-segment-joiner-BtAFQSA_.js.map +1 -0
- package/dist/text-stream-CpdiKrJB.js +272 -0
- package/dist/text-stream-CpdiKrJB.js.map +1 -0
- package/dist/tokens-nHAKcN6M.js +52 -0
- package/dist/tokens-nHAKcN6M.js.map +1 -0
- package/dist/tool-schema-CBjGPrsQ.js +31 -0
- package/dist/tool-schema-CBjGPrsQ.js.map +1 -0
- package/dist/types-B7LojTe4.d.ts +202 -0
- package/dist/types-_Faxb570.d.ts +439 -0
- package/dist/voice/client.d.ts +226 -0
- package/dist/voice/client.js +932 -0
- package/dist/voice/client.js.map +1 -0
- package/dist/voice/errors.d.ts +43 -0
- package/dist/voice/errors.js +41 -0
- package/dist/voice/errors.js.map +1 -0
- package/dist/voice/index.d.ts +271 -0
- package/dist/voice/index.js +1812 -0
- package/dist/voice/index.js.map +1 -0
- package/dist/voice/react.d.ts +167 -0
- package/dist/voice/react.js +234 -0
- package/dist/voice/react.js.map +1 -0
- package/dist/voice/sfu.d.ts +71 -0
- package/dist/voice/sfu.js +157 -0
- package/dist/voice/sfu.js.map +1 -0
- package/dist/voice/text.d.ts +6 -0
- package/dist/voice/text.js +2 -0
- package/dist/voice/types.d.ts +58 -0
- package/dist/voice/types.js +18 -0
- package/dist/voice/types.js.map +1 -0
- package/dist/voice/workers-ai.d.ts +136 -0
- package/dist/voice/workers-ai.js +568 -0
- package/dist/voice/workers-ai.js.map +1 -0
- package/dist/websockets/index.d.ts +192 -0
- package/dist/websockets/index.js +2 -0
- package/dist/websockets-DUfRHPRq.js +502 -0
- package/dist/websockets-DUfRHPRq.js.map +1 -0
- package/dist/workflow-types.d.ts +25 -25
- package/dist/workflows.d.ts +21 -21
- package/dist/workflows.js +1 -1
- package/docs/agent-class.md +2 -2
- package/docs/agent-tools.md +2 -1
- package/docs/channels.md +323 -0
- package/docs/chat-agents.md +6 -13
- package/docs/context.md +131 -0
- package/docs/index.md +15 -12
- package/docs/lifecycle.md +102 -55
- package/docs/long-running-agents.md +2 -2
- package/docs/mcp-servers.md +5 -1
- package/docs/resumable-streaming.md +1 -1
- package/docs/routing.md +105 -0
- package/docs/sessions.md +237 -871
- package/docs/streams.md +213 -0
- package/docs/sub-agents.md +184 -124
- package/docs/tasks.md +246 -0
- package/docs/voice.md +745 -0
- package/package.json +115 -13
- package/dist/capability-runner-CvHGZqUu.d.ts +0 -150
- package/dist/compaction-helpers-iiKMr2TQ.js +0 -340
- package/dist/compaction-helpers-iiKMr2TQ.js.map +0 -1
- package/dist/compaction-helpers-wUz6M3us.d.ts +0 -621
- package/dist/diagnostics-CaBjfz4J.js.map +0 -1
- package/dist/durable-object-lifecycle-D6nNQJJd.js +0 -862
- package/dist/durable-object-lifecycle-D6nNQJJd.js.map +0 -1
- package/dist/experimental/memory/session/index.d.ts +0 -671
- package/dist/experimental/memory/session/index.js +0 -2379
- package/dist/experimental/memory/session/index.js.map +0 -1
- package/dist/experimental/memory/utils/index.d.ts +0 -96
- package/dist/experimental/memory/utils/index.js +0 -79
- package/dist/experimental/memory/utils/index.js.map +0 -1
- package/dist/scheduler-CR9RHGos.js +0 -857
- package/dist/scheduler-CR9RHGos.js.map +0 -1
- package/dist/src-5W6JNKVb.js.map +0 -1
- package/dist/tool-output-truncation-CNnnGZQ3.js +0 -98
- package/dist/tool-output-truncation-CNnnGZQ3.js.map +0 -1
|
@@ -0,0 +1,1812 @@
|
|
|
1
|
+
import { i as _classPrivateFieldInitSpec, n as _classPrivateFieldSet2, r as _assertClassBrand, t as _classPrivateFieldGet2 } from "../classPrivateFieldGet2-DZBYAB34.js";
|
|
2
|
+
import { t as _classPrivateMethodInitSpec } from "../classPrivateMethodInitSpec-qMjJ6sHQ.js";
|
|
3
|
+
import { VOICE_PROTOCOL_VERSION } from "./types.js";
|
|
4
|
+
import { logVoiceError, toVoiceError, voiceErrorMessage } from "./errors.js";
|
|
5
|
+
import { n as ServerDiagnostics } from "../diagnostics-C4jcz3VK.js";
|
|
6
|
+
import { n as iterateTextEvents, r as SentenceChunker, t as iterateText } from "../text-stream-CpdiKrJB.js";
|
|
7
|
+
import { addSFUTracks, createSFUSession, createSFUWebSocketAdapter, decodeVarint, downsample48kStereoTo16kMono, encodePayloadToProtobuf, encodeVarint, extractPayloadFromProtobuf, renegotiateSFUSession, sfuFetch, upsample16kMonoTo48kStereo } from "./sfu.js";
|
|
8
|
+
import { WorkersAIFluxSTT, WorkersAINova3STT, WorkersAITTS } from "./workers-ai.js";
|
|
9
|
+
/**
|
|
10
|
+
* True when an error is the platform's signal that a connection (or its
|
|
11
|
+
* Durable Object) was torn down while an operation was in flight. A client
|
|
12
|
+
* can drop at any moment — including mid-`start_call` while a `keepAlive()`
|
|
13
|
+
* alarm write is still pending — and the runtime surfaces that as a
|
|
14
|
+
* retryable "Network connection lost." rejection (or a Durable Object reset).
|
|
15
|
+
* These are expected races during shutdown, not bugs.
|
|
16
|
+
*/
|
|
17
|
+
function isConnectionTeardownError(err) {
|
|
18
|
+
if (typeof err !== "object" || err === null) return false;
|
|
19
|
+
const e = err;
|
|
20
|
+
const message = typeof e.message === "string" ? e.message : "";
|
|
21
|
+
return e.retryable === true || message.includes("Network connection lost") || message.includes("Durable Object reset") || message.includes("Durable Object is overloaded") || message.includes("cannot access storage");
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Run a fire-and-forget task triggered by a WebSocket message so that it can
|
|
25
|
+
* never surface as an unhandled rejection. Voice lifecycle handlers (start
|
|
26
|
+
* call, end call, interrupt, transcript emission) do async work — storage
|
|
27
|
+
* writes for `keepAlive()`, user-defined hooks — but are dispatched without
|
|
28
|
+
* being awaited from the synchronous `onMessage` handler. If the connection
|
|
29
|
+
* is torn down before that work settles, the rejection would otherwise be
|
|
30
|
+
* unhandled. Expected teardown races are swallowed; anything else is logged.
|
|
31
|
+
*/
|
|
32
|
+
function runBackground(label, fn) {
|
|
33
|
+
Promise.resolve().then(fn).catch((error) => {
|
|
34
|
+
if (isConnectionTeardownError(error)) return;
|
|
35
|
+
logVoiceError({
|
|
36
|
+
component: "voice",
|
|
37
|
+
stage: "background_task",
|
|
38
|
+
message: `${label} failed`,
|
|
39
|
+
error: toVoiceError(error, `${label} failed`)
|
|
40
|
+
});
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
function sendVoiceJSON(connection, data, _logPrefix, _skipLog = false) {
|
|
44
|
+
const json = JSON.stringify(data);
|
|
45
|
+
connection.send(json);
|
|
46
|
+
}
|
|
47
|
+
var _audioBuffers = /* @__PURE__ */ new WeakMap();
|
|
48
|
+
var _transcriberSessions = /* @__PURE__ */ new WeakMap();
|
|
49
|
+
var _activePipeline = /* @__PURE__ */ new WeakMap();
|
|
50
|
+
/**
|
|
51
|
+
* Manages per-connection audio pipeline state for voice mixins.
|
|
52
|
+
* Owns the Maps for audio buffers, transcriber sessions, and abort controllers.
|
|
53
|
+
* Does not own pipeline orchestration — that stays in each mixin.
|
|
54
|
+
*/
|
|
55
|
+
var AudioConnectionManager = class {
|
|
56
|
+
constructor(_logPrefix) {
|
|
57
|
+
_classPrivateFieldInitSpec(this, _audioBuffers, /* @__PURE__ */ new Map());
|
|
58
|
+
_classPrivateFieldInitSpec(this, _transcriberSessions, /* @__PURE__ */ new Map());
|
|
59
|
+
_classPrivateFieldInitSpec(this, _activePipeline, /* @__PURE__ */ new Map());
|
|
60
|
+
}
|
|
61
|
+
initConnection(connectionId) {
|
|
62
|
+
if (!_classPrivateFieldGet2(_audioBuffers, this).has(connectionId)) _classPrivateFieldGet2(_audioBuffers, this).set(connectionId, []);
|
|
63
|
+
}
|
|
64
|
+
isInCall(connectionId) {
|
|
65
|
+
return _classPrivateFieldGet2(_audioBuffers, this).has(connectionId);
|
|
66
|
+
}
|
|
67
|
+
cleanup(connectionId) {
|
|
68
|
+
this.abortPipeline(connectionId);
|
|
69
|
+
_classPrivateFieldGet2(_audioBuffers, this).delete(connectionId);
|
|
70
|
+
this.closeTranscriberSession(connectionId);
|
|
71
|
+
}
|
|
72
|
+
bufferAudio(connectionId, chunk) {
|
|
73
|
+
const buffer = _classPrivateFieldGet2(_audioBuffers, this).get(connectionId);
|
|
74
|
+
if (!buffer) return;
|
|
75
|
+
buffer.push(chunk);
|
|
76
|
+
let totalBytes = 0;
|
|
77
|
+
for (const buf of buffer) totalBytes += buf.byteLength;
|
|
78
|
+
while (totalBytes > 96e4 && buffer.length > 1) totalBytes -= buffer.shift().byteLength;
|
|
79
|
+
const session = _classPrivateFieldGet2(_transcriberSessions, this).get(connectionId);
|
|
80
|
+
if (session) session.feed(chunk);
|
|
81
|
+
}
|
|
82
|
+
clearAudioBuffer(connectionId) {
|
|
83
|
+
if (_classPrivateFieldGet2(_audioBuffers, this).has(connectionId)) _classPrivateFieldGet2(_audioBuffers, this).set(connectionId, []);
|
|
84
|
+
}
|
|
85
|
+
hasTranscriberSession(connectionId) {
|
|
86
|
+
return _classPrivateFieldGet2(_transcriberSessions, this).has(connectionId);
|
|
87
|
+
}
|
|
88
|
+
startTranscriberSession(connectionId, transcriber, options) {
|
|
89
|
+
const hadSession = this.closeTranscriberSession(connectionId);
|
|
90
|
+
const session = transcriber.createSession(options);
|
|
91
|
+
_classPrivateFieldGet2(_transcriberSessions, this).set(connectionId, session);
|
|
92
|
+
const buffer = _classPrivateFieldGet2(_audioBuffers, this).get(connectionId);
|
|
93
|
+
if (!hadSession && buffer) for (const chunk of buffer) session.feed(chunk);
|
|
94
|
+
return session;
|
|
95
|
+
}
|
|
96
|
+
closeTranscriberSession(connectionId) {
|
|
97
|
+
const session = _classPrivateFieldGet2(_transcriberSessions, this).get(connectionId);
|
|
98
|
+
if (!session) return false;
|
|
99
|
+
session.close();
|
|
100
|
+
_classPrivateFieldGet2(_transcriberSessions, this).delete(connectionId);
|
|
101
|
+
return true;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Forward the agent's most recent spoken reply to the active transcriber
|
|
105
|
+
* session for conversational context carryover. No-op when there is no
|
|
106
|
+
* session or the provider does not implement `updateAgentContext`.
|
|
107
|
+
*/
|
|
108
|
+
updateAgentContext(connectionId, text) {
|
|
109
|
+
_classPrivateFieldGet2(_transcriberSessions, this).get(connectionId)?.updateAgentContext?.(text);
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Abort any in-flight pipeline and create a new AbortController.
|
|
113
|
+
* Returns the new AbortSignal.
|
|
114
|
+
*/
|
|
115
|
+
createPipelineAbort(connectionId) {
|
|
116
|
+
this.abortPipeline(connectionId);
|
|
117
|
+
const controller = new AbortController();
|
|
118
|
+
_classPrivateFieldGet2(_activePipeline, this).set(connectionId, controller);
|
|
119
|
+
return controller.signal;
|
|
120
|
+
}
|
|
121
|
+
abortPipeline(connectionId) {
|
|
122
|
+
const controller = _classPrivateFieldGet2(_activePipeline, this).get(connectionId);
|
|
123
|
+
if (!controller) return false;
|
|
124
|
+
controller.abort();
|
|
125
|
+
_classPrivateFieldGet2(_activePipeline, this).delete(connectionId);
|
|
126
|
+
return true;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Clear a pipeline abort controller only if it still matches the
|
|
130
|
+
* given signal. Prevents a finished pipeline from deleting a
|
|
131
|
+
* successor pipeline's controller in a concurrent scenario.
|
|
132
|
+
*/
|
|
133
|
+
clearPipelineAbort(connectionId, signal) {
|
|
134
|
+
if (signal) {
|
|
135
|
+
const controller = _classPrivateFieldGet2(_activePipeline, this).get(connectionId);
|
|
136
|
+
if (controller && controller.signal === signal) _classPrivateFieldGet2(_activePipeline, this).delete(connectionId);
|
|
137
|
+
} else _classPrivateFieldGet2(_activePipeline, this).delete(connectionId);
|
|
138
|
+
}
|
|
139
|
+
};
|
|
140
|
+
//#endregion
|
|
141
|
+
//#region src/voice/voice-input.ts
|
|
142
|
+
/**
|
|
143
|
+
* Voice-to-text input mixin. Adds STT-only voice input to an Agent class.
|
|
144
|
+
*
|
|
145
|
+
* Subclasses must set a `transcriber` property (or override `createTranscriber`).
|
|
146
|
+
* No TTS provider is needed. Override `onTranscript` to handle each
|
|
147
|
+
* transcribed utterance.
|
|
148
|
+
*
|
|
149
|
+
* @param Base - The Agent class to extend (e.g. `Agent`).
|
|
150
|
+
* @param voiceInputOptions - Optional pipeline configuration.
|
|
151
|
+
*
|
|
152
|
+
* @example
|
|
153
|
+
* ```typescript
|
|
154
|
+
* import { Agent } from "../index";
|
|
155
|
+
* import { withVoiceInput, WorkersAINova3STT } from "agents/voice";
|
|
156
|
+
*
|
|
157
|
+
* const InputAgent = withVoiceInput(Agent);
|
|
158
|
+
*
|
|
159
|
+
* class MyAgent extends InputAgent<Env> {
|
|
160
|
+
* transcriber = new WorkersAINova3STT(this.env.AI);
|
|
161
|
+
*
|
|
162
|
+
* onTranscript(text, connection) {
|
|
163
|
+
* console.log("User said:", text);
|
|
164
|
+
* }
|
|
165
|
+
* }
|
|
166
|
+
* ```
|
|
167
|
+
*/
|
|
168
|
+
function withVoiceInput(Base, voiceInputOptions) {
|
|
169
|
+
const diagnostics = new ServerDiagnostics(voiceInputOptions?.diagnostics?.browserConsole === true);
|
|
170
|
+
var _cm = /* @__PURE__ */ new WeakMap();
|
|
171
|
+
var _keepAliveDispose = /* @__PURE__ */ new WeakMap();
|
|
172
|
+
var _startupTokens = /* @__PURE__ */ new WeakMap();
|
|
173
|
+
var _callTokens = /* @__PURE__ */ new WeakMap();
|
|
174
|
+
var _turnSequence = /* @__PURE__ */ new WeakMap();
|
|
175
|
+
var _inputTurns = /* @__PURE__ */ new WeakMap();
|
|
176
|
+
var _activeTurns = /* @__PURE__ */ new WeakMap();
|
|
177
|
+
var _VoiceInputMixin_brand = /* @__PURE__ */ new WeakSet();
|
|
178
|
+
class VoiceInputMixin extends Base {
|
|
179
|
+
constructor(...args) {
|
|
180
|
+
super(...args);
|
|
181
|
+
_classPrivateMethodInitSpec(this, _VoiceInputMixin_brand);
|
|
182
|
+
_classPrivateFieldInitSpec(this, _cm, new AudioConnectionManager("VoiceInput"));
|
|
183
|
+
_classPrivateFieldInitSpec(this, _keepAliveDispose, /* @__PURE__ */ new Map());
|
|
184
|
+
_classPrivateFieldInitSpec(this, _startupTokens, /* @__PURE__ */ new Map());
|
|
185
|
+
_classPrivateFieldInitSpec(this, _callTokens, /* @__PURE__ */ new Map());
|
|
186
|
+
_classPrivateFieldInitSpec(this, _turnSequence, 0);
|
|
187
|
+
_classPrivateFieldInitSpec(this, _inputTurns, /* @__PURE__ */ new Map());
|
|
188
|
+
_classPrivateFieldInitSpec(this, _activeTurns, /* @__PURE__ */ new Map());
|
|
189
|
+
const _onConnect = this.onConnect?.bind(this);
|
|
190
|
+
const _onClose = this.onClose?.bind(this);
|
|
191
|
+
const _onMessage = this.onMessage?.bind(this);
|
|
192
|
+
this.onConnect = (connection, ...rest) => {
|
|
193
|
+
sendVoiceJSON(connection, {
|
|
194
|
+
type: "welcome",
|
|
195
|
+
protocol_version: 1,
|
|
196
|
+
...diagnostics.browserConsole ? { diagnostics: { browser_console: true } } : {}
|
|
197
|
+
}, "VoiceInput");
|
|
198
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _diagnose).call(this, connection, "connection.opened");
|
|
199
|
+
sendVoiceJSON(connection, {
|
|
200
|
+
type: "status",
|
|
201
|
+
status: "idle"
|
|
202
|
+
}, "VoiceInput");
|
|
203
|
+
return _onConnect?.(connection, ...rest);
|
|
204
|
+
};
|
|
205
|
+
this.onClose = (connection, ...rest) => {
|
|
206
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _diagnose).call(this, connection, "connection.closed", { in_call: _classPrivateFieldGet2(_cm, this).isInCall(connection.id) });
|
|
207
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _abortInputTurn).call(this, connection, "connection_closed");
|
|
208
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _requestActiveTurnAbort).call(this, connection, "connection_closed");
|
|
209
|
+
_classPrivateFieldGet2(_startupTokens, this).delete(connection.id);
|
|
210
|
+
_classPrivateFieldGet2(_callTokens, this).delete(connection.id);
|
|
211
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _releaseKeepAlive).call(this, connection.id);
|
|
212
|
+
_classPrivateFieldGet2(_cm, this).cleanup(connection.id);
|
|
213
|
+
return _onClose?.(connection, ...rest);
|
|
214
|
+
};
|
|
215
|
+
this.onMessage = (connection, message) => {
|
|
216
|
+
if (message instanceof ArrayBuffer) {
|
|
217
|
+
_classPrivateFieldGet2(_cm, this).bufferAudio(connection.id, message);
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
if (typeof message !== "string") return _onMessage?.(connection, message);
|
|
221
|
+
let parsed;
|
|
222
|
+
try {
|
|
223
|
+
parsed = JSON.parse(message);
|
|
224
|
+
} catch {
|
|
225
|
+
return _onMessage?.(connection, message);
|
|
226
|
+
}
|
|
227
|
+
if (_VOICE_MESSAGES._.has(parsed.type)) {
|
|
228
|
+
switch (parsed.type) {
|
|
229
|
+
case "hello": break;
|
|
230
|
+
case "start_call":
|
|
231
|
+
runBackground("start_call", () => _assertClassBrand(_VoiceInputMixin_brand, this, _handleStartCall).call(this, connection));
|
|
232
|
+
break;
|
|
233
|
+
case "end_call":
|
|
234
|
+
runBackground("end_call", () => _assertClassBrand(_VoiceInputMixin_brand, this, _handleEndCall).call(this, connection));
|
|
235
|
+
break;
|
|
236
|
+
case "start_of_speech":
|
|
237
|
+
case "end_of_speech": break;
|
|
238
|
+
case "interrupt":
|
|
239
|
+
runBackground("interrupt", () => _assertClassBrand(_VoiceInputMixin_brand, this, _handleInterrupt).call(this, connection));
|
|
240
|
+
break;
|
|
241
|
+
}
|
|
242
|
+
return;
|
|
243
|
+
}
|
|
244
|
+
return _onMessage?.(connection, message);
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
onTranscript(_text, _connection) {}
|
|
248
|
+
/**
|
|
249
|
+
* Override to create a transcriber dynamically per connection.
|
|
250
|
+
* Return null to fall back to the `transcriber` property.
|
|
251
|
+
*/
|
|
252
|
+
createTranscriber(_connection) {
|
|
253
|
+
return null;
|
|
254
|
+
}
|
|
255
|
+
beforeCallStart(_connection) {
|
|
256
|
+
return true;
|
|
257
|
+
}
|
|
258
|
+
onCallStart(_connection) {}
|
|
259
|
+
onCallEnd(_connection) {}
|
|
260
|
+
onInterrupt(_connection) {}
|
|
261
|
+
afterTranscribe(transcript, _connection) {
|
|
262
|
+
return transcript;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
function _diagnose(connection, event, data) {
|
|
266
|
+
diagnostics.emit(connection, event, data);
|
|
267
|
+
}
|
|
268
|
+
async function _handleStartCall(connection) {
|
|
269
|
+
if (_classPrivateFieldGet2(_cm, this).isInCall(connection.id)) {
|
|
270
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _diagnose).call(this, connection, "call.start_ignored", { reason: "already_active" });
|
|
271
|
+
return;
|
|
272
|
+
}
|
|
273
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _diagnose).call(this, connection, "call.starting");
|
|
274
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _abortInputTurn).call(this, connection, "call_restarted");
|
|
275
|
+
const startupToken = Symbol(connection.id);
|
|
276
|
+
_classPrivateFieldGet2(_startupTokens, this).set(connection.id, startupToken);
|
|
277
|
+
_classPrivateFieldGet2(_callTokens, this).set(connection.id, startupToken);
|
|
278
|
+
_classPrivateFieldGet2(_cm, this).initConnection(connection.id);
|
|
279
|
+
let startingTranscriber = false;
|
|
280
|
+
try {
|
|
281
|
+
const allowed = await this.beforeCallStart(connection);
|
|
282
|
+
if (!_assertClassBrand(_VoiceInputMixin_brand, this, _isCurrentStartup).call(this, connection.id, startupToken)) return;
|
|
283
|
+
if (!allowed) {
|
|
284
|
+
await _assertClassBrand(_VoiceInputMixin_brand, this, _handleStartupFailure).call(this, connection, startupToken, void 0, "Voice call was rejected", null);
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
const provider = this.createTranscriber(connection) ?? this.transcriber;
|
|
288
|
+
if (!provider) {
|
|
289
|
+
const message = "No transcriber configured. Set 'transcriber' on your VoiceInput subclass or override createTranscriber().";
|
|
290
|
+
logVoiceError({
|
|
291
|
+
component: "VoiceInput",
|
|
292
|
+
stage: "configuration",
|
|
293
|
+
message,
|
|
294
|
+
connectionId: connection.id,
|
|
295
|
+
error: /* @__PURE__ */ new Error(message)
|
|
296
|
+
});
|
|
297
|
+
await _assertClassBrand(_VoiceInputMixin_brand, this, _handleStartupFailure).call(this, connection, startupToken, void 0, message, null);
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
const dispose = await this.keepAlive();
|
|
301
|
+
if (!_assertClassBrand(_VoiceInputMixin_brand, this, _isCurrentStartup).call(this, connection.id, startupToken)) {
|
|
302
|
+
dispose();
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
_classPrivateFieldGet2(_keepAliveDispose, this).set(connection.id, dispose);
|
|
306
|
+
startingTranscriber = true;
|
|
307
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _diagnose).call(this, connection, "stt.starting");
|
|
308
|
+
await _classPrivateFieldGet2(_cm, this).startTranscriberSession(connection.id, provider, {
|
|
309
|
+
onInterim: (text) => {
|
|
310
|
+
if (_classPrivateFieldGet2(_callTokens, this).get(connection.id) !== startupToken) return;
|
|
311
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _getOrCreateInputTurn).call(this, connection).firstInterim(text.length);
|
|
312
|
+
sendVoiceJSON(connection, {
|
|
313
|
+
type: "transcript_interim",
|
|
314
|
+
text
|
|
315
|
+
}, "VoiceInput");
|
|
316
|
+
},
|
|
317
|
+
onSpeechStart: () => {
|
|
318
|
+
if (_classPrivateFieldGet2(_callTokens, this).get(connection.id) !== startupToken) return;
|
|
319
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _replaceInputTurn).call(this, connection).speechStarted();
|
|
320
|
+
},
|
|
321
|
+
onUtterance: (transcript) => {
|
|
322
|
+
if (_classPrivateFieldGet2(_callTokens, this).get(connection.id) !== startupToken) return;
|
|
323
|
+
const turn = _assertClassBrand(_VoiceInputMixin_brand, this, _takeInputTurn).call(this, connection);
|
|
324
|
+
turn.finalInput(transcript.length);
|
|
325
|
+
runBackground("emitTranscript", () => _assertClassBrand(_VoiceInputMixin_brand, this, _emitTranscript).call(this, connection, transcript, turn));
|
|
326
|
+
},
|
|
327
|
+
onFatalError: (error) => {
|
|
328
|
+
runBackground("transcriber_fatal", () => _assertClassBrand(_VoiceInputMixin_brand, this, _handleTranscriberFatal).call(this, connection, startupToken, error));
|
|
329
|
+
}
|
|
330
|
+
}).waitUntilReady?.();
|
|
331
|
+
startingTranscriber = false;
|
|
332
|
+
} catch (error) {
|
|
333
|
+
const clientMessage = startingTranscriber ? "Speech recognition failed to start" : "Voice input failed to start";
|
|
334
|
+
await _assertClassBrand(_VoiceInputMixin_brand, this, _handleStartupFailure).call(this, connection, startupToken, toVoiceError(error, clientMessage), clientMessage, startingTranscriber ? "transcriber_startup" : "call_startup", startingTranscriber ? {
|
|
335
|
+
code: "stt_startup_failed",
|
|
336
|
+
stage: "stt",
|
|
337
|
+
retryable: false
|
|
338
|
+
} : void 0);
|
|
339
|
+
return;
|
|
340
|
+
}
|
|
341
|
+
if (!_assertClassBrand(_VoiceInputMixin_brand, this, _isCurrentStartup).call(this, connection.id, startupToken)) return;
|
|
342
|
+
_classPrivateFieldGet2(_startupTokens, this).delete(connection.id);
|
|
343
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _diagnose).call(this, connection, "stt.ready");
|
|
344
|
+
sendVoiceJSON(connection, {
|
|
345
|
+
type: "status",
|
|
346
|
+
status: "listening"
|
|
347
|
+
}, "VoiceInput");
|
|
348
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _diagnose).call(this, connection, "call.ready");
|
|
349
|
+
await this.onCallStart(connection);
|
|
350
|
+
}
|
|
351
|
+
function _isCurrentStartup(connectionId, startupToken) {
|
|
352
|
+
return _classPrivateFieldGet2(_startupTokens, this).get(connectionId) === startupToken && _classPrivateFieldGet2(_cm, this).isInCall(connectionId);
|
|
353
|
+
}
|
|
354
|
+
async function _handleStartupFailure(connection, startupToken, error, clientMessage, logStage = "call_startup", structuredError) {
|
|
355
|
+
if (!_assertClassBrand(_VoiceInputMixin_brand, this, _isCurrentStartup).call(this, connection.id, startupToken)) return;
|
|
356
|
+
if (logStage && error !== void 0) logVoiceError({
|
|
357
|
+
component: "VoiceInput",
|
|
358
|
+
stage: logStage,
|
|
359
|
+
message: clientMessage,
|
|
360
|
+
connectionId: connection.id,
|
|
361
|
+
error
|
|
362
|
+
});
|
|
363
|
+
_classPrivateFieldGet2(_startupTokens, this).delete(connection.id);
|
|
364
|
+
if (_classPrivateFieldGet2(_callTokens, this).get(connection.id) === startupToken) _classPrivateFieldGet2(_callTokens, this).delete(connection.id);
|
|
365
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _diagnose).call(this, connection, "call.start_failed", {
|
|
366
|
+
stage: logStage ?? "authorization",
|
|
367
|
+
retryable: structuredError?.retryable ?? false,
|
|
368
|
+
...error === void 0 ? {} : { error }
|
|
369
|
+
});
|
|
370
|
+
sendVoiceJSON(connection, {
|
|
371
|
+
type: "error",
|
|
372
|
+
message: clientMessage,
|
|
373
|
+
...structuredError
|
|
374
|
+
}, "VoiceInput");
|
|
375
|
+
_classPrivateFieldGet2(_cm, this).cleanup(connection.id);
|
|
376
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _releaseKeepAlive).call(this, connection.id);
|
|
377
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _diagnose).call(this, connection, "cleanup.completed");
|
|
378
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _diagnose).call(this, connection, "call.ended", { reason: "startup_failed" });
|
|
379
|
+
sendVoiceJSON(connection, {
|
|
380
|
+
type: "status",
|
|
381
|
+
status: "idle"
|
|
382
|
+
}, "VoiceInput");
|
|
383
|
+
await this.onCallEnd(connection);
|
|
384
|
+
}
|
|
385
|
+
async function _handleTranscriberFatal(connection, callToken, error) {
|
|
386
|
+
if (_classPrivateFieldGet2(_callTokens, this).get(connection.id) !== callToken || !_classPrivateFieldGet2(_cm, this).isInCall(connection.id)) return;
|
|
387
|
+
const isStarting = _classPrivateFieldGet2(_startupTokens, this).get(connection.id) === callToken;
|
|
388
|
+
const message = isStarting ? "Speech recognition failed to start" : "Speech recognition connection was lost";
|
|
389
|
+
logVoiceError({
|
|
390
|
+
component: "VoiceInput",
|
|
391
|
+
stage: isStarting ? "transcriber_startup" : "transcriber_runtime",
|
|
392
|
+
message,
|
|
393
|
+
connectionId: connection.id,
|
|
394
|
+
error
|
|
395
|
+
});
|
|
396
|
+
_classPrivateFieldGet2(_startupTokens, this).delete(connection.id);
|
|
397
|
+
_classPrivateFieldGet2(_callTokens, this).delete(connection.id);
|
|
398
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _abortInputTurn).call(this, connection, "stt_fatal");
|
|
399
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _requestActiveTurnAbort).call(this, connection, "stt_fatal");
|
|
400
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _diagnose).call(this, connection, "stt.fatal", {
|
|
401
|
+
stage: isStarting ? "startup" : "runtime",
|
|
402
|
+
retryable: !isStarting,
|
|
403
|
+
error
|
|
404
|
+
});
|
|
405
|
+
sendVoiceJSON(connection, {
|
|
406
|
+
type: "error",
|
|
407
|
+
message,
|
|
408
|
+
code: isStarting ? "stt_startup_failed" : "stt_connection_lost",
|
|
409
|
+
stage: "stt",
|
|
410
|
+
retryable: !isStarting
|
|
411
|
+
}, "VoiceInput");
|
|
412
|
+
_classPrivateFieldGet2(_cm, this).cleanup(connection.id);
|
|
413
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _releaseKeepAlive).call(this, connection.id);
|
|
414
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _diagnose).call(this, connection, "cleanup.completed");
|
|
415
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _diagnose).call(this, connection, "call.ended", { reason: "stt_fatal" });
|
|
416
|
+
sendVoiceJSON(connection, {
|
|
417
|
+
type: "status",
|
|
418
|
+
status: "idle"
|
|
419
|
+
}, "VoiceInput");
|
|
420
|
+
await this.onCallEnd(connection);
|
|
421
|
+
}
|
|
422
|
+
function _createTurn(connection) {
|
|
423
|
+
var _this$turnSequence;
|
|
424
|
+
const turn = diagnostics.turn(connection, `turn_${_classPrivateFieldSet2(_turnSequence, this, (_this$turnSequence = _classPrivateFieldGet2(_turnSequence, this), ++_this$turnSequence)).toString(36)}`, "speech");
|
|
425
|
+
turn.emit("turn.started", { source: "speech" });
|
|
426
|
+
return turn;
|
|
427
|
+
}
|
|
428
|
+
function _replaceInputTurn(connection) {
|
|
429
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _abortInputTurn).call(this, connection, "replaced");
|
|
430
|
+
const turn = _assertClassBrand(_VoiceInputMixin_brand, this, _createTurn).call(this, connection);
|
|
431
|
+
_classPrivateFieldGet2(_inputTurns, this).set(connection.id, turn);
|
|
432
|
+
return turn;
|
|
433
|
+
}
|
|
434
|
+
function _getOrCreateInputTurn(connection) {
|
|
435
|
+
const current = _classPrivateFieldGet2(_inputTurns, this).get(connection.id);
|
|
436
|
+
if (current) return current;
|
|
437
|
+
const turn = _assertClassBrand(_VoiceInputMixin_brand, this, _createTurn).call(this, connection);
|
|
438
|
+
_classPrivateFieldGet2(_inputTurns, this).set(connection.id, turn);
|
|
439
|
+
return turn;
|
|
440
|
+
}
|
|
441
|
+
function _takeInputTurn(connection) {
|
|
442
|
+
const turn = _assertClassBrand(_VoiceInputMixin_brand, this, _getOrCreateInputTurn).call(this, connection);
|
|
443
|
+
_classPrivateFieldGet2(_inputTurns, this).delete(connection.id);
|
|
444
|
+
return turn;
|
|
445
|
+
}
|
|
446
|
+
function _abortInputTurn(connection, reason) {
|
|
447
|
+
const turn = _classPrivateFieldGet2(_inputTurns, this).get(connection.id);
|
|
448
|
+
if (!turn) return;
|
|
449
|
+
_classPrivateFieldGet2(_inputTurns, this).delete(connection.id);
|
|
450
|
+
turn.emit("turn.aborted", { reason });
|
|
451
|
+
turn.finish("aborted");
|
|
452
|
+
}
|
|
453
|
+
function _activateTurn(connection, turn) {
|
|
454
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _requestActiveTurnAbort).call(this, connection, "replaced");
|
|
455
|
+
const active = {
|
|
456
|
+
signal: _classPrivateFieldGet2(_cm, this).createPipelineAbort(connection.id),
|
|
457
|
+
turn
|
|
458
|
+
};
|
|
459
|
+
_classPrivateFieldGet2(_activeTurns, this).set(connection.id, active);
|
|
460
|
+
return active;
|
|
461
|
+
}
|
|
462
|
+
function _requestActiveTurnAbort(connection, reason) {
|
|
463
|
+
const active = _classPrivateFieldGet2(_activeTurns, this).get(connection.id);
|
|
464
|
+
if (!active) return;
|
|
465
|
+
active.turn.emit("turn.abort_requested", { reason });
|
|
466
|
+
}
|
|
467
|
+
function _clearActiveTurn(connectionId, active) {
|
|
468
|
+
if (_classPrivateFieldGet2(_activeTurns, this).get(connectionId) === active) _classPrivateFieldGet2(_activeTurns, this).delete(connectionId);
|
|
469
|
+
}
|
|
470
|
+
function _releaseKeepAlive(connectionId) {
|
|
471
|
+
const dispose = _classPrivateFieldGet2(_keepAliveDispose, this).get(connectionId);
|
|
472
|
+
if (dispose) {
|
|
473
|
+
dispose();
|
|
474
|
+
_classPrivateFieldGet2(_keepAliveDispose, this).delete(connectionId);
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
function _handleEndCall(connection) {
|
|
478
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _diagnose).call(this, connection, "call.ended", { reason: "requested" });
|
|
479
|
+
_classPrivateFieldGet2(_startupTokens, this).delete(connection.id);
|
|
480
|
+
_classPrivateFieldGet2(_callTokens, this).delete(connection.id);
|
|
481
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _abortInputTurn).call(this, connection, "call_ended");
|
|
482
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _requestActiveTurnAbort).call(this, connection, "call_ended");
|
|
483
|
+
_classPrivateFieldGet2(_cm, this).cleanup(connection.id);
|
|
484
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _releaseKeepAlive).call(this, connection.id);
|
|
485
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _diagnose).call(this, connection, "cleanup.completed");
|
|
486
|
+
sendVoiceJSON(connection, {
|
|
487
|
+
type: "status",
|
|
488
|
+
status: "idle"
|
|
489
|
+
}, "VoiceInput");
|
|
490
|
+
return this.onCallEnd(connection);
|
|
491
|
+
}
|
|
492
|
+
function _handleInterrupt(connection) {
|
|
493
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _abortInputTurn).call(this, connection, "client_interrupt");
|
|
494
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _requestActiveTurnAbort).call(this, connection, "client_interrupt");
|
|
495
|
+
const interrupted = _classPrivateFieldGet2(_cm, this).abortPipeline(connection.id);
|
|
496
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _diagnose).call(this, connection, "turn.interrupt_requested", { active_turn: interrupted });
|
|
497
|
+
_classPrivateFieldGet2(_cm, this).clearAudioBuffer(connection.id);
|
|
498
|
+
sendVoiceJSON(connection, {
|
|
499
|
+
type: "status",
|
|
500
|
+
status: "listening"
|
|
501
|
+
}, "VoiceInput");
|
|
502
|
+
return this.onInterrupt(connection);
|
|
503
|
+
}
|
|
504
|
+
async function _emitTranscript(connection, transcript, turn) {
|
|
505
|
+
const active = _assertClassBrand(_VoiceInputMixin_brand, this, _activateTurn).call(this, connection, turn);
|
|
506
|
+
const { signal } = active;
|
|
507
|
+
let outcome = "completed";
|
|
508
|
+
let restoreListening = true;
|
|
509
|
+
try {
|
|
510
|
+
const afterTranscribeStart = Date.now();
|
|
511
|
+
const userText = await this.afterTranscribe(transcript, connection);
|
|
512
|
+
turn.recordAfterTranscribe(Date.now() - afterTranscribeStart, userText ? "accepted" : "skipped", userText?.length ?? 0);
|
|
513
|
+
if (signal.aborted) return;
|
|
514
|
+
if (!userText) {
|
|
515
|
+
outcome = "skipped";
|
|
516
|
+
restoreListening = false;
|
|
517
|
+
return;
|
|
518
|
+
}
|
|
519
|
+
sendVoiceJSON(connection, {
|
|
520
|
+
type: "transcript_interim",
|
|
521
|
+
text: ""
|
|
522
|
+
}, "VoiceInput");
|
|
523
|
+
sendVoiceJSON(connection, {
|
|
524
|
+
type: "transcript",
|
|
525
|
+
role: "user",
|
|
526
|
+
text: userText
|
|
527
|
+
}, "VoiceInput");
|
|
528
|
+
await this.onTranscript(userText, connection);
|
|
529
|
+
} catch (error) {
|
|
530
|
+
if (signal.aborted) return;
|
|
531
|
+
const voiceError = toVoiceError(error, "Transcript processing failed");
|
|
532
|
+
outcome = "error";
|
|
533
|
+
turn.emit("turn.error", {
|
|
534
|
+
stage: "transcript",
|
|
535
|
+
error: voiceError
|
|
536
|
+
});
|
|
537
|
+
logVoiceError({
|
|
538
|
+
component: "VoiceInput",
|
|
539
|
+
stage: "transcript",
|
|
540
|
+
message: "Transcript processing failed",
|
|
541
|
+
connectionId: connection.id,
|
|
542
|
+
error: voiceError
|
|
543
|
+
});
|
|
544
|
+
sendVoiceJSON(connection, {
|
|
545
|
+
type: "error",
|
|
546
|
+
message: voiceErrorMessage(voiceError, "Transcript processing failed")
|
|
547
|
+
}, "VoiceInput");
|
|
548
|
+
} finally {
|
|
549
|
+
if (signal.aborted) {
|
|
550
|
+
outcome = "aborted";
|
|
551
|
+
turn.emit("turn.aborted");
|
|
552
|
+
}
|
|
553
|
+
turn.finish(outcome);
|
|
554
|
+
_classPrivateFieldGet2(_cm, this).clearPipelineAbort(connection.id, signal);
|
|
555
|
+
_assertClassBrand(_VoiceInputMixin_brand, this, _clearActiveTurn).call(this, connection.id, active);
|
|
556
|
+
if (restoreListening && _classPrivateFieldGet2(_cm, this).isInCall(connection.id)) sendVoiceJSON(connection, {
|
|
557
|
+
type: "status",
|
|
558
|
+
status: "listening"
|
|
559
|
+
}, "VoiceInput");
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
var _VOICE_MESSAGES = { _: /* @__PURE__ */ new Set([
|
|
563
|
+
"hello",
|
|
564
|
+
"start_call",
|
|
565
|
+
"end_call",
|
|
566
|
+
"start_of_speech",
|
|
567
|
+
"end_of_speech",
|
|
568
|
+
"interrupt"
|
|
569
|
+
]) };
|
|
570
|
+
return VoiceInputMixin;
|
|
571
|
+
}
|
|
572
|
+
//#endregion
|
|
573
|
+
//#region src/voice/index.ts
|
|
574
|
+
const DEFAULT_HISTORY_LIMIT = 20;
|
|
575
|
+
const DEFAULT_MAX_MESSAGE_COUNT = 1e3;
|
|
576
|
+
const DEFAULT_SAMPLE_RATE = 16e3;
|
|
577
|
+
var ModelStreamError = class extends Error {
|
|
578
|
+
constructor(streamError, partialOutput) {
|
|
579
|
+
super(streamError.message, { cause: streamError });
|
|
580
|
+
this.name = "ModelStreamError";
|
|
581
|
+
this.streamError = streamError;
|
|
582
|
+
this.partialOutput = partialOutput;
|
|
583
|
+
}
|
|
584
|
+
};
|
|
585
|
+
function completionOutcomeCode(finishReason, hasOutput) {
|
|
586
|
+
if (finishReason === "length") return "output_limit";
|
|
587
|
+
if (finishReason === "content-filter") return "content_filtered";
|
|
588
|
+
if (finishReason === "error") return "model_error";
|
|
589
|
+
return hasOutput ? null : "no_output";
|
|
590
|
+
}
|
|
591
|
+
function stableTurnOutcome(finishReason, hasOutput) {
|
|
592
|
+
return completionOutcomeCode(finishReason, hasOutput) ?? "completed";
|
|
593
|
+
}
|
|
594
|
+
function createCompletionOutcome(finishReason, partialOutput) {
|
|
595
|
+
const code = completionOutcomeCode(finishReason, partialOutput);
|
|
596
|
+
return code === null ? null : {
|
|
597
|
+
code,
|
|
598
|
+
stage: "llm",
|
|
599
|
+
...finishReason === void 0 ? {} : { finishReason },
|
|
600
|
+
partialOutput
|
|
601
|
+
};
|
|
602
|
+
}
|
|
603
|
+
/**
|
|
604
|
+
* Voice pipeline mixin. Adds the full voice pipeline to an Agent class.
|
|
605
|
+
*
|
|
606
|
+
* Subclasses must set a `transcriber` property (or override `createTranscriber`)
|
|
607
|
+
* and a `tts` provider property. The transcriber session is per-call — created
|
|
608
|
+
* at start_call and closed at end_call. The model handles turn detection.
|
|
609
|
+
*
|
|
610
|
+
* @param Base - The Agent class to extend (e.g. `Agent`).
|
|
611
|
+
* @param voiceOptions - Optional pipeline configuration.
|
|
612
|
+
*
|
|
613
|
+
* @example
|
|
614
|
+
* ```typescript
|
|
615
|
+
* import { Agent } from "../index";
|
|
616
|
+
* import { withVoice, WorkersAIFluxSTT, WorkersAITTS } from "agents/voice";
|
|
617
|
+
*
|
|
618
|
+
* const VoiceAgent = withVoice(Agent);
|
|
619
|
+
*
|
|
620
|
+
* class MyAgent extends VoiceAgent<Env> {
|
|
621
|
+
* transcriber = new WorkersAIFluxSTT(this.env.AI);
|
|
622
|
+
* tts = new WorkersAITTS(this.env.AI);
|
|
623
|
+
*
|
|
624
|
+
* async onTurn(transcript, context) {
|
|
625
|
+
* return "Hello! I heard you say: " + transcript;
|
|
626
|
+
* }
|
|
627
|
+
* }
|
|
628
|
+
* ```
|
|
629
|
+
*/
|
|
630
|
+
function withVoice(Base, voiceOptions) {
|
|
631
|
+
const opts = voiceOptions ?? {};
|
|
632
|
+
const diagnostics = new ServerDiagnostics(opts.diagnostics?.browserConsole === true);
|
|
633
|
+
function opt(key, fallback) {
|
|
634
|
+
return opts[key] ?? fallback;
|
|
635
|
+
}
|
|
636
|
+
var _cm = /* @__PURE__ */ new WeakMap();
|
|
637
|
+
var _keepAliveDispose = /* @__PURE__ */ new WeakMap();
|
|
638
|
+
var _startupTokens = /* @__PURE__ */ new WeakMap();
|
|
639
|
+
var _callTokens = /* @__PURE__ */ new WeakMap();
|
|
640
|
+
var _turnSequence = /* @__PURE__ */ new WeakMap();
|
|
641
|
+
var _inputTurns = /* @__PURE__ */ new WeakMap();
|
|
642
|
+
var _activeTurnDiagnostics = /* @__PURE__ */ new WeakMap();
|
|
643
|
+
var _schemaReady = /* @__PURE__ */ new WeakMap();
|
|
644
|
+
var _VoiceAgentMixin_brand = /* @__PURE__ */ new WeakSet();
|
|
645
|
+
class VoiceAgentMixin extends Base {
|
|
646
|
+
constructor(...args) {
|
|
647
|
+
super(...args);
|
|
648
|
+
_classPrivateMethodInitSpec(this, _VoiceAgentMixin_brand);
|
|
649
|
+
_classPrivateFieldInitSpec(this, _cm, new AudioConnectionManager("VoiceAgent"));
|
|
650
|
+
_classPrivateFieldInitSpec(this, _keepAliveDispose, /* @__PURE__ */ new Map());
|
|
651
|
+
_classPrivateFieldInitSpec(this, _startupTokens, /* @__PURE__ */ new Map());
|
|
652
|
+
_classPrivateFieldInitSpec(this, _callTokens, /* @__PURE__ */ new Map());
|
|
653
|
+
_classPrivateFieldInitSpec(this, _turnSequence, 0);
|
|
654
|
+
_classPrivateFieldInitSpec(this, _inputTurns, /* @__PURE__ */ new Map());
|
|
655
|
+
_classPrivateFieldInitSpec(this, _activeTurnDiagnostics, /* @__PURE__ */ new Map());
|
|
656
|
+
_classPrivateFieldInitSpec(this, _schemaReady, false);
|
|
657
|
+
const _onConnect = this.onConnect?.bind(this);
|
|
658
|
+
const _onClose = this.onClose?.bind(this);
|
|
659
|
+
const _onMessage = this.onMessage?.bind(this);
|
|
660
|
+
this.onConnect = (connection, ...rest) => {
|
|
661
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
662
|
+
type: "welcome",
|
|
663
|
+
protocol_version: 1,
|
|
664
|
+
...diagnostics.browserConsole ? { diagnostics: { browser_console: true } } : {}
|
|
665
|
+
});
|
|
666
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _diagnose).call(this, connection, "connection.opened");
|
|
667
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
668
|
+
type: "status",
|
|
669
|
+
status: "idle"
|
|
670
|
+
});
|
|
671
|
+
return _onConnect?.(connection, ...rest);
|
|
672
|
+
};
|
|
673
|
+
this.onClose = (connection, ...rest) => {
|
|
674
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _diagnose).call(this, connection, "connection.closed", { in_call: _classPrivateFieldGet2(_cm, this).isInCall(connection.id) });
|
|
675
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _requestActiveTurnAbort).call(this, connection, "turn.abort_requested", "connection_closed");
|
|
676
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _abortInputTurn).call(this, connection, "connection_closed");
|
|
677
|
+
_classPrivateFieldGet2(_activeTurnDiagnostics, this).delete(connection.id);
|
|
678
|
+
_classPrivateFieldGet2(_startupTokens, this).delete(connection.id);
|
|
679
|
+
_classPrivateFieldGet2(_callTokens, this).delete(connection.id);
|
|
680
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _releaseKeepAlive).call(this, connection.id);
|
|
681
|
+
_classPrivateFieldGet2(_cm, this).cleanup(connection.id);
|
|
682
|
+
return _onClose?.(connection, ...rest);
|
|
683
|
+
};
|
|
684
|
+
this.onMessage = (connection, message) => {
|
|
685
|
+
if (message instanceof ArrayBuffer) {
|
|
686
|
+
_classPrivateFieldGet2(_cm, this).bufferAudio(connection.id, message);
|
|
687
|
+
return;
|
|
688
|
+
}
|
|
689
|
+
if (typeof message !== "string") return _onMessage?.(connection, message);
|
|
690
|
+
let parsed;
|
|
691
|
+
try {
|
|
692
|
+
parsed = JSON.parse(message);
|
|
693
|
+
} catch {
|
|
694
|
+
return _onMessage?.(connection, message);
|
|
695
|
+
}
|
|
696
|
+
if (_VOICE_MESSAGES._.has(parsed.type)) {
|
|
697
|
+
switch (parsed.type) {
|
|
698
|
+
case "hello": break;
|
|
699
|
+
case "start_call":
|
|
700
|
+
runBackground("start_call", () => _assertClassBrand(_VoiceAgentMixin_brand, this, _handleStartCall).call(this, connection, parsed.preferred_format));
|
|
701
|
+
break;
|
|
702
|
+
case "end_call":
|
|
703
|
+
runBackground("end_call", () => _assertClassBrand(_VoiceAgentMixin_brand, this, _handleEndCall).call(this, connection));
|
|
704
|
+
break;
|
|
705
|
+
case "start_of_speech":
|
|
706
|
+
case "end_of_speech": break;
|
|
707
|
+
case "interrupt":
|
|
708
|
+
runBackground("interrupt", () => _assertClassBrand(_VoiceAgentMixin_brand, this, _handleInterrupt).call(this, connection));
|
|
709
|
+
break;
|
|
710
|
+
case "text_message": {
|
|
711
|
+
const text = parsed.text;
|
|
712
|
+
if (typeof text === "string") runBackground("text_message", () => _assertClassBrand(_VoiceAgentMixin_brand, this, _handleTextMessage).call(this, connection, text));
|
|
713
|
+
break;
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
return;
|
|
717
|
+
}
|
|
718
|
+
return _onMessage?.(connection, message);
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
onTurn(_transcript, _context) {
|
|
722
|
+
throw new Error("VoiceAgent subclass must implement onTurn(). Return a string, AI SDK stream, AsyncIterable<string>, or ReadableStream.");
|
|
723
|
+
}
|
|
724
|
+
/**
|
|
725
|
+
* Override to create a transcriber dynamically per connection.
|
|
726
|
+
* Useful for runtime model switching (e.g. Flux vs Nova 3 dropdown).
|
|
727
|
+
* Return null to fall back to the `transcriber` property.
|
|
728
|
+
*/
|
|
729
|
+
createTranscriber(_connection) {
|
|
730
|
+
return null;
|
|
731
|
+
}
|
|
732
|
+
beforeCallStart(_connection) {
|
|
733
|
+
return true;
|
|
734
|
+
}
|
|
735
|
+
onCallStart(_connection) {}
|
|
736
|
+
onCallEnd(_connection) {}
|
|
737
|
+
onInterrupt(_connection) {}
|
|
738
|
+
afterTranscribe(transcript, _connection) {
|
|
739
|
+
return transcript;
|
|
740
|
+
}
|
|
741
|
+
beforeSynthesize(text, _connection) {
|
|
742
|
+
return text;
|
|
743
|
+
}
|
|
744
|
+
afterSynthesize(audio, _text, _connection) {
|
|
745
|
+
return audio;
|
|
746
|
+
}
|
|
747
|
+
saveMessage(role, text) {
|
|
748
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _ensureSchema).call(this);
|
|
749
|
+
this.sql`
|
|
750
|
+
INSERT INTO cf_voice_messages (role, text, timestamp)
|
|
751
|
+
VALUES (${role}, ${text}, ${Date.now()})
|
|
752
|
+
`;
|
|
753
|
+
const maxMessages = opt("maxMessageCount", DEFAULT_MAX_MESSAGE_COUNT);
|
|
754
|
+
this.sql`
|
|
755
|
+
DELETE FROM cf_voice_messages
|
|
756
|
+
WHERE id NOT IN (
|
|
757
|
+
SELECT id FROM cf_voice_messages
|
|
758
|
+
ORDER BY id DESC LIMIT ${maxMessages}
|
|
759
|
+
)
|
|
760
|
+
`;
|
|
761
|
+
}
|
|
762
|
+
getConversationHistory(limit) {
|
|
763
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _ensureSchema).call(this);
|
|
764
|
+
const historyLimit = limit ?? opt("historyLimit", DEFAULT_HISTORY_LIMIT);
|
|
765
|
+
return this.sql`
|
|
766
|
+
SELECT role, text FROM cf_voice_messages
|
|
767
|
+
ORDER BY id DESC LIMIT ${historyLimit}
|
|
768
|
+
`.reverse().map((row) => ({
|
|
769
|
+
role: row.role,
|
|
770
|
+
content: row.text
|
|
771
|
+
}));
|
|
772
|
+
}
|
|
773
|
+
forceEndCall(connection) {
|
|
774
|
+
if (!_classPrivateFieldGet2(_cm, this).isInCall(connection.id)) return;
|
|
775
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _handleEndCall).call(this, connection);
|
|
776
|
+
}
|
|
777
|
+
async speak(connection, text) {
|
|
778
|
+
const signal = _classPrivateFieldGet2(_cm, this).createPipelineAbort(connection.id);
|
|
779
|
+
try {
|
|
780
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
781
|
+
type: "transcript_start",
|
|
782
|
+
role: "assistant"
|
|
783
|
+
});
|
|
784
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
785
|
+
type: "transcript_end",
|
|
786
|
+
text
|
|
787
|
+
});
|
|
788
|
+
const audio = await _assertClassBrand(_VoiceAgentMixin_brand, this, _synthesizeWithHooks).call(this, text, connection, signal);
|
|
789
|
+
if (audio && !signal.aborted) {
|
|
790
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
791
|
+
type: "status",
|
|
792
|
+
status: "speaking"
|
|
793
|
+
});
|
|
794
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _diagnose).call(this, connection, "audio.first_sent", { bytes: audio.byteLength });
|
|
795
|
+
connection.send(audio);
|
|
796
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _diagnose).call(this, connection, "audio.completed", { bytes: audio.byteLength });
|
|
797
|
+
}
|
|
798
|
+
if (!signal.aborted) {
|
|
799
|
+
_classPrivateFieldGet2(_cm, this).updateAgentContext(connection.id, text);
|
|
800
|
+
this.saveMessage("assistant", text);
|
|
801
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
802
|
+
type: "status",
|
|
803
|
+
status: "listening"
|
|
804
|
+
});
|
|
805
|
+
}
|
|
806
|
+
} finally {
|
|
807
|
+
_classPrivateFieldGet2(_cm, this).clearPipelineAbort(connection.id, signal);
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
async speakAll(text) {
|
|
811
|
+
this.saveMessage("assistant", text);
|
|
812
|
+
const connections = [...this.getConnections()];
|
|
813
|
+
if (connections.length === 0) return;
|
|
814
|
+
for (const connection of connections) {
|
|
815
|
+
const signal = _classPrivateFieldGet2(_cm, this).createPipelineAbort(connection.id);
|
|
816
|
+
try {
|
|
817
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
818
|
+
type: "transcript_start",
|
|
819
|
+
role: "assistant"
|
|
820
|
+
});
|
|
821
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
822
|
+
type: "transcript_end",
|
|
823
|
+
text
|
|
824
|
+
});
|
|
825
|
+
const audio = await _assertClassBrand(_VoiceAgentMixin_brand, this, _synthesizeWithHooks).call(this, text, connection, signal);
|
|
826
|
+
if (audio && !signal.aborted) {
|
|
827
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
828
|
+
type: "status",
|
|
829
|
+
status: "speaking"
|
|
830
|
+
});
|
|
831
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _diagnose).call(this, connection, "audio.first_sent", { bytes: audio.byteLength });
|
|
832
|
+
connection.send(audio);
|
|
833
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _diagnose).call(this, connection, "audio.completed", { bytes: audio.byteLength });
|
|
834
|
+
}
|
|
835
|
+
if (!signal.aborted) {
|
|
836
|
+
_classPrivateFieldGet2(_cm, this).updateAgentContext(connection.id, text);
|
|
837
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
838
|
+
type: "status",
|
|
839
|
+
status: "listening"
|
|
840
|
+
});
|
|
841
|
+
}
|
|
842
|
+
} finally {
|
|
843
|
+
_classPrivateFieldGet2(_cm, this).clearPipelineAbort(connection.id, signal);
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
function _ensureSchema() {
|
|
849
|
+
if (_classPrivateFieldGet2(_schemaReady, this)) return;
|
|
850
|
+
this.sql`
|
|
851
|
+
CREATE TABLE IF NOT EXISTS cf_voice_messages (
|
|
852
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
853
|
+
role TEXT NOT NULL,
|
|
854
|
+
text TEXT NOT NULL,
|
|
855
|
+
timestamp INTEGER NOT NULL
|
|
856
|
+
)
|
|
857
|
+
`;
|
|
858
|
+
_classPrivateFieldSet2(_schemaReady, this, true);
|
|
859
|
+
}
|
|
860
|
+
function _requireTTS() {
|
|
861
|
+
if (!this.tts) throw new Error("No TTS provider configured. Set 'tts' on your VoiceAgent subclass.");
|
|
862
|
+
return this.tts;
|
|
863
|
+
}
|
|
864
|
+
async function _synthesizeWithHooks(text, connection, signal, turn) {
|
|
865
|
+
const sentence = turn?.beginTtsSentence();
|
|
866
|
+
let sentenceOutcome = "completed";
|
|
867
|
+
try {
|
|
868
|
+
let textToSpeak;
|
|
869
|
+
try {
|
|
870
|
+
textToSpeak = await this.beforeSynthesize(text, connection);
|
|
871
|
+
} catch (error) {
|
|
872
|
+
const voiceError = toVoiceError(error, "TTS preparation failed");
|
|
873
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _emitTurnDiagnostic).call(this, connection, turn, "tts.failed", {
|
|
874
|
+
stage: "before_synthesize",
|
|
875
|
+
error: voiceError
|
|
876
|
+
});
|
|
877
|
+
throw voiceError;
|
|
878
|
+
}
|
|
879
|
+
if (!textToSpeak) {
|
|
880
|
+
sentenceOutcome = "skipped";
|
|
881
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _emitTurnDiagnostic).call(this, connection, turn, "tts.skipped", { reason: "before_synthesize" });
|
|
882
|
+
return null;
|
|
883
|
+
}
|
|
884
|
+
let tts;
|
|
885
|
+
try {
|
|
886
|
+
tts = _assertClassBrand(_VoiceAgentMixin_brand, this, _requireTTS).call(this);
|
|
887
|
+
} catch (error) {
|
|
888
|
+
const voiceError = toVoiceError(error, "TTS is not configured");
|
|
889
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _emitTurnDiagnostic).call(this, connection, turn, "tts.failed", {
|
|
890
|
+
stage: "configuration",
|
|
891
|
+
error: voiceError
|
|
892
|
+
});
|
|
893
|
+
throw voiceError;
|
|
894
|
+
}
|
|
895
|
+
const startedAt = Date.now();
|
|
896
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _emitTurnDiagnostic).call(this, connection, turn, "tts.started", { characters: textToSpeak.length });
|
|
897
|
+
sentence?.providerStarted();
|
|
898
|
+
try {
|
|
899
|
+
const rawAudio = await tts.synthesize(textToSpeak, signal);
|
|
900
|
+
const audio = await this.afterSynthesize(rawAudio, textToSpeak, connection);
|
|
901
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _emitTurnDiagnostic).call(this, connection, turn, "tts.completed", {
|
|
902
|
+
duration_ms: Date.now() - startedAt,
|
|
903
|
+
outcome: audio ? "audio" : "no_audio",
|
|
904
|
+
bytes: audio?.byteLength ?? 0
|
|
905
|
+
});
|
|
906
|
+
return audio;
|
|
907
|
+
} catch (error) {
|
|
908
|
+
const voiceError = toVoiceError(error, "TTS failed");
|
|
909
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _emitTurnDiagnostic).call(this, connection, turn, "tts.failed", {
|
|
910
|
+
duration_ms: Date.now() - startedAt,
|
|
911
|
+
error: voiceError
|
|
912
|
+
});
|
|
913
|
+
throw voiceError;
|
|
914
|
+
}
|
|
915
|
+
} catch (error) {
|
|
916
|
+
sentenceOutcome = "failed";
|
|
917
|
+
throw error;
|
|
918
|
+
} finally {
|
|
919
|
+
sentence?.settle(sentenceOutcome);
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
async function _handleStartCall(connection, _preferredFormat) {
|
|
923
|
+
if (_classPrivateFieldGet2(_cm, this).isInCall(connection.id)) {
|
|
924
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _diagnose).call(this, connection, "call.start_ignored", { reason: "already_active" });
|
|
925
|
+
return;
|
|
926
|
+
}
|
|
927
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _diagnose).call(this, connection, "call.starting");
|
|
928
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _abortInputTurn).call(this, connection, "call_restarted");
|
|
929
|
+
const startupToken = Symbol(connection.id);
|
|
930
|
+
_classPrivateFieldGet2(_startupTokens, this).set(connection.id, startupToken);
|
|
931
|
+
_classPrivateFieldGet2(_callTokens, this).set(connection.id, startupToken);
|
|
932
|
+
_classPrivateFieldGet2(_cm, this).initConnection(connection.id);
|
|
933
|
+
let provider;
|
|
934
|
+
try {
|
|
935
|
+
const allowed = await this.beforeCallStart(connection);
|
|
936
|
+
if (!_assertClassBrand(_VoiceAgentMixin_brand, this, _isCurrentStartup).call(this, connection.id, startupToken)) return;
|
|
937
|
+
if (!allowed) {
|
|
938
|
+
await _assertClassBrand(_VoiceAgentMixin_brand, this, _handleStartupFailure).call(this, connection, startupToken, void 0, "Voice call was rejected", null);
|
|
939
|
+
return;
|
|
940
|
+
}
|
|
941
|
+
provider = this.createTranscriber(connection) ?? this.transcriber;
|
|
942
|
+
if (!provider) {
|
|
943
|
+
const message = "No transcriber configured. Set 'transcriber' on your VoiceAgent subclass or override createTranscriber().";
|
|
944
|
+
logVoiceError({
|
|
945
|
+
component: "VoiceAgent",
|
|
946
|
+
stage: "configuration",
|
|
947
|
+
message,
|
|
948
|
+
connectionId: connection.id,
|
|
949
|
+
error: /* @__PURE__ */ new Error(message)
|
|
950
|
+
});
|
|
951
|
+
await _assertClassBrand(_VoiceAgentMixin_brand, this, _handleStartupFailure).call(this, connection, startupToken, void 0, message, null);
|
|
952
|
+
return;
|
|
953
|
+
}
|
|
954
|
+
const dispose = await this.keepAlive();
|
|
955
|
+
if (!_assertClassBrand(_VoiceAgentMixin_brand, this, _isCurrentStartup).call(this, connection.id, startupToken)) {
|
|
956
|
+
dispose();
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
_classPrivateFieldGet2(_keepAliveDispose, this).set(connection.id, dispose);
|
|
960
|
+
const configuredFormat = opt("audioFormat", "mp3");
|
|
961
|
+
const configuredSampleRate = opt("sampleRate", DEFAULT_SAMPLE_RATE);
|
|
962
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
963
|
+
type: "audio_config",
|
|
964
|
+
format: configuredFormat,
|
|
965
|
+
sampleRate: configuredSampleRate
|
|
966
|
+
});
|
|
967
|
+
} catch (error) {
|
|
968
|
+
await _assertClassBrand(_VoiceAgentMixin_brand, this, _handleStartupFailure).call(this, connection, startupToken, toVoiceError(error, "Voice call failed to start"), "Voice call failed to start");
|
|
969
|
+
return;
|
|
970
|
+
}
|
|
971
|
+
if (!provider) return;
|
|
972
|
+
let session;
|
|
973
|
+
try {
|
|
974
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _diagnose).call(this, connection, "stt.starting");
|
|
975
|
+
session = _classPrivateFieldGet2(_cm, this).startTranscriberSession(connection.id, provider, {
|
|
976
|
+
onInterim: (text) => {
|
|
977
|
+
if (_classPrivateFieldGet2(_callTokens, this).get(connection.id) !== startupToken) return;
|
|
978
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _getOrCreateInputTurn).call(this, connection).firstInterim(text.length);
|
|
979
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
980
|
+
type: "transcript_interim",
|
|
981
|
+
text
|
|
982
|
+
});
|
|
983
|
+
},
|
|
984
|
+
onSpeechStart: () => {
|
|
985
|
+
if (_classPrivateFieldGet2(_callTokens, this).get(connection.id) !== startupToken) return;
|
|
986
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _replaceInputTurn).call(this, connection).speechStarted();
|
|
987
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _handleBargeIn).call(this, connection);
|
|
988
|
+
},
|
|
989
|
+
onUtterance: (transcript) => {
|
|
990
|
+
if (_classPrivateFieldGet2(_callTokens, this).get(connection.id) !== startupToken) return;
|
|
991
|
+
const turn = _assertClassBrand(_VoiceAgentMixin_brand, this, _takeInputTurn).call(this, connection);
|
|
992
|
+
turn.finalInput(transcript.length);
|
|
993
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
994
|
+
type: "transcript_interim",
|
|
995
|
+
text: ""
|
|
996
|
+
});
|
|
997
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _runPipeline).call(this, connection, transcript, turn);
|
|
998
|
+
},
|
|
999
|
+
onFatalError: (error) => {
|
|
1000
|
+
runBackground("transcriber_fatal", () => _assertClassBrand(_VoiceAgentMixin_brand, this, _handleTranscriberFatal).call(this, connection, startupToken, error));
|
|
1001
|
+
}
|
|
1002
|
+
});
|
|
1003
|
+
await session.waitUntilReady?.();
|
|
1004
|
+
} catch (error) {
|
|
1005
|
+
await _assertClassBrand(_VoiceAgentMixin_brand, this, _handleTranscriberStartupFailure).call(this, connection, startupToken, toVoiceError(error, "Speech recognition failed to start"));
|
|
1006
|
+
return;
|
|
1007
|
+
}
|
|
1008
|
+
if (!_assertClassBrand(_VoiceAgentMixin_brand, this, _isCurrentStartup).call(this, connection.id, startupToken)) return;
|
|
1009
|
+
_classPrivateFieldGet2(_startupTokens, this).delete(connection.id);
|
|
1010
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _diagnose).call(this, connection, "stt.ready");
|
|
1011
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1012
|
+
type: "status",
|
|
1013
|
+
status: "listening"
|
|
1014
|
+
});
|
|
1015
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _diagnose).call(this, connection, "call.ready");
|
|
1016
|
+
await this.onCallStart(connection);
|
|
1017
|
+
}
|
|
1018
|
+
function _isCurrentStartup(connectionId, startupToken) {
|
|
1019
|
+
return _classPrivateFieldGet2(_startupTokens, this).get(connectionId) === startupToken && _classPrivateFieldGet2(_cm, this).isInCall(connectionId);
|
|
1020
|
+
}
|
|
1021
|
+
async function _handleTranscriberStartupFailure(connection, startupToken, error) {
|
|
1022
|
+
await _assertClassBrand(_VoiceAgentMixin_brand, this, _handleStartupFailure).call(this, connection, startupToken, error, "Speech recognition failed to start", "transcriber_startup", {
|
|
1023
|
+
code: "stt_startup_failed",
|
|
1024
|
+
stage: "stt",
|
|
1025
|
+
retryable: false
|
|
1026
|
+
});
|
|
1027
|
+
}
|
|
1028
|
+
async function _handleStartupFailure(connection, startupToken, error, clientMessage, logStage = "call_startup", structuredError) {
|
|
1029
|
+
if (!_assertClassBrand(_VoiceAgentMixin_brand, this, _isCurrentStartup).call(this, connection.id, startupToken)) return;
|
|
1030
|
+
if (logStage && error !== void 0) logVoiceError({
|
|
1031
|
+
component: "VoiceAgent",
|
|
1032
|
+
stage: logStage,
|
|
1033
|
+
message: clientMessage,
|
|
1034
|
+
connectionId: connection.id,
|
|
1035
|
+
error
|
|
1036
|
+
});
|
|
1037
|
+
_classPrivateFieldGet2(_startupTokens, this).delete(connection.id);
|
|
1038
|
+
if (_classPrivateFieldGet2(_callTokens, this).get(connection.id) === startupToken) _classPrivateFieldGet2(_callTokens, this).delete(connection.id);
|
|
1039
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _diagnose).call(this, connection, "call.start_failed", {
|
|
1040
|
+
stage: logStage ?? "authorization",
|
|
1041
|
+
retryable: structuredError?.retryable ?? false,
|
|
1042
|
+
...error === void 0 ? {} : { error }
|
|
1043
|
+
});
|
|
1044
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1045
|
+
type: "error",
|
|
1046
|
+
message: clientMessage,
|
|
1047
|
+
...structuredError
|
|
1048
|
+
});
|
|
1049
|
+
_classPrivateFieldGet2(_cm, this).cleanup(connection.id);
|
|
1050
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _releaseKeepAlive).call(this, connection.id);
|
|
1051
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _diagnose).call(this, connection, "cleanup.completed");
|
|
1052
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _diagnose).call(this, connection, "call.ended", { reason: "startup_failed" });
|
|
1053
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1054
|
+
type: "status",
|
|
1055
|
+
status: "idle"
|
|
1056
|
+
});
|
|
1057
|
+
await this.onCallEnd(connection);
|
|
1058
|
+
}
|
|
1059
|
+
async function _handleTranscriberFatal(connection, callToken, error) {
|
|
1060
|
+
if (_classPrivateFieldGet2(_callTokens, this).get(connection.id) !== callToken || !_classPrivateFieldGet2(_cm, this).isInCall(connection.id)) return;
|
|
1061
|
+
const isStarting = _classPrivateFieldGet2(_startupTokens, this).get(connection.id) === callToken;
|
|
1062
|
+
const message = isStarting ? "Speech recognition failed to start" : "Speech recognition connection was lost";
|
|
1063
|
+
logVoiceError({
|
|
1064
|
+
component: "VoiceAgent",
|
|
1065
|
+
stage: isStarting ? "transcriber_startup" : "transcriber_runtime",
|
|
1066
|
+
message,
|
|
1067
|
+
connectionId: connection.id,
|
|
1068
|
+
error
|
|
1069
|
+
});
|
|
1070
|
+
_classPrivateFieldGet2(_startupTokens, this).delete(connection.id);
|
|
1071
|
+
_classPrivateFieldGet2(_callTokens, this).delete(connection.id);
|
|
1072
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _abortInputTurn).call(this, connection, "stt_fatal");
|
|
1073
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _diagnose).call(this, connection, "stt.fatal", {
|
|
1074
|
+
stage: isStarting ? "startup" : "runtime",
|
|
1075
|
+
retryable: !isStarting,
|
|
1076
|
+
error
|
|
1077
|
+
});
|
|
1078
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1079
|
+
type: "error",
|
|
1080
|
+
message,
|
|
1081
|
+
code: isStarting ? "stt_startup_failed" : "stt_connection_lost",
|
|
1082
|
+
stage: "stt",
|
|
1083
|
+
retryable: !isStarting
|
|
1084
|
+
});
|
|
1085
|
+
_classPrivateFieldGet2(_cm, this).cleanup(connection.id);
|
|
1086
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _releaseKeepAlive).call(this, connection.id);
|
|
1087
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _diagnose).call(this, connection, "cleanup.completed");
|
|
1088
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _diagnose).call(this, connection, "call.ended", { reason: "stt_fatal" });
|
|
1089
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1090
|
+
type: "status",
|
|
1091
|
+
status: "idle"
|
|
1092
|
+
});
|
|
1093
|
+
await this.onCallEnd(connection);
|
|
1094
|
+
}
|
|
1095
|
+
function _releaseKeepAlive(connectionId) {
|
|
1096
|
+
const dispose = _classPrivateFieldGet2(_keepAliveDispose, this).get(connectionId);
|
|
1097
|
+
if (dispose) {
|
|
1098
|
+
dispose();
|
|
1099
|
+
_classPrivateFieldGet2(_keepAliveDispose, this).delete(connectionId);
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1102
|
+
function _handleEndCall(connection) {
|
|
1103
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _diagnose).call(this, connection, "call.ended", { reason: "requested" });
|
|
1104
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _requestActiveTurnAbort).call(this, connection, "turn.abort_requested", "call_ended");
|
|
1105
|
+
_classPrivateFieldGet2(_startupTokens, this).delete(connection.id);
|
|
1106
|
+
_classPrivateFieldGet2(_callTokens, this).delete(connection.id);
|
|
1107
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _abortInputTurn).call(this, connection, "call_ended");
|
|
1108
|
+
_classPrivateFieldGet2(_cm, this).cleanup(connection.id);
|
|
1109
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _releaseKeepAlive).call(this, connection.id);
|
|
1110
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _diagnose).call(this, connection, "cleanup.completed");
|
|
1111
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1112
|
+
type: "status",
|
|
1113
|
+
status: "idle"
|
|
1114
|
+
});
|
|
1115
|
+
return this.onCallEnd(connection);
|
|
1116
|
+
}
|
|
1117
|
+
function _handleInterrupt(connection) {
|
|
1118
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _abortInputTurn).call(this, connection, "client_interrupt");
|
|
1119
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _requestActiveTurnAbort).call(this, connection, "turn.interrupt_requested", "client_interrupt");
|
|
1120
|
+
_classPrivateFieldGet2(_cm, this).abortPipeline(connection.id);
|
|
1121
|
+
_classPrivateFieldGet2(_cm, this).clearAudioBuffer(connection.id);
|
|
1122
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1123
|
+
type: "status",
|
|
1124
|
+
status: "listening"
|
|
1125
|
+
});
|
|
1126
|
+
return this.onInterrupt(connection);
|
|
1127
|
+
}
|
|
1128
|
+
function _handleBargeIn(connection) {
|
|
1129
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _requestActiveTurnAbort).call(this, connection, "turn.abort_requested", "barge_in");
|
|
1130
|
+
if (!_classPrivateFieldGet2(_cm, this).abortPipeline(connection.id)) return;
|
|
1131
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, { type: "playback_interrupt" });
|
|
1132
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1133
|
+
type: "status",
|
|
1134
|
+
status: "listening"
|
|
1135
|
+
});
|
|
1136
|
+
this.onInterrupt(connection);
|
|
1137
|
+
}
|
|
1138
|
+
function _createTurn(connection, source) {
|
|
1139
|
+
var _this$turnSequence;
|
|
1140
|
+
const turn = diagnostics.turn(connection, `turn_${_classPrivateFieldSet2(_turnSequence, this, (_this$turnSequence = _classPrivateFieldGet2(_turnSequence, this), ++_this$turnSequence)).toString(36)}`, source);
|
|
1141
|
+
if (source === "text") turn.markTextInput();
|
|
1142
|
+
turn.emit("turn.started", { source });
|
|
1143
|
+
return turn;
|
|
1144
|
+
}
|
|
1145
|
+
function _replaceInputTurn(connection) {
|
|
1146
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _abortInputTurn).call(this, connection, "replaced");
|
|
1147
|
+
const turn = _assertClassBrand(_VoiceAgentMixin_brand, this, _createTurn).call(this, connection, "speech");
|
|
1148
|
+
_classPrivateFieldGet2(_inputTurns, this).set(connection.id, turn);
|
|
1149
|
+
return turn;
|
|
1150
|
+
}
|
|
1151
|
+
function _getOrCreateInputTurn(connection) {
|
|
1152
|
+
const current = _classPrivateFieldGet2(_inputTurns, this).get(connection.id);
|
|
1153
|
+
if (current) return current;
|
|
1154
|
+
const turn = _assertClassBrand(_VoiceAgentMixin_brand, this, _createTurn).call(this, connection, "speech");
|
|
1155
|
+
_classPrivateFieldGet2(_inputTurns, this).set(connection.id, turn);
|
|
1156
|
+
return turn;
|
|
1157
|
+
}
|
|
1158
|
+
function _takeInputTurn(connection) {
|
|
1159
|
+
const turn = _assertClassBrand(_VoiceAgentMixin_brand, this, _getOrCreateInputTurn).call(this, connection);
|
|
1160
|
+
_classPrivateFieldGet2(_inputTurns, this).delete(connection.id);
|
|
1161
|
+
return turn;
|
|
1162
|
+
}
|
|
1163
|
+
function _abortInputTurn(connection, reason) {
|
|
1164
|
+
const turn = _classPrivateFieldGet2(_inputTurns, this).get(connection.id);
|
|
1165
|
+
if (!turn) return;
|
|
1166
|
+
_classPrivateFieldGet2(_inputTurns, this).delete(connection.id);
|
|
1167
|
+
turn.emit("turn.aborted", { reason });
|
|
1168
|
+
turn.finish("aborted");
|
|
1169
|
+
}
|
|
1170
|
+
function _beginTurnDiagnostics(connection, source, turn = _assertClassBrand(_VoiceAgentMixin_brand, this, _createTurn).call(this, connection, source)) {
|
|
1171
|
+
const previous = _classPrivateFieldGet2(_activeTurnDiagnostics, this).get(connection.id);
|
|
1172
|
+
if (previous) {
|
|
1173
|
+
previous.turn.emit("turn.abort_requested", { reason: "replaced" });
|
|
1174
|
+
previous.model?.abort();
|
|
1175
|
+
}
|
|
1176
|
+
const active = {
|
|
1177
|
+
signal: _classPrivateFieldGet2(_cm, this).createPipelineAbort(connection.id),
|
|
1178
|
+
turn
|
|
1179
|
+
};
|
|
1180
|
+
_classPrivateFieldGet2(_activeTurnDiagnostics, this).set(connection.id, active);
|
|
1181
|
+
return active;
|
|
1182
|
+
}
|
|
1183
|
+
function _requestActiveTurnAbort(connection, event, reason) {
|
|
1184
|
+
const active = _classPrivateFieldGet2(_activeTurnDiagnostics, this).get(connection.id);
|
|
1185
|
+
if (!active) return;
|
|
1186
|
+
active.turn.emit(event, { reason });
|
|
1187
|
+
active.model?.abort();
|
|
1188
|
+
}
|
|
1189
|
+
function _clearActiveTurn(connectionId, active) {
|
|
1190
|
+
if (_classPrivateFieldGet2(_activeTurnDiagnostics, this).get(connectionId) === active) _classPrivateFieldGet2(_activeTurnDiagnostics, this).delete(connectionId);
|
|
1191
|
+
}
|
|
1192
|
+
function _emitTurnDiagnostic(connection, turn, event, data) {
|
|
1193
|
+
if (turn) turn.emit(event, data);
|
|
1194
|
+
else _assertClassBrand(_VoiceAgentMixin_brand, this, _diagnose).call(this, connection, event, data);
|
|
1195
|
+
}
|
|
1196
|
+
async function _handleTextMessage(connection, text) {
|
|
1197
|
+
if (!text || text.trim().length === 0) return;
|
|
1198
|
+
const userText = text.trim();
|
|
1199
|
+
const pipelineStart = Date.now();
|
|
1200
|
+
const active = _assertClassBrand(_VoiceAgentMixin_brand, this, _beginTurnDiagnostics).call(this, connection, "text");
|
|
1201
|
+
const { signal, turn } = active;
|
|
1202
|
+
let turnOutcome = "completed";
|
|
1203
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1204
|
+
type: "status",
|
|
1205
|
+
status: "thinking"
|
|
1206
|
+
});
|
|
1207
|
+
const priorMessages = this.getConversationHistory();
|
|
1208
|
+
this.saveMessage("user", userText);
|
|
1209
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1210
|
+
type: "transcript",
|
|
1211
|
+
role: "user",
|
|
1212
|
+
text: userText
|
|
1213
|
+
});
|
|
1214
|
+
try {
|
|
1215
|
+
const context = {
|
|
1216
|
+
connection,
|
|
1217
|
+
messages: priorMessages,
|
|
1218
|
+
signal
|
|
1219
|
+
};
|
|
1220
|
+
const model = turn.startModel();
|
|
1221
|
+
active.model = model;
|
|
1222
|
+
const turnResult = await this.onTurn(userText, context);
|
|
1223
|
+
if (signal.aborted) return;
|
|
1224
|
+
if (_classPrivateFieldGet2(_cm, this).isInCall(connection.id)) {
|
|
1225
|
+
const { text: fullText, finishReason } = await _assertClassBrand(_VoiceAgentMixin_brand, this, _streamResponse).call(this, connection, turnResult, pipelineStart, signal, turn, model);
|
|
1226
|
+
if (signal.aborted) return;
|
|
1227
|
+
const hasOutput = fullText.trim().length > 0;
|
|
1228
|
+
turnOutcome = stableTurnOutcome(finishReason, hasOutput);
|
|
1229
|
+
if (turnOutcome === "completed" && turn.hasTtsFailures) turnOutcome = "tts_error";
|
|
1230
|
+
if (hasOutput) {
|
|
1231
|
+
_classPrivateFieldGet2(_cm, this).updateAgentContext(connection.id, fullText);
|
|
1232
|
+
this.saveMessage("assistant", fullText);
|
|
1233
|
+
}
|
|
1234
|
+
const completionOutcome = createCompletionOutcome(finishReason, hasOutput);
|
|
1235
|
+
if (completionOutcome) _assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1236
|
+
type: "completion_outcome",
|
|
1237
|
+
...completionOutcome
|
|
1238
|
+
});
|
|
1239
|
+
if (!hasOutput) _assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1240
|
+
type: "error",
|
|
1241
|
+
message: "No response generated"
|
|
1242
|
+
});
|
|
1243
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1244
|
+
type: "status",
|
|
1245
|
+
status: "listening"
|
|
1246
|
+
});
|
|
1247
|
+
} else {
|
|
1248
|
+
let fullText = "";
|
|
1249
|
+
let pendingText = "";
|
|
1250
|
+
let transcriptStarted = false;
|
|
1251
|
+
const sendAssistantDelta = (token) => {
|
|
1252
|
+
if (!transcriptStarted) {
|
|
1253
|
+
pendingText += token;
|
|
1254
|
+
if (pendingText.trim().length === 0) return;
|
|
1255
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1256
|
+
type: "transcript_start",
|
|
1257
|
+
role: "assistant"
|
|
1258
|
+
});
|
|
1259
|
+
transcriptStarted = true;
|
|
1260
|
+
token = pendingText;
|
|
1261
|
+
pendingText = "";
|
|
1262
|
+
}
|
|
1263
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1264
|
+
type: "transcript_delta",
|
|
1265
|
+
text: token
|
|
1266
|
+
});
|
|
1267
|
+
};
|
|
1268
|
+
let finishReason;
|
|
1269
|
+
for await (const event of iterateTextEvents(turnResult)) {
|
|
1270
|
+
if (signal.aborted) break;
|
|
1271
|
+
model.observe(event);
|
|
1272
|
+
if (event.type === "finish") finishReason = event.finishReason;
|
|
1273
|
+
else if (event.type === "error") {
|
|
1274
|
+
if (transcriptStarted) _assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1275
|
+
type: "transcript_end",
|
|
1276
|
+
text: fullText
|
|
1277
|
+
});
|
|
1278
|
+
throw new ModelStreamError(event.error, fullText.trim().length > 0);
|
|
1279
|
+
} else if (event.type === "text") {
|
|
1280
|
+
fullText += event.text;
|
|
1281
|
+
sendAssistantDelta(event.text);
|
|
1282
|
+
}
|
|
1283
|
+
}
|
|
1284
|
+
const hasOutput = fullText.trim().length > 0;
|
|
1285
|
+
model.complete(hasOutput ? "output" : "no_output", finishReason);
|
|
1286
|
+
turnOutcome = stableTurnOutcome(finishReason, hasOutput);
|
|
1287
|
+
if (hasOutput) {
|
|
1288
|
+
if (transcriptStarted) _assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1289
|
+
type: "transcript_end",
|
|
1290
|
+
text: fullText
|
|
1291
|
+
});
|
|
1292
|
+
this.saveMessage("assistant", fullText);
|
|
1293
|
+
}
|
|
1294
|
+
const completionOutcome = createCompletionOutcome(finishReason, hasOutput);
|
|
1295
|
+
if (completionOutcome) _assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1296
|
+
type: "completion_outcome",
|
|
1297
|
+
...completionOutcome
|
|
1298
|
+
});
|
|
1299
|
+
if (!hasOutput) _assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1300
|
+
type: "error",
|
|
1301
|
+
message: "No response generated"
|
|
1302
|
+
});
|
|
1303
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1304
|
+
type: "status",
|
|
1305
|
+
status: "idle"
|
|
1306
|
+
});
|
|
1307
|
+
}
|
|
1308
|
+
} catch (error) {
|
|
1309
|
+
if (signal.aborted) return;
|
|
1310
|
+
turnOutcome = error instanceof ModelStreamError ? "model_error" : turn.hasTtsFailures ? "tts_error" : "error";
|
|
1311
|
+
const pipelineError = error instanceof ModelStreamError ? error.streamError : toVoiceError(error, "Text turn failed");
|
|
1312
|
+
active.model?.fail(pipelineError);
|
|
1313
|
+
turn.emit("turn.error", {
|
|
1314
|
+
stage: error instanceof ModelStreamError ? "model" : "pipeline",
|
|
1315
|
+
error: pipelineError
|
|
1316
|
+
});
|
|
1317
|
+
if (error instanceof ModelStreamError) _assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1318
|
+
type: "completion_outcome",
|
|
1319
|
+
code: "model_error",
|
|
1320
|
+
stage: "llm",
|
|
1321
|
+
partialOutput: error.partialOutput
|
|
1322
|
+
});
|
|
1323
|
+
logVoiceError({
|
|
1324
|
+
component: "VoiceAgent",
|
|
1325
|
+
stage: "text_pipeline",
|
|
1326
|
+
message: "Text pipeline failed",
|
|
1327
|
+
connectionId: connection.id,
|
|
1328
|
+
error: pipelineError
|
|
1329
|
+
});
|
|
1330
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1331
|
+
type: "error",
|
|
1332
|
+
message: voiceErrorMessage(pipelineError, "Text pipeline failed")
|
|
1333
|
+
});
|
|
1334
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1335
|
+
type: "status",
|
|
1336
|
+
status: _classPrivateFieldGet2(_cm, this).isInCall(connection.id) ? "listening" : "idle"
|
|
1337
|
+
});
|
|
1338
|
+
} finally {
|
|
1339
|
+
if (signal.aborted) {
|
|
1340
|
+
turnOutcome = "aborted";
|
|
1341
|
+
active.model?.abort();
|
|
1342
|
+
turn.emit("turn.aborted");
|
|
1343
|
+
}
|
|
1344
|
+
turn.finish(turnOutcome);
|
|
1345
|
+
_classPrivateFieldGet2(_cm, this).clearPipelineAbort(connection.id, signal);
|
|
1346
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _clearActiveTurn).call(this, connection.id, active);
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1349
|
+
async function _runPipeline(connection, transcript, turn) {
|
|
1350
|
+
const pipelineStart = Date.now();
|
|
1351
|
+
const active = _assertClassBrand(_VoiceAgentMixin_brand, this, _beginTurnDiagnostics).call(this, connection, "speech", turn);
|
|
1352
|
+
const { signal } = active;
|
|
1353
|
+
let turnOutcome = "completed";
|
|
1354
|
+
try {
|
|
1355
|
+
const afterTranscribeStart = Date.now();
|
|
1356
|
+
const userText = await this.afterTranscribe(transcript, connection);
|
|
1357
|
+
turn.recordAfterTranscribe(Date.now() - afterTranscribeStart, userText ? "accepted" : "skipped", userText?.length ?? 0);
|
|
1358
|
+
if (signal.aborted) return;
|
|
1359
|
+
if (!userText) {
|
|
1360
|
+
turnOutcome = "skipped";
|
|
1361
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1362
|
+
type: "status",
|
|
1363
|
+
status: "listening"
|
|
1364
|
+
});
|
|
1365
|
+
return;
|
|
1366
|
+
}
|
|
1367
|
+
const priorMessages = this.getConversationHistory();
|
|
1368
|
+
this.saveMessage("user", userText);
|
|
1369
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1370
|
+
type: "transcript",
|
|
1371
|
+
role: "user",
|
|
1372
|
+
text: userText
|
|
1373
|
+
});
|
|
1374
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1375
|
+
type: "status",
|
|
1376
|
+
status: "thinking"
|
|
1377
|
+
});
|
|
1378
|
+
const context = {
|
|
1379
|
+
connection,
|
|
1380
|
+
messages: priorMessages,
|
|
1381
|
+
signal
|
|
1382
|
+
};
|
|
1383
|
+
const model = turn.startModel();
|
|
1384
|
+
active.model = model;
|
|
1385
|
+
const turnResult = await this.onTurn(userText, context);
|
|
1386
|
+
if (signal.aborted) return;
|
|
1387
|
+
const { text: fullText, llmMs, ttsMs, firstAudioMs, finishReason } = await _assertClassBrand(_VoiceAgentMixin_brand, this, _streamResponse).call(this, connection, turnResult, pipelineStart, signal, turn, model);
|
|
1388
|
+
if (signal.aborted) return;
|
|
1389
|
+
const hasOutput = fullText.trim().length > 0;
|
|
1390
|
+
turnOutcome = stableTurnOutcome(finishReason, hasOutput);
|
|
1391
|
+
if (turnOutcome === "completed" && turn.hasTtsFailures) turnOutcome = "tts_error";
|
|
1392
|
+
if (!hasOutput) {
|
|
1393
|
+
const completionOutcome = createCompletionOutcome(finishReason, false);
|
|
1394
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1395
|
+
type: "completion_outcome",
|
|
1396
|
+
...completionOutcome
|
|
1397
|
+
});
|
|
1398
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1399
|
+
type: "error",
|
|
1400
|
+
message: "No response generated"
|
|
1401
|
+
});
|
|
1402
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1403
|
+
type: "status",
|
|
1404
|
+
status: "listening"
|
|
1405
|
+
});
|
|
1406
|
+
return;
|
|
1407
|
+
}
|
|
1408
|
+
const completionOutcome = createCompletionOutcome(finishReason, true);
|
|
1409
|
+
if (completionOutcome) _assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1410
|
+
type: "completion_outcome",
|
|
1411
|
+
...completionOutcome
|
|
1412
|
+
});
|
|
1413
|
+
const totalMs = Date.now() - pipelineStart;
|
|
1414
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1415
|
+
type: "metrics",
|
|
1416
|
+
llm_ms: llmMs,
|
|
1417
|
+
tts_ms: ttsMs,
|
|
1418
|
+
first_audio_ms: firstAudioMs,
|
|
1419
|
+
total_ms: totalMs
|
|
1420
|
+
});
|
|
1421
|
+
_classPrivateFieldGet2(_cm, this).updateAgentContext(connection.id, fullText);
|
|
1422
|
+
this.saveMessage("assistant", fullText);
|
|
1423
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1424
|
+
type: "status",
|
|
1425
|
+
status: "listening"
|
|
1426
|
+
});
|
|
1427
|
+
} catch (error) {
|
|
1428
|
+
if (signal.aborted) return;
|
|
1429
|
+
turnOutcome = error instanceof ModelStreamError ? "model_error" : turn.hasTtsFailures ? "tts_error" : "error";
|
|
1430
|
+
const pipelineError = error instanceof ModelStreamError ? error.streamError : toVoiceError(error, "Voice turn failed");
|
|
1431
|
+
active.model?.fail(pipelineError);
|
|
1432
|
+
turn.emit("turn.error", {
|
|
1433
|
+
stage: error instanceof ModelStreamError ? "model" : "pipeline",
|
|
1434
|
+
error: pipelineError
|
|
1435
|
+
});
|
|
1436
|
+
if (error instanceof ModelStreamError) _assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1437
|
+
type: "completion_outcome",
|
|
1438
|
+
code: "model_error",
|
|
1439
|
+
stage: "llm",
|
|
1440
|
+
partialOutput: error.partialOutput
|
|
1441
|
+
});
|
|
1442
|
+
logVoiceError({
|
|
1443
|
+
component: "VoiceAgent",
|
|
1444
|
+
stage: "pipeline",
|
|
1445
|
+
message: "Voice pipeline failed",
|
|
1446
|
+
connectionId: connection.id,
|
|
1447
|
+
error: pipelineError
|
|
1448
|
+
});
|
|
1449
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1450
|
+
type: "error",
|
|
1451
|
+
message: voiceErrorMessage(pipelineError, "Voice pipeline failed")
|
|
1452
|
+
});
|
|
1453
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1454
|
+
type: "status",
|
|
1455
|
+
status: "listening"
|
|
1456
|
+
});
|
|
1457
|
+
} finally {
|
|
1458
|
+
if (signal.aborted) {
|
|
1459
|
+
turnOutcome = "aborted";
|
|
1460
|
+
active.model?.abort();
|
|
1461
|
+
turn.emit("turn.aborted");
|
|
1462
|
+
}
|
|
1463
|
+
turn.finish(turnOutcome);
|
|
1464
|
+
_classPrivateFieldGet2(_cm, this).clearPipelineAbort(connection.id, signal);
|
|
1465
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _clearActiveTurn).call(this, connection.id, active);
|
|
1466
|
+
}
|
|
1467
|
+
}
|
|
1468
|
+
async function _streamResponse(connection, response, pipelineStart, signal, turn, model) {
|
|
1469
|
+
if (typeof response === "string") {
|
|
1470
|
+
const llmMs = model.elapsedMs();
|
|
1471
|
+
if (response.trim().length === 0) {
|
|
1472
|
+
model.complete("no_output");
|
|
1473
|
+
return {
|
|
1474
|
+
text: response,
|
|
1475
|
+
llmMs,
|
|
1476
|
+
ttsMs: 0,
|
|
1477
|
+
firstAudioMs: 0
|
|
1478
|
+
};
|
|
1479
|
+
}
|
|
1480
|
+
model.observe({
|
|
1481
|
+
type: "text",
|
|
1482
|
+
text: response
|
|
1483
|
+
});
|
|
1484
|
+
model.complete("output");
|
|
1485
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1486
|
+
type: "transcript_start",
|
|
1487
|
+
role: "assistant"
|
|
1488
|
+
});
|
|
1489
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1490
|
+
type: "transcript_end",
|
|
1491
|
+
text: response
|
|
1492
|
+
});
|
|
1493
|
+
const ttsStart = Date.now();
|
|
1494
|
+
let audio;
|
|
1495
|
+
try {
|
|
1496
|
+
audio = await _assertClassBrand(_VoiceAgentMixin_brand, this, _synthesizeWithHooks).call(this, response, connection, void 0, turn);
|
|
1497
|
+
} finally {
|
|
1498
|
+
turn.finishTts();
|
|
1499
|
+
}
|
|
1500
|
+
const ttsMs = Date.now() - ttsStart;
|
|
1501
|
+
let firstAudioMs = 0;
|
|
1502
|
+
if (audio && !signal.aborted) {
|
|
1503
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1504
|
+
type: "status",
|
|
1505
|
+
status: "speaking"
|
|
1506
|
+
});
|
|
1507
|
+
firstAudioMs = Date.now() - pipelineStart;
|
|
1508
|
+
turn.emit("audio.first_sent", {
|
|
1509
|
+
bytes: audio.byteLength,
|
|
1510
|
+
elapsed_ms: firstAudioMs
|
|
1511
|
+
});
|
|
1512
|
+
turn.audioSent();
|
|
1513
|
+
connection.send(audio);
|
|
1514
|
+
turn.emit("audio.completed", { bytes: audio.byteLength });
|
|
1515
|
+
}
|
|
1516
|
+
return {
|
|
1517
|
+
text: response,
|
|
1518
|
+
llmMs,
|
|
1519
|
+
ttsMs,
|
|
1520
|
+
firstAudioMs
|
|
1521
|
+
};
|
|
1522
|
+
}
|
|
1523
|
+
try {
|
|
1524
|
+
return await _assertClassBrand(_VoiceAgentMixin_brand, this, _streamingTTSPipeline).call(this, connection, iterateTextEvents(response), pipelineStart, signal, turn, model);
|
|
1525
|
+
} finally {
|
|
1526
|
+
turn.finishTts();
|
|
1527
|
+
}
|
|
1528
|
+
}
|
|
1529
|
+
async function _streamingTTSPipeline(connection, tokenStream, pipelineStart, signal, turn, model) {
|
|
1530
|
+
const chunker = new SentenceChunker();
|
|
1531
|
+
const ttsQueue = [];
|
|
1532
|
+
let fullText = "";
|
|
1533
|
+
let pendingTranscriptText = "";
|
|
1534
|
+
let transcriptStarted = false;
|
|
1535
|
+
let firstAudioSentAt = null;
|
|
1536
|
+
let firstTtsStartedAt = null;
|
|
1537
|
+
let cumulativeTtsMs = 0;
|
|
1538
|
+
let totalAudioBytes = 0;
|
|
1539
|
+
let skippedSentences = 0;
|
|
1540
|
+
let ttsFailures = 0;
|
|
1541
|
+
let finishReason;
|
|
1542
|
+
let streamComplete = false;
|
|
1543
|
+
let drainNotify = null;
|
|
1544
|
+
let drainPending = false;
|
|
1545
|
+
let drainedCount = 0;
|
|
1546
|
+
const drainWaiters = /* @__PURE__ */ new Map();
|
|
1547
|
+
const notifyDrain = () => {
|
|
1548
|
+
if (drainNotify) {
|
|
1549
|
+
const resolve = drainNotify;
|
|
1550
|
+
drainNotify = null;
|
|
1551
|
+
resolve();
|
|
1552
|
+
} else drainPending = true;
|
|
1553
|
+
};
|
|
1554
|
+
const notifyDrained = () => {
|
|
1555
|
+
for (const [target, waiters] of drainWaiters) {
|
|
1556
|
+
if (drainedCount < target) continue;
|
|
1557
|
+
drainWaiters.delete(target);
|
|
1558
|
+
for (const resolve of waiters) resolve();
|
|
1559
|
+
}
|
|
1560
|
+
};
|
|
1561
|
+
const waitForDrained = (target) => {
|
|
1562
|
+
if (drainedCount >= target) return Promise.resolve();
|
|
1563
|
+
return new Promise((resolve) => {
|
|
1564
|
+
const waiters = drainWaiters.get(target) ?? [];
|
|
1565
|
+
waiters.push(resolve);
|
|
1566
|
+
drainWaiters.set(target, waiters);
|
|
1567
|
+
});
|
|
1568
|
+
};
|
|
1569
|
+
const tts = _assertClassBrand(_VoiceAgentMixin_brand, this, _requireTTS).call(this);
|
|
1570
|
+
const hasStreamingTTS = typeof tts.synthesizeStream === "function";
|
|
1571
|
+
const drainPromise = (async () => {
|
|
1572
|
+
let i = 0;
|
|
1573
|
+
while (true) {
|
|
1574
|
+
while (i >= ttsQueue.length) {
|
|
1575
|
+
if (streamComplete && i >= ttsQueue.length) return;
|
|
1576
|
+
if (drainPending) {
|
|
1577
|
+
drainPending = false;
|
|
1578
|
+
continue;
|
|
1579
|
+
}
|
|
1580
|
+
await new Promise((r) => {
|
|
1581
|
+
drainNotify = r;
|
|
1582
|
+
});
|
|
1583
|
+
if (streamComplete && i >= ttsQueue.length) return;
|
|
1584
|
+
}
|
|
1585
|
+
if (signal.aborted) return;
|
|
1586
|
+
try {
|
|
1587
|
+
for await (const chunk of ttsQueue[i]) {
|
|
1588
|
+
if (signal.aborted) return;
|
|
1589
|
+
if (firstAudioSentAt === null) {
|
|
1590
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1591
|
+
type: "status",
|
|
1592
|
+
status: "speaking"
|
|
1593
|
+
});
|
|
1594
|
+
firstAudioSentAt = Date.now();
|
|
1595
|
+
turn.emit("audio.first_sent", {
|
|
1596
|
+
bytes: chunk.byteLength,
|
|
1597
|
+
elapsed_ms: firstAudioSentAt - pipelineStart
|
|
1598
|
+
});
|
|
1599
|
+
}
|
|
1600
|
+
totalAudioBytes += chunk.byteLength;
|
|
1601
|
+
turn.audioSent();
|
|
1602
|
+
connection.send(chunk);
|
|
1603
|
+
}
|
|
1604
|
+
} catch (error) {
|
|
1605
|
+
if (signal.aborted) return;
|
|
1606
|
+
const voiceError = toVoiceError(error, "TTS sentence failed");
|
|
1607
|
+
ttsFailures++;
|
|
1608
|
+
turn.emit("tts.failed", { error: voiceError });
|
|
1609
|
+
logVoiceError({
|
|
1610
|
+
component: "VoiceAgent",
|
|
1611
|
+
stage: "tts",
|
|
1612
|
+
message: "TTS failed for a sentence",
|
|
1613
|
+
connectionId: connection.id,
|
|
1614
|
+
error: voiceError
|
|
1615
|
+
});
|
|
1616
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1617
|
+
type: "error",
|
|
1618
|
+
message: voiceErrorMessage(voiceError, "TTS failed for a sentence")
|
|
1619
|
+
});
|
|
1620
|
+
}
|
|
1621
|
+
i++;
|
|
1622
|
+
drainedCount = i;
|
|
1623
|
+
notifyDrained();
|
|
1624
|
+
}
|
|
1625
|
+
})();
|
|
1626
|
+
const makeSentenceTTS = (sentence) => {
|
|
1627
|
+
const self = this;
|
|
1628
|
+
async function* generate() {
|
|
1629
|
+
const attempt = turn.beginTtsSentence();
|
|
1630
|
+
let sentenceOutcome = "completed";
|
|
1631
|
+
try {
|
|
1632
|
+
const text = await self.beforeSynthesize(sentence, connection);
|
|
1633
|
+
if (!text) {
|
|
1634
|
+
sentenceOutcome = "skipped";
|
|
1635
|
+
skippedSentences++;
|
|
1636
|
+
return;
|
|
1637
|
+
}
|
|
1638
|
+
if (firstTtsStartedAt === null) {
|
|
1639
|
+
firstTtsStartedAt = Date.now();
|
|
1640
|
+
turn.emit("tts.started", {
|
|
1641
|
+
mode: hasStreamingTTS ? "streaming" : "buffered",
|
|
1642
|
+
characters: text.length
|
|
1643
|
+
});
|
|
1644
|
+
}
|
|
1645
|
+
attempt.providerStarted();
|
|
1646
|
+
if (hasStreamingTTS) for await (const chunk of tts.synthesizeStream(text, signal)) {
|
|
1647
|
+
const processed = await self.afterSynthesize(chunk, text, connection);
|
|
1648
|
+
if (processed) yield processed;
|
|
1649
|
+
}
|
|
1650
|
+
else {
|
|
1651
|
+
const rawAudio = await tts.synthesize(text, signal);
|
|
1652
|
+
const processed = await self.afterSynthesize(rawAudio, text, connection);
|
|
1653
|
+
if (processed) yield processed;
|
|
1654
|
+
}
|
|
1655
|
+
} catch (error) {
|
|
1656
|
+
sentenceOutcome = "failed";
|
|
1657
|
+
throw error;
|
|
1658
|
+
} finally {
|
|
1659
|
+
cumulativeTtsMs += attempt.settle(sentenceOutcome);
|
|
1660
|
+
}
|
|
1661
|
+
}
|
|
1662
|
+
return eagerAsyncIterable(generate());
|
|
1663
|
+
};
|
|
1664
|
+
const enqueueSentence = (sentence) => {
|
|
1665
|
+
ttsQueue.push(makeSentenceTTS(sentence));
|
|
1666
|
+
notifyDrain();
|
|
1667
|
+
};
|
|
1668
|
+
const sendAssistantDelta = (token) => {
|
|
1669
|
+
if (!transcriptStarted) {
|
|
1670
|
+
pendingTranscriptText += token;
|
|
1671
|
+
if (pendingTranscriptText.trim().length === 0) return;
|
|
1672
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1673
|
+
type: "transcript_start",
|
|
1674
|
+
role: "assistant"
|
|
1675
|
+
});
|
|
1676
|
+
transcriptStarted = true;
|
|
1677
|
+
token = pendingTranscriptText;
|
|
1678
|
+
pendingTranscriptText = "";
|
|
1679
|
+
}
|
|
1680
|
+
_assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1681
|
+
type: "transcript_delta",
|
|
1682
|
+
text: token
|
|
1683
|
+
});
|
|
1684
|
+
};
|
|
1685
|
+
for await (const event of tokenStream) {
|
|
1686
|
+
if (signal.aborted) break;
|
|
1687
|
+
model.observe(event);
|
|
1688
|
+
if (event.type === "boundary") {
|
|
1689
|
+
for (const sentence of chunker.flush()) enqueueSentence(sentence);
|
|
1690
|
+
await waitForDrained(ttsQueue.length);
|
|
1691
|
+
continue;
|
|
1692
|
+
}
|
|
1693
|
+
if (event.type === "finish") {
|
|
1694
|
+
finishReason = event.finishReason;
|
|
1695
|
+
continue;
|
|
1696
|
+
}
|
|
1697
|
+
if (event.type === "error") {
|
|
1698
|
+
for (const sentence of chunker.flush()) enqueueSentence(sentence);
|
|
1699
|
+
await waitForDrained(ttsQueue.length);
|
|
1700
|
+
if (transcriptStarted) _assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1701
|
+
type: "transcript_end",
|
|
1702
|
+
text: fullText
|
|
1703
|
+
});
|
|
1704
|
+
streamComplete = true;
|
|
1705
|
+
notifyDrain();
|
|
1706
|
+
await drainPromise;
|
|
1707
|
+
throw new ModelStreamError(event.error, fullText.trim().length > 0);
|
|
1708
|
+
}
|
|
1709
|
+
if (event.type !== "text") continue;
|
|
1710
|
+
const token = event.text;
|
|
1711
|
+
fullText += token;
|
|
1712
|
+
sendAssistantDelta(token);
|
|
1713
|
+
const sentences = chunker.add(token);
|
|
1714
|
+
for (const sentence of sentences) enqueueSentence(sentence);
|
|
1715
|
+
}
|
|
1716
|
+
const llmMs = model.elapsedMs();
|
|
1717
|
+
model.complete(fullText.trim().length > 0 ? "output" : "no_output", finishReason);
|
|
1718
|
+
const remaining = chunker.flush();
|
|
1719
|
+
for (const sentence of remaining) enqueueSentence(sentence);
|
|
1720
|
+
streamComplete = true;
|
|
1721
|
+
notifyDrain();
|
|
1722
|
+
if (transcriptStarted) _assertClassBrand(_VoiceAgentMixin_brand, this, _sendJSON).call(this, connection, {
|
|
1723
|
+
type: "transcript_end",
|
|
1724
|
+
text: fullText
|
|
1725
|
+
});
|
|
1726
|
+
await drainPromise;
|
|
1727
|
+
if (firstTtsStartedAt === null) turn.emit("tts.skipped", {
|
|
1728
|
+
reason: fullText.trim().length === 0 ? "no_output" : ttsFailures > 0 ? "preparation_failed" : "before_synthesize",
|
|
1729
|
+
sentences: skippedSentences,
|
|
1730
|
+
failures: ttsFailures
|
|
1731
|
+
});
|
|
1732
|
+
else turn.emit("tts.completed", {
|
|
1733
|
+
duration_ms: Date.now() - firstTtsStartedAt,
|
|
1734
|
+
outcome: totalAudioBytes > 0 ? ttsFailures > 0 ? "partial" : "audio" : ttsFailures > 0 ? "failed" : "no_audio",
|
|
1735
|
+
bytes: totalAudioBytes,
|
|
1736
|
+
failures: ttsFailures,
|
|
1737
|
+
skipped_sentences: skippedSentences
|
|
1738
|
+
});
|
|
1739
|
+
if (totalAudioBytes > 0) turn.emit("audio.completed", { bytes: totalAudioBytes });
|
|
1740
|
+
const firstAudioMs = firstAudioSentAt ? firstAudioSentAt - pipelineStart : 0;
|
|
1741
|
+
return {
|
|
1742
|
+
text: fullText,
|
|
1743
|
+
llmMs,
|
|
1744
|
+
ttsMs: cumulativeTtsMs,
|
|
1745
|
+
firstAudioMs,
|
|
1746
|
+
...finishReason === void 0 ? {} : { finishReason }
|
|
1747
|
+
};
|
|
1748
|
+
}
|
|
1749
|
+
function _diagnose(connection, event, data) {
|
|
1750
|
+
diagnostics.emit(connection, event, data);
|
|
1751
|
+
}
|
|
1752
|
+
function _sendJSON(connection, data) {
|
|
1753
|
+
sendVoiceJSON(connection, data, "VoiceAgent", data.type === "transcript_delta");
|
|
1754
|
+
}
|
|
1755
|
+
var _VOICE_MESSAGES = { _: /* @__PURE__ */ new Set([
|
|
1756
|
+
"hello",
|
|
1757
|
+
"start_call",
|
|
1758
|
+
"end_call",
|
|
1759
|
+
"start_of_speech",
|
|
1760
|
+
"end_of_speech",
|
|
1761
|
+
"interrupt",
|
|
1762
|
+
"text_message"
|
|
1763
|
+
]) };
|
|
1764
|
+
return VoiceAgentMixin;
|
|
1765
|
+
}
|
|
1766
|
+
function eagerAsyncIterable(source) {
|
|
1767
|
+
const buffer = [];
|
|
1768
|
+
let finished = false;
|
|
1769
|
+
let error = null;
|
|
1770
|
+
let waitResolve = null;
|
|
1771
|
+
const notify = () => {
|
|
1772
|
+
if (waitResolve) {
|
|
1773
|
+
const resolve = waitResolve;
|
|
1774
|
+
waitResolve = null;
|
|
1775
|
+
resolve();
|
|
1776
|
+
}
|
|
1777
|
+
};
|
|
1778
|
+
(async () => {
|
|
1779
|
+
try {
|
|
1780
|
+
for await (const item of source) {
|
|
1781
|
+
buffer.push(item);
|
|
1782
|
+
notify();
|
|
1783
|
+
}
|
|
1784
|
+
} catch (err) {
|
|
1785
|
+
error = err;
|
|
1786
|
+
} finally {
|
|
1787
|
+
finished = true;
|
|
1788
|
+
notify();
|
|
1789
|
+
}
|
|
1790
|
+
})();
|
|
1791
|
+
return { [Symbol.asyncIterator]() {
|
|
1792
|
+
let index = 0;
|
|
1793
|
+
return { async next() {
|
|
1794
|
+
while (index >= buffer.length && !finished) await new Promise((r) => {
|
|
1795
|
+
waitResolve = r;
|
|
1796
|
+
});
|
|
1797
|
+
if (error) throw error;
|
|
1798
|
+
if (index >= buffer.length) return {
|
|
1799
|
+
done: true,
|
|
1800
|
+
value: void 0
|
|
1801
|
+
};
|
|
1802
|
+
return {
|
|
1803
|
+
done: false,
|
|
1804
|
+
value: buffer[index++]
|
|
1805
|
+
};
|
|
1806
|
+
} };
|
|
1807
|
+
} };
|
|
1808
|
+
}
|
|
1809
|
+
//#endregion
|
|
1810
|
+
export { SentenceChunker, VOICE_PROTOCOL_VERSION, WorkersAIFluxSTT, WorkersAINova3STT, WorkersAITTS, addSFUTracks, createSFUSession, createSFUWebSocketAdapter, decodeVarint, downsample48kStereoTo16kMono, encodePayloadToProtobuf, encodeVarint, extractPayloadFromProtobuf, iterateText, renegotiateSFUSession, sfuFetch, upsample16kMonoTo48kStereo, withVoice, withVoiceInput };
|
|
1811
|
+
|
|
1812
|
+
//# sourceMappingURL=index.js.map
|