@runtypelabs/voice 0.2.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +131 -6
- package/dist/index.cjs +211 -17
- package/dist/index.d.cts +24 -2
- package/dist/index.d.ts +24 -2
- package/dist/index.js +211 -17
- package/dist/persona.cjs +272 -21
- package/dist/persona.d.cts +23 -4
- package/dist/persona.d.ts +23 -4
- package/dist/persona.js +272 -21
- package/dist/react.cjs +261 -24
- package/dist/react.d.cts +7 -4
- package/dist/react.d.ts +7 -4
- package/dist/react.js +262 -25
- package/dist/types-DaXyPeiV.d.cts +95 -0
- package/dist/types-DaXyPeiV.d.ts +95 -0
- package/package.json +9 -10
- package/dist/types-9SfA7oHc.d.cts +0 -40
- package/dist/types-9SfA7oHc.d.ts +0 -40
package/README.md
CHANGED
|
@@ -46,7 +46,11 @@ Do not pass a server API key. Tokens use the WebSocket subprotocol, never the UR
|
|
|
46
46
|
`clientToken` can also be a function returning a token or a promise; it is called
|
|
47
47
|
once per new call. `startCall(tokenOverride)` overrides that call's token.
|
|
48
48
|
|
|
49
|
-
The hook ends the call on unmount and when `agentId
|
|
49
|
+
The hook ends the call on unmount and when `agentId`, `apiUrl`, `artifacts`, or
|
|
50
|
+
`sessionId` selects a different session. Copying the session ID reported by
|
|
51
|
+
`onSession` back into the hook preserves the active call, including after session
|
|
52
|
+
renewal. The snapshot it spreads carries `artifacts` and `session`
|
|
53
|
+
alongside `transcript`.
|
|
50
54
|
Changing `clientToken` affects the next call without interrupting the current one.
|
|
51
55
|
Startup and connection failures populate `error` and `status: 'error'`.
|
|
52
56
|
|
|
@@ -95,6 +99,67 @@ const config = {
|
|
|
95
99
|
}
|
|
96
100
|
```
|
|
97
101
|
|
|
102
|
+
### Artifacts
|
|
103
|
+
|
|
104
|
+
Set `artifacts: true` to declare the capability. The client then connects with
|
|
105
|
+
`clientCapabilities=partial_transcript,artifacts`, the server sends an additive
|
|
106
|
+
`artifact` message for every artifact the agent produces, and the client
|
|
107
|
+
assembles them into `snapshot.artifacts`. Leaving it off keeps an existing
|
|
108
|
+
embed's wire byte-identical.
|
|
109
|
+
|
|
110
|
+
```json
|
|
111
|
+
{
|
|
112
|
+
"type": "artifact",
|
|
113
|
+
"turnId": "turn-3",
|
|
114
|
+
"executionId": "exec_123",
|
|
115
|
+
"event": {
|
|
116
|
+
"type": "artifact_start",
|
|
117
|
+
"id": "art_1",
|
|
118
|
+
"artifactType": "markdown",
|
|
119
|
+
"title": "game.html",
|
|
120
|
+
"file": { "path": "game.html", "mimeType": "text/html", "language": "html" }
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
`event` is a unified `artifact_start`, `artifact_delta`, `artifact_update`, or
|
|
126
|
+
`artifact_complete` frame. A start opens a `streaming` record, deltas append to
|
|
127
|
+
its `content`, an update carries the component payload, and a complete settles
|
|
128
|
+
it. Artifacts of a turn the caller stopped or cancelled are dropped, the same
|
|
129
|
+
suppression the transcript applies. Ending the call keeps completed artifacts
|
|
130
|
+
and drops half-streamed ones. Artifacts are never spoken: file bodies arrive
|
|
131
|
+
only through these frames.
|
|
132
|
+
|
|
133
|
+
The Persona adapter adds `onArtifact`, which fires each time a record changes,
|
|
134
|
+
and `bindVoiceArtifactsToPersona` streams those records into an initialized
|
|
135
|
+
widget through its `upsertArtifact` handle:
|
|
136
|
+
|
|
137
|
+
```ts
|
|
138
|
+
import { bindVoiceArtifactsToPersona, createPersonaVoiceProvider } from '@runtypelabs/voice/persona'
|
|
139
|
+
|
|
140
|
+
const provider = createPersonaVoiceProvider({ agentId, clientToken, artifacts: true })
|
|
141
|
+
const widget = initAgentWidget({ ...config, features: { artifacts: { enabled: true } } })
|
|
142
|
+
const release = bindVoiceArtifactsToPersona(provider, widget)
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
The artifact id is the upsert id, so every frame updates the same record in
|
|
146
|
+
place, and only the settled record writes a transcript block. The pane opens
|
|
147
|
+
once on the first artifact; pass `{ showOnFirst: false }` to leave it closed.
|
|
148
|
+
Persona's `features.artifacts.enabled` must be on, because `showArtifacts()` is
|
|
149
|
+
a no-op otherwise. Call the returned function to stop delivery; the adapter's
|
|
150
|
+
`disconnect()` releases artifact callbacks along with the rest.
|
|
151
|
+
|
|
152
|
+
### Reusing a conversation
|
|
153
|
+
|
|
154
|
+
`snapshot.session` is `{ sessionId, conversationId }` once the server reports
|
|
155
|
+
them on `session_config`. It survives `endCall`, so the next `startCall` sends
|
|
156
|
+
the same `sessionId` and reattaches the same conversation. Pass the
|
|
157
|
+
`sessionId` option to name another client session instead, for example the one
|
|
158
|
+
a Persona chat widget already opened. Call `resetSession()` between calls when
|
|
159
|
+
the next caller should not inherit the conversation, as on a shared kiosk; it
|
|
160
|
+
is a no-op while a call is open. An older server omits both ids and `session`
|
|
161
|
+
stays null.
|
|
162
|
+
|
|
98
163
|
**Widget release prerequisite:** Persona 4.22.0's factory accepts this adapter,
|
|
99
164
|
but its built-in microphone controls incorrectly route custom providers to
|
|
100
165
|
browser dictation. Use a Persona release containing
|
|
@@ -108,8 +173,14 @@ also need that widget release and a bundled adapter supplied by their host.
|
|
|
108
173
|
The agent's saved mode arrives in `session_config` and remains authoritative:
|
|
109
174
|
|
|
110
175
|
The client requests `voiceProtocol=runtype-browser-v1` when connecting. The API routes
|
|
111
|
-
Cloudflare calls using this protocol to its PCM browser engine
|
|
112
|
-
|
|
176
|
+
Cloudflare calls using this protocol to its PCM browser engine only when the operator's
|
|
177
|
+
`enable-voice-browser-engine-cloudflare` gate is enabled. When disabled, the API rejects
|
|
178
|
+
new v1 calls with HTTP 503 (`VOICE_BROWSER_ENGINE_DISABLED`) before creating a session.
|
|
179
|
+
Operators can enable the gate for the organization or configure ElevenLabs voice.
|
|
180
|
+
Browsers expose rejected WebSocket handshakes as a connection failure, without the HTTP
|
|
181
|
+
response body. The legacy Cloudflare Durable Object uses a different protocol and
|
|
182
|
+
remains available to compatible older clients. Existing calls keep their selected
|
|
183
|
+
engine. ElevenLabs supports this client with either value of its portable-engine gate.
|
|
113
184
|
|
|
114
185
|
- `none`: speaking does not clear playback; `canCancel` is false.
|
|
115
186
|
- `cancel`: tap `cancelResponse()` to clear local playback immediately and send `cancel`, then speak again.
|
|
@@ -126,9 +197,14 @@ Pending Blob conversions and stale player events are invalidated.
|
|
|
126
197
|
`audio_end` means synthesis completed, while `speaking` continues until playback drains.
|
|
127
198
|
|
|
128
199
|
Persona's explicit `stopPlayback()` and the client's matching method also stop
|
|
129
|
-
playback in `none` mode. They discard the current reply
|
|
130
|
-
without sending a mode-disallowed cancellation or
|
|
131
|
-
|
|
200
|
+
playback in `none` mode. They discard the current reply's audio and captions
|
|
201
|
+
locally through `audio_end` without sending a mode-disallowed cancellation or
|
|
202
|
+
ending the call. Stopping playback does not stop the agent's execution: in
|
|
203
|
+
`none` mode the turn keeps running and its artifacts (`snapshot.artifacts`,
|
|
204
|
+
`onArtifact`) keep arriving, whether they began before or after the stop.
|
|
205
|
+
Artifacts are dropped only for a turn that was actually cancelled (`cancel` /
|
|
206
|
+
`barge-in`), interrupted by the server (`audio_clear`), or cut off by hanging
|
|
207
|
+
up. Ordinary `cancelResponse()` still respects the agent's mode. Disconnecting the Persona
|
|
132
208
|
adapter releases its callbacks; register callbacks again when reusing it.
|
|
133
209
|
When a server supplies `turnId` on final transcripts, the adapter forwards it
|
|
134
210
|
as Persona's optional fourth callback argument `{ turnId }`. Untagged servers
|
|
@@ -145,3 +221,52 @@ pnpm --filter @runtypelabs/voice test
|
|
|
145
221
|
pnpm --filter @runtypelabs/voice typecheck
|
|
146
222
|
pnpm --filter @runtypelabs/voice build
|
|
147
223
|
```
|
|
224
|
+
|
|
225
|
+
## Share a conversation with text
|
|
226
|
+
|
|
227
|
+
Initialize `/v1/client/init` before starting voice, including when voice is the
|
|
228
|
+
first user interaction. Request `durableRecovery: true` for a token whose durable
|
|
229
|
+
turn policy is enabled. Keep the returned `visitor.token` in your host's visitor
|
|
230
|
+
store, scoped to that client token. The server returns this secret only when it
|
|
231
|
+
mints the visitor; later init responses do not replace the stored value.
|
|
232
|
+
|
|
233
|
+
Pass the current text session ID and visitor credential to the voice client or
|
|
234
|
+
Persona adapter. For example, after your host initializes the text session:
|
|
235
|
+
|
|
236
|
+
```ts
|
|
237
|
+
const voice = createPersonaVoiceProvider({
|
|
238
|
+
agentId,
|
|
239
|
+
clientToken,
|
|
240
|
+
get sessionId() {
|
|
241
|
+
return currentSessionId
|
|
242
|
+
},
|
|
243
|
+
visitorToken: () => visitorStore.get(clientToken),
|
|
244
|
+
onSession: (session) => {
|
|
245
|
+
currentSessionId = session.sessionId
|
|
246
|
+
currentConversationId = session.conversationId
|
|
247
|
+
},
|
|
248
|
+
})
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
Use the returned session ID for subsequent text requests and send the same
|
|
252
|
+
credential in `X-Visitor-Token`. For Persona, use your integration's supported
|
|
253
|
+
session initialization and credential storage APIs. Do not read Persona's
|
|
254
|
+
internal local-storage keys. `onSession` reports IDs only; never log the visitor
|
|
255
|
+
secret. A configured getter that returns no credential fails before microphone
|
|
256
|
+
acquisition, so disable Start until initialization completes.
|
|
257
|
+
|
|
258
|
+
A valid visitor credential without a session ID creates a visitor-owned voice
|
|
259
|
+
conversation. An expired session ID can locate an existing conversation when
|
|
260
|
+
current visitor proof still authorizes it. An invalid explicit session, another
|
|
261
|
+
visitor's conversation, or a different agent returns an error instead of starting
|
|
262
|
+
a separate conversation. A legacy unowned record can be claimed only when its
|
|
263
|
+
stored initialization provenance matches the presented visitor. Old voice
|
|
264
|
+
records without that provenance require a fresh conversation; a public client
|
|
265
|
+
token does not authorize claiming them.
|
|
266
|
+
|
|
267
|
+
Both portable browser engines support this contract. ElevenLabs requires its
|
|
268
|
+
browser engine flag for shared sessions. Existing clients without visitor proof
|
|
269
|
+
retain their legacy new-call behavior. Visitor proof travels in WebSocket
|
|
270
|
+
headers, never URLs, and the selected protocol remains `runtype.bearer`.
|
|
271
|
+
Disconnecting still cancels the active voice turn; conversation reuse does not
|
|
272
|
+
resume unfinished voice work.
|
package/dist/index.cjs
CHANGED
|
@@ -169,9 +169,58 @@ function initialSnapshot() {
|
|
|
169
169
|
error: null,
|
|
170
170
|
errorDetails: void 0,
|
|
171
171
|
interruptionMode: "none",
|
|
172
|
-
canCancel: false
|
|
172
|
+
canCancel: false,
|
|
173
|
+
artifacts: [],
|
|
174
|
+
session: null
|
|
173
175
|
};
|
|
174
176
|
}
|
|
177
|
+
function readString(value) {
|
|
178
|
+
return typeof value === "string" && value ? value : void 0;
|
|
179
|
+
}
|
|
180
|
+
function readArtifactFile(value) {
|
|
181
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return void 0;
|
|
182
|
+
const file = value;
|
|
183
|
+
const path = readString(file.path);
|
|
184
|
+
const mimeType = readString(file.mimeType);
|
|
185
|
+
if (!path || !mimeType) return void 0;
|
|
186
|
+
const language = readString(file.language);
|
|
187
|
+
return { path, mimeType, ...language ? { language } : {} };
|
|
188
|
+
}
|
|
189
|
+
function applyArtifactFrame(existing, frame, identity) {
|
|
190
|
+
if (frame.type === "artifact_start") {
|
|
191
|
+
const title = readString(frame.title);
|
|
192
|
+
const component = readString(frame.component);
|
|
193
|
+
const file = readArtifactFile(frame.file);
|
|
194
|
+
return {
|
|
195
|
+
...identity,
|
|
196
|
+
artifactType: frame.artifactType === "component" ? "component" : "markdown",
|
|
197
|
+
...title ? { title } : {},
|
|
198
|
+
...file ? { file } : {},
|
|
199
|
+
...component ? { component } : {},
|
|
200
|
+
content: "",
|
|
201
|
+
status: "streaming"
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
if (!existing) return void 0;
|
|
205
|
+
if (frame.type === "artifact_delta") {
|
|
206
|
+
const delta = typeof frame.delta === "string" ? frame.delta : "";
|
|
207
|
+
return delta ? { ...existing, content: existing.content + delta } : void 0;
|
|
208
|
+
}
|
|
209
|
+
if (frame.type === "artifact_update") {
|
|
210
|
+
const component = readString(frame.component) ?? existing.component;
|
|
211
|
+
const props = frame.props && typeof frame.props === "object" && !Array.isArray(frame.props) ? frame.props : existing.props;
|
|
212
|
+
return {
|
|
213
|
+
...existing,
|
|
214
|
+
...component ? { component } : {},
|
|
215
|
+
...props ? { props } : {}
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
if (frame.type === "artifact_complete") return { ...existing, status: "complete" };
|
|
219
|
+
return void 0;
|
|
220
|
+
}
|
|
221
|
+
function readTurnId(msg) {
|
|
222
|
+
return typeof msg.turnId === "string" && msg.turnId ? msg.turnId : null;
|
|
223
|
+
}
|
|
175
224
|
var VoiceClient = class {
|
|
176
225
|
constructor(options) {
|
|
177
226
|
this.options = options;
|
|
@@ -192,6 +241,16 @@ var VoiceClient = class {
|
|
|
192
241
|
stoppedResponse = false;
|
|
193
242
|
responseEnded = true;
|
|
194
243
|
hasPendingAudio = false;
|
|
244
|
+
/**
|
|
245
|
+
* The assistant entry built from clause captions for the turn being spoken. The
|
|
246
|
+
* authoritative transcript replaces it; a stop commits it, because it captions
|
|
247
|
+
* audio the caller already heard.
|
|
248
|
+
*/
|
|
249
|
+
pendingAssistant = null;
|
|
250
|
+
/** Turn whose clause captions were already committed by a stop, so its authoritative text is redundant. */
|
|
251
|
+
committedAssistantTurnId = null;
|
|
252
|
+
activeTurnId = null;
|
|
253
|
+
suppressedArtifactTurnId = null;
|
|
195
254
|
getSnapshot = () => this.snapshot;
|
|
196
255
|
subscribe = (listener) => {
|
|
197
256
|
this.listeners.add(listener);
|
|
@@ -213,7 +272,7 @@ var VoiceClient = class {
|
|
|
213
272
|
if (this.snapshot.status !== "idle" && this.snapshot.status !== "error") return;
|
|
214
273
|
this.cleanup();
|
|
215
274
|
const generation = this.generation;
|
|
216
|
-
this.update({ ...initialSnapshot(), status: "connecting" });
|
|
275
|
+
this.update({ ...initialSnapshot(), session: this.snapshot.session, status: "connecting" });
|
|
217
276
|
try {
|
|
218
277
|
const token = tokenOverride ?? (typeof this.options.clientToken === "function" ? await this.options.clientToken() : this.options.clientToken);
|
|
219
278
|
if (generation !== this.generation) return;
|
|
@@ -228,6 +287,20 @@ var VoiceClient = class {
|
|
|
228
287
|
url.pathname = `${basePath}/ws/agents/${encodeURIComponent(this.options.agentId)}/voice`;
|
|
229
288
|
url.search = "";
|
|
230
289
|
url.searchParams.set("voiceProtocol", "runtype-browser-v1");
|
|
290
|
+
const capabilities = this.options.artifacts ? "partial_transcript,artifacts" : "partial_transcript";
|
|
291
|
+
url.searchParams.set("clientCapabilities", capabilities);
|
|
292
|
+
const sessionId = this.options.sessionId ?? this.snapshot.session?.sessionId;
|
|
293
|
+
if (sessionId) url.searchParams.set("sessionId", sessionId);
|
|
294
|
+
const visitorToken = typeof this.options.visitorToken === "function" ? await this.options.visitorToken() : this.options.visitorToken;
|
|
295
|
+
if (generation !== this.generation) return;
|
|
296
|
+
if (this.options.visitorToken !== void 0 && !visitorToken) {
|
|
297
|
+
throw new Error(
|
|
298
|
+
"Voice visitor credential unavailable. Initialize the client session first."
|
|
299
|
+
);
|
|
300
|
+
}
|
|
301
|
+
if (visitorToken && !/^cvt_[A-Za-z0-9_-]+$/.test(visitorToken)) {
|
|
302
|
+
throw new Error("Invalid voice visitor credential.");
|
|
303
|
+
}
|
|
231
304
|
url.hash = "";
|
|
232
305
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
233
306
|
audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
|
|
@@ -251,7 +324,11 @@ var VoiceClient = class {
|
|
|
251
324
|
return;
|
|
252
325
|
}
|
|
253
326
|
this.player = player;
|
|
254
|
-
const socket = new WebSocket(url.toString(), [
|
|
327
|
+
const socket = new WebSocket(url.toString(), [
|
|
328
|
+
"runtype.bearer",
|
|
329
|
+
token,
|
|
330
|
+
...visitorToken ? [visitorToken] : []
|
|
331
|
+
]);
|
|
255
332
|
socket.binaryType = "arraybuffer";
|
|
256
333
|
this.socket = socket;
|
|
257
334
|
socket.onopen = () => {
|
|
@@ -282,6 +359,11 @@ var VoiceClient = class {
|
|
|
282
359
|
this.cleanup();
|
|
283
360
|
this.update({ status: "idle", interimTranscript: null, audioLevel: 0, isMuted: false });
|
|
284
361
|
};
|
|
362
|
+
/** Forget the remembered conversation so the next startCall opens a new one. No-op during a call. */
|
|
363
|
+
resetSession = () => {
|
|
364
|
+
if (this.snapshot.status !== "idle" && this.snapshot.status !== "error") return;
|
|
365
|
+
this.update({ session: null });
|
|
366
|
+
};
|
|
285
367
|
toggleMute = () => {
|
|
286
368
|
this.update({ isMuted: !this.snapshot.isMuted, audioLevel: 0 });
|
|
287
369
|
};
|
|
@@ -296,9 +378,11 @@ var VoiceClient = class {
|
|
|
296
378
|
this.clearPlayback();
|
|
297
379
|
if (this.snapshot.interruptionMode === "none") {
|
|
298
380
|
this.stoppedResponse = !this.responseEnded;
|
|
381
|
+
this.commitClauseCaptions();
|
|
299
382
|
this.setStatus("listening");
|
|
300
383
|
return;
|
|
301
384
|
}
|
|
385
|
+
this.suppressedArtifactTurnId = this.activeTurnId;
|
|
302
386
|
this.awaitingClear = true;
|
|
303
387
|
this.setStatus("listening");
|
|
304
388
|
this.socket.send(JSON.stringify({ type: "cancel" }));
|
|
@@ -340,6 +424,82 @@ var VoiceClient = class {
|
|
|
340
424
|
this.awaitingClear = false;
|
|
341
425
|
this.stoppedResponse = false;
|
|
342
426
|
this.responseEnded = true;
|
|
427
|
+
this.commitClauseCaptions();
|
|
428
|
+
this.committedAssistantTurnId = null;
|
|
429
|
+
this.activeTurnId = null;
|
|
430
|
+
this.suppressedArtifactTurnId = null;
|
|
431
|
+
const settledArtifacts = this.snapshot.artifacts.filter(
|
|
432
|
+
(artifact) => artifact.status === "complete"
|
|
433
|
+
);
|
|
434
|
+
if (settledArtifacts.length !== this.snapshot.artifacts.length)
|
|
435
|
+
this.update({ artifacts: settledArtifacts });
|
|
436
|
+
}
|
|
437
|
+
/**
|
|
438
|
+
* Folds an `artifact` message into the snapshot. Frames of a turn the caller
|
|
439
|
+
* cancelled are dropped; a playback-only stop leaves the turn running, so its
|
|
440
|
+
* artifacts keep arriving and are kept.
|
|
441
|
+
*/
|
|
442
|
+
applyArtifactMessage(msg) {
|
|
443
|
+
if (this.awaitingClear) return;
|
|
444
|
+
const frame = msg.event;
|
|
445
|
+
if (!frame || typeof frame !== "object" || Array.isArray(frame)) return;
|
|
446
|
+
const event = frame;
|
|
447
|
+
const id = readString(event.id);
|
|
448
|
+
if (!id) return;
|
|
449
|
+
const turnId = readTurnId(msg);
|
|
450
|
+
if (turnId !== null && turnId === this.suppressedArtifactTurnId) return;
|
|
451
|
+
if (turnId !== null) this.activeTurnId = turnId;
|
|
452
|
+
const artifacts = this.snapshot.artifacts;
|
|
453
|
+
const index = artifacts.findIndex((artifact) => artifact.id === id);
|
|
454
|
+
const executionId = readString(msg.executionId);
|
|
455
|
+
const next = applyArtifactFrame(index === -1 ? void 0 : artifacts[index], event, {
|
|
456
|
+
id,
|
|
457
|
+
turnId,
|
|
458
|
+
...executionId ? { executionId } : {}
|
|
459
|
+
});
|
|
460
|
+
if (!next) return;
|
|
461
|
+
this.update({
|
|
462
|
+
artifacts: index === -1 ? [...artifacts, next] : [...artifacts.slice(0, index), next, ...artifacts.slice(index + 1)]
|
|
463
|
+
});
|
|
464
|
+
}
|
|
465
|
+
/** Settles the spoken clauses in place and ends the turn's claim on them. */
|
|
466
|
+
commitClauseCaptions() {
|
|
467
|
+
this.committedAssistantTurnId = this.pendingAssistant?.turnId ?? null;
|
|
468
|
+
const settled = this.pendingAssistant !== null;
|
|
469
|
+
this.pendingAssistant = null;
|
|
470
|
+
if (!settled) return;
|
|
471
|
+
const transcript = this.snapshot.transcript;
|
|
472
|
+
const last = transcript[transcript.length - 1];
|
|
473
|
+
if (last?.role !== "assistant" || last.partial !== true) return;
|
|
474
|
+
const { partial: _partial, ...committed } = last;
|
|
475
|
+
this.update({ transcript: [...transcript.slice(0, -1), committed] });
|
|
476
|
+
}
|
|
477
|
+
/** Extends the turn's spoken-so-far assistant entry, or opens one for a new turn. */
|
|
478
|
+
appendClauseCaption(text, turnId) {
|
|
479
|
+
const transcript = this.snapshot.transcript;
|
|
480
|
+
const last = transcript[transcript.length - 1];
|
|
481
|
+
if (this.pendingAssistant?.turnId === turnId && last?.role === "assistant") {
|
|
482
|
+
return {
|
|
483
|
+
interimTranscript: null,
|
|
484
|
+
transcript: [...transcript.slice(0, -1), { ...last, content: last.content + text }],
|
|
485
|
+
status: "speaking"
|
|
486
|
+
};
|
|
487
|
+
}
|
|
488
|
+
this.pendingAssistant = { turnId };
|
|
489
|
+
return {
|
|
490
|
+
interimTranscript: null,
|
|
491
|
+
transcript: [
|
|
492
|
+
...transcript,
|
|
493
|
+
{
|
|
494
|
+
role: "assistant",
|
|
495
|
+
content: text,
|
|
496
|
+
timestamp: Date.now(),
|
|
497
|
+
partial: true,
|
|
498
|
+
...turnId ? { turnId } : {}
|
|
499
|
+
}
|
|
500
|
+
],
|
|
501
|
+
status: "speaking"
|
|
502
|
+
};
|
|
343
503
|
}
|
|
344
504
|
clearPlayback() {
|
|
345
505
|
this.playbackRevision += 1;
|
|
@@ -374,32 +534,61 @@ var VoiceClient = class {
|
|
|
374
534
|
return;
|
|
375
535
|
}
|
|
376
536
|
switch (msg.type) {
|
|
377
|
-
case "session_config":
|
|
537
|
+
case "session_config": {
|
|
378
538
|
if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
|
|
379
539
|
this.update({ interruptionMode: msg.interruptionMode });
|
|
380
540
|
}
|
|
541
|
+
const sessionId = readString(msg.sessionId);
|
|
542
|
+
const conversationId = readString(msg.conversationId);
|
|
543
|
+
if (sessionId && conversationId) {
|
|
544
|
+
const session = { sessionId, conversationId };
|
|
545
|
+
this.update({ session });
|
|
546
|
+
this.options.onSession?.(session);
|
|
547
|
+
}
|
|
548
|
+
break;
|
|
549
|
+
}
|
|
550
|
+
case "artifact":
|
|
551
|
+
this.applyArtifactMessage(msg);
|
|
381
552
|
break;
|
|
382
553
|
case "transcript_interim":
|
|
383
554
|
this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
|
|
384
555
|
break;
|
|
385
|
-
case "
|
|
556
|
+
case "transcript_partial": {
|
|
557
|
+
if (msg.role !== "assistant" || typeof msg.text !== "string" || !msg.text) break;
|
|
558
|
+
this.activeTurnId = readTurnId(msg) ?? this.activeTurnId;
|
|
559
|
+
if (this.awaitingClear || this.stoppedResponse) break;
|
|
560
|
+
this.responseEnded = false;
|
|
561
|
+
this.update(this.appendClauseCaption(msg.text, readTurnId(msg)));
|
|
562
|
+
break;
|
|
563
|
+
}
|
|
564
|
+
case "transcript_final": {
|
|
386
565
|
if (typeof msg.text !== "string" || msg.role !== "user" && msg.role !== "assistant") break;
|
|
387
|
-
|
|
566
|
+
const role = msg.role;
|
|
567
|
+
const text = msg.text;
|
|
568
|
+
if (role === "assistant" && (this.awaitingClear || this.stoppedResponse)) break;
|
|
569
|
+
const turnId = readTurnId(msg);
|
|
570
|
+
if (turnId !== null) this.activeTurnId = turnId;
|
|
571
|
+
if (role === "assistant" && turnId !== null && turnId === this.committedAssistantTurnId)
|
|
572
|
+
break;
|
|
388
573
|
this.responseEnded = false;
|
|
574
|
+
const transcript = this.snapshot.transcript;
|
|
575
|
+
const last = transcript[transcript.length - 1];
|
|
576
|
+
const supersedes = role === "assistant" && this.pendingAssistant !== null && last?.role === "assistant" && (turnId === null || this.pendingAssistant.turnId === null || this.pendingAssistant.turnId === turnId);
|
|
577
|
+
this.pendingAssistant = null;
|
|
578
|
+
const entry = {
|
|
579
|
+
role,
|
|
580
|
+
content: text,
|
|
581
|
+
// INVARIANT: reuse the caption's timestamp so a keyed list does not remount the bubble.
|
|
582
|
+
timestamp: supersedes && last ? last.timestamp : Date.now(),
|
|
583
|
+
...turnId ? { turnId } : {}
|
|
584
|
+
};
|
|
389
585
|
this.update({
|
|
390
586
|
interimTranscript: null,
|
|
391
|
-
transcript: [
|
|
392
|
-
|
|
393
|
-
{
|
|
394
|
-
role: msg.role,
|
|
395
|
-
content: msg.text,
|
|
396
|
-
timestamp: Date.now(),
|
|
397
|
-
...typeof msg.turnId === "string" && msg.turnId ? { turnId: msg.turnId } : {}
|
|
398
|
-
}
|
|
399
|
-
],
|
|
400
|
-
status: this.awaitingClear ? this.snapshot.status : msg.role === "user" ? "thinking" : "speaking"
|
|
587
|
+
transcript: supersedes ? [...transcript.slice(0, -1), entry] : [...transcript, entry],
|
|
588
|
+
status: this.awaitingClear ? this.snapshot.status : role === "user" ? "thinking" : "speaking"
|
|
401
589
|
});
|
|
402
590
|
break;
|
|
591
|
+
}
|
|
403
592
|
case "audio_end": {
|
|
404
593
|
this.responseEnded = true;
|
|
405
594
|
if (this.stoppedResponse) {
|
|
@@ -416,10 +605,12 @@ var VoiceClient = class {
|
|
|
416
605
|
break;
|
|
417
606
|
}
|
|
418
607
|
case "audio_clear":
|
|
608
|
+
this.suppressedArtifactTurnId = this.activeTurnId ?? this.suppressedArtifactTurnId;
|
|
419
609
|
this.clearPlayback();
|
|
420
610
|
this.awaitingClear = false;
|
|
421
611
|
this.stoppedResponse = false;
|
|
422
612
|
this.responseEnded = true;
|
|
613
|
+
this.commitClauseCaptions();
|
|
423
614
|
this.setStatus("listening");
|
|
424
615
|
break;
|
|
425
616
|
case "metrics": {
|
|
@@ -432,8 +623,11 @@ var VoiceClient = class {
|
|
|
432
623
|
// @snake-case-ok: Existing voice wire contract.
|
|
433
624
|
firstAudioMs: number(msg.first_audio_ms),
|
|
434
625
|
// @snake-case-ok: Existing voice wire contract.
|
|
435
|
-
totalMs: number(msg.total_ms)
|
|
626
|
+
totalMs: number(msg.total_ms),
|
|
627
|
+
// @snake-case-ok: Existing voice wire contract.
|
|
628
|
+
firstSynthesisMs: number(msg.first_synthesis_ms),
|
|
436
629
|
// @snake-case-ok: Existing voice wire contract.
|
|
630
|
+
incremental: msg.incremental === true
|
|
437
631
|
}
|
|
438
632
|
});
|
|
439
633
|
break;
|
package/dist/index.d.cts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { V as VoiceClientOptions, a as VoiceSnapshot } from './types-
|
|
2
|
-
export { I as InterruptionMode, T as TranscriptEntry, b as
|
|
1
|
+
import { V as VoiceClientOptions, a as VoiceSnapshot } from './types-DaXyPeiV.cjs';
|
|
2
|
+
export { I as InterruptionMode, T as TranscriptEntry, b as VoiceArtifact, c as VoiceArtifactFile, d as VoiceMetrics, e as VoiceSession, f as VoiceStatus } from './types-DaXyPeiV.cjs';
|
|
3
3
|
|
|
4
4
|
/** Browser microphone and playback lifecycle for Runtype's voice WebSocket endpoint. */
|
|
5
5
|
declare class VoiceClient {
|
|
@@ -19,6 +19,16 @@ declare class VoiceClient {
|
|
|
19
19
|
private stoppedResponse;
|
|
20
20
|
private responseEnded;
|
|
21
21
|
private hasPendingAudio;
|
|
22
|
+
/**
|
|
23
|
+
* The assistant entry built from clause captions for the turn being spoken. The
|
|
24
|
+
* authoritative transcript replaces it; a stop commits it, because it captions
|
|
25
|
+
* audio the caller already heard.
|
|
26
|
+
*/
|
|
27
|
+
private pendingAssistant;
|
|
28
|
+
/** Turn whose clause captions were already committed by a stop, so its authoritative text is redundant. */
|
|
29
|
+
private committedAssistantTurnId;
|
|
30
|
+
private activeTurnId;
|
|
31
|
+
private suppressedArtifactTurnId;
|
|
22
32
|
constructor(options: VoiceClientOptions);
|
|
23
33
|
getSnapshot: () => VoiceSnapshot;
|
|
24
34
|
subscribe: (listener: () => void) => (() => void);
|
|
@@ -27,12 +37,24 @@ declare class VoiceClient {
|
|
|
27
37
|
/** Acquire microphone access and open a call. Invoke from a user gesture. Failures populate snapshot.error. */
|
|
28
38
|
startCall: (tokenOverride?: string) => Promise<void>;
|
|
29
39
|
endCall: () => void;
|
|
40
|
+
/** Forget the remembered conversation so the next startCall opens a new one. No-op during a call. */
|
|
41
|
+
resetSession: () => void;
|
|
30
42
|
toggleMute: () => void;
|
|
31
43
|
cancelResponse: () => void;
|
|
32
44
|
/** Stop the current reply explicitly, including local playback when interruptions are disabled. */
|
|
33
45
|
stopPlayback: () => void;
|
|
34
46
|
private fail;
|
|
35
47
|
private cleanup;
|
|
48
|
+
/**
|
|
49
|
+
* Folds an `artifact` message into the snapshot. Frames of a turn the caller
|
|
50
|
+
* cancelled are dropped; a playback-only stop leaves the turn running, so its
|
|
51
|
+
* artifacts keep arriving and are kept.
|
|
52
|
+
*/
|
|
53
|
+
private applyArtifactMessage;
|
|
54
|
+
/** Settles the spoken clauses in place and ends the turn's claim on them. */
|
|
55
|
+
private commitClauseCaptions;
|
|
56
|
+
/** Extends the turn's spoken-so-far assistant entry, or opens one for a new turn. */
|
|
57
|
+
private appendClauseCaption;
|
|
36
58
|
private clearPlayback;
|
|
37
59
|
private handleMessage;
|
|
38
60
|
private startCapture;
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { V as VoiceClientOptions, a as VoiceSnapshot } from './types-
|
|
2
|
-
export { I as InterruptionMode, T as TranscriptEntry, b as
|
|
1
|
+
import { V as VoiceClientOptions, a as VoiceSnapshot } from './types-DaXyPeiV.js';
|
|
2
|
+
export { I as InterruptionMode, T as TranscriptEntry, b as VoiceArtifact, c as VoiceArtifactFile, d as VoiceMetrics, e as VoiceSession, f as VoiceStatus } from './types-DaXyPeiV.js';
|
|
3
3
|
|
|
4
4
|
/** Browser microphone and playback lifecycle for Runtype's voice WebSocket endpoint. */
|
|
5
5
|
declare class VoiceClient {
|
|
@@ -19,6 +19,16 @@ declare class VoiceClient {
|
|
|
19
19
|
private stoppedResponse;
|
|
20
20
|
private responseEnded;
|
|
21
21
|
private hasPendingAudio;
|
|
22
|
+
/**
|
|
23
|
+
* The assistant entry built from clause captions for the turn being spoken. The
|
|
24
|
+
* authoritative transcript replaces it; a stop commits it, because it captions
|
|
25
|
+
* audio the caller already heard.
|
|
26
|
+
*/
|
|
27
|
+
private pendingAssistant;
|
|
28
|
+
/** Turn whose clause captions were already committed by a stop, so its authoritative text is redundant. */
|
|
29
|
+
private committedAssistantTurnId;
|
|
30
|
+
private activeTurnId;
|
|
31
|
+
private suppressedArtifactTurnId;
|
|
22
32
|
constructor(options: VoiceClientOptions);
|
|
23
33
|
getSnapshot: () => VoiceSnapshot;
|
|
24
34
|
subscribe: (listener: () => void) => (() => void);
|
|
@@ -27,12 +37,24 @@ declare class VoiceClient {
|
|
|
27
37
|
/** Acquire microphone access and open a call. Invoke from a user gesture. Failures populate snapshot.error. */
|
|
28
38
|
startCall: (tokenOverride?: string) => Promise<void>;
|
|
29
39
|
endCall: () => void;
|
|
40
|
+
/** Forget the remembered conversation so the next startCall opens a new one. No-op during a call. */
|
|
41
|
+
resetSession: () => void;
|
|
30
42
|
toggleMute: () => void;
|
|
31
43
|
cancelResponse: () => void;
|
|
32
44
|
/** Stop the current reply explicitly, including local playback when interruptions are disabled. */
|
|
33
45
|
stopPlayback: () => void;
|
|
34
46
|
private fail;
|
|
35
47
|
private cleanup;
|
|
48
|
+
/**
|
|
49
|
+
* Folds an `artifact` message into the snapshot. Frames of a turn the caller
|
|
50
|
+
* cancelled are dropped; a playback-only stop leaves the turn running, so its
|
|
51
|
+
* artifacts keep arriving and are kept.
|
|
52
|
+
*/
|
|
53
|
+
private applyArtifactMessage;
|
|
54
|
+
/** Settles the spoken clauses in place and ends the turn's claim on them. */
|
|
55
|
+
private commitClauseCaptions;
|
|
56
|
+
/** Extends the turn's spoken-so-far assistant entry, or opens one for a new turn. */
|
|
57
|
+
private appendClauseCaption;
|
|
36
58
|
private clearPlayback;
|
|
37
59
|
private handleMessage;
|
|
38
60
|
private startCapture;
|