nanocodex 0.6.4 → 0.6.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/README.md +37 -11
  2. package/browser/Agent.d.mts +1 -1
  3. package/browser/ElevenLabsPlayback.mjs +119 -0
  4. package/browser/InlineAgent.mjs +22 -17
  5. package/browser/Transport.d.mts +2 -0
  6. package/browser/Transport.mjs +3 -2
  7. package/browser/Voice.d.mts +4 -0
  8. package/browser/Voice.mjs +38 -6
  9. package/browser/VoiceSession.mjs +37 -7
  10. package/browser/host.d.mts +20 -4
  11. package/browser/host.mjs +156 -18
  12. package/cloud/types.d.mts +1 -0
  13. package/cloudflare/Agent.d.mts +14 -9
  14. package/cloudflare/Agent.mjs +354 -96
  15. package/cloudflare/agent-settings.d.mts +15 -0
  16. package/cloudflare/agent-settings.mjs +135 -0
  17. package/cloudflare/chat-reasoning.mjs +20 -0
  18. package/cloudflare/durable-placement.d.mts +12 -0
  19. package/cloudflare/durable-placement.mjs +82 -0
  20. package/cloudflare/egress.d.mts +31 -0
  21. package/cloudflare/gateway-binding-responses.mjs +84 -0
  22. package/cloudflare/gateway-responses.d.mts +48 -0
  23. package/cloudflare/gateway-responses.mjs +191 -0
  24. package/cloudflare/index.d.mts +6 -0
  25. package/cloudflare/index.mjs +3 -0
  26. package/cloudflare/managed-auth.d.mts +15 -0
  27. package/cloudflare/managed-auth.mjs +119 -0
  28. package/cloudflare/managed-live.d.mts +7 -0
  29. package/cloudflare/managed-live.mjs +56 -0
  30. package/cloudflare/prepared-connection.mjs +49 -0
  31. package/cloudflare/provider-stream.mjs +353 -0
  32. package/cloudflare/rpc.d.mts +9 -0
  33. package/cloudflare/rpc.mjs +14 -0
  34. package/cloudflare/tool-alias-diagnostic.mjs +21 -0
  35. package/cloudflare/tracing.d.mts +16 -0
  36. package/cloudflare/tracing.mjs +47 -0
  37. package/cloudflare/workers-ai-responses.d.mts +28 -0
  38. package/cloudflare/workers-ai-responses.mjs +399 -0
  39. package/host/Agent.d.mts +1 -1
  40. package/index.d.mts +2 -0
  41. package/internal.mjs +53 -14
  42. package/managed/Agent.d.mts +25 -69
  43. package/managed/Agent.mjs +82 -196
  44. package/managed/README.md +9 -38
  45. package/managed/Voice.mjs +5 -2
  46. package/managed/index.d.mts +0 -3
  47. package/managed/internal.mjs +2 -1
  48. package/node/Agent.d.mts +1 -1
  49. package/node/Agent.mjs +9 -3
  50. package/node/host.mjs +5 -0
  51. package/package.json +34 -2
  52. package/pkg-node/nanocodex.d.ts +22 -18
  53. package/pkg-node/nanocodex.js +118 -92
  54. package/pkg-node/nanocodex_bg.wasm.d.ts +6 -5
  55. package/pkg-web/.nanocodex-bindgen-stamp +7 -1
  56. package/pkg-web/nanocodex-build.json +1 -1
  57. package/pkg-web/nanocodex.d.ts +28 -23
  58. package/pkg-web/nanocodex.js +117 -91
  59. package/pkg-web/nanocodex_bg.js +117 -91
  60. package/pkg-web/nanocodex_bg.wasm +0 -0
  61. package/pkg-web/nanocodex_bg.wasm.d.ts +6 -5
  62. package/pkg-web/nanocodex_worker.js +1 -1
  63. package/runtime/before-compaction.mjs +49 -0
  64. package/runtime/code-evaluator.worker.mjs +2 -9
  65. package/runtime/managed-transport.mjs +10 -2
  66. package/runtime/mcp-runtime.mjs +20 -4
  67. package/runtime/quickjs-evaluator.mjs +11 -5
  68. package/runtime/response-controls.mjs +67 -4
  69. package/runtime/steer-receipt.mjs +13 -0
  70. package/runtime/subagent-routing.d.mts +38 -0
  71. package/runtime/subagent-routing.mjs +81 -0
  72. package/runtime/subagents.d.mts +1 -1
  73. package/runtime/tempo-provider.d.mts +1 -1
  74. package/runtime/tempo-provider.mjs +1 -1
  75. package/tools/environment.d.mts +17 -2
  76. package/tools/environment.mjs +30 -3
  77. package/tools/index.d.mts +2 -0
  78. package/types.d.mts +30 -2
package/README.md CHANGED
@@ -10,7 +10,7 @@ import { Actions, Agent, Transport } from "nanocodex/node";
10
10
 
11
11
  const agent = await Agent.create({
12
12
  transport: Transport.openAi({ apiKey: process.env.OPENAI_API_KEY }),
13
- model: "gpt-5.6-luna",
13
+ model: "gpt-6-luna",
14
14
  instructions: "You are a Rust coding agent. Preserve unrelated work and run relevant tests.",
15
15
  reasoningMode: "pro",
16
16
  thinking: "high",
@@ -236,8 +236,13 @@ requesting speech. Context and speech are split into provider-sized messages.
236
236
  These commands retain frames until sent and preserve them
237
237
  across reconnects when using the sideband transport. They are also methods on
238
238
  the resource and on
239
- `useVoice` from `nanocodex-react`. These settings use ChatGPT subscription voice;
240
- custom voices and Platform audio configuration are not accepted.
239
+ `useVoice` from `nanocodex-react`. Choose `outputProvider: "elevenlabs"` and `elevenLabsVoiceId` to synthesize
240
+ spoken output with an ElevenLabs account voice, including an instant clone.
241
+ The account voice settings panel connects the API key, lists voices, and uploads
242
+ cloning samples after explicit consent. Keys are encrypted on the server and
243
+ never included in saved voice settings. ChatGPT still owns live input and agent
244
+ handoffs; `voice` continues to select its built-in voice. The default output
245
+ provider is `openai`. Platform audio configuration is not accepted.
241
246
 
242
247
  `Voice.create(...)` remains the equivalent namespaced resource constructor, and
243
248
  `Voice.voices` is the exact ChatGPT V3 voice catalog. The constructor accepts a
@@ -247,6 +252,20 @@ normal browser Agent, an account-owned managed Agent, or a grant-scoped
247
252
  one-use ticket for that path. The browser binding never receives ChatGPT
248
253
  credentials or places its reusable grant bearer in a WebSocket URL.
249
254
 
255
+ ### Content-free Worker tracing
256
+
257
+ `nanocodex/cloudflare/tracing` wraps native `tracing.enterSpan(name, callback)`
258
+ with sanitized exception events. Use static operation names: they become
259
+ exception codes on failure. The original error is rethrown; its message and
260
+ stack are never passed to `recordException`.
261
+
262
+ `setSpanAttributes(span, attributes)` sets bulk metadata, while
263
+ `annotateActiveSpan(attributes)` annotates the current invocation or active
264
+ span without changing nesting. `recordSpanException(span, code)` handles
265
+ failures represented by results rather than thrown errors. Supply only static
266
+ codes and content-free attributes. On older workerd versions, these retain
267
+ individual attributes and `error.type` without requiring the September APIs.
268
+
250
269
  ### Durable Cloudflare Agent
251
270
 
252
271
  `nanocodex/cloudflare` is the standard Durable Object consumer. It keeps the
@@ -302,10 +321,12 @@ Cloudflare Agents default to direct tool mode because Workers prohibit dynamic
302
321
  `eval`/`new Function`. Caller-defined tools therefore work without a code
303
322
  evaluator. Select `toolMode: "code"` only when also supplying an evaluator that
304
323
  is explicitly compatible with the deployed Worker runtime. Runtime-owned
305
- Subagents are installed by default, including on a durable root. Clean children
306
- persist independent execution state under their own agent session IDs. The
307
- Rust task-tree registry remains in memory and is closed with the live root, so
308
- tree-local IDs and topology are not reconstructed from those agent states. Use
324
+ Subagents are installed by default, including on a durable root. All children are
325
+ ephemeral: their identities, topology, conversations, results, and routing exist
326
+ only for the lifetime of the root runtime. Root shutdown or restart discards the
327
+ entire child tree; only the root's own durable history resumes. Existing child
328
+ checkpoints from older versions are discarded. Within a live runtime, completed
329
+ children remain available for follow-up messages until closed. Use
309
330
  `Subagents.create({ maxConcurrency })` in `tools` to set an explicit finite
310
331
  concurrency limit. Active subagent turns are unlimited by default.
311
332
 
@@ -349,6 +370,11 @@ native and WASM consumers run the same Rust implementation and receive the
349
370
  same seven tools: `spawn_agent`, `submit_result`, `send_agent_message`,
350
371
  `list_agents`, `wait_agent`, `interrupt_agent`, and `close_agent`.
351
372
 
373
+ Children call `submit_result({output})`; the runtime supplies the trusted
374
+ instruction revision. The response is `{accepted: true, status: "accepted"}` or
375
+ `{accepted: false, status: "superseded"}`. Superseded submissions are normal
376
+ continuations: incorporate the updated instructions and submit again.
377
+
352
378
  Inside a caller-owned Worker or server isolate, host capabilities stay as
353
379
  ordinary functions without crossing another compatibility protocol:
354
380
 
@@ -686,7 +712,7 @@ const mpp = tempoProvider.session;
686
712
 
687
713
  const agent = await Agent.create({
688
714
  transport: Transport.mpp({ session: tempoProvider }),
689
- thinking: "none",
715
+ thinking: "low",
690
716
  fastMode: true,
691
717
  tools,
692
718
  });
@@ -736,8 +762,8 @@ of its wallet adapter, and constructs both payment paths from that provider's
736
762
  adapter-neutral `getMppxParameters()` contract. The lower-level
737
763
  `createTempoProvider({ session, payment })` remains available when the
738
764
  application constructs MPPx itself. Both explicitly select Tempo provider mode.
739
- In that mode Nanocodex automatically adds its built-in Mercator MCP and wraps it
740
- with the same wallet and payment policy. The provider also exposes an MPP-aware
765
+ In that mode Nanocodex automatically adds its built-in Mercator MCP at
766
+ `https://mercator.sh/mcp` and wraps it with the same wallet and payment policy. The provider also exposes an MPP-aware
741
767
  `fetch`; Mercator's paid REST handoffs use that same method rather than a second
742
768
  wallet or payment configuration. Its MCP transport remains wrapped at the MCP
743
769
  protocol layer, so browser requests do not need an `Accept-Payment` CORS header.
@@ -1155,7 +1181,7 @@ package manager or build step:
1155
1181
 
1156
1182
  ```html
1157
1183
  <script type="module">
1158
- import { Agent, Transport } from "https://cdn.jsdelivr.net/npm/nanocodex@0.6.4/host/index.mjs";
1184
+ import { Agent, Transport } from "https://cdn.jsdelivr.net/npm/nanocodex@0.6.6/host/index.mjs";
1159
1185
  const agent = await Agent.create({
1160
1186
  transport: Transport.hostManaged({
1161
1187
  websocketUrl: "/api/responses",
@@ -33,7 +33,7 @@ export declare namespace create {
33
33
  transport: ManagedTransport;
34
34
  tools?: Tools | undefined;
35
35
  }>;
36
- type Options = AgentOptions & WorkerToolExposureOptions & {
36
+ type Options = Omit<AgentOptions, "beforeCompaction"> & WorkerToolExposureOptions & {
37
37
  /** Precompiled browser module; WebAssembly modules are structured-clone-safe. */
38
38
  module?: WebAssembly.Module | undefined;
39
39
  /** Fixed browser workspace facts, including its AGENTS.md snapshot. */
@@ -0,0 +1,119 @@
1
+ /** Plays account-proxied PCM without exposing provider credentials to the browser. */
2
+ export class ElevenLabsPlayback {
3
+ #synthesize;
4
+ #onError;
5
+ #context;
6
+ #abort = new AbortController();
7
+ #queue = Promise.resolve();
8
+ #sources = new Set();
9
+ #nextTime = 0;
10
+ #caption;
11
+ #offset = 0;
12
+ #suppressedThrough = -1;
13
+ #pending = 0;
14
+ #pendingChars = 0;
15
+ #closed = false;
16
+
17
+ constructor(synthesize, onError) {
18
+ this.#synthesize = synthesize;
19
+ this.#onError = onError;
20
+ }
21
+
22
+ prime() {
23
+ if (this.#closed) return;
24
+ this.#context ??= new AudioContext({ sampleRate: 24000 });
25
+ return this.#context.resume();
26
+ }
27
+
28
+ transcript(entry) {
29
+ if (this.#closed || entry.speaker !== "assistant" || entry.id <= this.#suppressedThrough) return;
30
+ if (this.#caption !== undefined && entry.id < this.#caption) return;
31
+ if (entry.id !== this.#caption) { this.#caption = entry.id; this.#offset = 0; }
32
+ const remaining = entry.text.slice(this.#offset);
33
+ // Buffer partial words; emit complete sentences before the turn finishes.
34
+ const boundary = entry.is_partial ? [...remaining.matchAll(/[.!?\n](?:\s|$)/g)].at(-1) : undefined;
35
+ const length = entry.is_partial ? (boundary ? boundary.index + boundary[0].length : 0) : remaining.length;
36
+ const text = remaining.slice(0, length).trim();
37
+ this.#offset += length;
38
+ if (!text) return;
39
+ if (this.#pending >= 32 || this.#pendingChars + text.length > 32768) {
40
+ this.interrupt();
41
+ this.#onError(new Error("ElevenLabs playback queue exceeded its limit"));
42
+ return;
43
+ }
44
+ this.#pending++;
45
+ this.#pendingChars += text.length;
46
+ const signal = this.#abort.signal;
47
+ this.#queue = this.#queue.then(async () => {
48
+ if (signal.aborted) return;
49
+ this.#context ??= new AudioContext({ sampleRate: 24000 });
50
+ await this.#context.resume();
51
+ if (signal.aborted) return;
52
+ const response = await this.#synthesize(text, signal);
53
+ if (!response.ok) throw new Error(`ElevenLabs synthesis failed (${response.status})`);
54
+ const reader = response.body?.getReader();
55
+ if (!reader) throw new Error("ElevenLabs returned no audio stream");
56
+ const cancel = () => { void reader.cancel().catch(() => {}); };
57
+ signal.addEventListener("abort", cancel, { once: true });
58
+ let carry;
59
+ try {
60
+ while (!signal.aborted) {
61
+ const { value, done } = await reader.read();
62
+ if (done) break;
63
+ if (signal.aborted) break;
64
+ if (value.byteLength > 1024 * 1024) throw new Error("ElevenLabs audio chunk exceeded its limit");
65
+ const bytes = carry === undefined ? value : Uint8Array.from([carry, ...value]);
66
+ const size = bytes.length - bytes.length % 2;
67
+ carry = size < bytes.length ? bytes[size] : undefined;
68
+ if (!size) continue;
69
+ // Keep decoded/scheduled audio bounded even when synthesis outruns playback.
70
+ for (let offset = 0; offset < size && !signal.aborted; offset += 9600) {
71
+ while (!signal.aborted && (this.#nextTime - this.#context.currentTime > 2 || this.#sources.size >= 128)) {
72
+ await new Promise((resolve) => {
73
+ const done = () => { clearTimeout(timer); signal.removeEventListener("abort", done); resolve(); };
74
+ const timer = setTimeout(done, 25);
75
+ signal.addEventListener("abort", done, { once: true });
76
+ });
77
+ }
78
+ if (signal.aborted) break;
79
+ const length = Math.min(9600, size - offset);
80
+ const buffer = this.#context.createBuffer(1, length / 2, 24000);
81
+ const output = buffer.getChannelData(0);
82
+ const view = new DataView(bytes.buffer, bytes.byteOffset + offset, length);
83
+ for (let i = 0; i < output.length; i++) output[i] = view.getInt16(i * 2, true) / 32768;
84
+ const source = this.#context.createBufferSource();
85
+ source.buffer = buffer;
86
+ source.connect(this.#context.destination);
87
+ this.#sources.add(source);
88
+ source.onended = () => { this.#sources.delete(source); source.disconnect(); };
89
+ this.#nextTime = Math.max(this.#nextTime, this.#context.currentTime + 0.02);
90
+ source.start(this.#nextTime);
91
+ this.#nextTime += buffer.duration;
92
+ }
93
+ }
94
+ } finally { signal.removeEventListener("abort", cancel); await reader.cancel().catch(() => {}); reader.releaseLock(); }
95
+ }).catch((error) => { if (!signal.aborted) this.#onError(error); }).finally(() => {
96
+ if (!signal.aborted) { this.#pending--; this.#pendingChars -= text.length; }
97
+ });
98
+ }
99
+
100
+ interrupt() {
101
+ this.#suppressedThrough = Math.max(this.#suppressedThrough, this.#caption ?? -1);
102
+ this.#pending = 0;
103
+ this.#pendingChars = 0;
104
+ this.#abort.abort();
105
+ this.#abort = new AbortController();
106
+ this.#queue = Promise.resolve();
107
+ for (const source of this.#sources) { try { source.stop(); } catch {} source.disconnect(); }
108
+ this.#sources.clear();
109
+ this.#nextTime = 0;
110
+ // Caption IDs are monotonic: every late delta/final for this epoch stays silent.
111
+ }
112
+
113
+ close() {
114
+ this.#closed = true;
115
+ this.interrupt();
116
+ void this.#context?.close();
117
+ this.#context = undefined;
118
+ }
119
+ }
@@ -16,6 +16,7 @@ import {
16
16
  registerDefinitionHost,
17
17
  releaseDefinitionHost,
18
18
  releaseHostSession,
19
+ releaseHostSessions,
19
20
  toWasmConfig,
20
21
  } from "../internal.mjs";
21
22
  import { createBrowserHost } from "./host.mjs";
@@ -45,8 +46,10 @@ export async function create(options = {}) {
45
46
  thinking,
46
47
  reasoningMode,
47
48
  fastMode,
49
+ rawApiEvents,
48
50
  instructions,
49
51
  additionalInstructions,
52
+ beforeCompaction,
50
53
  sessionId,
51
54
  workspace,
52
55
  resume,
@@ -75,6 +78,7 @@ export async function create(options = {}) {
75
78
  websocketPreconnect,
76
79
  apiBaseUrl,
77
80
  websocketWarmup,
81
+ stateless,
78
82
  WebSocketImpl,
79
83
  createWebSocket,
80
84
  createResponse,
@@ -116,11 +120,14 @@ export async function create(options = {}) {
116
120
  hostManagedProtocol,
117
121
  mpp,
118
122
  onEvent: events.emit,
123
+ beforeCompaction,
119
124
  filesystem,
120
125
  filesystemTools,
121
126
  tools: hostTools,
122
127
  toolProviders,
123
128
  subagentSessions,
129
+ subagentRouting: internalRuntime?.subagentRouting,
130
+ onSocketTiming: internalRuntime?.onSocketTiming,
124
131
  toolMode,
125
132
  mcp: mcp === false
126
133
  ? undefined
@@ -163,8 +170,11 @@ export async function create(options = {}) {
163
170
  : "wss://openai.mpp.tempo.xyz/v1/responses"),
164
171
  apiBaseUrl,
165
172
  websocketWarmup,
173
+ stateless,
166
174
  subagents: subagentConfig,
175
+ subagentRouting: internalRuntime?.subagentRouting !== undefined,
167
176
  hostDefinitionId,
177
+ beforeCompaction: beforeCompaction !== undefined,
168
178
  ...config,
169
179
  durabilityHostId: durabilityOwner?.id,
170
180
  }));
@@ -176,29 +186,19 @@ export async function create(options = {}) {
176
186
  );
177
187
  if (cloudflareReservation !== undefined) {
178
188
  activateCloudflareAgentSession(cloudflareReservation);
179
- const restoredSubagents = subagentSessions?.restore?.() ?? [];
180
- if (restoredSubagents.length > 0) {
181
- const restoredHostContextRefs = Object.fromEntries(
182
- restoredSubagents.map((descriptor) => {
183
- const hostContextRef = subagentSessions?.hostContextRef?.(descriptor.sessionId);
184
- return [descriptor.sessionId, hostContextRef ?? null];
185
- }),
186
- );
187
- await raw.restoreSubagents(
188
- JSON.stringify(restoredSubagents),
189
- JSON.stringify(restoredHostContextRefs),
190
- );
191
- }
192
189
  }
193
190
  return raw;
194
191
  } catch (error) {
195
192
  const cleanupErrors = [];
196
193
  if (raw !== undefined) {
197
- try { await raw.shutdown(); }
194
+ try {
195
+ await raw.shutdown();
196
+ }
198
197
  catch (cleanupError) { cleanupErrors.push(cleanupError); }
199
198
  try { raw.free(); }
200
199
  catch (cleanupError) { cleanupErrors.push(cleanupError); }
201
200
  }
201
+ releaseHostSessions(host);
202
202
  try { durabilityOwner?.abandon(); }
203
203
  catch (cleanupError) { cleanupErrors.push(cleanupError); }
204
204
  try { await host.dispose(); }
@@ -212,16 +212,20 @@ export async function create(options = {}) {
212
212
  throw error;
213
213
  }
214
214
  },
215
+ async shutdown(raw) {
216
+ await raw.shutdown();
217
+ },
215
218
  subscribe: events.subscribe,
216
219
  adopt(raw) {
217
220
  host.retain();
218
221
  try {
219
- durabilityOwner?.retain();
222
+ // Adopted child handles are ephemeral and do not own the root store.
223
+ if (raw.sessionId === stableSessionId) durabilityOwner?.retain();
220
224
  bindHostSession(host, raw.sessionId, cloudflareReservation);
221
225
  events.addSource(raw);
222
226
  } catch (error) {
223
227
  events.removeSource(raw);
224
- durabilityOwner?.release();
228
+ if (raw.sessionId === stableSessionId) durabilityOwner?.release();
225
229
  releaseHost(host);
226
230
  throw error;
227
231
  }
@@ -230,7 +234,7 @@ export async function create(options = {}) {
230
234
  events.removeSource(raw);
231
235
  host.releaseSession(raw.sessionId);
232
236
  releaseHostSession(host, raw.sessionId);
233
- durabilityOwner?.release();
237
+ if (raw.sessionId === stableSessionId) durabilityOwner?.release();
234
238
  releaseHost(host);
235
239
  },
236
240
  decorate: (agent) => agent.extend(agentActions()),
@@ -242,6 +246,7 @@ export async function create(options = {}) {
242
246
  thinking,
243
247
  reasoningMode,
244
248
  fastMode,
249
+ rawApiEvents,
245
250
  instructions,
246
251
  additionalInstructions,
247
252
  sessionId: stableSessionId,
@@ -26,6 +26,8 @@ export type WorkerTransport = ResponsesTransport & Readonly<{
26
26
 
27
27
  type SharedEndpointOptions = Readonly<{
28
28
  apiBaseUrl?: string | undefined;
29
+ /** Use HTTPS with full history replay, no stored responses, and no WebSocket warmup or preconnect. */
30
+ stateless?: boolean | undefined;
29
31
  websocketUrl?: string | undefined;
30
32
  /** Open the persistent socket as soon as Agent.create returns. Defaults to true for hostManaged. */
31
33
  websocketPreconnect?: boolean | undefined;
@@ -25,12 +25,12 @@ export function chatGpt(options) {
25
25
 
26
26
  export function hostManaged(options = {}) {
27
27
  const websocketUrl = options.websocketUrl
28
- ?? (options.createWebSocket ? undefined : defaultHostManagedWebSocketUrl());
28
+ ?? (options.stateless === true || options.createWebSocket ? undefined : defaultHostManagedWebSocketUrl());
29
29
  return createResponsesTransport({
30
30
  hostAuth: true,
31
31
  hostManagedProtocol: true,
32
32
  ...connection({ ...options, websocketUrl }),
33
- websocketPreconnect: options.websocketPreconnect ?? true,
33
+ websocketPreconnect: options.stateless === true ? false : options.websocketPreconnect ?? true,
34
34
  });
35
35
  }
36
36
 
@@ -53,6 +53,7 @@ function connection(options = {}) {
53
53
  return {
54
54
  WebSocketImpl: options.WebSocketImpl,
55
55
  apiBaseUrl: options.apiBaseUrl,
56
+ stateless: options.stateless,
56
57
  createWebSocket: options.createWebSocket,
57
58
  createResponse: options.createResponse,
58
59
  websocketUrl: options.websocketUrl,
@@ -51,6 +51,8 @@ export type Event =
51
51
  | Readonly<{ type: "stopped" }>;
52
52
  export type Settings = Readonly<{
53
53
  voice?: VoiceName | undefined;
54
+ outputProvider?: "openai" | "elevenlabs" | undefined;
55
+ elevenLabsVoiceId?: string | undefined;
54
56
  /** Additional speaking preferences; base assistant instructions are retained. */
55
57
  instructions?: string | undefined;
56
58
  pace?: "slow" | "natural" | "fast" | undefined;
@@ -59,6 +61,8 @@ export type Settings = Readonly<{
59
61
  acknowledgements?: boolean | undefined;
60
62
  }>;
61
63
  export type Options = Settings & Readonly<{
64
+ /** Authenticated synthesis transport returning mono signed 16-bit little-endian PCM at 24 kHz. */
65
+ synthesize?(text: string, signal: AbortSignal): Promise<Response>;
62
66
  callUrl?: string | URL | undefined;
63
67
  sidebandUrl?(callId: string, sessionId: string): string | URL | Promise<string | URL>;
64
68
  captureMicrophone?(): Promise<MediaStream>;
package/browser/Voice.mjs CHANGED
@@ -91,6 +91,11 @@ export function create(agent, options = {}) {
91
91
  if (destroyed) throw new Error("voice resource is destroyed");
92
92
  const selectedVoice = parameters.voice ?? options.voice ?? defaultVoice;
93
93
  if (!voices.includes(selectedVoice)) throw new TypeError(`unsupported ChatGPT voice: ${selectedVoice}`);
94
+ const settings = voiceSettings({ ...options, ...parameters, voice: selectedVoice });
95
+ validateOutputSettings(settings);
96
+ if (agent.type === "connect" && settings.outputProvider === "elevenlabs" && !options.synthesize) {
97
+ throw new TypeError("Connect ElevenLabs output requires an explicit authorized synthesis transport");
98
+ }
94
99
  if (session) return startPromise;
95
100
  if (stopPromise) await stopPromise.catch(() => {});
96
101
  if (destroyed) throw new Error("voice resource is destroyed");
@@ -125,7 +130,14 @@ export function create(agent, options = {}) {
125
130
  core,
126
131
  sessionId,
127
132
  voice: selectedVoice,
128
- settings: voiceSettings({ ...options, ...parameters, voice: selectedVoice }),
133
+ synthesize: options.synthesize ?? (transport?.synthesize
134
+ ? (text, signal) => transport.synthesize(text, signal, settings.elevenLabsVoiceId)
135
+ : (text, signal) => fetch("/api/voice/elevenlabs/speech", {
136
+ method: "POST", credentials: "same-origin", signal,
137
+ headers: { "Content-Type": "application/json" },
138
+ body: JSON.stringify({ voice_id: settings.elevenLabsVoiceId, text, output_format: "pcm_24000" }),
139
+ })),
140
+ settings,
129
141
  ...(transport?.call === undefined ? {} : { call: transport.call }),
130
142
  ...(transport?.sidebandUrl === undefined ? {} : { sidebandUrl: transport.sidebandUrl }),
131
143
  ...(options.callUrl === undefined ? {} : { callUrl: options.callUrl }),
@@ -186,10 +198,17 @@ export function create(agent, options = {}) {
186
198
  // Media readiness updates the UI independently; start still waits for task
187
199
  // admission, and a later rejection closes media through the same error path.
188
200
  startPromise = next.start().then(publishReady).catch(async (cause) => {
189
- if (session === next) session = undefined;
190
- if (activeResources.get(agent) === resource) activeResources.delete(agent);
191
- cleanupWatcher();
192
- await next.close().catch(() => next.abort());
201
+ if (session === next) {
202
+ session = undefined;
203
+ if (activeResources.get(agent) === resource) activeResources.delete(agent);
204
+ cleanupWatcher();
205
+ }
206
+ // Keep replacement starts behind failed startup cleanup, just like stop().
207
+ // A stale attempt must never detach a replacement session's event watcher.
208
+ const closing = next.close().catch(() => next.abort());
209
+ if (generation === current) stopPromise = closing;
210
+ await closing;
211
+ if (stopPromise === closing) stopPromise = undefined;
193
212
  if (destroyed || generation !== current) return;
194
213
  if (attempt === 0 && cause?.code === "peer_connection_timeout") return start(parameters, 1);
195
214
  const error = cause instanceof Error ? cause : new Error(String(cause));
@@ -299,6 +318,9 @@ function validateOptions(options) {
299
318
  if (!options || typeof options !== "object" || Array.isArray(options)) {
300
319
  throw new TypeError("Voice.create options must be an object");
301
320
  }
321
+ if (options.synthesize !== undefined && typeof options.synthesize !== "function") {
322
+ throw new TypeError("voice synthesize must be a function");
323
+ }
302
324
  if (options.captureMicrophone !== undefined && typeof options.captureMicrophone !== "function") {
303
325
  throw new TypeError("voice captureMicrophone must be a function");
304
326
  }
@@ -317,6 +339,16 @@ function browserLocationOrigin() {
317
339
  }
318
340
 
319
341
  function voiceSettings(options) {
320
- return Object.fromEntries(["voice", "instructions", "pace", "updates", "handoffMode", "acknowledgements"]
342
+ return Object.fromEntries(["voice", "instructions", "pace", "updates", "handoffMode", "acknowledgements", "outputProvider", "elevenLabsVoiceId"]
321
343
  .filter((key) => options[key] !== undefined).map((key) => [key, options[key]]));
322
344
  }
345
+
346
+ function validateOutputSettings(settings) {
347
+ if (settings.outputProvider !== undefined && !["openai", "elevenlabs"].includes(settings.outputProvider)) {
348
+ throw new TypeError("unsupported voice output provider");
349
+ }
350
+ if (settings.outputProvider === "elevenlabs"
351
+ && (typeof settings.elevenLabsVoiceId !== "string" || !/^[A-Za-z0-9_-]{1,128}$/.test(settings.elevenLabsVoiceId))) {
352
+ throw new TypeError("ElevenLabs output requires a valid voice ID");
353
+ }
354
+ }
@@ -1,3 +1,5 @@
1
+ import { ElevenLabsPlayback } from "./ElevenLabsPlayback.mjs";
2
+
1
3
  export const MICROPHONE_CAPTURE_TIMEOUT_MS = 15_000;
2
4
  export const ICE_GATHERING_TIMEOUT_MS = 15_000;
3
5
  export const REALTIME_CALL_TIMEOUT_MS = 15_000;
@@ -85,8 +87,10 @@ export class BrowserVoiceSession {
85
87
  #microphone;
86
88
  #speaker;
87
89
  #playbackEnabled = false;
90
+ #elevenLabs;
88
91
  #muted = false;
89
92
  #inputGeneration = 0;
93
+ #playbackRevision = 0;
90
94
  #meterTimer;
91
95
  #backendReady;
92
96
  #resolveBackendReady;
@@ -102,10 +106,17 @@ export class BrowserVoiceSession {
102
106
 
103
107
  constructor(options) {
104
108
  this.#options = options;
109
+ if (options.settings?.outputProvider === "elevenlabs") {
110
+ if (typeof options.synthesize !== "function") throw new TypeError("ElevenLabs output requires an authenticated synthesis transport");
111
+ this.#elevenLabs = new ElevenLabsPlayback(options.synthesize, (error) => {
112
+ if (!this.#closed) options.onTerminated(error.message);
113
+ });
114
+ }
105
115
  this.#backendReady = new Promise((resolve) => { this.#resolveBackendReady = resolve; });
106
116
  }
107
117
 
108
118
  start() {
119
+ if (!this.#starting) void this.#elevenLabs?.prime()?.catch((error) => this.#options.onTerminated(error.message));
109
120
  this.#starting ??= this.#start();
110
121
  return this.#starting;
111
122
  }
@@ -140,7 +151,8 @@ export class BrowserVoiceSession {
140
151
  // Only delegated work waits for admission on the direct control path.
141
152
  const coreStartup = coreReady.then(async (core) => {
142
153
  if (this.#directControl) await this.#options.beforeAgentTurn?.();
143
- await core?.start();
154
+ const effects = await core?.start();
155
+ if (!this.#closed && !this.#closing.signal.aborted) await this.#apply(effects);
144
156
  return core;
145
157
  });
146
158
  this.#admission = coreStartup;
@@ -256,7 +268,7 @@ export class BrowserVoiceSession {
256
268
  const channel = this.#channel;
257
269
  channel.addEventListener("message", (event) => {
258
270
  if (this.#directControl && !this.#closed && this.#channel === channel) {
259
- this.#receiveControl(event.data, () => this.#channel === channel);
271
+ this.#receiveControl(event.data, () => this.#channel === channel && channel.readyState === "open");
260
272
  }
261
273
  });
262
274
  channel.addEventListener("close", () => {
@@ -271,7 +283,7 @@ export class BrowserVoiceSession {
271
283
  }
272
284
  const stream = event.streams[0] ?? new MediaStream([event.track]);
273
285
  this.#speaker ??= new SpeakerPlayback(new Audio(), this.#options.onStatus);
274
- this.#speaker.setEnabled(this.#playbackEnabled);
286
+ this.#speaker.setEnabled(this.#playbackEnabled && !this.#elevenLabs);
275
287
  this.#speaker.attach(stream);
276
288
  });
277
289
  peer.addEventListener("connectionstatechange", () => {
@@ -300,7 +312,9 @@ export class BrowserVoiceSession {
300
312
  }
301
313
 
302
314
  noteTypedInput() {
315
+ this.#playbackRevision++;
303
316
  this.#playbackEnabled = false;
317
+ this.#elevenLabs?.interrupt();
304
318
  this.#speaker?.setEnabled(false);
305
319
  return this.#applyLive(async () => {
306
320
  const core = await this.#options.core;
@@ -417,12 +431,19 @@ export class BrowserVoiceSession {
417
431
  for (const text of effects.undelivered_answers ?? []) this.#options.onUndeliveredAnswer?.(text);
418
432
  if (effects.input_generation !== undefined) {
419
433
  if (effects.input_generation < this.#inputGeneration) return;
434
+ if (effects.input_generation > this.#inputGeneration) {
435
+ this.#playbackRevision++;
436
+ this.#elevenLabs?.interrupt();
437
+ }
420
438
  this.#inputGeneration = effects.input_generation;
421
439
  }
422
440
  if (effects.playback_enabled === false) {
441
+ this.#playbackRevision++;
423
442
  this.#playbackEnabled = false;
443
+ this.#elevenLabs?.interrupt();
424
444
  this.#speaker?.setEnabled(false);
425
445
  }
446
+ const playbackRevision = this.#playbackRevision;
426
447
  let sent = 0;
427
448
  for (const frame of effects.frames ?? []) {
428
449
  if (this.#directControl && this.#channel?.readyState === "open") {
@@ -434,11 +455,15 @@ export class BrowserVoiceSession {
434
455
  }
435
456
  }
436
457
  if (effects.acknowledge_frames && sent > 0) await this.#core?.framesSent(sent);
437
- if (!this.#closed && effects.playback_enabled === true && sent === (effects.frames?.length ?? 0)) {
458
+ // A live interruption can overtake asynchronous frame acknowledgement.
459
+ // Keep its speaker gate authoritative while retaining accepted captions.
460
+ const currentPlayback = playbackRevision === this.#playbackRevision;
461
+ if (currentPlayback && !this.#closed && effects.playback_enabled === true && sent === (effects.frames?.length ?? 0)) {
438
462
  this.#playbackEnabled = true;
439
- this.#speaker?.setEnabled(true);
463
+ this.#speaker?.setEnabled(!this.#elevenLabs);
440
464
  }
441
465
  for (const entry of effects.transcripts ?? []) {
466
+ if (currentPlayback && this.#playbackEnabled && !this.#closed) this.#elevenLabs?.transcript(entry);
442
467
  this.#options.onTranscript(entry.speaker, entry.text, entry);
443
468
  }
444
469
  if (effects.status) this.#status(effects.status);
@@ -475,7 +500,7 @@ export class BrowserVoiceSession {
475
500
  let opened = false;
476
501
  sideband.addEventListener("message", (event) => {
477
502
  if (!this.#closed && generation === this.#sidebandGeneration) {
478
- this.#receiveControl(event.data, () => generation === this.#sidebandGeneration);
503
+ this.#receiveControl(event.data, () => generation === this.#sidebandGeneration && sideband.readyState === WebSocket.OPEN);
479
504
  }
480
505
  });
481
506
  sideband.addEventListener("close", () => {
@@ -501,7 +526,11 @@ export class BrowserVoiceSession {
501
526
  this.#applyLive(async () => {
502
527
  if (!this.#directControl) await this.#admission;
503
528
  if (this.#closed || !isCurrent()) return;
504
- if (await this.#core.requiresAgentAdmission(payload)) {
529
+ const requiresAdmission = await this.#core.requiresAgentAdmission(payload);
530
+ // Classification can yield while this transport closes or is replaced.
531
+ // Only requests accepted on the current connection may enter the queue.
532
+ if (this.#closed || !isCurrent()) return;
533
+ if (requiresAdmission) {
505
534
  // Retain accepted requests on close, but keep captions and interruptions
506
535
  // independent of the durable route and its ordered task queue.
507
536
  void this.#enqueue(async () => {
@@ -549,6 +578,7 @@ export class BrowserVoiceSession {
549
578
  }
550
579
 
551
580
  #stopBrowserMedia() {
581
+ this.#elevenLabs?.close();
552
582
  if (this.#meterTimer !== undefined) window.clearTimeout(this.#meterTimer);
553
583
  this.#meterTimer = undefined;
554
584
  this.#call?.abort();
@@ -75,12 +75,28 @@ export function createBrowserHost(options?: {
75
75
  mcp?: McpServers;
76
76
  codeEvaluator?: CodeEvaluator;
77
77
  toolMode?: "code" | "direct";
78
- /** @internal Durable host lifecycle for Rust-owned subagent descriptors. */
78
+ /** @internal Live host lifecycle for ephemeral Rust-owned subagents. */
79
+ subagentRouting?: Pick<import('../runtime/subagent-routing.mjs').SubagentRouting, 'resolve' | 'bind'>;
79
80
  subagentSessions?: {
80
- restore(): readonly SubagentToolContext[];
81
- bind(sessionId: string, descriptor: SubagentToolContext): void;
82
- release(sessionId: string): void;
81
+ bindingDescriptor?(sessionId: string, descriptor: SubagentToolContext, hostContextRef?: string): SubagentToolContext;
82
+ bind(sessionId: string, descriptor: SubagentToolContext, hostContextRef?: string): void;
83
+ release(sessionId: string, hostContextRef?: string): void;
83
84
  };
85
+ /** @internal Content-free summary, once when ownership closes; no public transport option. */
86
+ onSocketTiming?: (timing: {
87
+ message_count: number;
88
+ delivered_message_count: number;
89
+ buffered_message_count: number;
90
+ discarded_message_count: number;
91
+ queue_residence_total_ms: number;
92
+ queue_residence_max_ms: number;
93
+ provider_timings: Array<{
94
+ response_id?: string;
95
+ pre_inference_ms?: number;
96
+ engine_queue_max_ms?: number;
97
+ engine_service_ttft_total_ms?: number;
98
+ }>;
99
+ }) => void;
84
100
  maxQueuedMessages?: number;
85
101
  maxQueuedBytes?: number;
86
102
  maxBufferedSendBytes?: number;