nanocodex 0.6.4 → 0.6.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/README.md +23 -11
  2. package/browser/Agent.d.mts +1 -1
  3. package/browser/ElevenLabsPlayback.mjs +119 -0
  4. package/browser/InlineAgent.mjs +22 -17
  5. package/browser/Transport.d.mts +2 -0
  6. package/browser/Transport.mjs +3 -2
  7. package/browser/Voice.d.mts +4 -0
  8. package/browser/Voice.mjs +27 -2
  9. package/browser/VoiceSession.mjs +19 -3
  10. package/browser/host.d.mts +20 -4
  11. package/browser/host.mjs +156 -18
  12. package/cloudflare/Agent.d.mts +10 -9
  13. package/cloudflare/Agent.mjs +259 -95
  14. package/cloudflare/agent-settings.d.mts +15 -0
  15. package/cloudflare/agent-settings.mjs +135 -0
  16. package/cloudflare/chat-reasoning.mjs +20 -0
  17. package/cloudflare/durable-placement.d.mts +12 -0
  18. package/cloudflare/durable-placement.mjs +82 -0
  19. package/cloudflare/egress.d.mts +31 -0
  20. package/cloudflare/gateway-binding-responses.mjs +84 -0
  21. package/cloudflare/gateway-responses.d.mts +48 -0
  22. package/cloudflare/gateway-responses.mjs +191 -0
  23. package/cloudflare/index.d.mts +6 -0
  24. package/cloudflare/index.mjs +3 -0
  25. package/cloudflare/managed-auth.d.mts +15 -0
  26. package/cloudflare/managed-auth.mjs +113 -0
  27. package/cloudflare/managed-live.d.mts +7 -0
  28. package/cloudflare/managed-live.mjs +56 -0
  29. package/cloudflare/prepared-connection.mjs +49 -0
  30. package/cloudflare/provider-stream.mjs +353 -0
  31. package/cloudflare/rpc.d.mts +9 -0
  32. package/cloudflare/rpc.mjs +14 -0
  33. package/cloudflare/tool-alias-diagnostic.mjs +21 -0
  34. package/cloudflare/workers-ai-responses.d.mts +28 -0
  35. package/cloudflare/workers-ai-responses.mjs +398 -0
  36. package/host/Agent.d.mts +1 -1
  37. package/index.d.mts +2 -0
  38. package/internal.mjs +48 -14
  39. package/managed/Agent.d.mts +21 -68
  40. package/managed/Agent.mjs +61 -191
  41. package/managed/README.md +9 -38
  42. package/managed/Voice.mjs +2 -1
  43. package/managed/index.d.mts +0 -3
  44. package/node/Agent.d.mts +1 -1
  45. package/node/Agent.mjs +9 -3
  46. package/node/host.mjs +5 -0
  47. package/package.json +30 -2
  48. package/pkg-node/nanocodex.d.ts +8 -16
  49. package/pkg-node/nanocodex.js +96 -93
  50. package/pkg-node/nanocodex_bg.wasm.d.ts +4 -5
  51. package/pkg-web/.nanocodex-bindgen-stamp +7 -1
  52. package/pkg-web/nanocodex-build.json +1 -1
  53. package/pkg-web/nanocodex.d.ts +12 -21
  54. package/pkg-web/nanocodex.js +96 -92
  55. package/pkg-web/nanocodex_bg.js +96 -92
  56. package/pkg-web/nanocodex_bg.wasm +0 -0
  57. package/pkg-web/nanocodex_bg.wasm.d.ts +4 -5
  58. package/pkg-web/nanocodex_worker.js +1 -1
  59. package/runtime/before-compaction.mjs +49 -0
  60. package/runtime/code-evaluator.worker.mjs +2 -9
  61. package/runtime/managed-transport.mjs +10 -2
  62. package/runtime/mcp-runtime.mjs +15 -1
  63. package/runtime/quickjs-evaluator.mjs +11 -5
  64. package/runtime/response-controls.mjs +56 -4
  65. package/runtime/steer-receipt.mjs +13 -0
  66. package/runtime/subagent-routing.d.mts +38 -0
  67. package/runtime/subagent-routing.mjs +80 -0
  68. package/runtime/subagents.d.mts +1 -1
  69. package/runtime/tempo-provider.d.mts +1 -1
  70. package/runtime/tempo-provider.mjs +1 -1
  71. package/tools/environment.d.mts +8 -2
  72. package/tools/environment.mjs +16 -2
  73. package/tools/index.d.mts +2 -0
  74. package/types.d.mts +28 -1
package/README.md CHANGED
@@ -10,7 +10,7 @@ import { Actions, Agent, Transport } from "nanocodex/node";
10
10
 
11
11
  const agent = await Agent.create({
12
12
  transport: Transport.openAi({ apiKey: process.env.OPENAI_API_KEY }),
13
- model: "gpt-5.6-luna",
13
+ model: "gpt-6-luna",
14
14
  instructions: "You are a Rust coding agent. Preserve unrelated work and run relevant tests.",
15
15
  reasoningMode: "pro",
16
16
  thinking: "high",
@@ -236,8 +236,13 @@ requesting speech. Context and speech are split into provider-sized messages.
236
236
  These commands retain frames until sent and preserve them
237
237
  across reconnects when using the sideband transport. They are also methods on
238
238
  the resource and on
239
- `useVoice` from `nanocodex-react`. These settings use ChatGPT subscription voice;
240
- custom voices and Platform audio configuration are not accepted.
239
+ `useVoice` from `nanocodex-react`. Choose `outputProvider: "elevenlabs"` and `elevenLabsVoiceId` to synthesize
240
+ spoken output with an ElevenLabs account voice, including an instant clone.
241
+ The account voice settings panel connects the API key, lists voices, and uploads
242
+ cloning samples after explicit consent. Keys are encrypted on the server and
243
+ never included in saved voice settings. ChatGPT still owns live input and agent
244
+ handoffs; `voice` continues to select its built-in voice. The default output
245
+ provider is `openai`. Platform audio configuration is not accepted.
241
246
 
242
247
  `Voice.create(...)` remains the equivalent namespaced resource constructor, and
243
248
  `Voice.voices` is the exact ChatGPT V3 voice catalog. The constructor accepts a
@@ -302,10 +307,12 @@ Cloudflare Agents default to direct tool mode because Workers prohibit dynamic
302
307
  `eval`/`new Function`. Caller-defined tools therefore work without a code
303
308
  evaluator. Select `toolMode: "code"` only when also supplying an evaluator that
304
309
  is explicitly compatible with the deployed Worker runtime. Runtime-owned
305
- Subagents are installed by default, including on a durable root. Clean children
306
- persist independent execution state under their own agent session IDs. The
307
- Rust task-tree registry remains in memory and is closed with the live root, so
308
- tree-local IDs and topology are not reconstructed from those agent states. Use
310
+ Subagents are installed by default, including on a durable root. All children are
311
+ ephemeral: their identities, topology, conversations, results, and routing exist
312
+ only for the lifetime of the root runtime. Root shutdown or restart discards the
313
+ entire child tree; only the root's own durable history resumes. Existing child
314
+ checkpoints from older versions are discarded. Within a live runtime, completed
315
+ children remain available for follow-up messages until closed. Use
309
316
  `Subagents.create({ maxConcurrency })` in `tools` to set an explicit finite
310
317
  concurrency limit. Active subagent turns are unlimited by default.
311
318
 
@@ -349,6 +356,11 @@ native and WASM consumers run the same Rust implementation and receive the
349
356
  same seven tools: `spawn_agent`, `submit_result`, `send_agent_message`,
350
357
  `list_agents`, `wait_agent`, `interrupt_agent`, and `close_agent`.
351
358
 
359
+ Children call `submit_result({output})`; the runtime supplies the trusted
360
+ instruction revision. The response is `{accepted: true, status: "accepted"}` or
361
+ `{accepted: false, status: "superseded"}`. Superseded submissions are normal
362
+ continuations: incorporate the updated instructions and submit again.
363
+
352
364
  Inside a caller-owned Worker or server isolate, host capabilities stay as
353
365
  ordinary functions without crossing another compatibility protocol:
354
366
 
@@ -686,7 +698,7 @@ const mpp = tempoProvider.session;
686
698
 
687
699
  const agent = await Agent.create({
688
700
  transport: Transport.mpp({ session: tempoProvider }),
689
- thinking: "none",
701
+ thinking: "low",
690
702
  fastMode: true,
691
703
  tools,
692
704
  });
@@ -736,8 +748,8 @@ of its wallet adapter, and constructs both payment paths from that provider's
736
748
  adapter-neutral `getMppxParameters()` contract. The lower-level
737
749
  `createTempoProvider({ session, payment })` remains available when the
738
750
  application constructs MPPx itself. Both explicitly select Tempo provider mode.
739
- In that mode Nanocodex automatically adds its built-in Mercator MCP and wraps it
740
- with the same wallet and payment policy. The provider also exposes an MPP-aware
751
+ In that mode Nanocodex automatically adds its built-in Mercator MCP at
752
+ `https://mercator.sh/mcp` and wraps it with the same wallet and payment policy. The provider also exposes an MPP-aware
741
753
  `fetch`; Mercator's paid REST handoffs use that same method rather than a second
742
754
  wallet or payment configuration. Its MCP transport remains wrapped at the MCP
743
755
  protocol layer, so browser requests do not need an `Accept-Payment` CORS header.
@@ -1155,7 +1167,7 @@ package manager or build step:
1155
1167
 
1156
1168
  ```html
1157
1169
  <script type="module">
1158
- import { Agent, Transport } from "https://cdn.jsdelivr.net/npm/nanocodex@0.6.4/host/index.mjs";
1170
+ import { Agent, Transport } from "https://cdn.jsdelivr.net/npm/nanocodex@0.6.5/host/index.mjs";
1159
1171
  const agent = await Agent.create({
1160
1172
  transport: Transport.hostManaged({
1161
1173
  websocketUrl: "/api/responses",
@@ -33,7 +33,7 @@ export declare namespace create {
33
33
  transport: ManagedTransport;
34
34
  tools?: Tools | undefined;
35
35
  }>;
36
- type Options = AgentOptions & WorkerToolExposureOptions & {
36
+ type Options = Omit<AgentOptions, "beforeCompaction"> & WorkerToolExposureOptions & {
37
37
  /** Precompiled browser module; WebAssembly modules are structured-clone-safe. */
38
38
  module?: WebAssembly.Module | undefined;
39
39
  /** Fixed browser workspace facts, including its AGENTS.md snapshot. */
@@ -0,0 +1,119 @@
1
+ /** Plays account-proxied PCM without exposing provider credentials to the browser. */
2
+ export class ElevenLabsPlayback {
3
+ #synthesize;
4
+ #onError;
5
+ #context;
6
+ #abort = new AbortController();
7
+ #queue = Promise.resolve();
8
+ #sources = new Set();
9
+ #nextTime = 0;
10
+ #caption;
11
+ #offset = 0;
12
+ #suppressedThrough = -1;
13
+ #pending = 0;
14
+ #pendingChars = 0;
15
+ #closed = false;
16
+
17
+ constructor(synthesize, onError) {
18
+ this.#synthesize = synthesize;
19
+ this.#onError = onError;
20
+ }
21
+
22
+ prime() {
23
+ if (this.#closed) return;
24
+ this.#context ??= new AudioContext({ sampleRate: 24000 });
25
+ return this.#context.resume();
26
+ }
27
+
28
+ transcript(entry) {
29
+ if (this.#closed || entry.speaker !== "assistant" || entry.id <= this.#suppressedThrough) return;
30
+ if (this.#caption !== undefined && entry.id < this.#caption) return;
31
+ if (entry.id !== this.#caption) { this.#caption = entry.id; this.#offset = 0; }
32
+ const remaining = entry.text.slice(this.#offset);
33
+ // Buffer partial words; emit complete sentences before the turn finishes.
34
+ const boundary = entry.is_partial ? [...remaining.matchAll(/[.!?\n](?:\s|$)/g)].at(-1) : undefined;
35
+ const length = entry.is_partial ? (boundary ? boundary.index + boundary[0].length : 0) : remaining.length;
36
+ const text = remaining.slice(0, length).trim();
37
+ this.#offset += length;
38
+ if (!text) return;
39
+ if (this.#pending >= 32 || this.#pendingChars + text.length > 32768) {
40
+ this.interrupt();
41
+ this.#onError(new Error("ElevenLabs playback queue exceeded its limit"));
42
+ return;
43
+ }
44
+ this.#pending++;
45
+ this.#pendingChars += text.length;
46
+ const signal = this.#abort.signal;
47
+ this.#queue = this.#queue.then(async () => {
48
+ if (signal.aborted) return;
49
+ this.#context ??= new AudioContext({ sampleRate: 24000 });
50
+ await this.#context.resume();
51
+ if (signal.aborted) return;
52
+ const response = await this.#synthesize(text, signal);
53
+ if (!response.ok) throw new Error(`ElevenLabs synthesis failed (${response.status})`);
54
+ const reader = response.body?.getReader();
55
+ if (!reader) throw new Error("ElevenLabs returned no audio stream");
56
+ const cancel = () => { void reader.cancel().catch(() => {}); };
57
+ signal.addEventListener("abort", cancel, { once: true });
58
+ let carry;
59
+ try {
60
+ while (!signal.aborted) {
61
+ const { value, done } = await reader.read();
62
+ if (done) break;
63
+ if (signal.aborted) break;
64
+ if (value.byteLength > 1024 * 1024) throw new Error("ElevenLabs audio chunk exceeded its limit");
65
+ const bytes = carry === undefined ? value : Uint8Array.from([carry, ...value]);
66
+ const size = bytes.length - bytes.length % 2;
67
+ carry = size < bytes.length ? bytes[size] : undefined;
68
+ if (!size) continue;
69
+ // Keep decoded/scheduled audio bounded even when synthesis outruns playback.
70
+ for (let offset = 0; offset < size && !signal.aborted; offset += 9600) {
71
+ while (!signal.aborted && (this.#nextTime - this.#context.currentTime > 2 || this.#sources.size >= 128)) {
72
+ await new Promise((resolve) => {
73
+ const done = () => { clearTimeout(timer); signal.removeEventListener("abort", done); resolve(); };
74
+ const timer = setTimeout(done, 25);
75
+ signal.addEventListener("abort", done, { once: true });
76
+ });
77
+ }
78
+ if (signal.aborted) break;
79
+ const length = Math.min(9600, size - offset);
80
+ const buffer = this.#context.createBuffer(1, length / 2, 24000);
81
+ const output = buffer.getChannelData(0);
82
+ const view = new DataView(bytes.buffer, bytes.byteOffset + offset, length);
83
+ for (let i = 0; i < output.length; i++) output[i] = view.getInt16(i * 2, true) / 32768;
84
+ const source = this.#context.createBufferSource();
85
+ source.buffer = buffer;
86
+ source.connect(this.#context.destination);
87
+ this.#sources.add(source);
88
+ source.onended = () => { this.#sources.delete(source); source.disconnect(); };
89
+ this.#nextTime = Math.max(this.#nextTime, this.#context.currentTime + 0.02);
90
+ source.start(this.#nextTime);
91
+ this.#nextTime += buffer.duration;
92
+ }
93
+ }
94
+ } finally { signal.removeEventListener("abort", cancel); await reader.cancel().catch(() => {}); reader.releaseLock(); }
95
+ }).catch((error) => { if (!signal.aborted) this.#onError(error); }).finally(() => {
96
+ if (!signal.aborted) { this.#pending--; this.#pendingChars -= text.length; }
97
+ });
98
+ }
99
+
100
+ interrupt() {
101
+ this.#suppressedThrough = Math.max(this.#suppressedThrough, this.#caption ?? -1);
102
+ this.#pending = 0;
103
+ this.#pendingChars = 0;
104
+ this.#abort.abort();
105
+ this.#abort = new AbortController();
106
+ this.#queue = Promise.resolve();
107
+ for (const source of this.#sources) { try { source.stop(); } catch {} source.disconnect(); }
108
+ this.#sources.clear();
109
+ this.#nextTime = 0;
110
+ // Caption IDs are monotonic: every late delta/final for this epoch stays silent.
111
+ }
112
+
113
+ close() {
114
+ this.#closed = true;
115
+ this.interrupt();
116
+ void this.#context?.close();
117
+ this.#context = undefined;
118
+ }
119
+ }
@@ -16,6 +16,7 @@ import {
16
16
  registerDefinitionHost,
17
17
  releaseDefinitionHost,
18
18
  releaseHostSession,
19
+ releaseHostSessions,
19
20
  toWasmConfig,
20
21
  } from "../internal.mjs";
21
22
  import { createBrowserHost } from "./host.mjs";
@@ -45,8 +46,10 @@ export async function create(options = {}) {
45
46
  thinking,
46
47
  reasoningMode,
47
48
  fastMode,
49
+ rawApiEvents,
48
50
  instructions,
49
51
  additionalInstructions,
52
+ beforeCompaction,
50
53
  sessionId,
51
54
  workspace,
52
55
  resume,
@@ -75,6 +78,7 @@ export async function create(options = {}) {
75
78
  websocketPreconnect,
76
79
  apiBaseUrl,
77
80
  websocketWarmup,
81
+ stateless,
78
82
  WebSocketImpl,
79
83
  createWebSocket,
80
84
  createResponse,
@@ -116,11 +120,14 @@ export async function create(options = {}) {
116
120
  hostManagedProtocol,
117
121
  mpp,
118
122
  onEvent: events.emit,
123
+ beforeCompaction,
119
124
  filesystem,
120
125
  filesystemTools,
121
126
  tools: hostTools,
122
127
  toolProviders,
123
128
  subagentSessions,
129
+ subagentRouting: internalRuntime?.subagentRouting,
130
+ onSocketTiming: internalRuntime?.onSocketTiming,
124
131
  toolMode,
125
132
  mcp: mcp === false
126
133
  ? undefined
@@ -163,8 +170,11 @@ export async function create(options = {}) {
163
170
  : "wss://openai.mpp.tempo.xyz/v1/responses"),
164
171
  apiBaseUrl,
165
172
  websocketWarmup,
173
+ stateless,
166
174
  subagents: subagentConfig,
175
+ subagentRouting: internalRuntime?.subagentRouting !== undefined,
167
176
  hostDefinitionId,
177
+ beforeCompaction: beforeCompaction !== undefined,
168
178
  ...config,
169
179
  durabilityHostId: durabilityOwner?.id,
170
180
  }));
@@ -176,29 +186,19 @@ export async function create(options = {}) {
176
186
  );
177
187
  if (cloudflareReservation !== undefined) {
178
188
  activateCloudflareAgentSession(cloudflareReservation);
179
- const restoredSubagents = subagentSessions?.restore?.() ?? [];
180
- if (restoredSubagents.length > 0) {
181
- const restoredHostContextRefs = Object.fromEntries(
182
- restoredSubagents.map((descriptor) => {
183
- const hostContextRef = subagentSessions?.hostContextRef?.(descriptor.sessionId);
184
- return [descriptor.sessionId, hostContextRef ?? null];
185
- }),
186
- );
187
- await raw.restoreSubagents(
188
- JSON.stringify(restoredSubagents),
189
- JSON.stringify(restoredHostContextRefs),
190
- );
191
- }
192
189
  }
193
190
  return raw;
194
191
  } catch (error) {
195
192
  const cleanupErrors = [];
196
193
  if (raw !== undefined) {
197
- try { await raw.shutdown(); }
194
+ try {
195
+ await raw.shutdown();
196
+ }
198
197
  catch (cleanupError) { cleanupErrors.push(cleanupError); }
199
198
  try { raw.free(); }
200
199
  catch (cleanupError) { cleanupErrors.push(cleanupError); }
201
200
  }
201
+ releaseHostSessions(host);
202
202
  try { durabilityOwner?.abandon(); }
203
203
  catch (cleanupError) { cleanupErrors.push(cleanupError); }
204
204
  try { await host.dispose(); }
@@ -212,16 +212,20 @@ export async function create(options = {}) {
212
212
  throw error;
213
213
  }
214
214
  },
215
+ async shutdown(raw) {
216
+ await raw.shutdown();
217
+ },
215
218
  subscribe: events.subscribe,
216
219
  adopt(raw) {
217
220
  host.retain();
218
221
  try {
219
- durabilityOwner?.retain();
222
+ // Adopted child handles are ephemeral and do not own the root store.
223
+ if (raw.sessionId === stableSessionId) durabilityOwner?.retain();
220
224
  bindHostSession(host, raw.sessionId, cloudflareReservation);
221
225
  events.addSource(raw);
222
226
  } catch (error) {
223
227
  events.removeSource(raw);
224
- durabilityOwner?.release();
228
+ if (raw.sessionId === stableSessionId) durabilityOwner?.release();
225
229
  releaseHost(host);
226
230
  throw error;
227
231
  }
@@ -230,7 +234,7 @@ export async function create(options = {}) {
230
234
  events.removeSource(raw);
231
235
  host.releaseSession(raw.sessionId);
232
236
  releaseHostSession(host, raw.sessionId);
233
- durabilityOwner?.release();
237
+ if (raw.sessionId === stableSessionId) durabilityOwner?.release();
234
238
  releaseHost(host);
235
239
  },
236
240
  decorate: (agent) => agent.extend(agentActions()),
@@ -242,6 +246,7 @@ export async function create(options = {}) {
242
246
  thinking,
243
247
  reasoningMode,
244
248
  fastMode,
249
+ rawApiEvents,
245
250
  instructions,
246
251
  additionalInstructions,
247
252
  sessionId: stableSessionId,
@@ -26,6 +26,8 @@ export type WorkerTransport = ResponsesTransport & Readonly<{
26
26
 
27
27
  type SharedEndpointOptions = Readonly<{
28
28
  apiBaseUrl?: string | undefined;
29
+ /** Use HTTPS with full history replay, no stored responses, and no WebSocket warmup or preconnect. */
30
+ stateless?: boolean | undefined;
29
31
  websocketUrl?: string | undefined;
30
32
  /** Open the persistent socket as soon as Agent.create returns. Defaults to true for hostManaged. */
31
33
  websocketPreconnect?: boolean | undefined;
@@ -25,12 +25,12 @@ export function chatGpt(options) {
25
25
 
26
26
  export function hostManaged(options = {}) {
27
27
  const websocketUrl = options.websocketUrl
28
- ?? (options.createWebSocket ? undefined : defaultHostManagedWebSocketUrl());
28
+ ?? (options.stateless === true || options.createWebSocket ? undefined : defaultHostManagedWebSocketUrl());
29
29
  return createResponsesTransport({
30
30
  hostAuth: true,
31
31
  hostManagedProtocol: true,
32
32
  ...connection({ ...options, websocketUrl }),
33
- websocketPreconnect: options.websocketPreconnect ?? true,
33
+ websocketPreconnect: options.stateless === true ? false : options.websocketPreconnect ?? true,
34
34
  });
35
35
  }
36
36
 
@@ -53,6 +53,7 @@ function connection(options = {}) {
53
53
  return {
54
54
  WebSocketImpl: options.WebSocketImpl,
55
55
  apiBaseUrl: options.apiBaseUrl,
56
+ stateless: options.stateless,
56
57
  createWebSocket: options.createWebSocket,
57
58
  createResponse: options.createResponse,
58
59
  websocketUrl: options.websocketUrl,
@@ -51,6 +51,8 @@ export type Event =
51
51
  | Readonly<{ type: "stopped" }>;
52
52
  export type Settings = Readonly<{
53
53
  voice?: VoiceName | undefined;
54
+ outputProvider?: "openai" | "elevenlabs" | undefined;
55
+ elevenLabsVoiceId?: string | undefined;
54
56
  /** Additional speaking preferences; base assistant instructions are retained. */
55
57
  instructions?: string | undefined;
56
58
  pace?: "slow" | "natural" | "fast" | undefined;
@@ -59,6 +61,8 @@ export type Settings = Readonly<{
59
61
  acknowledgements?: boolean | undefined;
60
62
  }>;
61
63
  export type Options = Settings & Readonly<{
64
+ /** Authenticated synthesis transport returning mono signed 16-bit little-endian PCM at 24 kHz. */
65
+ synthesize?(text: string, signal: AbortSignal): Promise<Response>;
62
66
  callUrl?: string | URL | undefined;
63
67
  sidebandUrl?(callId: string, sessionId: string): string | URL | Promise<string | URL>;
64
68
  captureMicrophone?(): Promise<MediaStream>;
package/browser/Voice.mjs CHANGED
@@ -91,6 +91,11 @@ export function create(agent, options = {}) {
91
91
  if (destroyed) throw new Error("voice resource is destroyed");
92
92
  const selectedVoice = parameters.voice ?? options.voice ?? defaultVoice;
93
93
  if (!voices.includes(selectedVoice)) throw new TypeError(`unsupported ChatGPT voice: ${selectedVoice}`);
94
+ const settings = voiceSettings({ ...options, ...parameters, voice: selectedVoice });
95
+ validateOutputSettings(settings);
96
+ if (agent.type === "connect" && settings.outputProvider === "elevenlabs" && !options.synthesize) {
97
+ throw new TypeError("Connect ElevenLabs output requires an explicit authorized synthesis transport");
98
+ }
94
99
  if (session) return startPromise;
95
100
  if (stopPromise) await stopPromise.catch(() => {});
96
101
  if (destroyed) throw new Error("voice resource is destroyed");
@@ -125,7 +130,14 @@ export function create(agent, options = {}) {
125
130
  core,
126
131
  sessionId,
127
132
  voice: selectedVoice,
128
- settings: voiceSettings({ ...options, ...parameters, voice: selectedVoice }),
133
+ synthesize: options.synthesize ?? (transport?.synthesize
134
+ ? (text, signal) => transport.synthesize(text, signal, settings.elevenLabsVoiceId)
135
+ : (text, signal) => fetch("/api/voice/elevenlabs/speech", {
136
+ method: "POST", credentials: "same-origin", signal,
137
+ headers: { "Content-Type": "application/json" },
138
+ body: JSON.stringify({ voice_id: settings.elevenLabsVoiceId, text, output_format: "pcm_24000" }),
139
+ })),
140
+ settings,
129
141
  ...(transport?.call === undefined ? {} : { call: transport.call }),
130
142
  ...(transport?.sidebandUrl === undefined ? {} : { sidebandUrl: transport.sidebandUrl }),
131
143
  ...(options.callUrl === undefined ? {} : { callUrl: options.callUrl }),
@@ -299,6 +311,9 @@ function validateOptions(options) {
299
311
  if (!options || typeof options !== "object" || Array.isArray(options)) {
300
312
  throw new TypeError("Voice.create options must be an object");
301
313
  }
314
+ if (options.synthesize !== undefined && typeof options.synthesize !== "function") {
315
+ throw new TypeError("voice synthesize must be a function");
316
+ }
302
317
  if (options.captureMicrophone !== undefined && typeof options.captureMicrophone !== "function") {
303
318
  throw new TypeError("voice captureMicrophone must be a function");
304
319
  }
@@ -317,6 +332,16 @@ function browserLocationOrigin() {
317
332
  }
318
333
 
319
334
  function voiceSettings(options) {
320
- return Object.fromEntries(["voice", "instructions", "pace", "updates", "handoffMode", "acknowledgements"]
335
+ return Object.fromEntries(["voice", "instructions", "pace", "updates", "handoffMode", "acknowledgements", "outputProvider", "elevenLabsVoiceId"]
321
336
  .filter((key) => options[key] !== undefined).map((key) => [key, options[key]]));
322
337
  }
338
+
339
+ function validateOutputSettings(settings) {
340
+ if (settings.outputProvider !== undefined && !["openai", "elevenlabs"].includes(settings.outputProvider)) {
341
+ throw new TypeError("unsupported voice output provider");
342
+ }
343
+ if (settings.outputProvider === "elevenlabs"
344
+ && (typeof settings.elevenLabsVoiceId !== "string" || !/^[A-Za-z0-9_-]{1,128}$/.test(settings.elevenLabsVoiceId))) {
345
+ throw new TypeError("ElevenLabs output requires a valid voice ID");
346
+ }
347
+ }
@@ -1,3 +1,5 @@
1
+ import { ElevenLabsPlayback } from "./ElevenLabsPlayback.mjs";
2
+
1
3
  export const MICROPHONE_CAPTURE_TIMEOUT_MS = 15_000;
2
4
  export const ICE_GATHERING_TIMEOUT_MS = 15_000;
3
5
  export const REALTIME_CALL_TIMEOUT_MS = 15_000;
@@ -85,6 +87,7 @@ export class BrowserVoiceSession {
85
87
  #microphone;
86
88
  #speaker;
87
89
  #playbackEnabled = false;
90
+ #elevenLabs;
88
91
  #muted = false;
89
92
  #inputGeneration = 0;
90
93
  #meterTimer;
@@ -102,10 +105,17 @@ export class BrowserVoiceSession {
102
105
 
103
106
  constructor(options) {
104
107
  this.#options = options;
108
+ if (options.settings?.outputProvider === "elevenlabs") {
109
+ if (typeof options.synthesize !== "function") throw new TypeError("ElevenLabs output requires an authenticated synthesis transport");
110
+ this.#elevenLabs = new ElevenLabsPlayback(options.synthesize, (error) => {
111
+ if (!this.#closed) options.onTerminated(error.message);
112
+ });
113
+ }
105
114
  this.#backendReady = new Promise((resolve) => { this.#resolveBackendReady = resolve; });
106
115
  }
107
116
 
108
117
  start() {
118
+ if (!this.#starting) void this.#elevenLabs?.prime()?.catch((error) => this.#options.onTerminated(error.message));
109
119
  this.#starting ??= this.#start();
110
120
  return this.#starting;
111
121
  }
@@ -140,7 +150,8 @@ export class BrowserVoiceSession {
140
150
  // Only delegated work waits for admission on the direct control path.
141
151
  const coreStartup = coreReady.then(async (core) => {
142
152
  if (this.#directControl) await this.#options.beforeAgentTurn?.();
143
- await core?.start();
153
+ const effects = await core?.start();
154
+ if (!this.#closed && !this.#closing.signal.aborted) await this.#apply(effects);
144
155
  return core;
145
156
  });
146
157
  this.#admission = coreStartup;
@@ -271,7 +282,7 @@ export class BrowserVoiceSession {
271
282
  }
272
283
  const stream = event.streams[0] ?? new MediaStream([event.track]);
273
284
  this.#speaker ??= new SpeakerPlayback(new Audio(), this.#options.onStatus);
274
- this.#speaker.setEnabled(this.#playbackEnabled);
285
+ this.#speaker.setEnabled(this.#playbackEnabled && !this.#elevenLabs);
275
286
  this.#speaker.attach(stream);
276
287
  });
277
288
  peer.addEventListener("connectionstatechange", () => {
@@ -301,6 +312,7 @@ export class BrowserVoiceSession {
301
312
 
302
313
  noteTypedInput() {
303
314
  this.#playbackEnabled = false;
315
+ this.#elevenLabs?.interrupt();
304
316
  this.#speaker?.setEnabled(false);
305
317
  return this.#applyLive(async () => {
306
318
  const core = await this.#options.core;
@@ -417,10 +429,12 @@ export class BrowserVoiceSession {
417
429
  for (const text of effects.undelivered_answers ?? []) this.#options.onUndeliveredAnswer?.(text);
418
430
  if (effects.input_generation !== undefined) {
419
431
  if (effects.input_generation < this.#inputGeneration) return;
432
+ if (effects.input_generation > this.#inputGeneration) this.#elevenLabs?.interrupt();
420
433
  this.#inputGeneration = effects.input_generation;
421
434
  }
422
435
  if (effects.playback_enabled === false) {
423
436
  this.#playbackEnabled = false;
437
+ this.#elevenLabs?.interrupt();
424
438
  this.#speaker?.setEnabled(false);
425
439
  }
426
440
  let sent = 0;
@@ -436,9 +450,10 @@ export class BrowserVoiceSession {
436
450
  if (effects.acknowledge_frames && sent > 0) await this.#core?.framesSent(sent);
437
451
  if (!this.#closed && effects.playback_enabled === true && sent === (effects.frames?.length ?? 0)) {
438
452
  this.#playbackEnabled = true;
439
- this.#speaker?.setEnabled(true);
453
+ this.#speaker?.setEnabled(!this.#elevenLabs);
440
454
  }
441
455
  for (const entry of effects.transcripts ?? []) {
456
+ if (this.#playbackEnabled && !this.#closed) this.#elevenLabs?.transcript(entry);
442
457
  this.#options.onTranscript(entry.speaker, entry.text, entry);
443
458
  }
444
459
  if (effects.status) this.#status(effects.status);
@@ -549,6 +564,7 @@ export class BrowserVoiceSession {
549
564
  }
550
565
 
551
566
  #stopBrowserMedia() {
567
+ this.#elevenLabs?.close();
552
568
  if (this.#meterTimer !== undefined) window.clearTimeout(this.#meterTimer);
553
569
  this.#meterTimer = undefined;
554
570
  this.#call?.abort();
@@ -75,12 +75,28 @@ export function createBrowserHost(options?: {
75
75
  mcp?: McpServers;
76
76
  codeEvaluator?: CodeEvaluator;
77
77
  toolMode?: "code" | "direct";
78
- /** @internal Durable host lifecycle for Rust-owned subagent descriptors. */
78
+ /** @internal Live host lifecycle for ephemeral Rust-owned subagents. */
79
+ subagentRouting?: Pick<import('../runtime/subagent-routing.mjs').SubagentRouting, 'resolve' | 'bind'>;
79
80
  subagentSessions?: {
80
- restore(): readonly SubagentToolContext[];
81
- bind(sessionId: string, descriptor: SubagentToolContext): void;
82
- release(sessionId: string): void;
81
+ bindingDescriptor?(sessionId: string, descriptor: SubagentToolContext, hostContextRef?: string): SubagentToolContext;
82
+ bind(sessionId: string, descriptor: SubagentToolContext, hostContextRef?: string): void;
83
+ release(sessionId: string, hostContextRef?: string): void;
83
84
  };
85
+ /** @internal Content-free summary, once when ownership closes; no public transport option. */
86
+ onSocketTiming?: (timing: {
87
+ message_count: number;
88
+ delivered_message_count: number;
89
+ buffered_message_count: number;
90
+ discarded_message_count: number;
91
+ queue_residence_total_ms: number;
92
+ queue_residence_max_ms: number;
93
+ provider_timings: Array<{
94
+ response_id?: string;
95
+ pre_inference_ms?: number;
96
+ engine_queue_max_ms?: number;
97
+ engine_service_ttft_total_ms?: number;
98
+ }>;
99
+ }) => void;
84
100
  maxQueuedMessages?: number;
85
101
  maxQueuedBytes?: number;
86
102
  maxBufferedSendBytes?: number;