nanocodex 0.6.1 → 0.6.5

package/browser/Voice.mjs

@@ -91,6 +91,11 @@ export function create(agent, options = {}) {
     if (destroyed) throw new Error("voice resource is destroyed");
     const selectedVoice = parameters.voice ?? options.voice ?? defaultVoice;
     if (!voices.includes(selectedVoice)) throw new TypeError(`unsupported ChatGPT voice: ${selectedVoice}`);
+    const settings = voiceSettings({ ...options, ...parameters, voice: selectedVoice });
+    validateOutputSettings(settings);
+    if (agent.type === "connect" && settings.outputProvider === "elevenlabs" && !options.synthesize) {
+      throw new TypeError("Connect ElevenLabs output requires an explicit authorized synthesis transport");
+    }
     if (session) return startPromise;
     if (stopPromise) await stopPromise.catch(() => {});
     if (destroyed) throw new Error("voice resource is destroyed");
@@ -114,17 +119,32 @@ export function create(agent, options = {}) {
       : createBrowserVoice(agent, selectedVoice));
     const transport = managedTransport;
     let transcriptSequence = 0;
+    let publishedReady = false;
+    const publishReady = () => {
+      if (publishedReady || destroyed || session !== next || generation !== current) return;
+      publishedReady = true;
+      publish({ ...snapshot, error: undefined, status: "active", statusText: `Voice active (${selectedVoice})` });
+      emit(Object.freeze({ type: "started", voice: selectedVoice }));
+    };
     const next = new BrowserVoiceSession({
       core,
       sessionId,
       voice: selectedVoice,
-      settings: voiceSettings({ ...options, ...parameters, voice: selectedVoice }),
+      synthesize: options.synthesize ?? (transport?.synthesize
+        ? (text, signal) => transport.synthesize(text, signal, settings.elevenLabsVoiceId)
+        : (text, signal) => fetch("/api/voice/elevenlabs/speech", {
+          method: "POST", credentials: "same-origin", signal,
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify({ voice_id: settings.elevenLabsVoiceId, text, output_format: "pcm_24000" }),
+        })),
+      settings,
       ...(transport?.call === undefined ? {} : { call: transport.call }),
       ...(transport?.sidebandUrl === undefined ? {} : { sidebandUrl: transport.sidebandUrl }),
       ...(options.callUrl === undefined ? {} : { callUrl: options.callUrl }),
-      ...(options.sidebandUrl === undefined ? {} : { sidebandUrl: options.sidebandUrl }),
+      ...(options.sidebandUrl === undefined ? {} : { sidebandUrl: options.sidebandUrl, dataChannelControl: false }),
       ...(options.captureMicrophone === undefined ? {} : { captureMicrophone: options.captureMicrophone }),
       ...(options.beforeAgentTurn === undefined ? {} : { beforeAgentTurn: options.beforeAgentTurn }),
+      onReady: publishReady,
       onStatus(text) {
         if (session === next && generation === current && snapshot.status !== "idle" && snapshot.status !== "error") {
           publish({ ...snapshot, statusText: text });
@@ -175,11 +195,9 @@ export function create(agent, options = {}) {
     session = next;
     next.setMuted(snapshot.muted);
     observeAgentEvents(next);
-    startPromise = next.start().then(() => {
-      if (destroyed || session !== next || generation !== current) return;
-      publish({ ...snapshot, error: undefined, status: "active", statusText: `Voice active (${selectedVoice})` });
-      emit(Object.freeze({ type: "started", voice: selectedVoice }));
-    }).catch(async (cause) => {
+    // Media readiness updates the UI independently; start still waits for task
+    // admission, and a later rejection closes media through the same error path.
+    startPromise = next.start().then(publishReady).catch(async (cause) => {
       if (session === next) session = undefined;
       if (activeResources.get(agent) === resource) activeResources.delete(agent);
       cleanupWatcher();
@@ -293,6 +311,9 @@ function validateOptions(options) {
   if (!options || typeof options !== "object" || Array.isArray(options)) {
     throw new TypeError("Voice.create options must be an object");
   }
+  if (options.synthesize !== undefined && typeof options.synthesize !== "function") {
+    throw new TypeError("voice synthesize must be a function");
+  }
   if (options.captureMicrophone !== undefined && typeof options.captureMicrophone !== "function") {
     throw new TypeError("voice captureMicrophone must be a function");
   }
@@ -311,6 +332,16 @@ function browserLocationOrigin() {
 }
 
 function voiceSettings(options) {
-  return Object.fromEntries(["voice", "instructions", "pace", "updates", "handoffMode", "acknowledgements"]
+  return Object.fromEntries(["voice", "instructions", "pace", "updates", "handoffMode", "acknowledgements", "outputProvider", "elevenLabsVoiceId"]
     .filter((key) => options[key] !== undefined).map((key) => [key, options[key]]));
 }
+
+function validateOutputSettings(settings) {
+  if (settings.outputProvider !== undefined && !["openai", "elevenlabs"].includes(settings.outputProvider)) {
+    throw new TypeError("unsupported voice output provider");
+  }
+  if (settings.outputProvider === "elevenlabs"
+    && (typeof settings.elevenLabsVoiceId !== "string" || !/^[A-Za-z0-9_-]{1,128}$/.test(settings.elevenLabsVoiceId))) {
+    throw new TypeError("ElevenLabs output requires a valid voice ID");
+  }
+}

package/browser/VoiceSession.mjs

@@ -1,3 +1,5 @@
+import { ElevenLabsPlayback } from "./ElevenLabsPlayback.mjs";
+
 export const MICROPHONE_CAPTURE_TIMEOUT_MS = 15_000;
 export const ICE_GATHERING_TIMEOUT_MS = 15_000;
 export const REALTIME_CALL_TIMEOUT_MS = 15_000;
@@ -76,6 +78,7 @@ export class BrowserVoiceSession {
   #admission;
   #peer;
   #channel;
+  #directControl = false;
   #sideband;
   #sidebandUrl;
   #sidebandCallId;
@@ -84,6 +87,7 @@ export class BrowserVoiceSession {
   #microphone;
   #speaker;
   #playbackEnabled = false;
+  #elevenLabs;
   #muted = false;
   #inputGeneration = 0;
   #meterTimer;
@@ -101,10 +105,17 @@ export class BrowserVoiceSession {
 
   constructor(options) {
     this.#options = options;
+    if (options.settings?.outputProvider === "elevenlabs") {
+      if (typeof options.synthesize !== "function") throw new TypeError("ElevenLabs output requires an authenticated synthesis transport");
+      this.#elevenLabs = new ElevenLabsPlayback(options.synthesize, (error) => {
+        if (!this.#closed) options.onTerminated(error.message);
+      });
+    }
     this.#backendReady = new Promise((resolve) => { this.#resolveBackendReady = resolve; });
   }
 
   start() {
+    if (!this.#starting) void this.#elevenLabs?.prime()?.catch((error) => this.#options.onTerminated(error.message));
     this.#starting ??= this.#start();
     return this.#starting;
   }
@@ -129,15 +140,18 @@ export class BrowserVoiceSession {
     const coreReady = Promise.resolve(this.#options.core).then(async (core) => {
       if (this.#closed || this.#closing.signal.aborted) { core.free(); return; }
       this.#core = core;
+      this.#directControl = core.dataChannelControl === true && this.#options.dataChannelControl !== false;
       if (this.#options.settings) await core.configure(JSON.stringify(this.#options.settings));
-      await this.#options.beforeAgentTurn?.();
+      if (!this.#directControl) await this.#options.beforeAgentTurn?.();
       if (this.#closed || this.#closing.signal.aborted) return;
       return core;
     });
-    // Managed Rust can deliver authoritative context after SDP negotiation.
-    // Start both immediately, but fence incoming control events on admission.
+    // Media and live conversation do not depend on durable task admission.
+    // Only delegated work waits for admission on the direct control path.
     const coreStartup = coreReady.then(async (core) => {
-      await core?.start();
+      if (this.#directControl) await this.#options.beforeAgentTurn?.();
+      const effects = await core?.start();
+      if (!this.#closed && !this.#closing.signal.aborted) await this.#apply(effects);
       return core;
     });
     this.#admission = coreStartup;
@@ -203,7 +217,7 @@ export class BrowserVoiceSession {
     const completed = JSON.parse(await core.completeCall(callResponse.body, callResponse.location));
     if (this.#closed || peer.signalingState === "closed") return;
     this.#sidebandCallId = completed.call_id;
-    this.#sidebandUrl = this.#options.sidebandUrl
+    this.#sidebandUrl = this.#directControl || this.#options.sidebandUrl
       ? undefined
       : String(await core.sidebandUrl(completed.call_id));
     if (this.#closed) return;
@@ -215,7 +229,7 @@ export class BrowserVoiceSession {
         }, { signal: this.#closing.signal, timeoutMs: PEER_CONNECTION_TIMEOUT_MS,
           timeoutError: new VoiceError("peer_connection_timeout", "Voice media did not connect in time."),
           onTimeout: () => { peer.close(); } }),
-        this.#openSideband().then(() => withStartupDeadline(() => this.#backendReady, { signal: this.#closing.signal,
+        (this.#directControl ? this.#openDataChannel() : this.#openSideband()).then(() => withStartupDeadline(() => this.#backendReady, { signal: this.#closing.signal,
           timeoutMs: SIDEBAND_OPEN_TIMEOUT_MS,
           timeoutError: new VoiceError("session_ready_timeout", "The Realtime session did not become ready in time.") })),
       ]);
@@ -227,6 +241,7 @@ export class BrowserVoiceSession {
     if (this.#closed) return;
     this.#sampleLevels();
     this.#status(`Voice active (${this.#options.voice})`);
+    this.#options.onReady?.();
   }
 
   async #prepareMedia(capture) {
@@ -249,6 +264,17 @@ export class BrowserVoiceSession {
     this.#peer = peer;
     for (const track of microphone.getAudioTracks()) peer.addTrack(track, microphone);
     this.#channel = peer.createDataChannel("oai-events");
+    const channel = this.#channel;
+    channel.addEventListener("message", (event) => {
+      if (this.#directControl && !this.#closed && this.#channel === channel) {
+        this.#receiveControl(event.data, () => this.#channel === channel);
+      }
+    });
+    channel.addEventListener("close", () => {
+      if (this.#directControl && !this.#closed && !this.#closing.signal.aborted && this.#channel === channel) {
+        this.#options.onTerminated("Voice control connection closed — tap Voice to reconnect");
+      }
+    });
     peer.addEventListener("track", (event) => {
       if (this.#closed || this.#closing.signal.aborted || this.#peer !== peer) {
         event.track.stop();
@@ -256,7 +282,7 @@ export class BrowserVoiceSession {
       }
       const stream = event.streams[0] ?? new MediaStream([event.track]);
       this.#speaker ??= new SpeakerPlayback(new Audio(), this.#options.onStatus);
-      this.#speaker.setEnabled(this.#playbackEnabled);
+      this.#speaker.setEnabled(this.#playbackEnabled && !this.#elevenLabs);
       this.#speaker.attach(stream);
     });
     peer.addEventListener("connectionstatechange", () => {
@@ -286,6 +312,7 @@ export class BrowserVoiceSession {
 
   noteTypedInput() {
     this.#playbackEnabled = false;
+    this.#elevenLabs?.interrupt();
     this.#speaker?.setEnabled(false);
     return this.#applyLive(async () => {
       const core = await this.#options.core;
@@ -402,15 +429,20 @@ export class BrowserVoiceSession {
     for (const text of effects.undelivered_answers ?? []) this.#options.onUndeliveredAnswer?.(text);
     if (effects.input_generation !== undefined) {
       if (effects.input_generation < this.#inputGeneration) return;
+      if (effects.input_generation > this.#inputGeneration) this.#elevenLabs?.interrupt();
       this.#inputGeneration = effects.input_generation;
     }
     if (effects.playback_enabled === false) {
       this.#playbackEnabled = false;
+      this.#elevenLabs?.interrupt();
       this.#speaker?.setEnabled(false);
     }
     let sent = 0;
     for (const frame of effects.frames ?? []) {
-      if (this.#sideband?.readyState === WebSocket.OPEN) {
+      if (this.#directControl && this.#channel?.readyState === "open") {
+        this.#channel.send(frame);
+        sent += 1;
+      } else if (!this.#directControl && this.#sideband?.readyState === WebSocket.OPEN) {
         this.#sideband.send(frame);
         sent += 1;
       }
@@ -418,9 +450,10 @@ export class BrowserVoiceSession {
     if (effects.acknowledge_frames && sent > 0) await this.#core?.framesSent(sent);
     if (!this.#closed && effects.playback_enabled === true && sent === (effects.frames?.length ?? 0)) {
       this.#playbackEnabled = true;
-      this.#speaker?.setEnabled(true);
+      this.#speaker?.setEnabled(!this.#elevenLabs);
     }
     for (const entry of effects.transcripts ?? []) {
+      if (this.#playbackEnabled && !this.#closed) this.#elevenLabs?.transcript(entry);
       this.#options.onTranscript(entry.speaker, entry.text, entry);
     }
     if (effects.status) this.#status(effects.status);
@@ -457,20 +490,7 @@ export class BrowserVoiceSession {
     let opened = false;
     sideband.addEventListener("message", (event) => {
       if (!this.#closed && generation === this.#sidebandGeneration) {
-        this.#applyLive(async () => {
-          await this.#admission;
-          if (this.#closed || generation !== this.#sidebandGeneration) return;
-          if (await this.#core.requiresAgentAdmission(event.data)) {
-            // Only delegations wait for durable admission. Speech deltas and
-            // agent output must continue while that independent request waits.
-            void this.#enqueue(async () => {
-              await this.#options.beforeAgentTurn?.();
-              return this.#core.realtimeMessage(event.data);
-            }, true).catch(() => {});
-            return;
-          }
-          return this.#core.realtimeMessage(event.data);
-        });
+        this.#receiveControl(event.data, () => generation === this.#sidebandGeneration);
       }
     });
     sideband.addEventListener("close", () => {
@@ -492,6 +512,46 @@ export class BrowserVoiceSession {
     }
   }
 
+  #receiveControl(payload, isCurrent) {
+    this.#applyLive(async () => {
+      if (!this.#directControl) await this.#admission;
+      if (this.#closed || !isCurrent()) return;
+      if (await this.#core.requiresAgentAdmission(payload)) {
+        // Retain accepted requests on close, but keep captions and interruptions
+        // independent of the durable route and its ordered task queue.
+        void this.#enqueue(async () => {
+          await this.#admission;
+          await this.#options.beforeAgentTurn?.();
+          return this.#core.realtimeMessage(payload);
+        }, true).catch(() => {});
+        return;
+      }
+      return this.#core.realtimeMessage(payload);
+    });
+  }
+
+  async #openDataChannel() {
+    const channel = this.#channel;
+    await withStartupDeadline(() => new Promise((resolve, reject) => {
+      const cleanup = () => {
+        channel.removeEventListener("open", opened);
+        channel.removeEventListener("close", closed);
+        this.#closing.signal.removeEventListener("abort", closed);
+      };
+      const opened = () => { cleanup(); resolve(); };
+      const closed = () => { cleanup(); reject(new Error("voice data channel closed before opening")); };
+      channel.addEventListener("open", opened);
+      channel.addEventListener("close", closed);
+      this.#closing.signal.addEventListener("abort", closed, { once: true });
+      if (channel.readyState === "open") opened();
+      else if (channel.readyState === "closed" || this.#closing.signal.aborted) closed();
+    }), {
+      signal: this.#closing.signal, timeoutMs: SIDEBAND_OPEN_TIMEOUT_MS,
+      timeoutError: new VoiceError("data_channel_open_timeout", "The Realtime voice data channel did not open in time."),
+    });
+    if (!this.#closed && this.#channel === channel) await this.#applyLive(() => this.#core.sidebandOpened());
+  }
+
   #status(message) {
     if (!this.#closed) this.#options.onStatus(message);
   }
@@ -504,6 +564,7 @@ export class BrowserVoiceSession {
   }
 
   #stopBrowserMedia() {
+    this.#elevenLabs?.close();
     if (this.#meterTimer !== undefined) window.clearTimeout(this.#meterTimer);
     this.#meterTimer = undefined;
     this.#call?.abort();
@@ -512,8 +573,13 @@ export class BrowserVoiceSession {
     this.#flushTimer = undefined;
     if (this.#reconnectTimer !== undefined) window.clearTimeout(this.#reconnectTimer);
     this.#reconnectTimer = undefined;
-    this.#channel?.close();
+    // Match native close: release provider media before durable tail cleanup.
+    if (this.#directControl && this.#channel?.readyState === "open") {
+      try { this.#channel.send('{"type":"session.close"}'); } catch {}
+    }
+    const channel = this.#channel;
     this.#channel = undefined;
+    channel?.close();
     this.#peer?.close();
     this.#peer = undefined;
     stopStream(this.#microphone);

package/browser/WorkerAgent.mjs

@@ -1317,7 +1317,8 @@ function serializeConfig(options) {
       ...connection
     } = setup;
     if (hostAuth === true) {
-      if (typeof connection.createWebSocket === "function"
+      if (typeof connection.createResponse === "function"
+        || typeof connection.createWebSocket === "function"
         || typeof connection.WebSocketImpl === "function") {
         throw new TypeError("Worker Agent host-managed transport callbacks must live inside a custom Worker");
       }

package/browser/host.d.mts

@@ -41,6 +41,12 @@ export type BrowserWebSocketRequest = BrowserWebSocketMetadata & (
     }
 );
 
+/** One streaming HTTPS Responses request. The host must preserve cancellation. */
+export type BrowserHttpRequest = Exclude<BrowserWebSocketRequest, { authorization: "preconnect" }> & {
+  body: string;
+  signal: AbortSignal;
+};
+
 export type BrowserWebSocketConnection = {
   socket: WebSocket;
   status?: number | undefined;
@@ -59,6 +65,7 @@ export function createBrowserHost(options?: {
     sessionId: string,
     request: BrowserWebSocketRequest,
   ) => WebSocket | BrowserWebSocketConnection | Promise<WebSocket | BrowserWebSocketConnection>;
+  createResponse?: (endpoint: string, sessionId: string, request: BrowserHttpRequest) => Promise<Response>;
   filesystem?: Workspace;
   filesystemTools?: boolean;
   onEvent?: (eventJson: string) => void;
@@ -68,12 +75,28 @@ export function createBrowserHost(options?: {
   mcp?: McpServers;
   codeEvaluator?: CodeEvaluator;
   toolMode?: "code" | "direct";
-  /** @internal Durable host lifecycle for Rust-owned subagent descriptors. */
+  /** @internal Live host lifecycle for ephemeral Rust-owned subagents. */
+  subagentRouting?: Pick<import('../runtime/subagent-routing.mjs').SubagentRouting, 'resolve' | 'bind'>;
   subagentSessions?: {
-    restore(): readonly SubagentToolContext[];
-    bind(sessionId: string, descriptor: SubagentToolContext): void;
-    release(sessionId: string): void;
+    bindingDescriptor?(sessionId: string, descriptor: SubagentToolContext, hostContextRef?: string): SubagentToolContext;
+    bind(sessionId: string, descriptor: SubagentToolContext, hostContextRef?: string): void;
+    release(sessionId: string, hostContextRef?: string): void;
   };
+  /** @internal Content-free summary, once when ownership closes; no public transport option. */
+  onSocketTiming?: (timing: {
+    message_count: number;
+    delivered_message_count: number;
+    buffered_message_count: number;
+    discarded_message_count: number;
+    queue_residence_total_ms: number;
+    queue_residence_max_ms: number;
+    provider_timings: Array<{
+      response_id?: string;
+      pre_inference_ms?: number;
+      engine_queue_max_ms?: number;
+      engine_service_ttft_total_ms?: number;
+    }>;
+  }) => void;
   maxQueuedMessages?: number;
   maxQueuedBytes?: number;
   maxBufferedSendBytes?: number;