nanocodex 0.6.1 → 0.6.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/README.md +158 -6
  2. package/browser/InlineAgent.mjs +2 -0
  3. package/browser/Transport.d.mts +3 -0
  4. package/browser/Transport.mjs +1 -0
  5. package/browser/Voice.d.mts +2 -0
  6. package/browser/Voice.mjs +12 -6
  7. package/browser/VoiceSession.mjs +71 -21
  8. package/browser/WorkerAgent.mjs +2 -1
  9. package/browser/host.d.mts +7 -0
  10. package/browser/host.mjs +19 -0
  11. package/cloud/Decorator.d.mts +9 -0
  12. package/cloud/Decorator.mjs +9 -0
  13. package/cloud/Transport.mjs +6 -0
  14. package/cloud/actions/connection.mjs +3 -0
  15. package/cloud/actions/connector.d.mts +18 -0
  16. package/cloud/actions/connector.mjs +22 -0
  17. package/cloud/actions/index.d.mts +2 -0
  18. package/cloud/actions/index.mjs +2 -0
  19. package/cloud/internal.mjs +5 -0
  20. package/cloud/types.d.mts +2 -0
  21. package/cloudflare/Agent.mjs +7 -1
  22. package/cloudflare/egress.d.mts +3 -1
  23. package/cloudflare/egress.mjs +14 -0
  24. package/cloudflare/managed-access.d.mts +23 -0
  25. package/cloudflare/managed-access.mjs +120 -0
  26. package/internal.mjs +27 -0
  27. package/managed/Access.d.mts +5 -0
  28. package/managed/Access.mjs +66 -0
  29. package/managed/Agent.d.mts +16 -7
  30. package/managed/Agent.mjs +28 -7
  31. package/managed/Voice.mjs +1 -0
  32. package/managed/index.d.mts +1 -0
  33. package/managed/index.mjs +1 -0
  34. package/node/host.mjs +12 -0
  35. package/package.json +11 -2
  36. package/pkg-node/nanocodex.js +55 -40
  37. package/pkg-node/nanocodex_bg.wasm.d.ts +3 -3
  38. package/pkg-web/.nanocodex-bindgen-stamp +1 -1
  39. package/pkg-web/nanocodex-build.json +1 -1
  40. package/pkg-web/nanocodex.d.ts +3 -3
  41. package/pkg-web/nanocodex.js +55 -40
  42. package/pkg-web/nanocodex_bg.js +55 -40
  43. package/pkg-web/nanocodex_bg.wasm +0 -0
  44. package/pkg-web/nanocodex_bg.wasm.d.ts +3 -3
  45. package/runtime/code-evaluator.worker.mjs +28 -22
  46. package/runtime/managed-transport.mjs +1 -0
  47. package/runtime/quickjs-evaluator.mjs +59 -17
  48. package/runtime/response-controls.mjs +41 -24
  49. package/runtime/responses-http.mjs +69 -0
  50. package/tools/browser/accountInfo.mjs +53 -8
  51. package/tools/browser/browserShell.mjs +2 -2
  52. package/tools/browser/index.mjs +3 -3
  53. package/tools/environment.d.mts +34 -0
  54. package/tools/environment.mjs +59 -0
package/README.md CHANGED
@@ -80,6 +80,23 @@ Worker, Durable Object, or application proxy that owns rotating credentials.
80
80
  Authentication modes are constructors rather than a union of mutually
81
81
  exclusive fields on `Agent.create`.
82
82
 
83
+ ### Personal memories and caller context
84
+
85
+ Managed agents can keep user preferences separate from team knowledge:
86
+
87
+ ```js
88
+ import { Agent } from "nanocodex/managed";
89
+
90
+ await Agent.memory({ operation: "scan", query: "my preferences" }, { scope: "personal" });
91
+ await Agent.memory({ operation: "put", content: "I prefer concise replies." }, { scope: "personal" });
92
+ ```
93
+
94
+ The default scope remains `team`. Personal memories belong to the authenticated
95
+ user within their organization, across teams; use the same scope when reading,
96
+ replacing, or deleting a memory. Managed clients also accept descriptive
97
+ `requestOrigin: { client: "desktop", hand: "user:HAND_ID", cwd: "/HAND_ID" }`.
98
+ Caller context is included once at startup and never grants execution authority.
99
+
83
100
  ### Compose and place tools
84
101
 
85
102
  `createTools` owns one deterministic tool recipe. Custom functions, a portable
@@ -158,8 +175,11 @@ workspace or MCP configuration to an Agent that already receives them through
158
175
  `Tools`.
159
176
 
160
177
  Browser consumers can attach Codex's ChatGPT Realtime voice lifecycle to the
161
- same retained Agent. The resource owns microphone, speaker, WebRTC, sideband,
178
+ same retained Agent. The resource owns microphone, speaker, WebRTC control,
162
179
  and delegation cleanup; stopping voice does not cancel an active coding turn.
180
+ Managed and Connect voice use the WebRTC data channel for live events and
181
+ commands. Speech and captions can flow while durable startup completes;
182
+ delegated work still waits for admission. Local Agents retain the sideband path.
163
183
  Snapshots update each speaker's transcript row as speech arrives, using a stable
164
184
  `id` and `isPartial` flag. Completion replaces that row. `transcript.delta` events
165
185
  carry the current partial text; `transcript` events retain completed-turn semantics.
@@ -214,15 +234,17 @@ subscription adapter (which treats all roles as context), and
214
234
  `appendContext(voice, text)` adds background commentary without
215
235
  requesting speech. Context and speech are split into provider-sized messages.
216
236
  These commands retain frames until sent and preserve them
217
- across a sideband reconnect. They are also methods on the resource and on
237
+ across reconnects when using the sideband transport. They are also methods on
238
+ the resource and on
218
239
  `useVoice` from `nanocodex-react`. These settings use ChatGPT subscription voice;
219
240
  custom voices and Platform audio configuration are not accepted.
220
241
 
221
242
  `Voice.create(...)` remains the equivalent namespaced resource constructor, and
222
243
  `Voice.voices` is the exact ChatGPT V3 voice catalog. The constructor accepts a
223
244
  normal browser Agent, an account-owned managed Agent, or a grant-scoped
224
- `ConnectAgent`. Authentication stays in the owning host routes; Connect uses a
225
- fresh one-use sideband ticket, and the browser binding never receives ChatGPT
245
+ `ConnectAgent`. Authentication stays in the owning host routes. An explicit
246
+ `sidebandUrl` override selects the sideband transport; Connect uses a fresh
247
+ one-use ticket for that path. The browser binding never receives ChatGPT
226
248
  credentials or places its reusable grant bearer in a WebSocket URL.
227
249
 
228
250
  ### Durable Cloudflare Agent
@@ -500,7 +522,7 @@ use the individual factories in server-side Cloudflare Workers. Vite integration
500
522
  is provided separately by `nanocodex-vite`.
501
523
 
502
524
  The browser composition includes native `browseX` public X browsing, advertised
503
- by `accountInfo().apis` without an X connector. The embedding app serves
525
+ by `environment().apis` without an X connector. The embedding app serves
504
526
  `/api/tools/x/browse` and `/api/tools/x/convert`; Nanocodex's account app forwards
505
527
  these requests to the private X Worker.
506
528
 
@@ -1133,7 +1155,7 @@ package manager or build step:
1133
1155
 
1134
1156
  ```html
1135
1157
  <script type="module">
1136
- import { Agent, Transport } from "https://cdn.jsdelivr.net/npm/nanocodex@0.6.1/host/index.mjs";
1158
+ import { Agent, Transport } from "https://cdn.jsdelivr.net/npm/nanocodex@0.6.4/host/index.mjs";
1137
1159
  const agent = await Agent.create({
1138
1160
  transport: Transport.hostManaged({
1139
1161
  websocketUrl: "/api/responses",
@@ -1183,3 +1205,133 @@ inspect `agent.configuration()`, `environment()`, `usage()`, `requests()`,
1183
1205
  `artifacts`, `webhook`, and `requiredActions`. See the
1184
1206
  [managed configuration and operations guide](../../docs/MANAGED_AGENT_CONFIGURATION.md)
1185
1207
  for examples, authorization, delivery semantics, and runtime limits.
1208
+
1209
+
1210
+ ## Call connected services from your app
1211
+
1212
+ After Nanocodex Connect login, the app and its agents share the same approved
1213
+ service capabilities and exact account selections. Request the services during
1214
+ login and call them directly without creating an agent or running a turn:
1215
+
1216
+ ```js
1217
+ const connection = await client.connection.connect({
1218
+ capabilities: { cloudAccounts: { spotify: true, soundcloud: true } },
1219
+ });
1220
+ const response = await client.connectors.spotify.request({
1221
+ path: "/v1/me/playlists?limit=20",
1222
+ // Select an approved account when more than one is connected:
1223
+ connectionId: connection.grant.connectorConnections.spotify[0],
1224
+ });
1225
+ if (!response.ok) throw new Error(`Spotify returned ${response.status}`);
1226
+ const playlists = await response.json();
1227
+ ```
1228
+
1229
+ `client` is a `Client.create(...)` instance from `nanocodex/connect`. For a dynamic service, use
1230
+ `client.connectors.request({ connector: "spotify", path: "/v1/me/playlists" })`.
1231
+ The standalone form is `Actions.connectors.request(client, options)`.
1232
+ Every API service has a scoped entry point, including `client.connectors.gmail`
1233
+ and `client.connectors.soundcloud`. The original singular `client.connector.request`
1234
+ and `Actions.connector.request` remain supported. Requests support a provider
1235
+ path, HTTP method, optional `connectionId`, JSON object `body`, and abort `signal`.
1236
+ The response preserves provider HTTP status, body, and pagination/rate-limit
1237
+ headers. Writes are never automatically retried.
1238
+
1239
+ The underlying API is `POST /v1/connectors/:connector/request` at the configured
1240
+ Connect API origin, authenticated with the app's Connect grant bearer token and
1241
+ `X-Nanocodex-App-Id`. Browser calls must come from the grant's approved app origin.
1242
+ Its JSON body uses `path`, `method`, optional `connection_id`, and optional `body`.
1243
+ It requires no conversation or thread ID. Both this endpoint and agent egress
1244
+ check current grant revocation, expiry, approved service, and exact account IDs.
1245
+ Connecting an account alone does not grant it to every app; the user must approve
1246
+ that service in the app's Connect flow. Provider tokens remain in the broker.
1247
+ ChatGPT is a model connector and has no generic HTTP endpoint here.
1248
+
1249
+
1250
+ ### Use a provider SDK with a custom base URL
1251
+
1252
+ Native SDKs can use normal HTTP through
1253
+ `https://nanocodex.gakonst.workers.dev/connectors/<service>/<provider-path>`.
1254
+ All 13 API capabilities share this route. Append the provider's existing path:
1255
+
1256
+ | Connector | Example route |
1257
+ | --- | --- |
1258
+ | Spotify | `/connectors/spotify/v1/me/playlists` |
1259
+ | SoundCloud | `/connectors/soundcloud/me/playlists` |
1260
+ | Gmail | `/connectors/gmail/gmail/v1/users/me/messages` |
1261
+ | Google Drive | `/connectors/gdrive/drive/v3/files` |
1262
+ | Google Calendar | `/connectors/gcalendar/calendar/v3/calendars/primary/events` |
1263
+ | Google Tasks | `/connectors/gtasks/tasks/v1/users/@me/lists` |
1264
+ | Google Docs | `/connectors/gdocs/v1/documents/{id}` |
1265
+ | Google Sheets | `/connectors/gsheets/v4/spreadsheets/{id}` |
1266
+ | Google Slides | `/connectors/gslides/v1/presentations/{id}` |
1267
+ | Google Contacts | `/connectors/gcontacts/v1/people/me/connections?personFields=names` |
1268
+ | GitHub | `/connectors/github/user` |
1269
+ | Slack | `/connectors/slack/api/auth.test` |
1270
+ | X | `/connectors/x/2/users/me` |
1271
+
1272
+ Provider path restrictions and upstream OAuth permissions still apply. Google
1273
+ services are individually opted in even though they share a Google login.
1274
+ ChatGPT and remote MCP connections use their existing separate protocols.
1275
+ The same routes are available on the configured Connect API origin.
1276
+
1277
+ Send the app's **Nanocodex Connect grant token** as the bearer token. Native
1278
+ requests need no extra app identity headers: the token identifies its approved
1279
+ app and user. SDK routes also accept `Authorization: OAuth <grant-token>`
1280
+ (SoundCloud) and `Authorization: token <grant-token>` (GitHub). Tokens must stay in
1281
+ headers, not query parameters. Browser Origins and any explicit app ID must match that grant.
1282
+ The grant must already include the requested service; this route cannot connect
1283
+ accounts or enlarge the grant. If multiple accounts were approved, set
1284
+ `X-Nanocodex-Connector-Connection` to the selected approved connection ID.
1285
+
1286
+ For example, with [Spotipy](https://github.com/spotipy-dev/spotipy):
1287
+
1288
+ ```python
1289
+ import spotipy
1290
+
1291
+ spotify = spotipy.Spotify(auth=nanocodex_grant_token, retries=0, status_retries=0)
1292
+ spotify.prefix = "https://nanocodex.gakonst.workers.dev/connectors/spotify/v1/"
1293
+ page = spotify.current_user_playlists(limit=20)
1294
+ next_page = spotify.next(page)
1295
+ ```
1296
+
1297
+ For Gmail with Google's [Python API client](https://github.com/googleapis/google-api-python-client):
1298
+
1299
+ ```python
1300
+ from google.oauth2.credentials import Credentials
1301
+ from googleapiclient.discovery import build
1302
+
1303
+ gmail = build(
1304
+ "gmail", "v1",
1305
+ credentials=Credentials(token=nanocodex_grant_token),
1306
+ client_options={
1307
+ "api_endpoint": "https://nanocodex.gakonst.workers.dev/connectors/gmail/",
1308
+ },
1309
+ cache_discovery=False,
1310
+ )
1311
+ messages = gmail.users().messages().list(userId="me").execute(num_retries=0)
1312
+ ```
1313
+
1314
+ Use `userId="me"`; Gmail routes are restricted to the connected user. These
1315
+ credentials hold only the Connect grant; renew an expired grant through Nanocodex
1316
+ Connect instead of asking the provider SDK to refresh it with Google or Spotify.
1317
+ SoundCloud clients use `/connectors/soundcloud/` as their base URL and the same
1318
+ Connect grant, using [SoundCloud's OAuth header format](https://developers.soundcloud.com/docs).
1319
+
1320
+ The proxy preserves request methods, query parameters, and body bytes. Provider
1321
+ status codes, JSON errors, rate-limit headers, and non-JSON bodies are preserved.
1322
+ API links in JSON, pagination Link headers, and same-provider redirects are
1323
+ rewritten through the proxy; artwork and public share URLs stay unchanged.
1324
+ Cross-provider redirects are rejected. JSON link rewriting is bounded to 16 MiB
1325
+ per response; use provider pagination for larger collections. Spotify GETs share
1326
+ one-second read results and the broker's registration-wide rate-limit cooldown.
1327
+ The broker retries a rate-limited Spotify GET at most twice within a ten-second
1328
+ cooldown-wait budget; longer limits return 429 with the remaining `Retry-After`.
1329
+ No writes are automatically retried. Disable SDK write retries to avoid repeating
1330
+ a successful write after an ambiguous network failure.
1331
+
1332
+ See the runnable [Python example](../../examples/python/spotify_proxy.py).
1333
+ The integration test uses real Spotipy 2.26.0 and google-api-python-client 2.192.0
1334
+ against the Connect Worker with fixture provider responses, covering reads,
1335
+ writes, pagination, rate limits, and revoked grants. Worker tests cover routing
1336
+ and service denial for all 13 API connectors. SDKs that hardcode their API host need a custom transport instead of only
1337
+ a base URL setting.
@@ -77,6 +77,7 @@ export async function create(options = {}) {
77
77
  websocketWarmup,
78
78
  WebSocketImpl,
79
79
  createWebSocket,
80
+ createResponse,
80
81
  } = resolveResponsesTransport(transport ?? defaultHostManagedTransport());
81
82
  const subagentsEnabled = internalRuntime?.subagentsEnabled;
82
83
  if (subagentsEnabled !== undefined && typeof subagentsEnabled !== "boolean") {
@@ -109,6 +110,7 @@ export async function create(options = {}) {
109
110
  const host = createBrowserHost({
110
111
  WebSocketImpl,
111
112
  createWebSocket,
113
+ createResponse,
112
114
  hostAuth: hostAuth === true
113
115
  || (apiKey === undefined && mpp === undefined && subscription === undefined),
114
116
  hostManagedProtocol,
@@ -4,6 +4,7 @@ import type {
4
4
  } from "../types.mjs";
5
5
  import type { Options as ManagedClientOptions } from "../managed/Agent.mjs";
6
6
  import type {
7
+ BrowserHttpRequest,
7
8
  BrowserWebSocketConnection,
8
9
  BrowserWebSocketRequest,
9
10
  } from "./host.mjs";
@@ -34,9 +35,11 @@ type SharedEndpointOptions = Readonly<{
34
35
  type WorkerEndpointOptions = SharedEndpointOptions & Readonly<{
35
36
  WebSocketImpl?: never;
36
37
  createWebSocket?: never;
38
+ createResponse?: never;
37
39
  }>;
38
40
 
39
41
  type EndpointOptions = SharedEndpointOptions & Readonly<{
42
+ createResponse?(endpoint: string, sessionId: string, request: BrowserHttpRequest): Promise<Response>;
40
43
  WebSocketImpl?: typeof WebSocket | undefined;
41
44
  createWebSocket?(
42
45
  endpoint: string,
@@ -54,6 +54,7 @@ function connection(options = {}) {
54
54
  WebSocketImpl: options.WebSocketImpl,
55
55
  apiBaseUrl: options.apiBaseUrl,
56
56
  createWebSocket: options.createWebSocket,
57
+ createResponse: options.createResponse,
57
58
  websocketUrl: options.websocketUrl,
58
59
  websocketPreconnect: options.websocketPreconnect,
59
60
  websocketWarmup: options.websocketWarmup,
@@ -34,6 +34,7 @@ export type Snapshot = Readonly<{
34
34
  microphoneLevel: number;
35
35
  speakerLevel: number;
36
36
  error: Error | undefined;
37
+ /** Active means media and live control are ready; task admission may still be pending. */
37
38
  status: "idle" | "connecting" | "active" | "error";
38
39
  statusText: string | undefined;
39
40
  /** Latest 200 rows, retained across stop/start. Subscribe to events for longer history. */
@@ -78,6 +79,7 @@ export type Voice = Readonly<{
78
79
  destroy(): Promise<void>;
79
80
  getSnapshot(): Snapshot;
80
81
  onEvent(listener: (event: Event) => void): () => void;
82
+ /** Resolves after media and task admission are ready. The active snapshot may arrive first. */
81
83
  start(options?: Settings): Promise<void>;
82
84
  stop(): Promise<void>;
83
85
  subscribe(listener: () => void): () => void;
package/browser/Voice.mjs CHANGED
@@ -114,6 +114,13 @@ export function create(agent, options = {}) {
114
114
  : createBrowserVoice(agent, selectedVoice));
115
115
  const transport = managedTransport;
116
116
  let transcriptSequence = 0;
117
+ let publishedReady = false;
118
+ const publishReady = () => {
119
+ if (publishedReady || destroyed || session !== next || generation !== current) return;
120
+ publishedReady = true;
121
+ publish({ ...snapshot, error: undefined, status: "active", statusText: `Voice active (${selectedVoice})` });
122
+ emit(Object.freeze({ type: "started", voice: selectedVoice }));
123
+ };
117
124
  const next = new BrowserVoiceSession({
118
125
  core,
119
126
  sessionId,
@@ -122,9 +129,10 @@ export function create(agent, options = {}) {
122
129
  ...(transport?.call === undefined ? {} : { call: transport.call }),
123
130
  ...(transport?.sidebandUrl === undefined ? {} : { sidebandUrl: transport.sidebandUrl }),
124
131
  ...(options.callUrl === undefined ? {} : { callUrl: options.callUrl }),
125
- ...(options.sidebandUrl === undefined ? {} : { sidebandUrl: options.sidebandUrl }),
132
+ ...(options.sidebandUrl === undefined ? {} : { sidebandUrl: options.sidebandUrl, dataChannelControl: false }),
126
133
  ...(options.captureMicrophone === undefined ? {} : { captureMicrophone: options.captureMicrophone }),
127
134
  ...(options.beforeAgentTurn === undefined ? {} : { beforeAgentTurn: options.beforeAgentTurn }),
135
+ onReady: publishReady,
128
136
  onStatus(text) {
129
137
  if (session === next && generation === current && snapshot.status !== "idle" && snapshot.status !== "error") {
130
138
  publish({ ...snapshot, statusText: text });
@@ -175,11 +183,9 @@ export function create(agent, options = {}) {
175
183
  session = next;
176
184
  next.setMuted(snapshot.muted);
177
185
  observeAgentEvents(next);
178
- startPromise = next.start().then(() => {
179
- if (destroyed || session !== next || generation !== current) return;
180
- publish({ ...snapshot, error: undefined, status: "active", statusText: `Voice active (${selectedVoice})` });
181
- emit(Object.freeze({ type: "started", voice: selectedVoice }));
182
- }).catch(async (cause) => {
186
+ // Media readiness updates the UI independently; start still waits for task
187
+ // admission, and a later rejection closes media through the same error path.
188
+ startPromise = next.start().then(publishReady).catch(async (cause) => {
183
189
  if (session === next) session = undefined;
184
190
  if (activeResources.get(agent) === resource) activeResources.delete(agent);
185
191
  cleanupWatcher();
@@ -76,6 +76,7 @@ export class BrowserVoiceSession {
76
76
  #admission;
77
77
  #peer;
78
78
  #channel;
79
+ #directControl = false;
79
80
  #sideband;
80
81
  #sidebandUrl;
81
82
  #sidebandCallId;
@@ -129,14 +130,16 @@ export class BrowserVoiceSession {
129
130
  const coreReady = Promise.resolve(this.#options.core).then(async (core) => {
130
131
  if (this.#closed || this.#closing.signal.aborted) { core.free(); return; }
131
132
  this.#core = core;
133
+ this.#directControl = core.dataChannelControl === true && this.#options.dataChannelControl !== false;
132
134
  if (this.#options.settings) await core.configure(JSON.stringify(this.#options.settings));
133
- await this.#options.beforeAgentTurn?.();
135
+ if (!this.#directControl) await this.#options.beforeAgentTurn?.();
134
136
  if (this.#closed || this.#closing.signal.aborted) return;
135
137
  return core;
136
138
  });
137
- // Managed Rust can deliver authoritative context after SDP negotiation.
138
- // Start both immediately, but fence incoming control events on admission.
139
+ // Media and live conversation do not depend on durable task admission.
140
+ // Only delegated work waits for admission on the direct control path.
139
141
  const coreStartup = coreReady.then(async (core) => {
142
+ if (this.#directControl) await this.#options.beforeAgentTurn?.();
140
143
  await core?.start();
141
144
  return core;
142
145
  });
@@ -203,7 +206,7 @@ export class BrowserVoiceSession {
203
206
  const completed = JSON.parse(await core.completeCall(callResponse.body, callResponse.location));
204
207
  if (this.#closed || peer.signalingState === "closed") return;
205
208
  this.#sidebandCallId = completed.call_id;
206
- this.#sidebandUrl = this.#options.sidebandUrl
209
+ this.#sidebandUrl = this.#directControl || this.#options.sidebandUrl
207
210
  ? undefined
208
211
  : String(await core.sidebandUrl(completed.call_id));
209
212
  if (this.#closed) return;
@@ -215,7 +218,7 @@ export class BrowserVoiceSession {
215
218
  }, { signal: this.#closing.signal, timeoutMs: PEER_CONNECTION_TIMEOUT_MS,
216
219
  timeoutError: new VoiceError("peer_connection_timeout", "Voice media did not connect in time."),
217
220
  onTimeout: () => { peer.close(); } }),
218
- this.#openSideband().then(() => withStartupDeadline(() => this.#backendReady, { signal: this.#closing.signal,
221
+ (this.#directControl ? this.#openDataChannel() : this.#openSideband()).then(() => withStartupDeadline(() => this.#backendReady, { signal: this.#closing.signal,
219
222
  timeoutMs: SIDEBAND_OPEN_TIMEOUT_MS,
220
223
  timeoutError: new VoiceError("session_ready_timeout", "The Realtime session did not become ready in time.") })),
221
224
  ]);
@@ -227,6 +230,7 @@ export class BrowserVoiceSession {
227
230
  if (this.#closed) return;
228
231
  this.#sampleLevels();
229
232
  this.#status(`Voice active (${this.#options.voice})`);
233
+ this.#options.onReady?.();
230
234
  }
231
235
 
232
236
  async #prepareMedia(capture) {
@@ -249,6 +253,17 @@ export class BrowserVoiceSession {
249
253
  this.#peer = peer;
250
254
  for (const track of microphone.getAudioTracks()) peer.addTrack(track, microphone);
251
255
  this.#channel = peer.createDataChannel("oai-events");
256
+ const channel = this.#channel;
257
+ channel.addEventListener("message", (event) => {
258
+ if (this.#directControl && !this.#closed && this.#channel === channel) {
259
+ this.#receiveControl(event.data, () => this.#channel === channel);
260
+ }
261
+ });
262
+ channel.addEventListener("close", () => {
263
+ if (this.#directControl && !this.#closed && !this.#closing.signal.aborted && this.#channel === channel) {
264
+ this.#options.onTerminated("Voice control connection closed — tap Voice to reconnect");
265
+ }
266
+ });
252
267
  peer.addEventListener("track", (event) => {
253
268
  if (this.#closed || this.#closing.signal.aborted || this.#peer !== peer) {
254
269
  event.track.stop();
@@ -410,7 +425,10 @@ export class BrowserVoiceSession {
410
425
  }
411
426
  let sent = 0;
412
427
  for (const frame of effects.frames ?? []) {
413
- if (this.#sideband?.readyState === WebSocket.OPEN) {
428
+ if (this.#directControl && this.#channel?.readyState === "open") {
429
+ this.#channel.send(frame);
430
+ sent += 1;
431
+ } else if (!this.#directControl && this.#sideband?.readyState === WebSocket.OPEN) {
414
432
  this.#sideband.send(frame);
415
433
  sent += 1;
416
434
  }
@@ -457,20 +475,7 @@ export class BrowserVoiceSession {
457
475
  let opened = false;
458
476
  sideband.addEventListener("message", (event) => {
459
477
  if (!this.#closed && generation === this.#sidebandGeneration) {
460
- this.#applyLive(async () => {
461
- await this.#admission;
462
- if (this.#closed || generation !== this.#sidebandGeneration) return;
463
- if (await this.#core.requiresAgentAdmission(event.data)) {
464
- // Only delegations wait for durable admission. Speech deltas and
465
- // agent output must continue while that independent request waits.
466
- void this.#enqueue(async () => {
467
- await this.#options.beforeAgentTurn?.();
468
- return this.#core.realtimeMessage(event.data);
469
- }, true).catch(() => {});
470
- return;
471
- }
472
- return this.#core.realtimeMessage(event.data);
473
- });
478
+ this.#receiveControl(event.data, () => generation === this.#sidebandGeneration);
474
479
  }
475
480
  });
476
481
  sideband.addEventListener("close", () => {
@@ -492,6 +497,46 @@ export class BrowserVoiceSession {
492
497
  }
493
498
  }
494
499
 
500
+ #receiveControl(payload, isCurrent) {
501
+ this.#applyLive(async () => {
502
+ if (!this.#directControl) await this.#admission;
503
+ if (this.#closed || !isCurrent()) return;
504
+ if (await this.#core.requiresAgentAdmission(payload)) {
505
+ // Retain accepted requests on close, but keep captions and interruptions
506
+ // independent of the durable route and its ordered task queue.
507
+ void this.#enqueue(async () => {
508
+ await this.#admission;
509
+ await this.#options.beforeAgentTurn?.();
510
+ return this.#core.realtimeMessage(payload);
511
+ }, true).catch(() => {});
512
+ return;
513
+ }
514
+ return this.#core.realtimeMessage(payload);
515
+ });
516
+ }
517
+
518
+ async #openDataChannel() {
519
+ const channel = this.#channel;
520
+ await withStartupDeadline(() => new Promise((resolve, reject) => {
521
+ const cleanup = () => {
522
+ channel.removeEventListener("open", opened);
523
+ channel.removeEventListener("close", closed);
524
+ this.#closing.signal.removeEventListener("abort", closed);
525
+ };
526
+ const opened = () => { cleanup(); resolve(); };
527
+ const closed = () => { cleanup(); reject(new Error("voice data channel closed before opening")); };
528
+ channel.addEventListener("open", opened);
529
+ channel.addEventListener("close", closed);
530
+ this.#closing.signal.addEventListener("abort", closed, { once: true });
531
+ if (channel.readyState === "open") opened();
532
+ else if (channel.readyState === "closed" || this.#closing.signal.aborted) closed();
533
+ }), {
534
+ signal: this.#closing.signal, timeoutMs: SIDEBAND_OPEN_TIMEOUT_MS,
535
+ timeoutError: new VoiceError("data_channel_open_timeout", "The Realtime voice data channel did not open in time."),
536
+ });
537
+ if (!this.#closed && this.#channel === channel) await this.#applyLive(() => this.#core.sidebandOpened());
538
+ }
539
+
495
540
  #status(message) {
496
541
  if (!this.#closed) this.#options.onStatus(message);
497
542
  }
@@ -512,8 +557,13 @@ export class BrowserVoiceSession {
512
557
  this.#flushTimer = undefined;
513
558
  if (this.#reconnectTimer !== undefined) window.clearTimeout(this.#reconnectTimer);
514
559
  this.#reconnectTimer = undefined;
515
- this.#channel?.close();
560
+ // Match native close: release provider media before durable tail cleanup.
561
+ if (this.#directControl && this.#channel?.readyState === "open") {
562
+ try { this.#channel.send('{"type":"session.close"}'); } catch {}
563
+ }
564
+ const channel = this.#channel;
516
565
  this.#channel = undefined;
566
+ channel?.close();
517
567
  this.#peer?.close();
518
568
  this.#peer = undefined;
519
569
  stopStream(this.#microphone);
@@ -1317,7 +1317,8 @@ function serializeConfig(options) {
1317
1317
  ...connection
1318
1318
  } = setup;
1319
1319
  if (hostAuth === true) {
1320
- if (typeof connection.createWebSocket === "function"
1320
+ if (typeof connection.createResponse === "function"
1321
+ || typeof connection.createWebSocket === "function"
1321
1322
  || typeof connection.WebSocketImpl === "function") {
1322
1323
  throw new TypeError("Worker Agent host-managed transport callbacks must live inside a custom Worker");
1323
1324
  }
@@ -41,6 +41,12 @@ export type BrowserWebSocketRequest = BrowserWebSocketMetadata & (
41
41
  }
42
42
  );
43
43
 
44
+ /** One streaming HTTPS Responses request. The host must preserve cancellation. */
45
+ export type BrowserHttpRequest = Exclude<BrowserWebSocketRequest, { authorization: "preconnect" }> & {
46
+ body: string;
47
+ signal: AbortSignal;
48
+ };
49
+
44
50
  export type BrowserWebSocketConnection = {
45
51
  socket: WebSocket;
46
52
  status?: number | undefined;
@@ -59,6 +65,7 @@ export function createBrowserHost(options?: {
59
65
  sessionId: string,
60
66
  request: BrowserWebSocketRequest,
61
67
  ) => WebSocket | BrowserWebSocketConnection | Promise<WebSocket | BrowserWebSocketConnection>;
68
+ createResponse?: (endpoint: string, sessionId: string, request: BrowserHttpRequest) => Promise<Response>;
62
69
  filesystem?: Workspace;
63
70
  filesystemTools?: boolean;
64
71
  onEvent?: (eventJson: string) => void;
package/browser/host.mjs CHANGED
@@ -1,3 +1,4 @@
1
+ import { createResponsesHttp, responsesHttpHeaders } from "../runtime/responses-http.mjs";
1
2
  import { createCodeRuntime, toolResult } from "../runtime/code-runtime.mjs";
2
3
  import {
3
4
  toolRouterBrand,
@@ -45,6 +46,19 @@ export function createBrowserHost(options = {}) {
45
46
  if (!options.mpp && !createWebSocket) {
46
47
  throw new Error("WebSocket is unavailable in this runtime");
47
48
  }
49
+ const http = createResponsesHttp((endpoint, apiKey, sessionId, metadata, body, signal) => {
50
+ if (disposal) throw new Error("Nanocodex host is already disposed");
51
+ if (options.mpp) throw JSON.stringify({ kind: "transport", detail: "MPP HTTPS transport is unavailable", reconnectable: false });
52
+ if (options.createResponse) {
53
+ const authorization = options.hostAuth
54
+ ? { authorization: "host_managed" }
55
+ : { authorization: "bearer", bearerToken: apiKey };
56
+ return options.createResponse(endpoint, sessionId, { ...metadata, ...authorization, body, signal });
57
+ }
58
+ if (options.hostAuth) throw JSON.stringify({ kind: "transport", detail: "host-managed HTTPS requires createResponse", reconnectable: false });
59
+ return fetch(endpoint, { method: "POST", headers: responsesHttpHeaders(apiKey, sessionId, metadata),
60
+ body, signal, redirect: "error" });
61
+ });
48
62
  const connections = new Map();
49
63
  const openingAttempts = new Set();
50
64
  const connectingConnections = new Set();
@@ -375,6 +389,7 @@ export function createBrowserHost(options = {}) {
375
389
 
376
390
  function dispose() {
377
391
  if (disposal) return disposal;
392
+ http.dispose();
378
393
  disposalError = new Error("Nanocodex host was disposed during WebSocket connection");
379
394
  disposal = Promise.resolve().then(async () => {
380
395
  const cleanups = [];
@@ -480,6 +495,10 @@ export function createBrowserHost(options = {}) {
480
495
  if (references > 0) references -= 1;
481
496
  return references === 0 ? dispose() : Promise.resolve();
482
497
  },
498
+ httpOpen: http.httpOpen,
499
+ httpReady: http.httpReady,
500
+ httpNext: http.httpNext,
501
+ httpClose: http.httpClose,
483
502
  connect,
484
503
  preconnect,
485
504
  send,
@@ -1,5 +1,6 @@
1
1
  import type { logout } from "./actions/account.mjs";
2
2
  import type { create as createAgent } from "./actions/agent.mjs";
3
+ import type { request as connectorRequest } from "./actions/connector.mjs";
3
4
  import type { connect, disconnect, reconnect } from "./actions/connection.mjs";
4
5
  import type { revoke } from "./actions/grant.mjs";
5
6
  import type { fund, getConfig } from "./actions/machineUsd.mjs";
@@ -11,6 +12,14 @@ import type { Connection, HostConnection } from "./types.mjs";
11
12
  export type ConnectActions<connection extends Connection | HostConnection = Connection> = {
12
13
  account: { logout(): logout.ReturnType };
13
14
  agent: { create(options: createAgent.Options): Promise<createAgent.ReturnType> };
15
+ connector: { request(options: connectorRequest.Options): connectorRequest.ReturnType };
16
+ connectors: {
17
+ request(options: connectorRequest.Options): connectorRequest.ReturnType;
18
+ } & {
19
+ readonly [service in connectorRequest.Options["connector"]]: {
20
+ request(options: Omit<connectorRequest.Options, "connector">): connectorRequest.ReturnType;
21
+ };
22
+ };
14
23
  connection: {
15
24
  connect(options: connect.Options): Promise<connection>;
16
25
  disconnect(options?: disconnect.Options | undefined): disconnect.ReturnType;
@@ -1,5 +1,7 @@
1
+ import { API_CONNECTORS } from "./internal.mjs";
1
2
  import * as account from "./actions/account.mjs";
2
3
  import * as agent from "./actions/agent.mjs";
4
+ import * as connector from "./actions/connector.mjs";
3
5
  import * as connection from "./actions/connection.mjs";
4
6
  import * as grant from "./actions/grant.mjs";
5
7
  import * as machineUsd from "./actions/machineUsd.mjs";
@@ -14,6 +16,13 @@ export function connectActions() {
14
16
  agent: {
15
17
  create: (options) => agent.create(client, options),
16
18
  },
19
+ connector: { request: (options) => connector.request(client, options) },
20
+ connectors: {
21
+ request: (options) => connector.request(client, options),
22
+ ...Object.fromEntries(API_CONNECTORS.map(service => [service, {
23
+ request: (options) => connector.request(client, { ...options, connector: service }),
24
+ }])),
25
+ },
17
26
  connection: {
18
27
  connect: (options) => connection.connect(client, options),
19
28
  disconnect: (options) => connection.disconnect(client, options),
@@ -376,6 +376,9 @@ function requestedConnectors(value) {
376
376
  "gcontacts",
377
377
  "slack",
378
378
  "x",
379
+ "spotify",
380
+ "soundcloud",
381
+ "link",
379
382
  "chatgpt",
380
383
  ];
381
384
  if (value.some((provider) => !supported.includes(provider))) {
@@ -413,6 +416,9 @@ function connectorName(provider) {
413
416
  gcontacts: "Google Contacts",
414
417
  slack: "Slack",
415
418
  x: "X",
419
+ spotify: "Spotify",
420
+ soundcloud: "SoundCloud",
421
+ link: "Stripe Link",
416
422
  chatgpt: "ChatGPT",
417
423
  })[provider];
418
424
  }
@@ -22,6 +22,9 @@ const CLOUD_ACCOUNT_PROVIDERS = Object.freeze([
22
22
  "gcontacts",
23
23
  "slack",
24
24
  "x",
25
+ "spotify",
26
+ "soundcloud",
27
+ "link",
25
28
  "chatgpt",
26
29
  ]);
27
30
  const CONNECTOR_RESOURCE_PREFIX = "urn:nanocodex:connector:";
@@ -0,0 +1,18 @@
1
+ import type { Client } from "../Client.mjs";
2
+ import type { CloudAccount } from "../types.mjs";
3
+
4
+ export declare namespace request {
5
+ type Options = Readonly<{
6
+ connector: Exclude<CloudAccount, "chatgpt">;
7
+ path: string;
8
+ method?: "GET" | "HEAD" | "POST" | "PUT" | "PATCH" | "DELETE" | "OPTIONS" | undefined;
9
+ /** Required when the grant includes more than one account for this service. */
10
+ connectionId?: string | undefined;
11
+ body?: Record<string, unknown> | undefined;
12
+ signal?: AbortSignal | undefined;
13
+ }>;
14
+ type ReturnType = Promise<Response>;
15
+ }
16
+
17
+ /** Returns the provider response (including HTTP failures). Never retries writes. */
18
+ export function request(client: Client, options: request.Options): request.ReturnType;