nanocodex 0.6.1 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +158 -6
- package/browser/InlineAgent.mjs +2 -0
- package/browser/Transport.d.mts +3 -0
- package/browser/Transport.mjs +1 -0
- package/browser/Voice.d.mts +2 -0
- package/browser/Voice.mjs +12 -6
- package/browser/VoiceSession.mjs +71 -21
- package/browser/WorkerAgent.mjs +2 -1
- package/browser/host.d.mts +7 -0
- package/browser/host.mjs +19 -0
- package/cloud/Decorator.d.mts +9 -0
- package/cloud/Decorator.mjs +9 -0
- package/cloud/Transport.mjs +6 -0
- package/cloud/actions/connection.mjs +3 -0
- package/cloud/actions/connector.d.mts +18 -0
- package/cloud/actions/connector.mjs +22 -0
- package/cloud/actions/index.d.mts +2 -0
- package/cloud/actions/index.mjs +2 -0
- package/cloud/internal.mjs +5 -0
- package/cloud/types.d.mts +2 -0
- package/cloudflare/Agent.mjs +7 -1
- package/cloudflare/egress.d.mts +3 -1
- package/cloudflare/egress.mjs +14 -0
- package/cloudflare/managed-access.d.mts +23 -0
- package/cloudflare/managed-access.mjs +120 -0
- package/internal.mjs +27 -0
- package/managed/Access.d.mts +5 -0
- package/managed/Access.mjs +66 -0
- package/managed/Agent.d.mts +16 -7
- package/managed/Agent.mjs +28 -7
- package/managed/Voice.mjs +1 -0
- package/managed/index.d.mts +1 -0
- package/managed/index.mjs +1 -0
- package/node/host.mjs +12 -0
- package/package.json +11 -2
- package/pkg-node/nanocodex.js +55 -40
- package/pkg-node/nanocodex_bg.wasm.d.ts +3 -3
- package/pkg-web/.nanocodex-bindgen-stamp +1 -1
- package/pkg-web/nanocodex-build.json +1 -1
- package/pkg-web/nanocodex.d.ts +3 -3
- package/pkg-web/nanocodex.js +55 -40
- package/pkg-web/nanocodex_bg.js +55 -40
- package/pkg-web/nanocodex_bg.wasm +0 -0
- package/pkg-web/nanocodex_bg.wasm.d.ts +3 -3
- package/runtime/code-evaluator.worker.mjs +28 -22
- package/runtime/managed-transport.mjs +1 -0
- package/runtime/quickjs-evaluator.mjs +59 -17
- package/runtime/response-controls.mjs +41 -24
- package/runtime/responses-http.mjs +69 -0
- package/tools/browser/accountInfo.mjs +53 -8
- package/tools/browser/browserShell.mjs +2 -2
- package/tools/browser/index.mjs +3 -3
- package/tools/environment.d.mts +34 -0
- package/tools/environment.mjs +59 -0
package/README.md
CHANGED
|
@@ -80,6 +80,23 @@ Worker, Durable Object, or application proxy that owns rotating credentials.
|
|
|
80
80
|
Authentication modes are constructors rather than a union of mutually
|
|
81
81
|
exclusive fields on `Agent.create`.
|
|
82
82
|
|
|
83
|
+
### Personal memories and caller context
|
|
84
|
+
|
|
85
|
+
Managed agents can keep user preferences separate from team knowledge:
|
|
86
|
+
|
|
87
|
+
```js
|
|
88
|
+
import { Agent } from "nanocodex/managed";
|
|
89
|
+
|
|
90
|
+
await Agent.memory({ operation: "scan", query: "my preferences" }, { scope: "personal" });
|
|
91
|
+
await Agent.memory({ operation: "put", content: "I prefer concise replies." }, { scope: "personal" });
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
The default scope remains `team`. Personal memories belong to the authenticated
|
|
95
|
+
user within their organization, across teams; use the same scope when reading,
|
|
96
|
+
replacing, or deleting a memory. Managed clients also accept descriptive
|
|
97
|
+
`requestOrigin: { client: "desktop", hand: "user:HAND_ID", cwd: "/HAND_ID" }`.
|
|
98
|
+
Caller context is included once at startup and never grants execution authority.
|
|
99
|
+
|
|
83
100
|
### Compose and place tools
|
|
84
101
|
|
|
85
102
|
`createTools` owns one deterministic tool recipe. Custom functions, a portable
|
|
@@ -158,8 +175,11 @@ workspace or MCP configuration to an Agent that already receives them through
|
|
|
158
175
|
`Tools`.
|
|
159
176
|
|
|
160
177
|
Browser consumers can attach Codex's ChatGPT Realtime voice lifecycle to the
|
|
161
|
-
same retained Agent. The resource owns microphone, speaker, WebRTC
|
|
178
|
+
same retained Agent. The resource owns microphone, speaker, WebRTC control,
|
|
162
179
|
and delegation cleanup; stopping voice does not cancel an active coding turn.
|
|
180
|
+
Managed and Connect voice use the WebRTC data channel for live events and
|
|
181
|
+
commands. Speech and captions can flow while durable startup completes;
|
|
182
|
+
delegated work still waits for admission. Local Agents retain the sideband path.
|
|
163
183
|
Snapshots update each speaker's transcript row as speech arrives, using a stable
|
|
164
184
|
`id` and `isPartial` flag. Completion replaces that row. `transcript.delta` events
|
|
165
185
|
carry the current partial text; `transcript` events retain completed-turn semantics.
|
|
@@ -214,15 +234,17 @@ subscription adapter (which treats all roles as context), and
|
|
|
214
234
|
`appendContext(voice, text)` adds background commentary without
|
|
215
235
|
requesting speech. Context and speech are split into provider-sized messages.
|
|
216
236
|
These commands retain frames until sent and preserve them
|
|
217
|
-
across
|
|
237
|
+
across reconnects when using the sideband transport. They are also methods on
|
|
238
|
+
the resource and on
|
|
218
239
|
`useVoice` from `nanocodex-react`. These settings use ChatGPT subscription voice;
|
|
219
240
|
custom voices and Platform audio configuration are not accepted.
|
|
220
241
|
|
|
221
242
|
`Voice.create(...)` remains the equivalent namespaced resource constructor, and
|
|
222
243
|
`Voice.voices` is the exact ChatGPT V3 voice catalog. The constructor accepts a
|
|
223
244
|
normal browser Agent, an account-owned managed Agent, or a grant-scoped
|
|
224
|
-
`ConnectAgent`. Authentication stays in the owning host routes
|
|
225
|
-
|
|
245
|
+
`ConnectAgent`. Authentication stays in the owning host routes. An explicit
|
|
246
|
+
`sidebandUrl` override selects the sideband transport; Connect uses a fresh
|
|
247
|
+
one-use ticket for that path. The browser binding never receives ChatGPT
|
|
226
248
|
credentials or places its reusable grant bearer in a WebSocket URL.
|
|
227
249
|
|
|
228
250
|
### Durable Cloudflare Agent
|
|
@@ -500,7 +522,7 @@ use the individual factories in server-side Cloudflare Workers. Vite integration
|
|
|
500
522
|
is provided separately by `nanocodex-vite`.
|
|
501
523
|
|
|
502
524
|
The browser composition includes native `browseX` public X browsing, advertised
|
|
503
|
-
by `
|
|
525
|
+
by `environment().apis` without an X connector. The embedding app serves
|
|
504
526
|
`/api/tools/x/browse` and `/api/tools/x/convert`; Nanocodex's account app forwards
|
|
505
527
|
these requests to the private X Worker.
|
|
506
528
|
|
|
@@ -1133,7 +1155,7 @@ package manager or build step:
|
|
|
1133
1155
|
|
|
1134
1156
|
```html
|
|
1135
1157
|
<script type="module">
|
|
1136
|
-
import { Agent, Transport } from "https://cdn.jsdelivr.net/npm/nanocodex@0.6.
|
|
1158
|
+
import { Agent, Transport } from "https://cdn.jsdelivr.net/npm/nanocodex@0.6.4/host/index.mjs";
|
|
1137
1159
|
const agent = await Agent.create({
|
|
1138
1160
|
transport: Transport.hostManaged({
|
|
1139
1161
|
websocketUrl: "/api/responses",
|
|
@@ -1183,3 +1205,133 @@ inspect `agent.configuration()`, `environment()`, `usage()`, `requests()`,
|
|
|
1183
1205
|
`artifacts`, `webhook`, and `requiredActions`. See the
|
|
1184
1206
|
[managed configuration and operations guide](../../docs/MANAGED_AGENT_CONFIGURATION.md)
|
|
1185
1207
|
for examples, authorization, delivery semantics, and runtime limits.
|
|
1208
|
+
|
|
1209
|
+
|
|
1210
|
+
## Call connected services from your app
|
|
1211
|
+
|
|
1212
|
+
After Nanocodex Connect login, the app and its agents share the same approved
|
|
1213
|
+
service capabilities and exact account selections. Request the services during
|
|
1214
|
+
login and call them directly without creating an agent or running a turn:
|
|
1215
|
+
|
|
1216
|
+
```js
|
|
1217
|
+
const connection = await client.connection.connect({
|
|
1218
|
+
capabilities: { cloudAccounts: { spotify: true, soundcloud: true } },
|
|
1219
|
+
});
|
|
1220
|
+
const response = await client.connectors.spotify.request({
|
|
1221
|
+
path: "/v1/me/playlists?limit=20",
|
|
1222
|
+
// Select an approved account when more than one is connected:
|
|
1223
|
+
connectionId: connection.grant.connectorConnections.spotify[0],
|
|
1224
|
+
});
|
|
1225
|
+
if (!response.ok) throw new Error(`Spotify returned ${response.status}`);
|
|
1226
|
+
const playlists = await response.json();
|
|
1227
|
+
```
|
|
1228
|
+
|
|
1229
|
+
`client` is a `Client.create(...)` instance from `nanocodex/connect`. For a dynamic service, use
|
|
1230
|
+
`client.connectors.request({ connector: "spotify", path: "/v1/me/playlists" })`.
|
|
1231
|
+
The standalone form is `Actions.connectors.request(client, options)`.
|
|
1232
|
+
Every API service has a scoped entry point, including `client.connectors.gmail`
|
|
1233
|
+
and `client.connectors.soundcloud`. The original singular `client.connector.request`
|
|
1234
|
+
and `Actions.connector.request` remain supported. Requests support a provider
|
|
1235
|
+
path, HTTP method, optional `connectionId`, JSON object `body`, and abort `signal`.
|
|
1236
|
+
The response preserves provider HTTP status, body, and pagination/rate-limit
|
|
1237
|
+
headers. Writes are never automatically retried.
|
|
1238
|
+
|
|
1239
|
+
The underlying API is `POST /v1/connectors/:connector/request` at the configured
|
|
1240
|
+
Connect API origin, authenticated with the app's Connect grant bearer token and
|
|
1241
|
+
`X-Nanocodex-App-Id`. Browser calls must come from the grant's approved app origin.
|
|
1242
|
+
Its JSON body uses `path`, `method`, optional `connection_id`, and optional `body`.
|
|
1243
|
+
It requires no conversation or thread ID. Both this endpoint and agent egress
|
|
1244
|
+
check current grant revocation, expiry, approved service, and exact account IDs.
|
|
1245
|
+
Connecting an account alone does not grant it to every app; the user must approve
|
|
1246
|
+
that service in the app's Connect flow. Provider tokens remain in the broker.
|
|
1247
|
+
ChatGPT is a model connector and has no generic HTTP endpoint here.
|
|
1248
|
+
|
|
1249
|
+
|
|
1250
|
+
### Use a provider SDK with a custom base URL
|
|
1251
|
+
|
|
1252
|
+
Native SDKs can use normal HTTP through
|
|
1253
|
+
`https://nanocodex.gakonst.workers.dev/connectors/<service>/<provider-path>`.
|
|
1254
|
+
All 13 API capabilities share this route. Append the provider's existing path:
|
|
1255
|
+
|
|
1256
|
+
| Connector | Example route |
|
|
1257
|
+
| --- | --- |
|
|
1258
|
+
| Spotify | `/connectors/spotify/v1/me/playlists` |
|
|
1259
|
+
| SoundCloud | `/connectors/soundcloud/me/playlists` |
|
|
1260
|
+
| Gmail | `/connectors/gmail/gmail/v1/users/me/messages` |
|
|
1261
|
+
| Google Drive | `/connectors/gdrive/drive/v3/files` |
|
|
1262
|
+
| Google Calendar | `/connectors/gcalendar/calendar/v3/calendars/primary/events` |
|
|
1263
|
+
| Google Tasks | `/connectors/gtasks/tasks/v1/users/@me/lists` |
|
|
1264
|
+
| Google Docs | `/connectors/gdocs/v1/documents/{id}` |
|
|
1265
|
+
| Google Sheets | `/connectors/gsheets/v4/spreadsheets/{id}` |
|
|
1266
|
+
| Google Slides | `/connectors/gslides/v1/presentations/{id}` |
|
|
1267
|
+
| Google Contacts | `/connectors/gcontacts/v1/people/me/connections?personFields=names` |
|
|
1268
|
+
| GitHub | `/connectors/github/user` |
|
|
1269
|
+
| Slack | `/connectors/slack/api/auth.test` |
|
|
1270
|
+
| X | `/connectors/x/2/users/me` |
|
|
1271
|
+
|
|
1272
|
+
Provider path restrictions and upstream OAuth permissions still apply. Google
|
|
1273
|
+
services are individually opted in even though they share a Google login.
|
|
1274
|
+
ChatGPT and remote MCP connections use their existing separate protocols.
|
|
1275
|
+
The same routes are available on the configured Connect API origin.
|
|
1276
|
+
|
|
1277
|
+
Send the app's **Nanocodex Connect grant token** as the bearer token. Native
|
|
1278
|
+
requests need no extra app identity headers: the token identifies its approved
|
|
1279
|
+
app and user. SDK routes also accept `Authorization: OAuth <grant-token>`
|
|
1280
|
+
(SoundCloud) and `Authorization: token <grant-token>` (GitHub). Tokens must stay in
|
|
1281
|
+
headers, not query parameters. Browser Origins and any explicit app ID must match that grant.
|
|
1282
|
+
The grant must already include the requested service; this route cannot connect
|
|
1283
|
+
accounts or enlarge the grant. If multiple accounts were approved, set
|
|
1284
|
+
`X-Nanocodex-Connector-Connection` to the selected approved connection ID.
|
|
1285
|
+
|
|
1286
|
+
For example, with [Spotipy](https://github.com/spotipy-dev/spotipy):
|
|
1287
|
+
|
|
1288
|
+
```python
|
|
1289
|
+
import spotipy
|
|
1290
|
+
|
|
1291
|
+
spotify = spotipy.Spotify(auth=nanocodex_grant_token, retries=0, status_retries=0)
|
|
1292
|
+
spotify.prefix = "https://nanocodex.gakonst.workers.dev/connectors/spotify/v1/"
|
|
1293
|
+
page = spotify.current_user_playlists(limit=20)
|
|
1294
|
+
next_page = spotify.next(page)
|
|
1295
|
+
```
|
|
1296
|
+
|
|
1297
|
+
For Gmail with Google's [Python API client](https://github.com/googleapis/google-api-python-client):
|
|
1298
|
+
|
|
1299
|
+
```python
|
|
1300
|
+
from google.oauth2.credentials import Credentials
|
|
1301
|
+
from googleapiclient.discovery import build
|
|
1302
|
+
|
|
1303
|
+
gmail = build(
|
|
1304
|
+
"gmail", "v1",
|
|
1305
|
+
credentials=Credentials(token=nanocodex_grant_token),
|
|
1306
|
+
client_options={
|
|
1307
|
+
"api_endpoint": "https://nanocodex.gakonst.workers.dev/connectors/gmail/",
|
|
1308
|
+
},
|
|
1309
|
+
cache_discovery=False,
|
|
1310
|
+
)
|
|
1311
|
+
messages = gmail.users().messages().list(userId="me").execute(num_retries=0)
|
|
1312
|
+
```
|
|
1313
|
+
|
|
1314
|
+
Use `userId="me"`; Gmail routes are restricted to the connected user. These
|
|
1315
|
+
credentials hold only the Connect grant; renew an expired grant through Nanocodex
|
|
1316
|
+
Connect instead of asking the provider SDK to refresh it with Google or Spotify.
|
|
1317
|
+
SoundCloud clients use `/connectors/soundcloud/` as their base URL and the same
|
|
1318
|
+
Connect grant, using [SoundCloud's OAuth header format](https://developers.soundcloud.com/docs).
|
|
1319
|
+
|
|
1320
|
+
The proxy preserves request methods, query parameters, and body bytes. Provider
|
|
1321
|
+
status codes, JSON errors, rate-limit headers, and non-JSON bodies are preserved.
|
|
1322
|
+
API links in JSON, pagination Link headers, and same-provider redirects are
|
|
1323
|
+
rewritten through the proxy; artwork and public share URLs stay unchanged.
|
|
1324
|
+
Cross-provider redirects are rejected. JSON link rewriting is bounded to 16 MiB
|
|
1325
|
+
per response; use provider pagination for larger collections. Spotify GETs share
|
|
1326
|
+
one-second read results and the broker's registration-wide rate-limit cooldown.
|
|
1327
|
+
The broker retries a rate-limited Spotify GET at most twice within a ten-second
|
|
1328
|
+
cooldown-wait budget; longer limits return 429 with the remaining `Retry-After`.
|
|
1329
|
+
No writes are automatically retried. Disable SDK write retries to avoid repeating
|
|
1330
|
+
a successful write after an ambiguous network failure.
|
|
1331
|
+
|
|
1332
|
+
See the runnable [Python example](../../examples/python/spotify_proxy.py).
|
|
1333
|
+
The integration test uses real Spotipy 2.26.0 and google-api-python-client 2.192.0
|
|
1334
|
+
against the Connect Worker with fixture provider responses, covering reads,
|
|
1335
|
+
writes, pagination, rate limits, and revoked grants. Worker tests cover routing
|
|
1336
|
+
and service denial for all 13 API connectors. SDKs that hardcode their API host need a custom transport instead of only
|
|
1337
|
+
a base URL setting.
|
package/browser/InlineAgent.mjs
CHANGED
|
@@ -77,6 +77,7 @@ export async function create(options = {}) {
|
|
|
77
77
|
websocketWarmup,
|
|
78
78
|
WebSocketImpl,
|
|
79
79
|
createWebSocket,
|
|
80
|
+
createResponse,
|
|
80
81
|
} = resolveResponsesTransport(transport ?? defaultHostManagedTransport());
|
|
81
82
|
const subagentsEnabled = internalRuntime?.subagentsEnabled;
|
|
82
83
|
if (subagentsEnabled !== undefined && typeof subagentsEnabled !== "boolean") {
|
|
@@ -109,6 +110,7 @@ export async function create(options = {}) {
|
|
|
109
110
|
const host = createBrowserHost({
|
|
110
111
|
WebSocketImpl,
|
|
111
112
|
createWebSocket,
|
|
113
|
+
createResponse,
|
|
112
114
|
hostAuth: hostAuth === true
|
|
113
115
|
|| (apiKey === undefined && mpp === undefined && subscription === undefined),
|
|
114
116
|
hostManagedProtocol,
|
package/browser/Transport.d.mts
CHANGED
|
@@ -4,6 +4,7 @@ import type {
|
|
|
4
4
|
} from "../types.mjs";
|
|
5
5
|
import type { Options as ManagedClientOptions } from "../managed/Agent.mjs";
|
|
6
6
|
import type {
|
|
7
|
+
BrowserHttpRequest,
|
|
7
8
|
BrowserWebSocketConnection,
|
|
8
9
|
BrowserWebSocketRequest,
|
|
9
10
|
} from "./host.mjs";
|
|
@@ -34,9 +35,11 @@ type SharedEndpointOptions = Readonly<{
|
|
|
34
35
|
type WorkerEndpointOptions = SharedEndpointOptions & Readonly<{
|
|
35
36
|
WebSocketImpl?: never;
|
|
36
37
|
createWebSocket?: never;
|
|
38
|
+
createResponse?: never;
|
|
37
39
|
}>;
|
|
38
40
|
|
|
39
41
|
type EndpointOptions = SharedEndpointOptions & Readonly<{
|
|
42
|
+
createResponse?(endpoint: string, sessionId: string, request: BrowserHttpRequest): Promise<Response>;
|
|
40
43
|
WebSocketImpl?: typeof WebSocket | undefined;
|
|
41
44
|
createWebSocket?(
|
|
42
45
|
endpoint: string,
|
package/browser/Transport.mjs
CHANGED
|
@@ -54,6 +54,7 @@ function connection(options = {}) {
|
|
|
54
54
|
WebSocketImpl: options.WebSocketImpl,
|
|
55
55
|
apiBaseUrl: options.apiBaseUrl,
|
|
56
56
|
createWebSocket: options.createWebSocket,
|
|
57
|
+
createResponse: options.createResponse,
|
|
57
58
|
websocketUrl: options.websocketUrl,
|
|
58
59
|
websocketPreconnect: options.websocketPreconnect,
|
|
59
60
|
websocketWarmup: options.websocketWarmup,
|
package/browser/Voice.d.mts
CHANGED
|
@@ -34,6 +34,7 @@ export type Snapshot = Readonly<{
|
|
|
34
34
|
microphoneLevel: number;
|
|
35
35
|
speakerLevel: number;
|
|
36
36
|
error: Error | undefined;
|
|
37
|
+
/** Active means media and live control are ready; task admission may still be pending. */
|
|
37
38
|
status: "idle" | "connecting" | "active" | "error";
|
|
38
39
|
statusText: string | undefined;
|
|
39
40
|
/** Latest 200 rows, retained across stop/start. Subscribe to events for longer history. */
|
|
@@ -78,6 +79,7 @@ export type Voice = Readonly<{
|
|
|
78
79
|
destroy(): Promise<void>;
|
|
79
80
|
getSnapshot(): Snapshot;
|
|
80
81
|
onEvent(listener: (event: Event) => void): () => void;
|
|
82
|
+
/** Resolves after media and task admission are ready. The active snapshot may arrive first. */
|
|
81
83
|
start(options?: Settings): Promise<void>;
|
|
82
84
|
stop(): Promise<void>;
|
|
83
85
|
subscribe(listener: () => void): () => void;
|
package/browser/Voice.mjs
CHANGED
|
@@ -114,6 +114,13 @@ export function create(agent, options = {}) {
|
|
|
114
114
|
: createBrowserVoice(agent, selectedVoice));
|
|
115
115
|
const transport = managedTransport;
|
|
116
116
|
let transcriptSequence = 0;
|
|
117
|
+
let publishedReady = false;
|
|
118
|
+
const publishReady = () => {
|
|
119
|
+
if (publishedReady || destroyed || session !== next || generation !== current) return;
|
|
120
|
+
publishedReady = true;
|
|
121
|
+
publish({ ...snapshot, error: undefined, status: "active", statusText: `Voice active (${selectedVoice})` });
|
|
122
|
+
emit(Object.freeze({ type: "started", voice: selectedVoice }));
|
|
123
|
+
};
|
|
117
124
|
const next = new BrowserVoiceSession({
|
|
118
125
|
core,
|
|
119
126
|
sessionId,
|
|
@@ -122,9 +129,10 @@ export function create(agent, options = {}) {
|
|
|
122
129
|
...(transport?.call === undefined ? {} : { call: transport.call }),
|
|
123
130
|
...(transport?.sidebandUrl === undefined ? {} : { sidebandUrl: transport.sidebandUrl }),
|
|
124
131
|
...(options.callUrl === undefined ? {} : { callUrl: options.callUrl }),
|
|
125
|
-
...(options.sidebandUrl === undefined ? {} : { sidebandUrl: options.sidebandUrl }),
|
|
132
|
+
...(options.sidebandUrl === undefined ? {} : { sidebandUrl: options.sidebandUrl, dataChannelControl: false }),
|
|
126
133
|
...(options.captureMicrophone === undefined ? {} : { captureMicrophone: options.captureMicrophone }),
|
|
127
134
|
...(options.beforeAgentTurn === undefined ? {} : { beforeAgentTurn: options.beforeAgentTurn }),
|
|
135
|
+
onReady: publishReady,
|
|
128
136
|
onStatus(text) {
|
|
129
137
|
if (session === next && generation === current && snapshot.status !== "idle" && snapshot.status !== "error") {
|
|
130
138
|
publish({ ...snapshot, statusText: text });
|
|
@@ -175,11 +183,9 @@ export function create(agent, options = {}) {
|
|
|
175
183
|
session = next;
|
|
176
184
|
next.setMuted(snapshot.muted);
|
|
177
185
|
observeAgentEvents(next);
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
emit(Object.freeze({ type: "started", voice: selectedVoice }));
|
|
182
|
-
}).catch(async (cause) => {
|
|
186
|
+
// Media readiness updates the UI independently; start still waits for task
|
|
187
|
+
// admission, and a later rejection closes media through the same error path.
|
|
188
|
+
startPromise = next.start().then(publishReady).catch(async (cause) => {
|
|
183
189
|
if (session === next) session = undefined;
|
|
184
190
|
if (activeResources.get(agent) === resource) activeResources.delete(agent);
|
|
185
191
|
cleanupWatcher();
|
package/browser/VoiceSession.mjs
CHANGED
|
@@ -76,6 +76,7 @@ export class BrowserVoiceSession {
|
|
|
76
76
|
#admission;
|
|
77
77
|
#peer;
|
|
78
78
|
#channel;
|
|
79
|
+
#directControl = false;
|
|
79
80
|
#sideband;
|
|
80
81
|
#sidebandUrl;
|
|
81
82
|
#sidebandCallId;
|
|
@@ -129,14 +130,16 @@ export class BrowserVoiceSession {
|
|
|
129
130
|
const coreReady = Promise.resolve(this.#options.core).then(async (core) => {
|
|
130
131
|
if (this.#closed || this.#closing.signal.aborted) { core.free(); return; }
|
|
131
132
|
this.#core = core;
|
|
133
|
+
this.#directControl = core.dataChannelControl === true && this.#options.dataChannelControl !== false;
|
|
132
134
|
if (this.#options.settings) await core.configure(JSON.stringify(this.#options.settings));
|
|
133
|
-
await this.#options.beforeAgentTurn?.();
|
|
135
|
+
if (!this.#directControl) await this.#options.beforeAgentTurn?.();
|
|
134
136
|
if (this.#closed || this.#closing.signal.aborted) return;
|
|
135
137
|
return core;
|
|
136
138
|
});
|
|
137
|
-
//
|
|
138
|
-
//
|
|
139
|
+
// Media and live conversation do not depend on durable task admission.
|
|
140
|
+
// Only delegated work waits for admission on the direct control path.
|
|
139
141
|
const coreStartup = coreReady.then(async (core) => {
|
|
142
|
+
if (this.#directControl) await this.#options.beforeAgentTurn?.();
|
|
140
143
|
await core?.start();
|
|
141
144
|
return core;
|
|
142
145
|
});
|
|
@@ -203,7 +206,7 @@ export class BrowserVoiceSession {
|
|
|
203
206
|
const completed = JSON.parse(await core.completeCall(callResponse.body, callResponse.location));
|
|
204
207
|
if (this.#closed || peer.signalingState === "closed") return;
|
|
205
208
|
this.#sidebandCallId = completed.call_id;
|
|
206
|
-
this.#sidebandUrl = this.#options.sidebandUrl
|
|
209
|
+
this.#sidebandUrl = this.#directControl || this.#options.sidebandUrl
|
|
207
210
|
? undefined
|
|
208
211
|
: String(await core.sidebandUrl(completed.call_id));
|
|
209
212
|
if (this.#closed) return;
|
|
@@ -215,7 +218,7 @@ export class BrowserVoiceSession {
|
|
|
215
218
|
}, { signal: this.#closing.signal, timeoutMs: PEER_CONNECTION_TIMEOUT_MS,
|
|
216
219
|
timeoutError: new VoiceError("peer_connection_timeout", "Voice media did not connect in time."),
|
|
217
220
|
onTimeout: () => { peer.close(); } }),
|
|
218
|
-
this.#openSideband().then(() => withStartupDeadline(() => this.#backendReady, { signal: this.#closing.signal,
|
|
221
|
+
(this.#directControl ? this.#openDataChannel() : this.#openSideband()).then(() => withStartupDeadline(() => this.#backendReady, { signal: this.#closing.signal,
|
|
219
222
|
timeoutMs: SIDEBAND_OPEN_TIMEOUT_MS,
|
|
220
223
|
timeoutError: new VoiceError("session_ready_timeout", "The Realtime session did not become ready in time.") })),
|
|
221
224
|
]);
|
|
@@ -227,6 +230,7 @@ export class BrowserVoiceSession {
|
|
|
227
230
|
if (this.#closed) return;
|
|
228
231
|
this.#sampleLevels();
|
|
229
232
|
this.#status(`Voice active (${this.#options.voice})`);
|
|
233
|
+
this.#options.onReady?.();
|
|
230
234
|
}
|
|
231
235
|
|
|
232
236
|
async #prepareMedia(capture) {
|
|
@@ -249,6 +253,17 @@ export class BrowserVoiceSession {
|
|
|
249
253
|
this.#peer = peer;
|
|
250
254
|
for (const track of microphone.getAudioTracks()) peer.addTrack(track, microphone);
|
|
251
255
|
this.#channel = peer.createDataChannel("oai-events");
|
|
256
|
+
const channel = this.#channel;
|
|
257
|
+
channel.addEventListener("message", (event) => {
|
|
258
|
+
if (this.#directControl && !this.#closed && this.#channel === channel) {
|
|
259
|
+
this.#receiveControl(event.data, () => this.#channel === channel);
|
|
260
|
+
}
|
|
261
|
+
});
|
|
262
|
+
channel.addEventListener("close", () => {
|
|
263
|
+
if (this.#directControl && !this.#closed && !this.#closing.signal.aborted && this.#channel === channel) {
|
|
264
|
+
this.#options.onTerminated("Voice control connection closed — tap Voice to reconnect");
|
|
265
|
+
}
|
|
266
|
+
});
|
|
252
267
|
peer.addEventListener("track", (event) => {
|
|
253
268
|
if (this.#closed || this.#closing.signal.aborted || this.#peer !== peer) {
|
|
254
269
|
event.track.stop();
|
|
@@ -410,7 +425,10 @@ export class BrowserVoiceSession {
|
|
|
410
425
|
}
|
|
411
426
|
let sent = 0;
|
|
412
427
|
for (const frame of effects.frames ?? []) {
|
|
413
|
-
if (this.#
|
|
428
|
+
if (this.#directControl && this.#channel?.readyState === "open") {
|
|
429
|
+
this.#channel.send(frame);
|
|
430
|
+
sent += 1;
|
|
431
|
+
} else if (!this.#directControl && this.#sideband?.readyState === WebSocket.OPEN) {
|
|
414
432
|
this.#sideband.send(frame);
|
|
415
433
|
sent += 1;
|
|
416
434
|
}
|
|
@@ -457,20 +475,7 @@ export class BrowserVoiceSession {
|
|
|
457
475
|
let opened = false;
|
|
458
476
|
sideband.addEventListener("message", (event) => {
|
|
459
477
|
if (!this.#closed && generation === this.#sidebandGeneration) {
|
|
460
|
-
this.#
|
|
461
|
-
await this.#admission;
|
|
462
|
-
if (this.#closed || generation !== this.#sidebandGeneration) return;
|
|
463
|
-
if (await this.#core.requiresAgentAdmission(event.data)) {
|
|
464
|
-
// Only delegations wait for durable admission. Speech deltas and
|
|
465
|
-
// agent output must continue while that independent request waits.
|
|
466
|
-
void this.#enqueue(async () => {
|
|
467
|
-
await this.#options.beforeAgentTurn?.();
|
|
468
|
-
return this.#core.realtimeMessage(event.data);
|
|
469
|
-
}, true).catch(() => {});
|
|
470
|
-
return;
|
|
471
|
-
}
|
|
472
|
-
return this.#core.realtimeMessage(event.data);
|
|
473
|
-
});
|
|
478
|
+
this.#receiveControl(event.data, () => generation === this.#sidebandGeneration);
|
|
474
479
|
}
|
|
475
480
|
});
|
|
476
481
|
sideband.addEventListener("close", () => {
|
|
@@ -492,6 +497,46 @@ export class BrowserVoiceSession {
|
|
|
492
497
|
}
|
|
493
498
|
}
|
|
494
499
|
|
|
500
|
+
#receiveControl(payload, isCurrent) {
|
|
501
|
+
this.#applyLive(async () => {
|
|
502
|
+
if (!this.#directControl) await this.#admission;
|
|
503
|
+
if (this.#closed || !isCurrent()) return;
|
|
504
|
+
if (await this.#core.requiresAgentAdmission(payload)) {
|
|
505
|
+
// Retain accepted requests on close, but keep captions and interruptions
|
|
506
|
+
// independent of the durable route and its ordered task queue.
|
|
507
|
+
void this.#enqueue(async () => {
|
|
508
|
+
await this.#admission;
|
|
509
|
+
await this.#options.beforeAgentTurn?.();
|
|
510
|
+
return this.#core.realtimeMessage(payload);
|
|
511
|
+
}, true).catch(() => {});
|
|
512
|
+
return;
|
|
513
|
+
}
|
|
514
|
+
return this.#core.realtimeMessage(payload);
|
|
515
|
+
});
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
async #openDataChannel() {
|
|
519
|
+
const channel = this.#channel;
|
|
520
|
+
await withStartupDeadline(() => new Promise((resolve, reject) => {
|
|
521
|
+
const cleanup = () => {
|
|
522
|
+
channel.removeEventListener("open", opened);
|
|
523
|
+
channel.removeEventListener("close", closed);
|
|
524
|
+
this.#closing.signal.removeEventListener("abort", closed);
|
|
525
|
+
};
|
|
526
|
+
const opened = () => { cleanup(); resolve(); };
|
|
527
|
+
const closed = () => { cleanup(); reject(new Error("voice data channel closed before opening")); };
|
|
528
|
+
channel.addEventListener("open", opened);
|
|
529
|
+
channel.addEventListener("close", closed);
|
|
530
|
+
this.#closing.signal.addEventListener("abort", closed, { once: true });
|
|
531
|
+
if (channel.readyState === "open") opened();
|
|
532
|
+
else if (channel.readyState === "closed" || this.#closing.signal.aborted) closed();
|
|
533
|
+
}), {
|
|
534
|
+
signal: this.#closing.signal, timeoutMs: SIDEBAND_OPEN_TIMEOUT_MS,
|
|
535
|
+
timeoutError: new VoiceError("data_channel_open_timeout", "The Realtime voice data channel did not open in time."),
|
|
536
|
+
});
|
|
537
|
+
if (!this.#closed && this.#channel === channel) await this.#applyLive(() => this.#core.sidebandOpened());
|
|
538
|
+
}
|
|
539
|
+
|
|
495
540
|
#status(message) {
|
|
496
541
|
if (!this.#closed) this.#options.onStatus(message);
|
|
497
542
|
}
|
|
@@ -512,8 +557,13 @@ export class BrowserVoiceSession {
|
|
|
512
557
|
this.#flushTimer = undefined;
|
|
513
558
|
if (this.#reconnectTimer !== undefined) window.clearTimeout(this.#reconnectTimer);
|
|
514
559
|
this.#reconnectTimer = undefined;
|
|
515
|
-
|
|
560
|
+
// Match native close: release provider media before durable tail cleanup.
|
|
561
|
+
if (this.#directControl && this.#channel?.readyState === "open") {
|
|
562
|
+
try { this.#channel.send('{"type":"session.close"}'); } catch {}
|
|
563
|
+
}
|
|
564
|
+
const channel = this.#channel;
|
|
516
565
|
this.#channel = undefined;
|
|
566
|
+
channel?.close();
|
|
517
567
|
this.#peer?.close();
|
|
518
568
|
this.#peer = undefined;
|
|
519
569
|
stopStream(this.#microphone);
|
package/browser/WorkerAgent.mjs
CHANGED
|
@@ -1317,7 +1317,8 @@ function serializeConfig(options) {
|
|
|
1317
1317
|
...connection
|
|
1318
1318
|
} = setup;
|
|
1319
1319
|
if (hostAuth === true) {
|
|
1320
|
-
if (typeof connection.
|
|
1320
|
+
if (typeof connection.createResponse === "function"
|
|
1321
|
+
|| typeof connection.createWebSocket === "function"
|
|
1321
1322
|
|| typeof connection.WebSocketImpl === "function") {
|
|
1322
1323
|
throw new TypeError("Worker Agent host-managed transport callbacks must live inside a custom Worker");
|
|
1323
1324
|
}
|
package/browser/host.d.mts
CHANGED
|
@@ -41,6 +41,12 @@ export type BrowserWebSocketRequest = BrowserWebSocketMetadata & (
|
|
|
41
41
|
}
|
|
42
42
|
);
|
|
43
43
|
|
|
44
|
+
/** One streaming HTTPS Responses request. The host must preserve cancellation. */
|
|
45
|
+
export type BrowserHttpRequest = Exclude<BrowserWebSocketRequest, { authorization: "preconnect" }> & {
|
|
46
|
+
body: string;
|
|
47
|
+
signal: AbortSignal;
|
|
48
|
+
};
|
|
49
|
+
|
|
44
50
|
export type BrowserWebSocketConnection = {
|
|
45
51
|
socket: WebSocket;
|
|
46
52
|
status?: number | undefined;
|
|
@@ -59,6 +65,7 @@ export function createBrowserHost(options?: {
|
|
|
59
65
|
sessionId: string,
|
|
60
66
|
request: BrowserWebSocketRequest,
|
|
61
67
|
) => WebSocket | BrowserWebSocketConnection | Promise<WebSocket | BrowserWebSocketConnection>;
|
|
68
|
+
createResponse?: (endpoint: string, sessionId: string, request: BrowserHttpRequest) => Promise<Response>;
|
|
62
69
|
filesystem?: Workspace;
|
|
63
70
|
filesystemTools?: boolean;
|
|
64
71
|
onEvent?: (eventJson: string) => void;
|
package/browser/host.mjs
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createResponsesHttp, responsesHttpHeaders } from "../runtime/responses-http.mjs";
|
|
1
2
|
import { createCodeRuntime, toolResult } from "../runtime/code-runtime.mjs";
|
|
2
3
|
import {
|
|
3
4
|
toolRouterBrand,
|
|
@@ -45,6 +46,19 @@ export function createBrowserHost(options = {}) {
|
|
|
45
46
|
if (!options.mpp && !createWebSocket) {
|
|
46
47
|
throw new Error("WebSocket is unavailable in this runtime");
|
|
47
48
|
}
|
|
49
|
+
const http = createResponsesHttp((endpoint, apiKey, sessionId, metadata, body, signal) => {
|
|
50
|
+
if (disposal) throw new Error("Nanocodex host is already disposed");
|
|
51
|
+
if (options.mpp) throw JSON.stringify({ kind: "transport", detail: "MPP HTTPS transport is unavailable", reconnectable: false });
|
|
52
|
+
if (options.createResponse) {
|
|
53
|
+
const authorization = options.hostAuth
|
|
54
|
+
? { authorization: "host_managed" }
|
|
55
|
+
: { authorization: "bearer", bearerToken: apiKey };
|
|
56
|
+
return options.createResponse(endpoint, sessionId, { ...metadata, ...authorization, body, signal });
|
|
57
|
+
}
|
|
58
|
+
if (options.hostAuth) throw JSON.stringify({ kind: "transport", detail: "host-managed HTTPS requires createResponse", reconnectable: false });
|
|
59
|
+
return fetch(endpoint, { method: "POST", headers: responsesHttpHeaders(apiKey, sessionId, metadata),
|
|
60
|
+
body, signal, redirect: "error" });
|
|
61
|
+
});
|
|
48
62
|
const connections = new Map();
|
|
49
63
|
const openingAttempts = new Set();
|
|
50
64
|
const connectingConnections = new Set();
|
|
@@ -375,6 +389,7 @@ export function createBrowserHost(options = {}) {
|
|
|
375
389
|
|
|
376
390
|
function dispose() {
|
|
377
391
|
if (disposal) return disposal;
|
|
392
|
+
http.dispose();
|
|
378
393
|
disposalError = new Error("Nanocodex host was disposed during WebSocket connection");
|
|
379
394
|
disposal = Promise.resolve().then(async () => {
|
|
380
395
|
const cleanups = [];
|
|
@@ -480,6 +495,10 @@ export function createBrowserHost(options = {}) {
|
|
|
480
495
|
if (references > 0) references -= 1;
|
|
481
496
|
return references === 0 ? dispose() : Promise.resolve();
|
|
482
497
|
},
|
|
498
|
+
httpOpen: http.httpOpen,
|
|
499
|
+
httpReady: http.httpReady,
|
|
500
|
+
httpNext: http.httpNext,
|
|
501
|
+
httpClose: http.httpClose,
|
|
483
502
|
connect,
|
|
484
503
|
preconnect,
|
|
485
504
|
send,
|
package/cloud/Decorator.d.mts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { logout } from "./actions/account.mjs";
|
|
2
2
|
import type { create as createAgent } from "./actions/agent.mjs";
|
|
3
|
+
import type { request as connectorRequest } from "./actions/connector.mjs";
|
|
3
4
|
import type { connect, disconnect, reconnect } from "./actions/connection.mjs";
|
|
4
5
|
import type { revoke } from "./actions/grant.mjs";
|
|
5
6
|
import type { fund, getConfig } from "./actions/machineUsd.mjs";
|
|
@@ -11,6 +12,14 @@ import type { Connection, HostConnection } from "./types.mjs";
|
|
|
11
12
|
export type ConnectActions<connection extends Connection | HostConnection = Connection> = {
|
|
12
13
|
account: { logout(): logout.ReturnType };
|
|
13
14
|
agent: { create(options: createAgent.Options): Promise<createAgent.ReturnType> };
|
|
15
|
+
connector: { request(options: connectorRequest.Options): connectorRequest.ReturnType };
|
|
16
|
+
connectors: {
|
|
17
|
+
request(options: connectorRequest.Options): connectorRequest.ReturnType;
|
|
18
|
+
} & {
|
|
19
|
+
readonly [service in connectorRequest.Options["connector"]]: {
|
|
20
|
+
request(options: Omit<connectorRequest.Options, "connector">): connectorRequest.ReturnType;
|
|
21
|
+
};
|
|
22
|
+
};
|
|
14
23
|
connection: {
|
|
15
24
|
connect(options: connect.Options): Promise<connection>;
|
|
16
25
|
disconnect(options?: disconnect.Options | undefined): disconnect.ReturnType;
|
package/cloud/Decorator.mjs
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
import { API_CONNECTORS } from "./internal.mjs";
|
|
1
2
|
import * as account from "./actions/account.mjs";
|
|
2
3
|
import * as agent from "./actions/agent.mjs";
|
|
4
|
+
import * as connector from "./actions/connector.mjs";
|
|
3
5
|
import * as connection from "./actions/connection.mjs";
|
|
4
6
|
import * as grant from "./actions/grant.mjs";
|
|
5
7
|
import * as machineUsd from "./actions/machineUsd.mjs";
|
|
@@ -14,6 +16,13 @@ export function connectActions() {
|
|
|
14
16
|
agent: {
|
|
15
17
|
create: (options) => agent.create(client, options),
|
|
16
18
|
},
|
|
19
|
+
connector: { request: (options) => connector.request(client, options) },
|
|
20
|
+
connectors: {
|
|
21
|
+
request: (options) => connector.request(client, options),
|
|
22
|
+
...Object.fromEntries(API_CONNECTORS.map(service => [service, {
|
|
23
|
+
request: (options) => connector.request(client, { ...options, connector: service }),
|
|
24
|
+
}])),
|
|
25
|
+
},
|
|
17
26
|
connection: {
|
|
18
27
|
connect: (options) => connection.connect(client, options),
|
|
19
28
|
disconnect: (options) => connection.disconnect(client, options),
|
package/cloud/Transport.mjs
CHANGED
|
@@ -376,6 +376,9 @@ function requestedConnectors(value) {
|
|
|
376
376
|
"gcontacts",
|
|
377
377
|
"slack",
|
|
378
378
|
"x",
|
|
379
|
+
"spotify",
|
|
380
|
+
"soundcloud",
|
|
381
|
+
"link",
|
|
379
382
|
"chatgpt",
|
|
380
383
|
];
|
|
381
384
|
if (value.some((provider) => !supported.includes(provider))) {
|
|
@@ -413,6 +416,9 @@ function connectorName(provider) {
|
|
|
413
416
|
gcontacts: "Google Contacts",
|
|
414
417
|
slack: "Slack",
|
|
415
418
|
x: "X",
|
|
419
|
+
spotify: "Spotify",
|
|
420
|
+
soundcloud: "SoundCloud",
|
|
421
|
+
link: "Stripe Link",
|
|
416
422
|
chatgpt: "ChatGPT",
|
|
417
423
|
})[provider];
|
|
418
424
|
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { Client } from "../Client.mjs";
|
|
2
|
+
import type { CloudAccount } from "../types.mjs";
|
|
3
|
+
|
|
4
|
+
export declare namespace request {
|
|
5
|
+
type Options = Readonly<{
|
|
6
|
+
connector: Exclude<CloudAccount, "chatgpt">;
|
|
7
|
+
path: string;
|
|
8
|
+
method?: "GET" | "HEAD" | "POST" | "PUT" | "PATCH" | "DELETE" | "OPTIONS" | undefined;
|
|
9
|
+
/** Required when the grant includes more than one account for this service. */
|
|
10
|
+
connectionId?: string | undefined;
|
|
11
|
+
body?: Record<string, unknown> | undefined;
|
|
12
|
+
signal?: AbortSignal | undefined;
|
|
13
|
+
}>;
|
|
14
|
+
type ReturnType = Promise<Response>;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** Returns the provider response (including HTTP failures). Never retries writes. */
|
|
18
|
+
export function request(client: Client, options: request.Options): request.ReturnType;
|