nanocodex 0.6.4 → 0.6.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -11
- package/browser/Agent.d.mts +1 -1
- package/browser/ElevenLabsPlayback.mjs +119 -0
- package/browser/InlineAgent.mjs +22 -17
- package/browser/Transport.d.mts +2 -0
- package/browser/Transport.mjs +3 -2
- package/browser/Voice.d.mts +4 -0
- package/browser/Voice.mjs +38 -6
- package/browser/VoiceSession.mjs +37 -7
- package/browser/host.d.mts +20 -4
- package/browser/host.mjs +156 -18
- package/cloud/types.d.mts +1 -0
- package/cloudflare/Agent.d.mts +14 -9
- package/cloudflare/Agent.mjs +354 -96
- package/cloudflare/agent-settings.d.mts +15 -0
- package/cloudflare/agent-settings.mjs +135 -0
- package/cloudflare/chat-reasoning.mjs +20 -0
- package/cloudflare/durable-placement.d.mts +12 -0
- package/cloudflare/durable-placement.mjs +82 -0
- package/cloudflare/egress.d.mts +31 -0
- package/cloudflare/gateway-binding-responses.mjs +84 -0
- package/cloudflare/gateway-responses.d.mts +48 -0
- package/cloudflare/gateway-responses.mjs +191 -0
- package/cloudflare/index.d.mts +6 -0
- package/cloudflare/index.mjs +3 -0
- package/cloudflare/managed-auth.d.mts +15 -0
- package/cloudflare/managed-auth.mjs +119 -0
- package/cloudflare/managed-live.d.mts +7 -0
- package/cloudflare/managed-live.mjs +56 -0
- package/cloudflare/prepared-connection.mjs +49 -0
- package/cloudflare/provider-stream.mjs +353 -0
- package/cloudflare/rpc.d.mts +9 -0
- package/cloudflare/rpc.mjs +14 -0
- package/cloudflare/tool-alias-diagnostic.mjs +21 -0
- package/cloudflare/tracing.d.mts +16 -0
- package/cloudflare/tracing.mjs +47 -0
- package/cloudflare/workers-ai-responses.d.mts +28 -0
- package/cloudflare/workers-ai-responses.mjs +399 -0
- package/host/Agent.d.mts +1 -1
- package/index.d.mts +2 -0
- package/internal.mjs +53 -14
- package/managed/Agent.d.mts +25 -69
- package/managed/Agent.mjs +82 -196
- package/managed/README.md +9 -38
- package/managed/Voice.mjs +5 -2
- package/managed/index.d.mts +0 -3
- package/managed/internal.mjs +2 -1
- package/node/Agent.d.mts +1 -1
- package/node/Agent.mjs +9 -3
- package/node/host.mjs +5 -0
- package/package.json +34 -2
- package/pkg-node/nanocodex.d.ts +22 -18
- package/pkg-node/nanocodex.js +118 -92
- package/pkg-node/nanocodex_bg.wasm.d.ts +6 -5
- package/pkg-web/.nanocodex-bindgen-stamp +7 -1
- package/pkg-web/nanocodex-build.json +1 -1
- package/pkg-web/nanocodex.d.ts +28 -23
- package/pkg-web/nanocodex.js +117 -91
- package/pkg-web/nanocodex_bg.js +117 -91
- package/pkg-web/nanocodex_bg.wasm +0 -0
- package/pkg-web/nanocodex_bg.wasm.d.ts +6 -5
- package/pkg-web/nanocodex_worker.js +1 -1
- package/runtime/before-compaction.mjs +49 -0
- package/runtime/code-evaluator.worker.mjs +2 -9
- package/runtime/managed-transport.mjs +10 -2
- package/runtime/mcp-runtime.mjs +20 -4
- package/runtime/quickjs-evaluator.mjs +11 -5
- package/runtime/response-controls.mjs +67 -4
- package/runtime/steer-receipt.mjs +13 -0
- package/runtime/subagent-routing.d.mts +38 -0
- package/runtime/subagent-routing.mjs +81 -0
- package/runtime/subagents.d.mts +1 -1
- package/runtime/tempo-provider.d.mts +1 -1
- package/runtime/tempo-provider.mjs +1 -1
- package/tools/environment.d.mts +17 -2
- package/tools/environment.mjs +30 -3
- package/tools/index.d.mts +2 -0
- package/types.d.mts +30 -2
package/README.md
CHANGED
|
@@ -10,7 +10,7 @@ import { Actions, Agent, Transport } from "nanocodex/node";
|
|
|
10
10
|
|
|
11
11
|
const agent = await Agent.create({
|
|
12
12
|
transport: Transport.openAi({ apiKey: process.env.OPENAI_API_KEY }),
|
|
13
|
-
model: "gpt-
|
|
13
|
+
model: "gpt-6-luna",
|
|
14
14
|
instructions: "You are a Rust coding agent. Preserve unrelated work and run relevant tests.",
|
|
15
15
|
reasoningMode: "pro",
|
|
16
16
|
thinking: "high",
|
|
@@ -236,8 +236,13 @@ requesting speech. Context and speech are split into provider-sized messages.
|
|
|
236
236
|
These commands retain frames until sent and preserve them
|
|
237
237
|
across reconnects when using the sideband transport. They are also methods on
|
|
238
238
|
the resource and on
|
|
239
|
-
`useVoice` from `nanocodex-react`.
|
|
240
|
-
|
|
239
|
+
`useVoice` from `nanocodex-react`. Choose `outputProvider: "elevenlabs"` and `elevenLabsVoiceId` to synthesize
|
|
240
|
+
spoken output with an ElevenLabs account voice, including an instant clone.
|
|
241
|
+
The account voice settings panel connects the API key, lists voices, and uploads
|
|
242
|
+
cloning samples after explicit consent. Keys are encrypted on the server and
|
|
243
|
+
never included in saved voice settings. ChatGPT still owns live input and agent
|
|
244
|
+
handoffs; `voice` continues to select its built-in voice. The default output
|
|
245
|
+
provider is `openai`. Platform audio configuration is not accepted.
|
|
241
246
|
|
|
242
247
|
`Voice.create(...)` remains the equivalent namespaced resource constructor, and
|
|
243
248
|
`Voice.voices` is the exact ChatGPT V3 voice catalog. The constructor accepts a
|
|
@@ -247,6 +252,20 @@ normal browser Agent, an account-owned managed Agent, or a grant-scoped
|
|
|
247
252
|
one-use ticket for that path. The browser binding never receives ChatGPT
|
|
248
253
|
credentials or places its reusable grant bearer in a WebSocket URL.
|
|
249
254
|
|
|
255
|
+
### Content-free Worker tracing
|
|
256
|
+
|
|
257
|
+
`nanocodex/cloudflare/tracing` wraps native `tracing.enterSpan(name, callback)`
|
|
258
|
+
with sanitized exception events. Use static operation names: they become
|
|
259
|
+
exception codes on failure. The original error is rethrown; its message and
|
|
260
|
+
stack are never passed to `recordException`.
|
|
261
|
+
|
|
262
|
+
`setSpanAttributes(span, attributes)` sets bulk metadata, while
|
|
263
|
+
`annotateActiveSpan(attributes)` annotates the current invocation or active
|
|
264
|
+
span without changing nesting. `recordSpanException(span, code)` handles
|
|
265
|
+
failures represented by results rather than thrown errors. Supply only static
|
|
266
|
+
codes and content-free attributes. On older workerd versions, these retain
|
|
267
|
+
individual attributes and `error.type` without requiring the September APIs.
|
|
268
|
+
|
|
250
269
|
### Durable Cloudflare Agent
|
|
251
270
|
|
|
252
271
|
`nanocodex/cloudflare` is the standard Durable Object consumer. It keeps the
|
|
@@ -302,10 +321,12 @@ Cloudflare Agents default to direct tool mode because Workers prohibit dynamic
|
|
|
302
321
|
`eval`/`new Function`. Caller-defined tools therefore work without a code
|
|
303
322
|
evaluator. Select `toolMode: "code"` only when also supplying an evaluator that
|
|
304
323
|
is explicitly compatible with the deployed Worker runtime. Runtime-owned
|
|
305
|
-
Subagents are installed by default, including on a durable root.
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
324
|
+
Subagents are installed by default, including on a durable root. All children are
|
|
325
|
+
ephemeral: their identities, topology, conversations, results, and routing exist
|
|
326
|
+
only for the lifetime of the root runtime. Root shutdown or restart discards the
|
|
327
|
+
entire child tree; only the root's own durable history resumes. Existing child
|
|
328
|
+
checkpoints from older versions are discarded. Within a live runtime, completed
|
|
329
|
+
children remain available for follow-up messages until closed. Use
|
|
309
330
|
`Subagents.create({ maxConcurrency })` in `tools` to set an explicit finite
|
|
310
331
|
concurrency limit. Active subagent turns are unlimited by default.
|
|
311
332
|
|
|
@@ -349,6 +370,11 @@ native and WASM consumers run the same Rust implementation and receive the
|
|
|
349
370
|
same seven tools: `spawn_agent`, `submit_result`, `send_agent_message`,
|
|
350
371
|
`list_agents`, `wait_agent`, `interrupt_agent`, and `close_agent`.
|
|
351
372
|
|
|
373
|
+
Children call `submit_result({output})`; the runtime supplies the trusted
|
|
374
|
+
instruction revision. The response is `{accepted: true, status: "accepted"}` or
|
|
375
|
+
`{accepted: false, status: "superseded"}`. Superseded submissions are normal
|
|
376
|
+
continuations: incorporate the updated instructions and submit again.
|
|
377
|
+
|
|
352
378
|
Inside a caller-owned Worker or server isolate, host capabilities stay as
|
|
353
379
|
ordinary functions without crossing another compatibility protocol:
|
|
354
380
|
|
|
@@ -686,7 +712,7 @@ const mpp = tempoProvider.session;
|
|
|
686
712
|
|
|
687
713
|
const agent = await Agent.create({
|
|
688
714
|
transport: Transport.mpp({ session: tempoProvider }),
|
|
689
|
-
thinking: "
|
|
715
|
+
thinking: "low",
|
|
690
716
|
fastMode: true,
|
|
691
717
|
tools,
|
|
692
718
|
});
|
|
@@ -736,8 +762,8 @@ of its wallet adapter, and constructs both payment paths from that provider's
|
|
|
736
762
|
adapter-neutral `getMppxParameters()` contract. The lower-level
|
|
737
763
|
`createTempoProvider({ session, payment })` remains available when the
|
|
738
764
|
application constructs MPPx itself. Both explicitly select Tempo provider mode.
|
|
739
|
-
In that mode Nanocodex automatically adds its built-in Mercator MCP
|
|
740
|
-
with the same wallet and payment policy. The provider also exposes an MPP-aware
|
|
765
|
+
In that mode Nanocodex automatically adds its built-in Mercator MCP at
|
|
766
|
+
`https://mercator.sh/mcp` and wraps it with the same wallet and payment policy. The provider also exposes an MPP-aware
|
|
741
767
|
`fetch`; Mercator's paid REST handoffs use that same method rather than a second
|
|
742
768
|
wallet or payment configuration. Its MCP transport remains wrapped at the MCP
|
|
743
769
|
protocol layer, so browser requests do not need an `Accept-Payment` CORS header.
|
|
@@ -1155,7 +1181,7 @@ package manager or build step:
|
|
|
1155
1181
|
|
|
1156
1182
|
```html
|
|
1157
1183
|
<script type="module">
|
|
1158
|
-
import { Agent, Transport } from "https://cdn.jsdelivr.net/npm/nanocodex@0.6.
|
|
1184
|
+
import { Agent, Transport } from "https://cdn.jsdelivr.net/npm/nanocodex@0.6.6/host/index.mjs";
|
|
1159
1185
|
const agent = await Agent.create({
|
|
1160
1186
|
transport: Transport.hostManaged({
|
|
1161
1187
|
websocketUrl: "/api/responses",
|
package/browser/Agent.d.mts
CHANGED
|
@@ -33,7 +33,7 @@ export declare namespace create {
|
|
|
33
33
|
transport: ManagedTransport;
|
|
34
34
|
tools?: Tools | undefined;
|
|
35
35
|
}>;
|
|
36
|
-
type Options = AgentOptions & WorkerToolExposureOptions & {
|
|
36
|
+
type Options = Omit<AgentOptions, "beforeCompaction"> & WorkerToolExposureOptions & {
|
|
37
37
|
/** Precompiled browser module; WebAssembly modules are structured-clone-safe. */
|
|
38
38
|
module?: WebAssembly.Module | undefined;
|
|
39
39
|
/** Fixed browser workspace facts, including its AGENTS.md snapshot. */
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/** Plays account-proxied PCM without exposing provider credentials to the browser. */
|
|
2
|
+
export class ElevenLabsPlayback {
|
|
3
|
+
#synthesize;
|
|
4
|
+
#onError;
|
|
5
|
+
#context;
|
|
6
|
+
#abort = new AbortController();
|
|
7
|
+
#queue = Promise.resolve();
|
|
8
|
+
#sources = new Set();
|
|
9
|
+
#nextTime = 0;
|
|
10
|
+
#caption;
|
|
11
|
+
#offset = 0;
|
|
12
|
+
#suppressedThrough = -1;
|
|
13
|
+
#pending = 0;
|
|
14
|
+
#pendingChars = 0;
|
|
15
|
+
#closed = false;
|
|
16
|
+
|
|
17
|
+
constructor(synthesize, onError) {
|
|
18
|
+
this.#synthesize = synthesize;
|
|
19
|
+
this.#onError = onError;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
prime() {
|
|
23
|
+
if (this.#closed) return;
|
|
24
|
+
this.#context ??= new AudioContext({ sampleRate: 24000 });
|
|
25
|
+
return this.#context.resume();
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
transcript(entry) {
|
|
29
|
+
if (this.#closed || entry.speaker !== "assistant" || entry.id <= this.#suppressedThrough) return;
|
|
30
|
+
if (this.#caption !== undefined && entry.id < this.#caption) return;
|
|
31
|
+
if (entry.id !== this.#caption) { this.#caption = entry.id; this.#offset = 0; }
|
|
32
|
+
const remaining = entry.text.slice(this.#offset);
|
|
33
|
+
// Buffer partial words; emit complete sentences before the turn finishes.
|
|
34
|
+
const boundary = entry.is_partial ? [...remaining.matchAll(/[.!?\n](?:\s|$)/g)].at(-1) : undefined;
|
|
35
|
+
const length = entry.is_partial ? (boundary ? boundary.index + boundary[0].length : 0) : remaining.length;
|
|
36
|
+
const text = remaining.slice(0, length).trim();
|
|
37
|
+
this.#offset += length;
|
|
38
|
+
if (!text) return;
|
|
39
|
+
if (this.#pending >= 32 || this.#pendingChars + text.length > 32768) {
|
|
40
|
+
this.interrupt();
|
|
41
|
+
this.#onError(new Error("ElevenLabs playback queue exceeded its limit"));
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
this.#pending++;
|
|
45
|
+
this.#pendingChars += text.length;
|
|
46
|
+
const signal = this.#abort.signal;
|
|
47
|
+
this.#queue = this.#queue.then(async () => {
|
|
48
|
+
if (signal.aborted) return;
|
|
49
|
+
this.#context ??= new AudioContext({ sampleRate: 24000 });
|
|
50
|
+
await this.#context.resume();
|
|
51
|
+
if (signal.aborted) return;
|
|
52
|
+
const response = await this.#synthesize(text, signal);
|
|
53
|
+
if (!response.ok) throw new Error(`ElevenLabs synthesis failed (${response.status})`);
|
|
54
|
+
const reader = response.body?.getReader();
|
|
55
|
+
if (!reader) throw new Error("ElevenLabs returned no audio stream");
|
|
56
|
+
const cancel = () => { void reader.cancel().catch(() => {}); };
|
|
57
|
+
signal.addEventListener("abort", cancel, { once: true });
|
|
58
|
+
let carry;
|
|
59
|
+
try {
|
|
60
|
+
while (!signal.aborted) {
|
|
61
|
+
const { value, done } = await reader.read();
|
|
62
|
+
if (done) break;
|
|
63
|
+
if (signal.aborted) break;
|
|
64
|
+
if (value.byteLength > 1024 * 1024) throw new Error("ElevenLabs audio chunk exceeded its limit");
|
|
65
|
+
const bytes = carry === undefined ? value : Uint8Array.from([carry, ...value]);
|
|
66
|
+
const size = bytes.length - bytes.length % 2;
|
|
67
|
+
carry = size < bytes.length ? bytes[size] : undefined;
|
|
68
|
+
if (!size) continue;
|
|
69
|
+
// Keep decoded/scheduled audio bounded even when synthesis outruns playback.
|
|
70
|
+
for (let offset = 0; offset < size && !signal.aborted; offset += 9600) {
|
|
71
|
+
while (!signal.aborted && (this.#nextTime - this.#context.currentTime > 2 || this.#sources.size >= 128)) {
|
|
72
|
+
await new Promise((resolve) => {
|
|
73
|
+
const done = () => { clearTimeout(timer); signal.removeEventListener("abort", done); resolve(); };
|
|
74
|
+
const timer = setTimeout(done, 25);
|
|
75
|
+
signal.addEventListener("abort", done, { once: true });
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
if (signal.aborted) break;
|
|
79
|
+
const length = Math.min(9600, size - offset);
|
|
80
|
+
const buffer = this.#context.createBuffer(1, length / 2, 24000);
|
|
81
|
+
const output = buffer.getChannelData(0);
|
|
82
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset + offset, length);
|
|
83
|
+
for (let i = 0; i < output.length; i++) output[i] = view.getInt16(i * 2, true) / 32768;
|
|
84
|
+
const source = this.#context.createBufferSource();
|
|
85
|
+
source.buffer = buffer;
|
|
86
|
+
source.connect(this.#context.destination);
|
|
87
|
+
this.#sources.add(source);
|
|
88
|
+
source.onended = () => { this.#sources.delete(source); source.disconnect(); };
|
|
89
|
+
this.#nextTime = Math.max(this.#nextTime, this.#context.currentTime + 0.02);
|
|
90
|
+
source.start(this.#nextTime);
|
|
91
|
+
this.#nextTime += buffer.duration;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
} finally { signal.removeEventListener("abort", cancel); await reader.cancel().catch(() => {}); reader.releaseLock(); }
|
|
95
|
+
}).catch((error) => { if (!signal.aborted) this.#onError(error); }).finally(() => {
|
|
96
|
+
if (!signal.aborted) { this.#pending--; this.#pendingChars -= text.length; }
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
interrupt() {
|
|
101
|
+
this.#suppressedThrough = Math.max(this.#suppressedThrough, this.#caption ?? -1);
|
|
102
|
+
this.#pending = 0;
|
|
103
|
+
this.#pendingChars = 0;
|
|
104
|
+
this.#abort.abort();
|
|
105
|
+
this.#abort = new AbortController();
|
|
106
|
+
this.#queue = Promise.resolve();
|
|
107
|
+
for (const source of this.#sources) { try { source.stop(); } catch {} source.disconnect(); }
|
|
108
|
+
this.#sources.clear();
|
|
109
|
+
this.#nextTime = 0;
|
|
110
|
+
// Caption IDs are monotonic: every late delta/final for this epoch stays silent.
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
close() {
|
|
114
|
+
this.#closed = true;
|
|
115
|
+
this.interrupt();
|
|
116
|
+
void this.#context?.close();
|
|
117
|
+
this.#context = undefined;
|
|
118
|
+
}
|
|
119
|
+
}
|
package/browser/InlineAgent.mjs
CHANGED
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
registerDefinitionHost,
|
|
17
17
|
releaseDefinitionHost,
|
|
18
18
|
releaseHostSession,
|
|
19
|
+
releaseHostSessions,
|
|
19
20
|
toWasmConfig,
|
|
20
21
|
} from "../internal.mjs";
|
|
21
22
|
import { createBrowserHost } from "./host.mjs";
|
|
@@ -45,8 +46,10 @@ export async function create(options = {}) {
|
|
|
45
46
|
thinking,
|
|
46
47
|
reasoningMode,
|
|
47
48
|
fastMode,
|
|
49
|
+
rawApiEvents,
|
|
48
50
|
instructions,
|
|
49
51
|
additionalInstructions,
|
|
52
|
+
beforeCompaction,
|
|
50
53
|
sessionId,
|
|
51
54
|
workspace,
|
|
52
55
|
resume,
|
|
@@ -75,6 +78,7 @@ export async function create(options = {}) {
|
|
|
75
78
|
websocketPreconnect,
|
|
76
79
|
apiBaseUrl,
|
|
77
80
|
websocketWarmup,
|
|
81
|
+
stateless,
|
|
78
82
|
WebSocketImpl,
|
|
79
83
|
createWebSocket,
|
|
80
84
|
createResponse,
|
|
@@ -116,11 +120,14 @@ export async function create(options = {}) {
|
|
|
116
120
|
hostManagedProtocol,
|
|
117
121
|
mpp,
|
|
118
122
|
onEvent: events.emit,
|
|
123
|
+
beforeCompaction,
|
|
119
124
|
filesystem,
|
|
120
125
|
filesystemTools,
|
|
121
126
|
tools: hostTools,
|
|
122
127
|
toolProviders,
|
|
123
128
|
subagentSessions,
|
|
129
|
+
subagentRouting: internalRuntime?.subagentRouting,
|
|
130
|
+
onSocketTiming: internalRuntime?.onSocketTiming,
|
|
124
131
|
toolMode,
|
|
125
132
|
mcp: mcp === false
|
|
126
133
|
? undefined
|
|
@@ -163,8 +170,11 @@ export async function create(options = {}) {
|
|
|
163
170
|
: "wss://openai.mpp.tempo.xyz/v1/responses"),
|
|
164
171
|
apiBaseUrl,
|
|
165
172
|
websocketWarmup,
|
|
173
|
+
stateless,
|
|
166
174
|
subagents: subagentConfig,
|
|
175
|
+
subagentRouting: internalRuntime?.subagentRouting !== undefined,
|
|
167
176
|
hostDefinitionId,
|
|
177
|
+
beforeCompaction: beforeCompaction !== undefined,
|
|
168
178
|
...config,
|
|
169
179
|
durabilityHostId: durabilityOwner?.id,
|
|
170
180
|
}));
|
|
@@ -176,29 +186,19 @@ export async function create(options = {}) {
|
|
|
176
186
|
);
|
|
177
187
|
if (cloudflareReservation !== undefined) {
|
|
178
188
|
activateCloudflareAgentSession(cloudflareReservation);
|
|
179
|
-
const restoredSubagents = subagentSessions?.restore?.() ?? [];
|
|
180
|
-
if (restoredSubagents.length > 0) {
|
|
181
|
-
const restoredHostContextRefs = Object.fromEntries(
|
|
182
|
-
restoredSubagents.map((descriptor) => {
|
|
183
|
-
const hostContextRef = subagentSessions?.hostContextRef?.(descriptor.sessionId);
|
|
184
|
-
return [descriptor.sessionId, hostContextRef ?? null];
|
|
185
|
-
}),
|
|
186
|
-
);
|
|
187
|
-
await raw.restoreSubagents(
|
|
188
|
-
JSON.stringify(restoredSubagents),
|
|
189
|
-
JSON.stringify(restoredHostContextRefs),
|
|
190
|
-
);
|
|
191
|
-
}
|
|
192
189
|
}
|
|
193
190
|
return raw;
|
|
194
191
|
} catch (error) {
|
|
195
192
|
const cleanupErrors = [];
|
|
196
193
|
if (raw !== undefined) {
|
|
197
|
-
try {
|
|
194
|
+
try {
|
|
195
|
+
await raw.shutdown();
|
|
196
|
+
}
|
|
198
197
|
catch (cleanupError) { cleanupErrors.push(cleanupError); }
|
|
199
198
|
try { raw.free(); }
|
|
200
199
|
catch (cleanupError) { cleanupErrors.push(cleanupError); }
|
|
201
200
|
}
|
|
201
|
+
releaseHostSessions(host);
|
|
202
202
|
try { durabilityOwner?.abandon(); }
|
|
203
203
|
catch (cleanupError) { cleanupErrors.push(cleanupError); }
|
|
204
204
|
try { await host.dispose(); }
|
|
@@ -212,16 +212,20 @@ export async function create(options = {}) {
|
|
|
212
212
|
throw error;
|
|
213
213
|
}
|
|
214
214
|
},
|
|
215
|
+
async shutdown(raw) {
|
|
216
|
+
await raw.shutdown();
|
|
217
|
+
},
|
|
215
218
|
subscribe: events.subscribe,
|
|
216
219
|
adopt(raw) {
|
|
217
220
|
host.retain();
|
|
218
221
|
try {
|
|
219
|
-
|
|
222
|
+
// Adopted child handles are ephemeral and do not own the root store.
|
|
223
|
+
if (raw.sessionId === stableSessionId) durabilityOwner?.retain();
|
|
220
224
|
bindHostSession(host, raw.sessionId, cloudflareReservation);
|
|
221
225
|
events.addSource(raw);
|
|
222
226
|
} catch (error) {
|
|
223
227
|
events.removeSource(raw);
|
|
224
|
-
durabilityOwner?.release();
|
|
228
|
+
if (raw.sessionId === stableSessionId) durabilityOwner?.release();
|
|
225
229
|
releaseHost(host);
|
|
226
230
|
throw error;
|
|
227
231
|
}
|
|
@@ -230,7 +234,7 @@ export async function create(options = {}) {
|
|
|
230
234
|
events.removeSource(raw);
|
|
231
235
|
host.releaseSession(raw.sessionId);
|
|
232
236
|
releaseHostSession(host, raw.sessionId);
|
|
233
|
-
durabilityOwner?.release();
|
|
237
|
+
if (raw.sessionId === stableSessionId) durabilityOwner?.release();
|
|
234
238
|
releaseHost(host);
|
|
235
239
|
},
|
|
236
240
|
decorate: (agent) => agent.extend(agentActions()),
|
|
@@ -242,6 +246,7 @@ export async function create(options = {}) {
|
|
|
242
246
|
thinking,
|
|
243
247
|
reasoningMode,
|
|
244
248
|
fastMode,
|
|
249
|
+
rawApiEvents,
|
|
245
250
|
instructions,
|
|
246
251
|
additionalInstructions,
|
|
247
252
|
sessionId: stableSessionId,
|
package/browser/Transport.d.mts
CHANGED
|
@@ -26,6 +26,8 @@ export type WorkerTransport = ResponsesTransport & Readonly<{
|
|
|
26
26
|
|
|
27
27
|
type SharedEndpointOptions = Readonly<{
|
|
28
28
|
apiBaseUrl?: string | undefined;
|
|
29
|
+
/** Use HTTPS with full history replay, no stored responses, and no WebSocket warmup or preconnect. */
|
|
30
|
+
stateless?: boolean | undefined;
|
|
29
31
|
websocketUrl?: string | undefined;
|
|
30
32
|
/** Open the persistent socket as soon as Agent.create returns. Defaults to true for hostManaged. */
|
|
31
33
|
websocketPreconnect?: boolean | undefined;
|
package/browser/Transport.mjs
CHANGED
|
@@ -25,12 +25,12 @@ export function chatGpt(options) {
|
|
|
25
25
|
|
|
26
26
|
export function hostManaged(options = {}) {
|
|
27
27
|
const websocketUrl = options.websocketUrl
|
|
28
|
-
?? (options.createWebSocket ? undefined : defaultHostManagedWebSocketUrl());
|
|
28
|
+
?? (options.stateless === true || options.createWebSocket ? undefined : defaultHostManagedWebSocketUrl());
|
|
29
29
|
return createResponsesTransport({
|
|
30
30
|
hostAuth: true,
|
|
31
31
|
hostManagedProtocol: true,
|
|
32
32
|
...connection({ ...options, websocketUrl }),
|
|
33
|
-
websocketPreconnect: options.websocketPreconnect ?? true,
|
|
33
|
+
websocketPreconnect: options.stateless === true ? false : options.websocketPreconnect ?? true,
|
|
34
34
|
});
|
|
35
35
|
}
|
|
36
36
|
|
|
@@ -53,6 +53,7 @@ function connection(options = {}) {
|
|
|
53
53
|
return {
|
|
54
54
|
WebSocketImpl: options.WebSocketImpl,
|
|
55
55
|
apiBaseUrl: options.apiBaseUrl,
|
|
56
|
+
stateless: options.stateless,
|
|
56
57
|
createWebSocket: options.createWebSocket,
|
|
57
58
|
createResponse: options.createResponse,
|
|
58
59
|
websocketUrl: options.websocketUrl,
|
package/browser/Voice.d.mts
CHANGED
|
@@ -51,6 +51,8 @@ export type Event =
|
|
|
51
51
|
| Readonly<{ type: "stopped" }>;
|
|
52
52
|
export type Settings = Readonly<{
|
|
53
53
|
voice?: VoiceName | undefined;
|
|
54
|
+
outputProvider?: "openai" | "elevenlabs" | undefined;
|
|
55
|
+
elevenLabsVoiceId?: string | undefined;
|
|
54
56
|
/** Additional speaking preferences; base assistant instructions are retained. */
|
|
55
57
|
instructions?: string | undefined;
|
|
56
58
|
pace?: "slow" | "natural" | "fast" | undefined;
|
|
@@ -59,6 +61,8 @@ export type Settings = Readonly<{
|
|
|
59
61
|
acknowledgements?: boolean | undefined;
|
|
60
62
|
}>;
|
|
61
63
|
export type Options = Settings & Readonly<{
|
|
64
|
+
/** Authenticated synthesis transport returning mono signed 16-bit little-endian PCM at 24 kHz. */
|
|
65
|
+
synthesize?(text: string, signal: AbortSignal): Promise<Response>;
|
|
62
66
|
callUrl?: string | URL | undefined;
|
|
63
67
|
sidebandUrl?(callId: string, sessionId: string): string | URL | Promise<string | URL>;
|
|
64
68
|
captureMicrophone?(): Promise<MediaStream>;
|
package/browser/Voice.mjs
CHANGED
|
@@ -91,6 +91,11 @@ export function create(agent, options = {}) {
|
|
|
91
91
|
if (destroyed) throw new Error("voice resource is destroyed");
|
|
92
92
|
const selectedVoice = parameters.voice ?? options.voice ?? defaultVoice;
|
|
93
93
|
if (!voices.includes(selectedVoice)) throw new TypeError(`unsupported ChatGPT voice: ${selectedVoice}`);
|
|
94
|
+
const settings = voiceSettings({ ...options, ...parameters, voice: selectedVoice });
|
|
95
|
+
validateOutputSettings(settings);
|
|
96
|
+
if (agent.type === "connect" && settings.outputProvider === "elevenlabs" && !options.synthesize) {
|
|
97
|
+
throw new TypeError("Connect ElevenLabs output requires an explicit authorized synthesis transport");
|
|
98
|
+
}
|
|
94
99
|
if (session) return startPromise;
|
|
95
100
|
if (stopPromise) await stopPromise.catch(() => {});
|
|
96
101
|
if (destroyed) throw new Error("voice resource is destroyed");
|
|
@@ -125,7 +130,14 @@ export function create(agent, options = {}) {
|
|
|
125
130
|
core,
|
|
126
131
|
sessionId,
|
|
127
132
|
voice: selectedVoice,
|
|
128
|
-
|
|
133
|
+
synthesize: options.synthesize ?? (transport?.synthesize
|
|
134
|
+
? (text, signal) => transport.synthesize(text, signal, settings.elevenLabsVoiceId)
|
|
135
|
+
: (text, signal) => fetch("/api/voice/elevenlabs/speech", {
|
|
136
|
+
method: "POST", credentials: "same-origin", signal,
|
|
137
|
+
headers: { "Content-Type": "application/json" },
|
|
138
|
+
body: JSON.stringify({ voice_id: settings.elevenLabsVoiceId, text, output_format: "pcm_24000" }),
|
|
139
|
+
})),
|
|
140
|
+
settings,
|
|
129
141
|
...(transport?.call === undefined ? {} : { call: transport.call }),
|
|
130
142
|
...(transport?.sidebandUrl === undefined ? {} : { sidebandUrl: transport.sidebandUrl }),
|
|
131
143
|
...(options.callUrl === undefined ? {} : { callUrl: options.callUrl }),
|
|
@@ -186,10 +198,17 @@ export function create(agent, options = {}) {
|
|
|
186
198
|
// Media readiness updates the UI independently; start still waits for task
|
|
187
199
|
// admission, and a later rejection closes media through the same error path.
|
|
188
200
|
startPromise = next.start().then(publishReady).catch(async (cause) => {
|
|
189
|
-
if (session === next)
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
201
|
+
if (session === next) {
|
|
202
|
+
session = undefined;
|
|
203
|
+
if (activeResources.get(agent) === resource) activeResources.delete(agent);
|
|
204
|
+
cleanupWatcher();
|
|
205
|
+
}
|
|
206
|
+
// Keep replacement starts behind failed startup cleanup, just like stop().
|
|
207
|
+
// A stale attempt must never detach a replacement session's event watcher.
|
|
208
|
+
const closing = next.close().catch(() => next.abort());
|
|
209
|
+
if (generation === current) stopPromise = closing;
|
|
210
|
+
await closing;
|
|
211
|
+
if (stopPromise === closing) stopPromise = undefined;
|
|
193
212
|
if (destroyed || generation !== current) return;
|
|
194
213
|
if (attempt === 0 && cause?.code === "peer_connection_timeout") return start(parameters, 1);
|
|
195
214
|
const error = cause instanceof Error ? cause : new Error(String(cause));
|
|
@@ -299,6 +318,9 @@ function validateOptions(options) {
|
|
|
299
318
|
if (!options || typeof options !== "object" || Array.isArray(options)) {
|
|
300
319
|
throw new TypeError("Voice.create options must be an object");
|
|
301
320
|
}
|
|
321
|
+
if (options.synthesize !== undefined && typeof options.synthesize !== "function") {
|
|
322
|
+
throw new TypeError("voice synthesize must be a function");
|
|
323
|
+
}
|
|
302
324
|
if (options.captureMicrophone !== undefined && typeof options.captureMicrophone !== "function") {
|
|
303
325
|
throw new TypeError("voice captureMicrophone must be a function");
|
|
304
326
|
}
|
|
@@ -317,6 +339,16 @@ function browserLocationOrigin() {
|
|
|
317
339
|
}
|
|
318
340
|
|
|
319
341
|
function voiceSettings(options) {
|
|
320
|
-
return Object.fromEntries(["voice", "instructions", "pace", "updates", "handoffMode", "acknowledgements"]
|
|
342
|
+
return Object.fromEntries(["voice", "instructions", "pace", "updates", "handoffMode", "acknowledgements", "outputProvider", "elevenLabsVoiceId"]
|
|
321
343
|
.filter((key) => options[key] !== undefined).map((key) => [key, options[key]]));
|
|
322
344
|
}
|
|
345
|
+
|
|
346
|
+
function validateOutputSettings(settings) {
|
|
347
|
+
if (settings.outputProvider !== undefined && !["openai", "elevenlabs"].includes(settings.outputProvider)) {
|
|
348
|
+
throw new TypeError("unsupported voice output provider");
|
|
349
|
+
}
|
|
350
|
+
if (settings.outputProvider === "elevenlabs"
|
|
351
|
+
&& (typeof settings.elevenLabsVoiceId !== "string" || !/^[A-Za-z0-9_-]{1,128}$/.test(settings.elevenLabsVoiceId))) {
|
|
352
|
+
throw new TypeError("ElevenLabs output requires a valid voice ID");
|
|
353
|
+
}
|
|
354
|
+
}
|
package/browser/VoiceSession.mjs
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { ElevenLabsPlayback } from "./ElevenLabsPlayback.mjs";
|
|
2
|
+
|
|
1
3
|
export const MICROPHONE_CAPTURE_TIMEOUT_MS = 15_000;
|
|
2
4
|
export const ICE_GATHERING_TIMEOUT_MS = 15_000;
|
|
3
5
|
export const REALTIME_CALL_TIMEOUT_MS = 15_000;
|
|
@@ -85,8 +87,10 @@ export class BrowserVoiceSession {
|
|
|
85
87
|
#microphone;
|
|
86
88
|
#speaker;
|
|
87
89
|
#playbackEnabled = false;
|
|
90
|
+
#elevenLabs;
|
|
88
91
|
#muted = false;
|
|
89
92
|
#inputGeneration = 0;
|
|
93
|
+
#playbackRevision = 0;
|
|
90
94
|
#meterTimer;
|
|
91
95
|
#backendReady;
|
|
92
96
|
#resolveBackendReady;
|
|
@@ -102,10 +106,17 @@ export class BrowserVoiceSession {
|
|
|
102
106
|
|
|
103
107
|
constructor(options) {
|
|
104
108
|
this.#options = options;
|
|
109
|
+
if (options.settings?.outputProvider === "elevenlabs") {
|
|
110
|
+
if (typeof options.synthesize !== "function") throw new TypeError("ElevenLabs output requires an authenticated synthesis transport");
|
|
111
|
+
this.#elevenLabs = new ElevenLabsPlayback(options.synthesize, (error) => {
|
|
112
|
+
if (!this.#closed) options.onTerminated(error.message);
|
|
113
|
+
});
|
|
114
|
+
}
|
|
105
115
|
this.#backendReady = new Promise((resolve) => { this.#resolveBackendReady = resolve; });
|
|
106
116
|
}
|
|
107
117
|
|
|
108
118
|
start() {
|
|
119
|
+
if (!this.#starting) void this.#elevenLabs?.prime()?.catch((error) => this.#options.onTerminated(error.message));
|
|
109
120
|
this.#starting ??= this.#start();
|
|
110
121
|
return this.#starting;
|
|
111
122
|
}
|
|
@@ -140,7 +151,8 @@ export class BrowserVoiceSession {
|
|
|
140
151
|
// Only delegated work waits for admission on the direct control path.
|
|
141
152
|
const coreStartup = coreReady.then(async (core) => {
|
|
142
153
|
if (this.#directControl) await this.#options.beforeAgentTurn?.();
|
|
143
|
-
await core?.start();
|
|
154
|
+
const effects = await core?.start();
|
|
155
|
+
if (!this.#closed && !this.#closing.signal.aborted) await this.#apply(effects);
|
|
144
156
|
return core;
|
|
145
157
|
});
|
|
146
158
|
this.#admission = coreStartup;
|
|
@@ -256,7 +268,7 @@ export class BrowserVoiceSession {
|
|
|
256
268
|
const channel = this.#channel;
|
|
257
269
|
channel.addEventListener("message", (event) => {
|
|
258
270
|
if (this.#directControl && !this.#closed && this.#channel === channel) {
|
|
259
|
-
this.#receiveControl(event.data, () => this.#channel === channel);
|
|
271
|
+
this.#receiveControl(event.data, () => this.#channel === channel && channel.readyState === "open");
|
|
260
272
|
}
|
|
261
273
|
});
|
|
262
274
|
channel.addEventListener("close", () => {
|
|
@@ -271,7 +283,7 @@ export class BrowserVoiceSession {
|
|
|
271
283
|
}
|
|
272
284
|
const stream = event.streams[0] ?? new MediaStream([event.track]);
|
|
273
285
|
this.#speaker ??= new SpeakerPlayback(new Audio(), this.#options.onStatus);
|
|
274
|
-
this.#speaker.setEnabled(this.#playbackEnabled);
|
|
286
|
+
this.#speaker.setEnabled(this.#playbackEnabled && !this.#elevenLabs);
|
|
275
287
|
this.#speaker.attach(stream);
|
|
276
288
|
});
|
|
277
289
|
peer.addEventListener("connectionstatechange", () => {
|
|
@@ -300,7 +312,9 @@ export class BrowserVoiceSession {
|
|
|
300
312
|
}
|
|
301
313
|
|
|
302
314
|
noteTypedInput() {
|
|
315
|
+
this.#playbackRevision++;
|
|
303
316
|
this.#playbackEnabled = false;
|
|
317
|
+
this.#elevenLabs?.interrupt();
|
|
304
318
|
this.#speaker?.setEnabled(false);
|
|
305
319
|
return this.#applyLive(async () => {
|
|
306
320
|
const core = await this.#options.core;
|
|
@@ -417,12 +431,19 @@ export class BrowserVoiceSession {
|
|
|
417
431
|
for (const text of effects.undelivered_answers ?? []) this.#options.onUndeliveredAnswer?.(text);
|
|
418
432
|
if (effects.input_generation !== undefined) {
|
|
419
433
|
if (effects.input_generation < this.#inputGeneration) return;
|
|
434
|
+
if (effects.input_generation > this.#inputGeneration) {
|
|
435
|
+
this.#playbackRevision++;
|
|
436
|
+
this.#elevenLabs?.interrupt();
|
|
437
|
+
}
|
|
420
438
|
this.#inputGeneration = effects.input_generation;
|
|
421
439
|
}
|
|
422
440
|
if (effects.playback_enabled === false) {
|
|
441
|
+
this.#playbackRevision++;
|
|
423
442
|
this.#playbackEnabled = false;
|
|
443
|
+
this.#elevenLabs?.interrupt();
|
|
424
444
|
this.#speaker?.setEnabled(false);
|
|
425
445
|
}
|
|
446
|
+
const playbackRevision = this.#playbackRevision;
|
|
426
447
|
let sent = 0;
|
|
427
448
|
for (const frame of effects.frames ?? []) {
|
|
428
449
|
if (this.#directControl && this.#channel?.readyState === "open") {
|
|
@@ -434,11 +455,15 @@ export class BrowserVoiceSession {
|
|
|
434
455
|
}
|
|
435
456
|
}
|
|
436
457
|
if (effects.acknowledge_frames && sent > 0) await this.#core?.framesSent(sent);
|
|
437
|
-
|
|
458
|
+
// A live interruption can overtake asynchronous frame acknowledgement.
|
|
459
|
+
// Keep its speaker gate authoritative while retaining accepted captions.
|
|
460
|
+
const currentPlayback = playbackRevision === this.#playbackRevision;
|
|
461
|
+
if (currentPlayback && !this.#closed && effects.playback_enabled === true && sent === (effects.frames?.length ?? 0)) {
|
|
438
462
|
this.#playbackEnabled = true;
|
|
439
|
-
this.#speaker?.setEnabled(
|
|
463
|
+
this.#speaker?.setEnabled(!this.#elevenLabs);
|
|
440
464
|
}
|
|
441
465
|
for (const entry of effects.transcripts ?? []) {
|
|
466
|
+
if (currentPlayback && this.#playbackEnabled && !this.#closed) this.#elevenLabs?.transcript(entry);
|
|
442
467
|
this.#options.onTranscript(entry.speaker, entry.text, entry);
|
|
443
468
|
}
|
|
444
469
|
if (effects.status) this.#status(effects.status);
|
|
@@ -475,7 +500,7 @@ export class BrowserVoiceSession {
|
|
|
475
500
|
let opened = false;
|
|
476
501
|
sideband.addEventListener("message", (event) => {
|
|
477
502
|
if (!this.#closed && generation === this.#sidebandGeneration) {
|
|
478
|
-
this.#receiveControl(event.data, () => generation === this.#sidebandGeneration);
|
|
503
|
+
this.#receiveControl(event.data, () => generation === this.#sidebandGeneration && sideband.readyState === WebSocket.OPEN);
|
|
479
504
|
}
|
|
480
505
|
});
|
|
481
506
|
sideband.addEventListener("close", () => {
|
|
@@ -501,7 +526,11 @@ export class BrowserVoiceSession {
|
|
|
501
526
|
this.#applyLive(async () => {
|
|
502
527
|
if (!this.#directControl) await this.#admission;
|
|
503
528
|
if (this.#closed || !isCurrent()) return;
|
|
504
|
-
|
|
529
|
+
const requiresAdmission = await this.#core.requiresAgentAdmission(payload);
|
|
530
|
+
// Classification can yield while this transport closes or is replaced.
|
|
531
|
+
// Only requests accepted on the current connection may enter the queue.
|
|
532
|
+
if (this.#closed || !isCurrent()) return;
|
|
533
|
+
if (requiresAdmission) {
|
|
505
534
|
// Retain accepted requests on close, but keep captions and interruptions
|
|
506
535
|
// independent of the durable route and its ordered task queue.
|
|
507
536
|
void this.#enqueue(async () => {
|
|
@@ -549,6 +578,7 @@ export class BrowserVoiceSession {
|
|
|
549
578
|
}
|
|
550
579
|
|
|
551
580
|
#stopBrowserMedia() {
|
|
581
|
+
this.#elevenLabs?.close();
|
|
552
582
|
if (this.#meterTimer !== undefined) window.clearTimeout(this.#meterTimer);
|
|
553
583
|
this.#meterTimer = undefined;
|
|
554
584
|
this.#call?.abort();
|
package/browser/host.d.mts
CHANGED
|
@@ -75,12 +75,28 @@ export function createBrowserHost(options?: {
|
|
|
75
75
|
mcp?: McpServers;
|
|
76
76
|
codeEvaluator?: CodeEvaluator;
|
|
77
77
|
toolMode?: "code" | "direct";
|
|
78
|
-
/** @internal
|
|
78
|
+
/** @internal Live host lifecycle for ephemeral Rust-owned subagents. */
|
|
79
|
+
subagentRouting?: Pick<import('../runtime/subagent-routing.mjs').SubagentRouting, 'resolve' | 'bind'>;
|
|
79
80
|
subagentSessions?: {
|
|
80
|
-
|
|
81
|
-
bind(sessionId: string, descriptor: SubagentToolContext): void;
|
|
82
|
-
release(sessionId: string): void;
|
|
81
|
+
bindingDescriptor?(sessionId: string, descriptor: SubagentToolContext, hostContextRef?: string): SubagentToolContext;
|
|
82
|
+
bind(sessionId: string, descriptor: SubagentToolContext, hostContextRef?: string): void;
|
|
83
|
+
release(sessionId: string, hostContextRef?: string): void;
|
|
83
84
|
};
|
|
85
|
+
/** @internal Content-free summary, once when ownership closes; no public transport option. */
|
|
86
|
+
onSocketTiming?: (timing: {
|
|
87
|
+
message_count: number;
|
|
88
|
+
delivered_message_count: number;
|
|
89
|
+
buffered_message_count: number;
|
|
90
|
+
discarded_message_count: number;
|
|
91
|
+
queue_residence_total_ms: number;
|
|
92
|
+
queue_residence_max_ms: number;
|
|
93
|
+
provider_timings: Array<{
|
|
94
|
+
response_id?: string;
|
|
95
|
+
pre_inference_ms?: number;
|
|
96
|
+
engine_queue_max_ms?: number;
|
|
97
|
+
engine_service_ttft_total_ms?: number;
|
|
98
|
+
}>;
|
|
99
|
+
}) => void;
|
|
84
100
|
maxQueuedMessages?: number;
|
|
85
101
|
maxQueuedBytes?: number;
|
|
86
102
|
maxBufferedSendBytes?: number;
|