nanocodex 0.6.4 → 0.6.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -11
- package/browser/Agent.d.mts +1 -1
- package/browser/ElevenLabsPlayback.mjs +119 -0
- package/browser/InlineAgent.mjs +22 -17
- package/browser/Transport.d.mts +2 -0
- package/browser/Transport.mjs +3 -2
- package/browser/Voice.d.mts +4 -0
- package/browser/Voice.mjs +27 -2
- package/browser/VoiceSession.mjs +19 -3
- package/browser/host.d.mts +20 -4
- package/browser/host.mjs +156 -18
- package/cloudflare/Agent.d.mts +10 -9
- package/cloudflare/Agent.mjs +259 -95
- package/cloudflare/agent-settings.d.mts +15 -0
- package/cloudflare/agent-settings.mjs +135 -0
- package/cloudflare/chat-reasoning.mjs +20 -0
- package/cloudflare/durable-placement.d.mts +12 -0
- package/cloudflare/durable-placement.mjs +82 -0
- package/cloudflare/egress.d.mts +31 -0
- package/cloudflare/gateway-binding-responses.mjs +84 -0
- package/cloudflare/gateway-responses.d.mts +48 -0
- package/cloudflare/gateway-responses.mjs +191 -0
- package/cloudflare/index.d.mts +6 -0
- package/cloudflare/index.mjs +3 -0
- package/cloudflare/managed-auth.d.mts +15 -0
- package/cloudflare/managed-auth.mjs +113 -0
- package/cloudflare/managed-live.d.mts +7 -0
- package/cloudflare/managed-live.mjs +56 -0
- package/cloudflare/prepared-connection.mjs +49 -0
- package/cloudflare/provider-stream.mjs +353 -0
- package/cloudflare/rpc.d.mts +9 -0
- package/cloudflare/rpc.mjs +14 -0
- package/cloudflare/tool-alias-diagnostic.mjs +21 -0
- package/cloudflare/workers-ai-responses.d.mts +28 -0
- package/cloudflare/workers-ai-responses.mjs +398 -0
- package/host/Agent.d.mts +1 -1
- package/index.d.mts +2 -0
- package/internal.mjs +48 -14
- package/managed/Agent.d.mts +21 -68
- package/managed/Agent.mjs +61 -191
- package/managed/README.md +9 -38
- package/managed/Voice.mjs +2 -1
- package/managed/index.d.mts +0 -3
- package/node/Agent.d.mts +1 -1
- package/node/Agent.mjs +9 -3
- package/node/host.mjs +5 -0
- package/package.json +30 -2
- package/pkg-node/nanocodex.d.ts +8 -16
- package/pkg-node/nanocodex.js +96 -93
- package/pkg-node/nanocodex_bg.wasm.d.ts +4 -5
- package/pkg-web/.nanocodex-bindgen-stamp +7 -1
- package/pkg-web/nanocodex-build.json +1 -1
- package/pkg-web/nanocodex.d.ts +12 -21
- package/pkg-web/nanocodex.js +96 -92
- package/pkg-web/nanocodex_bg.js +96 -92
- package/pkg-web/nanocodex_bg.wasm +0 -0
- package/pkg-web/nanocodex_bg.wasm.d.ts +4 -5
- package/pkg-web/nanocodex_worker.js +1 -1
- package/runtime/before-compaction.mjs +49 -0
- package/runtime/code-evaluator.worker.mjs +2 -9
- package/runtime/managed-transport.mjs +10 -2
- package/runtime/mcp-runtime.mjs +15 -1
- package/runtime/quickjs-evaluator.mjs +11 -5
- package/runtime/response-controls.mjs +56 -4
- package/runtime/steer-receipt.mjs +13 -0
- package/runtime/subagent-routing.d.mts +38 -0
- package/runtime/subagent-routing.mjs +80 -0
- package/runtime/subagents.d.mts +1 -1
- package/runtime/tempo-provider.d.mts +1 -1
- package/runtime/tempo-provider.mjs +1 -1
- package/tools/environment.d.mts +8 -2
- package/tools/environment.mjs +16 -2
- package/tools/index.d.mts +2 -0
- package/types.d.mts +28 -1
package/README.md
CHANGED
|
@@ -10,7 +10,7 @@ import { Actions, Agent, Transport } from "nanocodex/node";
|
|
|
10
10
|
|
|
11
11
|
const agent = await Agent.create({
|
|
12
12
|
transport: Transport.openAi({ apiKey: process.env.OPENAI_API_KEY }),
|
|
13
|
-
model: "gpt-
|
|
13
|
+
model: "gpt-6-luna",
|
|
14
14
|
instructions: "You are a Rust coding agent. Preserve unrelated work and run relevant tests.",
|
|
15
15
|
reasoningMode: "pro",
|
|
16
16
|
thinking: "high",
|
|
@@ -236,8 +236,13 @@ requesting speech. Context and speech are split into provider-sized messages.
|
|
|
236
236
|
These commands retain frames until sent and preserve them
|
|
237
237
|
across reconnects when using the sideband transport. They are also methods on
|
|
238
238
|
the resource and on
|
|
239
|
-
`useVoice` from `nanocodex-react`.
|
|
240
|
-
|
|
239
|
+
`useVoice` from `nanocodex-react`. Choose `outputProvider: "elevenlabs"` and `elevenLabsVoiceId` to synthesize
|
|
240
|
+
spoken output with an ElevenLabs account voice, including an instant clone.
|
|
241
|
+
The account voice settings panel connects the API key, lists voices, and uploads
|
|
242
|
+
cloning samples after explicit consent. Keys are encrypted on the server and
|
|
243
|
+
never included in saved voice settings. ChatGPT still owns live input and agent
|
|
244
|
+
handoffs; `voice` continues to select its built-in voice. The default output
|
|
245
|
+
provider is `openai`. Platform audio configuration is not accepted.
|
|
241
246
|
|
|
242
247
|
`Voice.create(...)` remains the equivalent namespaced resource constructor, and
|
|
243
248
|
`Voice.voices` is the exact ChatGPT V3 voice catalog. The constructor accepts a
|
|
@@ -302,10 +307,12 @@ Cloudflare Agents default to direct tool mode because Workers prohibit dynamic
|
|
|
302
307
|
`eval`/`new Function`. Caller-defined tools therefore work without a code
|
|
303
308
|
evaluator. Select `toolMode: "code"` only when also supplying an evaluator that
|
|
304
309
|
is explicitly compatible with the deployed Worker runtime. Runtime-owned
|
|
305
|
-
Subagents are installed by default, including on a durable root.
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
310
|
+
Subagents are installed by default, including on a durable root. All children are
|
|
311
|
+
ephemeral: their identities, topology, conversations, results, and routing exist
|
|
312
|
+
only for the lifetime of the root runtime. Root shutdown or restart discards the
|
|
313
|
+
entire child tree; only the root's own durable history resumes. Existing child
|
|
314
|
+
checkpoints from older versions are discarded. Within a live runtime, completed
|
|
315
|
+
children remain available for follow-up messages until closed. Use
|
|
309
316
|
`Subagents.create({ maxConcurrency })` in `tools` to set an explicit finite
|
|
310
317
|
concurrency limit. Active subagent turns are unlimited by default.
|
|
311
318
|
|
|
@@ -349,6 +356,11 @@ native and WASM consumers run the same Rust implementation and receive the
|
|
|
349
356
|
same seven tools: `spawn_agent`, `submit_result`, `send_agent_message`,
|
|
350
357
|
`list_agents`, `wait_agent`, `interrupt_agent`, and `close_agent`.
|
|
351
358
|
|
|
359
|
+
Children call `submit_result({output})`; the runtime supplies the trusted
|
|
360
|
+
instruction revision. The response is `{accepted: true, status: "accepted"}` or
|
|
361
|
+
`{accepted: false, status: "superseded"}`. Superseded submissions are normal
|
|
362
|
+
continuations: incorporate the updated instructions and submit again.
|
|
363
|
+
|
|
352
364
|
Inside a caller-owned Worker or server isolate, host capabilities stay as
|
|
353
365
|
ordinary functions without crossing another compatibility protocol:
|
|
354
366
|
|
|
@@ -686,7 +698,7 @@ const mpp = tempoProvider.session;
|
|
|
686
698
|
|
|
687
699
|
const agent = await Agent.create({
|
|
688
700
|
transport: Transport.mpp({ session: tempoProvider }),
|
|
689
|
-
thinking: "
|
|
701
|
+
thinking: "low",
|
|
690
702
|
fastMode: true,
|
|
691
703
|
tools,
|
|
692
704
|
});
|
|
@@ -736,8 +748,8 @@ of its wallet adapter, and constructs both payment paths from that provider's
|
|
|
736
748
|
adapter-neutral `getMppxParameters()` contract. The lower-level
|
|
737
749
|
`createTempoProvider({ session, payment })` remains available when the
|
|
738
750
|
application constructs MPPx itself. Both explicitly select Tempo provider mode.
|
|
739
|
-
In that mode Nanocodex automatically adds its built-in Mercator MCP
|
|
740
|
-
with the same wallet and payment policy. The provider also exposes an MPP-aware
|
|
751
|
+
In that mode Nanocodex automatically adds its built-in Mercator MCP at
|
|
752
|
+
`https://mercator.sh/mcp` and wraps it with the same wallet and payment policy. The provider also exposes an MPP-aware
|
|
741
753
|
`fetch`; Mercator's paid REST handoffs use that same method rather than a second
|
|
742
754
|
wallet or payment configuration. Its MCP transport remains wrapped at the MCP
|
|
743
755
|
protocol layer, so browser requests do not need an `Accept-Payment` CORS header.
|
|
@@ -1155,7 +1167,7 @@ package manager or build step:
|
|
|
1155
1167
|
|
|
1156
1168
|
```html
|
|
1157
1169
|
<script type="module">
|
|
1158
|
-
import { Agent, Transport } from "https://cdn.jsdelivr.net/npm/nanocodex@0.6.
|
|
1170
|
+
import { Agent, Transport } from "https://cdn.jsdelivr.net/npm/nanocodex@0.6.5/host/index.mjs";
|
|
1159
1171
|
const agent = await Agent.create({
|
|
1160
1172
|
transport: Transport.hostManaged({
|
|
1161
1173
|
websocketUrl: "/api/responses",
|
package/browser/Agent.d.mts
CHANGED
|
@@ -33,7 +33,7 @@ export declare namespace create {
|
|
|
33
33
|
transport: ManagedTransport;
|
|
34
34
|
tools?: Tools | undefined;
|
|
35
35
|
}>;
|
|
36
|
-
type Options = AgentOptions & WorkerToolExposureOptions & {
|
|
36
|
+
type Options = Omit<AgentOptions, "beforeCompaction"> & WorkerToolExposureOptions & {
|
|
37
37
|
/** Precompiled browser module; WebAssembly modules are structured-clone-safe. */
|
|
38
38
|
module?: WebAssembly.Module | undefined;
|
|
39
39
|
/** Fixed browser workspace facts, including its AGENTS.md snapshot. */
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/** Plays account-proxied PCM without exposing provider credentials to the browser. */
|
|
2
|
+
export class ElevenLabsPlayback {
|
|
3
|
+
#synthesize;
|
|
4
|
+
#onError;
|
|
5
|
+
#context;
|
|
6
|
+
#abort = new AbortController();
|
|
7
|
+
#queue = Promise.resolve();
|
|
8
|
+
#sources = new Set();
|
|
9
|
+
#nextTime = 0;
|
|
10
|
+
#caption;
|
|
11
|
+
#offset = 0;
|
|
12
|
+
#suppressedThrough = -1;
|
|
13
|
+
#pending = 0;
|
|
14
|
+
#pendingChars = 0;
|
|
15
|
+
#closed = false;
|
|
16
|
+
|
|
17
|
+
constructor(synthesize, onError) {
|
|
18
|
+
this.#synthesize = synthesize;
|
|
19
|
+
this.#onError = onError;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
prime() {
|
|
23
|
+
if (this.#closed) return;
|
|
24
|
+
this.#context ??= new AudioContext({ sampleRate: 24000 });
|
|
25
|
+
return this.#context.resume();
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
transcript(entry) {
|
|
29
|
+
if (this.#closed || entry.speaker !== "assistant" || entry.id <= this.#suppressedThrough) return;
|
|
30
|
+
if (this.#caption !== undefined && entry.id < this.#caption) return;
|
|
31
|
+
if (entry.id !== this.#caption) { this.#caption = entry.id; this.#offset = 0; }
|
|
32
|
+
const remaining = entry.text.slice(this.#offset);
|
|
33
|
+
// Buffer partial words; emit complete sentences before the turn finishes.
|
|
34
|
+
const boundary = entry.is_partial ? [...remaining.matchAll(/[.!?\n](?:\s|$)/g)].at(-1) : undefined;
|
|
35
|
+
const length = entry.is_partial ? (boundary ? boundary.index + boundary[0].length : 0) : remaining.length;
|
|
36
|
+
const text = remaining.slice(0, length).trim();
|
|
37
|
+
this.#offset += length;
|
|
38
|
+
if (!text) return;
|
|
39
|
+
if (this.#pending >= 32 || this.#pendingChars + text.length > 32768) {
|
|
40
|
+
this.interrupt();
|
|
41
|
+
this.#onError(new Error("ElevenLabs playback queue exceeded its limit"));
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
this.#pending++;
|
|
45
|
+
this.#pendingChars += text.length;
|
|
46
|
+
const signal = this.#abort.signal;
|
|
47
|
+
this.#queue = this.#queue.then(async () => {
|
|
48
|
+
if (signal.aborted) return;
|
|
49
|
+
this.#context ??= new AudioContext({ sampleRate: 24000 });
|
|
50
|
+
await this.#context.resume();
|
|
51
|
+
if (signal.aborted) return;
|
|
52
|
+
const response = await this.#synthesize(text, signal);
|
|
53
|
+
if (!response.ok) throw new Error(`ElevenLabs synthesis failed (${response.status})`);
|
|
54
|
+
const reader = response.body?.getReader();
|
|
55
|
+
if (!reader) throw new Error("ElevenLabs returned no audio stream");
|
|
56
|
+
const cancel = () => { void reader.cancel().catch(() => {}); };
|
|
57
|
+
signal.addEventListener("abort", cancel, { once: true });
|
|
58
|
+
let carry;
|
|
59
|
+
try {
|
|
60
|
+
while (!signal.aborted) {
|
|
61
|
+
const { value, done } = await reader.read();
|
|
62
|
+
if (done) break;
|
|
63
|
+
if (signal.aborted) break;
|
|
64
|
+
if (value.byteLength > 1024 * 1024) throw new Error("ElevenLabs audio chunk exceeded its limit");
|
|
65
|
+
const bytes = carry === undefined ? value : Uint8Array.from([carry, ...value]);
|
|
66
|
+
const size = bytes.length - bytes.length % 2;
|
|
67
|
+
carry = size < bytes.length ? bytes[size] : undefined;
|
|
68
|
+
if (!size) continue;
|
|
69
|
+
// Keep decoded/scheduled audio bounded even when synthesis outruns playback.
|
|
70
|
+
for (let offset = 0; offset < size && !signal.aborted; offset += 9600) {
|
|
71
|
+
while (!signal.aborted && (this.#nextTime - this.#context.currentTime > 2 || this.#sources.size >= 128)) {
|
|
72
|
+
await new Promise((resolve) => {
|
|
73
|
+
const done = () => { clearTimeout(timer); signal.removeEventListener("abort", done); resolve(); };
|
|
74
|
+
const timer = setTimeout(done, 25);
|
|
75
|
+
signal.addEventListener("abort", done, { once: true });
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
if (signal.aborted) break;
|
|
79
|
+
const length = Math.min(9600, size - offset);
|
|
80
|
+
const buffer = this.#context.createBuffer(1, length / 2, 24000);
|
|
81
|
+
const output = buffer.getChannelData(0);
|
|
82
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset + offset, length);
|
|
83
|
+
for (let i = 0; i < output.length; i++) output[i] = view.getInt16(i * 2, true) / 32768;
|
|
84
|
+
const source = this.#context.createBufferSource();
|
|
85
|
+
source.buffer = buffer;
|
|
86
|
+
source.connect(this.#context.destination);
|
|
87
|
+
this.#sources.add(source);
|
|
88
|
+
source.onended = () => { this.#sources.delete(source); source.disconnect(); };
|
|
89
|
+
this.#nextTime = Math.max(this.#nextTime, this.#context.currentTime + 0.02);
|
|
90
|
+
source.start(this.#nextTime);
|
|
91
|
+
this.#nextTime += buffer.duration;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
} finally { signal.removeEventListener("abort", cancel); await reader.cancel().catch(() => {}); reader.releaseLock(); }
|
|
95
|
+
}).catch((error) => { if (!signal.aborted) this.#onError(error); }).finally(() => {
|
|
96
|
+
if (!signal.aborted) { this.#pending--; this.#pendingChars -= text.length; }
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
interrupt() {
|
|
101
|
+
this.#suppressedThrough = Math.max(this.#suppressedThrough, this.#caption ?? -1);
|
|
102
|
+
this.#pending = 0;
|
|
103
|
+
this.#pendingChars = 0;
|
|
104
|
+
this.#abort.abort();
|
|
105
|
+
this.#abort = new AbortController();
|
|
106
|
+
this.#queue = Promise.resolve();
|
|
107
|
+
for (const source of this.#sources) { try { source.stop(); } catch {} source.disconnect(); }
|
|
108
|
+
this.#sources.clear();
|
|
109
|
+
this.#nextTime = 0;
|
|
110
|
+
// Caption IDs are monotonic: every late delta/final for this epoch stays silent.
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
close() {
|
|
114
|
+
this.#closed = true;
|
|
115
|
+
this.interrupt();
|
|
116
|
+
void this.#context?.close();
|
|
117
|
+
this.#context = undefined;
|
|
118
|
+
}
|
|
119
|
+
}
|
package/browser/InlineAgent.mjs
CHANGED
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
registerDefinitionHost,
|
|
17
17
|
releaseDefinitionHost,
|
|
18
18
|
releaseHostSession,
|
|
19
|
+
releaseHostSessions,
|
|
19
20
|
toWasmConfig,
|
|
20
21
|
} from "../internal.mjs";
|
|
21
22
|
import { createBrowserHost } from "./host.mjs";
|
|
@@ -45,8 +46,10 @@ export async function create(options = {}) {
|
|
|
45
46
|
thinking,
|
|
46
47
|
reasoningMode,
|
|
47
48
|
fastMode,
|
|
49
|
+
rawApiEvents,
|
|
48
50
|
instructions,
|
|
49
51
|
additionalInstructions,
|
|
52
|
+
beforeCompaction,
|
|
50
53
|
sessionId,
|
|
51
54
|
workspace,
|
|
52
55
|
resume,
|
|
@@ -75,6 +78,7 @@ export async function create(options = {}) {
|
|
|
75
78
|
websocketPreconnect,
|
|
76
79
|
apiBaseUrl,
|
|
77
80
|
websocketWarmup,
|
|
81
|
+
stateless,
|
|
78
82
|
WebSocketImpl,
|
|
79
83
|
createWebSocket,
|
|
80
84
|
createResponse,
|
|
@@ -116,11 +120,14 @@ export async function create(options = {}) {
|
|
|
116
120
|
hostManagedProtocol,
|
|
117
121
|
mpp,
|
|
118
122
|
onEvent: events.emit,
|
|
123
|
+
beforeCompaction,
|
|
119
124
|
filesystem,
|
|
120
125
|
filesystemTools,
|
|
121
126
|
tools: hostTools,
|
|
122
127
|
toolProviders,
|
|
123
128
|
subagentSessions,
|
|
129
|
+
subagentRouting: internalRuntime?.subagentRouting,
|
|
130
|
+
onSocketTiming: internalRuntime?.onSocketTiming,
|
|
124
131
|
toolMode,
|
|
125
132
|
mcp: mcp === false
|
|
126
133
|
? undefined
|
|
@@ -163,8 +170,11 @@ export async function create(options = {}) {
|
|
|
163
170
|
: "wss://openai.mpp.tempo.xyz/v1/responses"),
|
|
164
171
|
apiBaseUrl,
|
|
165
172
|
websocketWarmup,
|
|
173
|
+
stateless,
|
|
166
174
|
subagents: subagentConfig,
|
|
175
|
+
subagentRouting: internalRuntime?.subagentRouting !== undefined,
|
|
167
176
|
hostDefinitionId,
|
|
177
|
+
beforeCompaction: beforeCompaction !== undefined,
|
|
168
178
|
...config,
|
|
169
179
|
durabilityHostId: durabilityOwner?.id,
|
|
170
180
|
}));
|
|
@@ -176,29 +186,19 @@ export async function create(options = {}) {
|
|
|
176
186
|
);
|
|
177
187
|
if (cloudflareReservation !== undefined) {
|
|
178
188
|
activateCloudflareAgentSession(cloudflareReservation);
|
|
179
|
-
const restoredSubagents = subagentSessions?.restore?.() ?? [];
|
|
180
|
-
if (restoredSubagents.length > 0) {
|
|
181
|
-
const restoredHostContextRefs = Object.fromEntries(
|
|
182
|
-
restoredSubagents.map((descriptor) => {
|
|
183
|
-
const hostContextRef = subagentSessions?.hostContextRef?.(descriptor.sessionId);
|
|
184
|
-
return [descriptor.sessionId, hostContextRef ?? null];
|
|
185
|
-
}),
|
|
186
|
-
);
|
|
187
|
-
await raw.restoreSubagents(
|
|
188
|
-
JSON.stringify(restoredSubagents),
|
|
189
|
-
JSON.stringify(restoredHostContextRefs),
|
|
190
|
-
);
|
|
191
|
-
}
|
|
192
189
|
}
|
|
193
190
|
return raw;
|
|
194
191
|
} catch (error) {
|
|
195
192
|
const cleanupErrors = [];
|
|
196
193
|
if (raw !== undefined) {
|
|
197
|
-
try {
|
|
194
|
+
try {
|
|
195
|
+
await raw.shutdown();
|
|
196
|
+
}
|
|
198
197
|
catch (cleanupError) { cleanupErrors.push(cleanupError); }
|
|
199
198
|
try { raw.free(); }
|
|
200
199
|
catch (cleanupError) { cleanupErrors.push(cleanupError); }
|
|
201
200
|
}
|
|
201
|
+
releaseHostSessions(host);
|
|
202
202
|
try { durabilityOwner?.abandon(); }
|
|
203
203
|
catch (cleanupError) { cleanupErrors.push(cleanupError); }
|
|
204
204
|
try { await host.dispose(); }
|
|
@@ -212,16 +212,20 @@ export async function create(options = {}) {
|
|
|
212
212
|
throw error;
|
|
213
213
|
}
|
|
214
214
|
},
|
|
215
|
+
async shutdown(raw) {
|
|
216
|
+
await raw.shutdown();
|
|
217
|
+
},
|
|
215
218
|
subscribe: events.subscribe,
|
|
216
219
|
adopt(raw) {
|
|
217
220
|
host.retain();
|
|
218
221
|
try {
|
|
219
|
-
|
|
222
|
+
// Adopted child handles are ephemeral and do not own the root store.
|
|
223
|
+
if (raw.sessionId === stableSessionId) durabilityOwner?.retain();
|
|
220
224
|
bindHostSession(host, raw.sessionId, cloudflareReservation);
|
|
221
225
|
events.addSource(raw);
|
|
222
226
|
} catch (error) {
|
|
223
227
|
events.removeSource(raw);
|
|
224
|
-
durabilityOwner?.release();
|
|
228
|
+
if (raw.sessionId === stableSessionId) durabilityOwner?.release();
|
|
225
229
|
releaseHost(host);
|
|
226
230
|
throw error;
|
|
227
231
|
}
|
|
@@ -230,7 +234,7 @@ export async function create(options = {}) {
|
|
|
230
234
|
events.removeSource(raw);
|
|
231
235
|
host.releaseSession(raw.sessionId);
|
|
232
236
|
releaseHostSession(host, raw.sessionId);
|
|
233
|
-
durabilityOwner?.release();
|
|
237
|
+
if (raw.sessionId === stableSessionId) durabilityOwner?.release();
|
|
234
238
|
releaseHost(host);
|
|
235
239
|
},
|
|
236
240
|
decorate: (agent) => agent.extend(agentActions()),
|
|
@@ -242,6 +246,7 @@ export async function create(options = {}) {
|
|
|
242
246
|
thinking,
|
|
243
247
|
reasoningMode,
|
|
244
248
|
fastMode,
|
|
249
|
+
rawApiEvents,
|
|
245
250
|
instructions,
|
|
246
251
|
additionalInstructions,
|
|
247
252
|
sessionId: stableSessionId,
|
package/browser/Transport.d.mts
CHANGED
|
@@ -26,6 +26,8 @@ export type WorkerTransport = ResponsesTransport & Readonly<{
|
|
|
26
26
|
|
|
27
27
|
type SharedEndpointOptions = Readonly<{
|
|
28
28
|
apiBaseUrl?: string | undefined;
|
|
29
|
+
/** Use HTTPS with full history replay, no stored responses, and no WebSocket warmup or preconnect. */
|
|
30
|
+
stateless?: boolean | undefined;
|
|
29
31
|
websocketUrl?: string | undefined;
|
|
30
32
|
/** Open the persistent socket as soon as Agent.create returns. Defaults to true for hostManaged. */
|
|
31
33
|
websocketPreconnect?: boolean | undefined;
|
package/browser/Transport.mjs
CHANGED
|
@@ -25,12 +25,12 @@ export function chatGpt(options) {
|
|
|
25
25
|
|
|
26
26
|
export function hostManaged(options = {}) {
|
|
27
27
|
const websocketUrl = options.websocketUrl
|
|
28
|
-
?? (options.createWebSocket ? undefined : defaultHostManagedWebSocketUrl());
|
|
28
|
+
?? (options.stateless === true || options.createWebSocket ? undefined : defaultHostManagedWebSocketUrl());
|
|
29
29
|
return createResponsesTransport({
|
|
30
30
|
hostAuth: true,
|
|
31
31
|
hostManagedProtocol: true,
|
|
32
32
|
...connection({ ...options, websocketUrl }),
|
|
33
|
-
websocketPreconnect: options.websocketPreconnect ?? true,
|
|
33
|
+
websocketPreconnect: options.stateless === true ? false : options.websocketPreconnect ?? true,
|
|
34
34
|
});
|
|
35
35
|
}
|
|
36
36
|
|
|
@@ -53,6 +53,7 @@ function connection(options = {}) {
|
|
|
53
53
|
return {
|
|
54
54
|
WebSocketImpl: options.WebSocketImpl,
|
|
55
55
|
apiBaseUrl: options.apiBaseUrl,
|
|
56
|
+
stateless: options.stateless,
|
|
56
57
|
createWebSocket: options.createWebSocket,
|
|
57
58
|
createResponse: options.createResponse,
|
|
58
59
|
websocketUrl: options.websocketUrl,
|
package/browser/Voice.d.mts
CHANGED
|
@@ -51,6 +51,8 @@ export type Event =
|
|
|
51
51
|
| Readonly<{ type: "stopped" }>;
|
|
52
52
|
export type Settings = Readonly<{
|
|
53
53
|
voice?: VoiceName | undefined;
|
|
54
|
+
outputProvider?: "openai" | "elevenlabs" | undefined;
|
|
55
|
+
elevenLabsVoiceId?: string | undefined;
|
|
54
56
|
/** Additional speaking preferences; base assistant instructions are retained. */
|
|
55
57
|
instructions?: string | undefined;
|
|
56
58
|
pace?: "slow" | "natural" | "fast" | undefined;
|
|
@@ -59,6 +61,8 @@ export type Settings = Readonly<{
|
|
|
59
61
|
acknowledgements?: boolean | undefined;
|
|
60
62
|
}>;
|
|
61
63
|
export type Options = Settings & Readonly<{
|
|
64
|
+
/** Authenticated synthesis transport returning mono signed 16-bit little-endian PCM at 24 kHz. */
|
|
65
|
+
synthesize?(text: string, signal: AbortSignal): Promise<Response>;
|
|
62
66
|
callUrl?: string | URL | undefined;
|
|
63
67
|
sidebandUrl?(callId: string, sessionId: string): string | URL | Promise<string | URL>;
|
|
64
68
|
captureMicrophone?(): Promise<MediaStream>;
|
package/browser/Voice.mjs
CHANGED
|
@@ -91,6 +91,11 @@ export function create(agent, options = {}) {
|
|
|
91
91
|
if (destroyed) throw new Error("voice resource is destroyed");
|
|
92
92
|
const selectedVoice = parameters.voice ?? options.voice ?? defaultVoice;
|
|
93
93
|
if (!voices.includes(selectedVoice)) throw new TypeError(`unsupported ChatGPT voice: ${selectedVoice}`);
|
|
94
|
+
const settings = voiceSettings({ ...options, ...parameters, voice: selectedVoice });
|
|
95
|
+
validateOutputSettings(settings);
|
|
96
|
+
if (agent.type === "connect" && settings.outputProvider === "elevenlabs" && !options.synthesize) {
|
|
97
|
+
throw new TypeError("Connect ElevenLabs output requires an explicit authorized synthesis transport");
|
|
98
|
+
}
|
|
94
99
|
if (session) return startPromise;
|
|
95
100
|
if (stopPromise) await stopPromise.catch(() => {});
|
|
96
101
|
if (destroyed) throw new Error("voice resource is destroyed");
|
|
@@ -125,7 +130,14 @@ export function create(agent, options = {}) {
|
|
|
125
130
|
core,
|
|
126
131
|
sessionId,
|
|
127
132
|
voice: selectedVoice,
|
|
128
|
-
|
|
133
|
+
synthesize: options.synthesize ?? (transport?.synthesize
|
|
134
|
+
? (text, signal) => transport.synthesize(text, signal, settings.elevenLabsVoiceId)
|
|
135
|
+
: (text, signal) => fetch("/api/voice/elevenlabs/speech", {
|
|
136
|
+
method: "POST", credentials: "same-origin", signal,
|
|
137
|
+
headers: { "Content-Type": "application/json" },
|
|
138
|
+
body: JSON.stringify({ voice_id: settings.elevenLabsVoiceId, text, output_format: "pcm_24000" }),
|
|
139
|
+
})),
|
|
140
|
+
settings,
|
|
129
141
|
...(transport?.call === undefined ? {} : { call: transport.call }),
|
|
130
142
|
...(transport?.sidebandUrl === undefined ? {} : { sidebandUrl: transport.sidebandUrl }),
|
|
131
143
|
...(options.callUrl === undefined ? {} : { callUrl: options.callUrl }),
|
|
@@ -299,6 +311,9 @@ function validateOptions(options) {
|
|
|
299
311
|
if (!options || typeof options !== "object" || Array.isArray(options)) {
|
|
300
312
|
throw new TypeError("Voice.create options must be an object");
|
|
301
313
|
}
|
|
314
|
+
if (options.synthesize !== undefined && typeof options.synthesize !== "function") {
|
|
315
|
+
throw new TypeError("voice synthesize must be a function");
|
|
316
|
+
}
|
|
302
317
|
if (options.captureMicrophone !== undefined && typeof options.captureMicrophone !== "function") {
|
|
303
318
|
throw new TypeError("voice captureMicrophone must be a function");
|
|
304
319
|
}
|
|
@@ -317,6 +332,16 @@ function browserLocationOrigin() {
|
|
|
317
332
|
}
|
|
318
333
|
|
|
319
334
|
function voiceSettings(options) {
|
|
320
|
-
return Object.fromEntries(["voice", "instructions", "pace", "updates", "handoffMode", "acknowledgements"]
|
|
335
|
+
return Object.fromEntries(["voice", "instructions", "pace", "updates", "handoffMode", "acknowledgements", "outputProvider", "elevenLabsVoiceId"]
|
|
321
336
|
.filter((key) => options[key] !== undefined).map((key) => [key, options[key]]));
|
|
322
337
|
}
|
|
338
|
+
|
|
339
|
+
function validateOutputSettings(settings) {
|
|
340
|
+
if (settings.outputProvider !== undefined && !["openai", "elevenlabs"].includes(settings.outputProvider)) {
|
|
341
|
+
throw new TypeError("unsupported voice output provider");
|
|
342
|
+
}
|
|
343
|
+
if (settings.outputProvider === "elevenlabs"
|
|
344
|
+
&& (typeof settings.elevenLabsVoiceId !== "string" || !/^[A-Za-z0-9_-]{1,128}$/.test(settings.elevenLabsVoiceId))) {
|
|
345
|
+
throw new TypeError("ElevenLabs output requires a valid voice ID");
|
|
346
|
+
}
|
|
347
|
+
}
|
package/browser/VoiceSession.mjs
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { ElevenLabsPlayback } from "./ElevenLabsPlayback.mjs";
|
|
2
|
+
|
|
1
3
|
export const MICROPHONE_CAPTURE_TIMEOUT_MS = 15_000;
|
|
2
4
|
export const ICE_GATHERING_TIMEOUT_MS = 15_000;
|
|
3
5
|
export const REALTIME_CALL_TIMEOUT_MS = 15_000;
|
|
@@ -85,6 +87,7 @@ export class BrowserVoiceSession {
|
|
|
85
87
|
#microphone;
|
|
86
88
|
#speaker;
|
|
87
89
|
#playbackEnabled = false;
|
|
90
|
+
#elevenLabs;
|
|
88
91
|
#muted = false;
|
|
89
92
|
#inputGeneration = 0;
|
|
90
93
|
#meterTimer;
|
|
@@ -102,10 +105,17 @@ export class BrowserVoiceSession {
|
|
|
102
105
|
|
|
103
106
|
constructor(options) {
|
|
104
107
|
this.#options = options;
|
|
108
|
+
if (options.settings?.outputProvider === "elevenlabs") {
|
|
109
|
+
if (typeof options.synthesize !== "function") throw new TypeError("ElevenLabs output requires an authenticated synthesis transport");
|
|
110
|
+
this.#elevenLabs = new ElevenLabsPlayback(options.synthesize, (error) => {
|
|
111
|
+
if (!this.#closed) options.onTerminated(error.message);
|
|
112
|
+
});
|
|
113
|
+
}
|
|
105
114
|
this.#backendReady = new Promise((resolve) => { this.#resolveBackendReady = resolve; });
|
|
106
115
|
}
|
|
107
116
|
|
|
108
117
|
start() {
|
|
118
|
+
if (!this.#starting) void this.#elevenLabs?.prime()?.catch((error) => this.#options.onTerminated(error.message));
|
|
109
119
|
this.#starting ??= this.#start();
|
|
110
120
|
return this.#starting;
|
|
111
121
|
}
|
|
@@ -140,7 +150,8 @@ export class BrowserVoiceSession {
|
|
|
140
150
|
// Only delegated work waits for admission on the direct control path.
|
|
141
151
|
const coreStartup = coreReady.then(async (core) => {
|
|
142
152
|
if (this.#directControl) await this.#options.beforeAgentTurn?.();
|
|
143
|
-
await core?.start();
|
|
153
|
+
const effects = await core?.start();
|
|
154
|
+
if (!this.#closed && !this.#closing.signal.aborted) await this.#apply(effects);
|
|
144
155
|
return core;
|
|
145
156
|
});
|
|
146
157
|
this.#admission = coreStartup;
|
|
@@ -271,7 +282,7 @@ export class BrowserVoiceSession {
|
|
|
271
282
|
}
|
|
272
283
|
const stream = event.streams[0] ?? new MediaStream([event.track]);
|
|
273
284
|
this.#speaker ??= new SpeakerPlayback(new Audio(), this.#options.onStatus);
|
|
274
|
-
this.#speaker.setEnabled(this.#playbackEnabled);
|
|
285
|
+
this.#speaker.setEnabled(this.#playbackEnabled && !this.#elevenLabs);
|
|
275
286
|
this.#speaker.attach(stream);
|
|
276
287
|
});
|
|
277
288
|
peer.addEventListener("connectionstatechange", () => {
|
|
@@ -301,6 +312,7 @@ export class BrowserVoiceSession {
|
|
|
301
312
|
|
|
302
313
|
noteTypedInput() {
|
|
303
314
|
this.#playbackEnabled = false;
|
|
315
|
+
this.#elevenLabs?.interrupt();
|
|
304
316
|
this.#speaker?.setEnabled(false);
|
|
305
317
|
return this.#applyLive(async () => {
|
|
306
318
|
const core = await this.#options.core;
|
|
@@ -417,10 +429,12 @@ export class BrowserVoiceSession {
|
|
|
417
429
|
for (const text of effects.undelivered_answers ?? []) this.#options.onUndeliveredAnswer?.(text);
|
|
418
430
|
if (effects.input_generation !== undefined) {
|
|
419
431
|
if (effects.input_generation < this.#inputGeneration) return;
|
|
432
|
+
if (effects.input_generation > this.#inputGeneration) this.#elevenLabs?.interrupt();
|
|
420
433
|
this.#inputGeneration = effects.input_generation;
|
|
421
434
|
}
|
|
422
435
|
if (effects.playback_enabled === false) {
|
|
423
436
|
this.#playbackEnabled = false;
|
|
437
|
+
this.#elevenLabs?.interrupt();
|
|
424
438
|
this.#speaker?.setEnabled(false);
|
|
425
439
|
}
|
|
426
440
|
let sent = 0;
|
|
@@ -436,9 +450,10 @@ export class BrowserVoiceSession {
|
|
|
436
450
|
if (effects.acknowledge_frames && sent > 0) await this.#core?.framesSent(sent);
|
|
437
451
|
if (!this.#closed && effects.playback_enabled === true && sent === (effects.frames?.length ?? 0)) {
|
|
438
452
|
this.#playbackEnabled = true;
|
|
439
|
-
this.#speaker?.setEnabled(
|
|
453
|
+
this.#speaker?.setEnabled(!this.#elevenLabs);
|
|
440
454
|
}
|
|
441
455
|
for (const entry of effects.transcripts ?? []) {
|
|
456
|
+
if (this.#playbackEnabled && !this.#closed) this.#elevenLabs?.transcript(entry);
|
|
442
457
|
this.#options.onTranscript(entry.speaker, entry.text, entry);
|
|
443
458
|
}
|
|
444
459
|
if (effects.status) this.#status(effects.status);
|
|
@@ -549,6 +564,7 @@ export class BrowserVoiceSession {
|
|
|
549
564
|
}
|
|
550
565
|
|
|
551
566
|
#stopBrowserMedia() {
|
|
567
|
+
this.#elevenLabs?.close();
|
|
552
568
|
if (this.#meterTimer !== undefined) window.clearTimeout(this.#meterTimer);
|
|
553
569
|
this.#meterTimer = undefined;
|
|
554
570
|
this.#call?.abort();
|
package/browser/host.d.mts
CHANGED
|
@@ -75,12 +75,28 @@ export function createBrowserHost(options?: {
|
|
|
75
75
|
mcp?: McpServers;
|
|
76
76
|
codeEvaluator?: CodeEvaluator;
|
|
77
77
|
toolMode?: "code" | "direct";
|
|
78
|
-
/** @internal
|
|
78
|
+
/** @internal Live host lifecycle for ephemeral Rust-owned subagents. */
|
|
79
|
+
subagentRouting?: Pick<import('../runtime/subagent-routing.mjs').SubagentRouting, 'resolve' | 'bind'>;
|
|
79
80
|
subagentSessions?: {
|
|
80
|
-
|
|
81
|
-
bind(sessionId: string, descriptor: SubagentToolContext): void;
|
|
82
|
-
release(sessionId: string): void;
|
|
81
|
+
bindingDescriptor?(sessionId: string, descriptor: SubagentToolContext, hostContextRef?: string): SubagentToolContext;
|
|
82
|
+
bind(sessionId: string, descriptor: SubagentToolContext, hostContextRef?: string): void;
|
|
83
|
+
release(sessionId: string, hostContextRef?: string): void;
|
|
83
84
|
};
|
|
85
|
+
/** @internal Content-free summary, once when ownership closes; no public transport option. */
|
|
86
|
+
onSocketTiming?: (timing: {
|
|
87
|
+
message_count: number;
|
|
88
|
+
delivered_message_count: number;
|
|
89
|
+
buffered_message_count: number;
|
|
90
|
+
discarded_message_count: number;
|
|
91
|
+
queue_residence_total_ms: number;
|
|
92
|
+
queue_residence_max_ms: number;
|
|
93
|
+
provider_timings: Array<{
|
|
94
|
+
response_id?: string;
|
|
95
|
+
pre_inference_ms?: number;
|
|
96
|
+
engine_queue_max_ms?: number;
|
|
97
|
+
engine_service_ttft_total_ms?: number;
|
|
98
|
+
}>;
|
|
99
|
+
}) => void;
|
|
84
100
|
maxQueuedMessages?: number;
|
|
85
101
|
maxQueuedBytes?: number;
|
|
86
102
|
maxBufferedSendBytes?: number;
|