@opengeni/sdk 0.36.1 → 0.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -2
- package/dist/chunk-DXDL7EEW.js +573 -0
- package/dist/chunk-DXDL7EEW.js.map +1 -0
- package/dist/chunk-OINSI33U.js +97 -0
- package/dist/chunk-OINSI33U.js.map +1 -0
- package/dist/{chunk-B7E4FPNZ.js → chunk-SSQPCFEN.js} +159 -108
- package/dist/chunk-SSQPCFEN.js.map +1 -0
- package/dist/chunk-XBBPYTI5.js +2018 -0
- package/dist/chunk-XBBPYTI5.js.map +1 -0
- package/dist/client.d.ts +39 -4
- package/dist/codex-realtime-controller.d.ts +124 -0
- package/dist/codex-realtime-controller.js +18 -0
- package/dist/codex-realtime-controller.js.map +1 -0
- package/dist/codex-realtime-lifecycle.d.ts +18 -0
- package/dist/codex-realtime-v3-wire.d.ts +86 -0
- package/dist/codex-realtime-v3.d.ts +52 -0
- package/dist/codex-realtime.d.ts +68 -0
- package/dist/core.js +6 -4
- package/dist/gateway-realtime-transport.d.ts +2 -0
- package/dist/gateway-realtime-transport.js +7 -0
- package/dist/gateway-realtime-transport.js.map +1 -0
- package/dist/index.d.ts +13 -4
- package/dist/index.js +40 -6
- package/dist/index.js.map +1 -1
- package/dist/realtime.d.ts +45 -0
- package/dist/realtime.js +73 -0
- package/dist/realtime.js.map +1 -0
- package/dist/types.d.ts +225 -6
- package/dist/workspace-instruction-policies.d.ts +12 -0
- package/dist/workspace-state.d.ts +70 -4
- package/package.json +14 -1
- package/src/client.ts +248 -20
- package/src/codex-realtime-controller.ts +1465 -0
- package/src/codex-realtime-lifecycle.ts +97 -0
- package/src/codex-realtime-v3-wire.ts +349 -0
- package/src/codex-realtime-v3.ts +575 -0
- package/src/codex-realtime.ts +411 -0
- package/src/gateway-realtime-transport.ts +650 -0
- package/src/index.ts +81 -0
- package/src/realtime.ts +172 -0
- package/src/types.ts +280 -6
- package/src/workspace-instruction-policies.ts +13 -0
- package/src/workspace-state.ts +82 -4
- package/dist/chunk-B7E4FPNZ.js.map +0 -1
package/README.md
CHANGED
|
@@ -37,6 +37,56 @@ for await (const event of client.streamEvents(workspaceId, session.id)) {
|
|
|
37
37
|
}
|
|
38
38
|
```
|
|
39
39
|
|
|
40
|
+
## Realtime browser controller (`@opengeni/sdk/realtime`)
|
|
41
|
+
|
|
42
|
+
The public realtime subpath owns the provider-neutral browser controller and
|
|
43
|
+
the existing Codex Live, WebRTC/V3, and AI Gateway transports. It selects the
|
|
44
|
+
transport from the catalog model without changing the backend API, durable
|
|
45
|
+
ledger, delegation, context, or recovery semantics:
|
|
46
|
+
|
|
47
|
+
```ts
|
|
48
|
+
import { OpenGeniClient } from "@opengeni/sdk";
|
|
49
|
+
import type { SessionRealtimeClientLike } from "@opengeni/sdk/realtime";
|
|
50
|
+
|
|
51
|
+
const client = new OpenGeniClient({ baseUrl: "/opengeni-api" });
|
|
52
|
+
const realtimeClient: SessionRealtimeClientLike = client;
|
|
53
|
+
const catalog = await realtimeClient.getWorkspaceRealtimeModelCatalog(workspaceId);
|
|
54
|
+
const model = catalog.models.find((candidate) => candidate.available)?.id;
|
|
55
|
+
if (!model) throw new Error("No realtime model is available");
|
|
56
|
+
|
|
57
|
+
// Lazy import keeps the base SDK entry safe for server and non-realtime hosts.
|
|
58
|
+
const { createSessionRealtimeController } = await import("@opengeni/sdk/realtime");
|
|
59
|
+
const controller = createSessionRealtimeController({
|
|
60
|
+
client: realtimeClient,
|
|
61
|
+
workspaceId,
|
|
62
|
+
sessionId,
|
|
63
|
+
model,
|
|
64
|
+
remoteAudio,
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
const unsubscribe = controller.subscribe((snapshot) => {
|
|
68
|
+
console.log(snapshot.status, snapshot.microphone, snapshot.diagnostic);
|
|
69
|
+
});
|
|
70
|
+
await controller.start();
|
|
71
|
+
|
|
72
|
+
// Later:
|
|
73
|
+
await controller.stop();
|
|
74
|
+
unsubscribe();
|
|
75
|
+
controller.close();
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
`SessionRealtimeClientLike` is the exact proxy-friendly backend surface:
|
|
79
|
+
catalog, begin, Codex/Gateway negotiation, activation, heartbeat, ledger sync,
|
|
80
|
+
and end. Existing `OpenGeniClient` methods remain the implementation. Current
|
|
81
|
+
Codex-named controller and transport exports remain available as compatibility
|
|
82
|
+
aliases, but new integrations should use the provider-neutral names.
|
|
83
|
+
|
|
84
|
+
Do not put API credentials in browser bundles. Browser hosts should either use
|
|
85
|
+
the deployment's normal browser authentication or expose these same methods
|
|
86
|
+
through a tenant-scoped, same-origin proxy. The SDK does not move persistence,
|
|
87
|
+
prompt construction, context processing, delegation, or provider credentials
|
|
88
|
+
out of `apps/api`, `apps/worker`, or `packages/db`.
|
|
89
|
+
|
|
40
90
|
## Workspace artifacts
|
|
41
91
|
|
|
42
92
|
Workspace artifacts are generic, immutable HTML publications. The SDK does not
|
|
@@ -239,6 +289,11 @@ const paused = await client.getQueue(workspaceId, sessionId);
|
|
|
239
289
|
await client.resumeSession(workspaceId, sessionId, {
|
|
240
290
|
expectedControlEtag: paused.effectiveControl.controlEtag,
|
|
241
291
|
});
|
|
292
|
+
// Cancel is irreversible: it drains and fences this session subtree.
|
|
293
|
+
await client.cancelSession(workspaceId, sessionId, {
|
|
294
|
+
reason: "host record deleted",
|
|
295
|
+
clientEventId: crypto.randomUUID(),
|
|
296
|
+
});
|
|
242
297
|
await client.sendApprovalDecision(workspaceId, sessionId, { approvalId, decision: "approve" });
|
|
243
298
|
```
|
|
244
299
|
|
|
@@ -348,14 +403,14 @@ Every public endpoint group has typed methods:
|
|
|
348
403
|
| Group | Methods |
|
|
349
404
|
| --- | --- |
|
|
350
405
|
| Access + workspaces | `getAccessContext`, `listWorkspaces`, `createWorkspace`, `getWorkspace`, `updateWorkspace` |
|
|
351
|
-
| Sessions + events | `createSession`, `listSessions`, `getSession`, `updateSession`, `listEvents`, `sendEvent`, `sendMessage`, `steerMessage`, `pauseSession`, `resumeSession`, `sendApprovalDecision`, `streamEvents`, `openEventStream` |
|
|
406
|
+
| Sessions + events | `createSession`, `listSessions`, `getSession`, `updateSession`, `listEvents`, `sendEvent`, `sendMessage`, `steerMessage`, `pauseSession`, `resumeSession`, `cancelSession`, `sendApprovalDecision`, `streamEvents`, `openEventStream` |
|
|
352
407
|
| Machines (bring-your-own-compute) | `listMachines`, `machineMetricsSeries`, `swapActiveSandbox`, `mintEnrollToken`, `lookupDeviceEnrollment`, `approveDeviceEnrollment`, `denyDeviceEnrollment` |
|
|
353
408
|
| Turn queue | `getQueue`, `moveQueueItem`, `editQueueItem`, `steerQueueItem`, `deleteQueueItem` |
|
|
354
409
|
| Goal | `getGoal`, `updateGoal`, `pauseGoal`, `resumeGoal` |
|
|
355
410
|
| Scheduled tasks | `createScheduledTask`, `listScheduledTasks`, `getScheduledTask`, `updateScheduledTask`, `pauseScheduledTask`, `resumeScheduledTask`, `triggerScheduledTask`, `deleteScheduledTask`, `listScheduledTaskRuns` |
|
|
356
411
|
| Variable sets | `listVariable sets`, `createVariable set`, `getVariable set`, `updateVariable set`, `deleteVariable set`, `setVariable setVariable`, `deleteVariable setVariable` (values are write-only) |
|
|
357
412
|
| Files | `uploadFile`, `beginFileUpload`, `completeFileUpload`, `getFile`, `createFileDownloadUrl` |
|
|
358
|
-
| Documents | `createDocumentBase`, `listDocumentBases`, `getDocumentBase`, `addDocument`, `listDocuments`, `reindexDocument`, `searchDocuments` |
|
|
413
|
+
| Documents | `createDocumentBase`, `listDocumentBases`, `getDocumentBase`, `addDocument`, `listDocuments`, `reindexDocument`, `searchDocuments`, `searchKnowledge` (effective organization + workspace + immutable initiating-user personal scope) |
|
|
359
414
|
| Packs | `listPacks`, `registerPack`, `getPack`, `enablePack`, `deletePack`, `listPackInstallations` |
|
|
360
415
|
| Capabilities | `listCapabilities`, `createCapability`, `enableCapability`, `disableCapability`, `discoverMcpCapabilities` |
|
|
361
416
|
| GitHub | `getGitHubApp`, `githubConnectUrl`, `listGitHubRepositories`, `syncGitHubRepositories`, `createGitHubAppManifest` |
|
|
@@ -0,0 +1,573 @@
|
|
|
1
|
+
// src/gateway-realtime-transport.ts
|
|
2
|
+
var AUDIO_SAMPLE_RATE = 24e3;
|
|
3
|
+
var DELEGATION_TOOL = "delegate_to_session";
|
|
4
|
+
function createGatewayRealtimeTransportStarter() {
|
|
5
|
+
return async (input) => {
|
|
6
|
+
const client = input.client;
|
|
7
|
+
if (!client.negotiateGatewayRealtime) {
|
|
8
|
+
throw new Error("The OpenGeni client does not support AI Gateway realtime");
|
|
9
|
+
}
|
|
10
|
+
const answer = await client.negotiateGatewayRealtime(
|
|
11
|
+
input.workspaceId,
|
|
12
|
+
input.sessionId,
|
|
13
|
+
{
|
|
14
|
+
realtimeId: input.realtimeId,
|
|
15
|
+
operationId: input.operationId,
|
|
16
|
+
browserInstanceId: input.browserInstanceId,
|
|
17
|
+
ownerKey: input.ownerKey,
|
|
18
|
+
expectedVersion: input.expectedVersion,
|
|
19
|
+
expectedConnectionEpoch: input.expectedConnectionEpoch,
|
|
20
|
+
rotate: input.rotate
|
|
21
|
+
},
|
|
22
|
+
{ signal: input.signal }
|
|
23
|
+
);
|
|
24
|
+
throwIfAborted(input.signal);
|
|
25
|
+
const websocket = new WebSocket(answer.url, [
|
|
26
|
+
"ai-gateway-realtime.v1",
|
|
27
|
+
`ai-gateway-auth.${answer.token}`
|
|
28
|
+
]);
|
|
29
|
+
const channel = new GatewayRealtimeDataChannel(
|
|
30
|
+
(payload) => handleBridgeOutbound(websocket, payload)
|
|
31
|
+
);
|
|
32
|
+
input.onEventsCreated(channel.asRtcDataChannel());
|
|
33
|
+
const audio = new GatewayRealtimeAudio({
|
|
34
|
+
onAudio: (encoded) => send(websocket, { type: "input-audio-append", audio: encoded }),
|
|
35
|
+
onAudibleOutputState: input.onAudibleOutputState
|
|
36
|
+
});
|
|
37
|
+
let stopped = false;
|
|
38
|
+
let currentOutputItemId = null;
|
|
39
|
+
const finalizedAssistantItems = /* @__PURE__ */ new Set();
|
|
40
|
+
const microphoneTracks = input.media.getAudioTracks();
|
|
41
|
+
const onMicrophoneEnded = () => {
|
|
42
|
+
if (!stopped) input.onMicrophoneEnded();
|
|
43
|
+
};
|
|
44
|
+
for (const track of microphoneTracks) track.addEventListener?.("ended", onMicrophoneEnded);
|
|
45
|
+
const onMessage = (event) => {
|
|
46
|
+
if (typeof event.data !== "string") return;
|
|
47
|
+
let parsed;
|
|
48
|
+
try {
|
|
49
|
+
parsed = JSON.parse(event.data);
|
|
50
|
+
} catch {
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
if (!isRecord(parsed) || typeof parsed.type !== "string") return;
|
|
54
|
+
handleGatewayEvent({
|
|
55
|
+
event: parsed,
|
|
56
|
+
channel,
|
|
57
|
+
websocket,
|
|
58
|
+
audio,
|
|
59
|
+
finalizedAssistantItems,
|
|
60
|
+
getCurrentOutputItemId: () => currentOutputItemId,
|
|
61
|
+
setCurrentOutputItemId: (value) => {
|
|
62
|
+
currentOutputItemId = value;
|
|
63
|
+
}
|
|
64
|
+
});
|
|
65
|
+
};
|
|
66
|
+
const onClose = () => {
|
|
67
|
+
if (!stopped) input.onConnectionHealth("closed");
|
|
68
|
+
};
|
|
69
|
+
const onError = () => {
|
|
70
|
+
if (!stopped) input.onConnectionHealth("failed");
|
|
71
|
+
};
|
|
72
|
+
websocket.addEventListener("message", onMessage);
|
|
73
|
+
websocket.addEventListener("close", onClose);
|
|
74
|
+
websocket.addEventListener("error", onError);
|
|
75
|
+
await waitForWebSocketOpen(websocket, input.signal);
|
|
76
|
+
channel.open();
|
|
77
|
+
send(websocket, {
|
|
78
|
+
type: "session-update",
|
|
79
|
+
config: {
|
|
80
|
+
instructions: answer.instructions,
|
|
81
|
+
outputModalities: ["audio"],
|
|
82
|
+
inputAudioFormat: { type: "audio/pcm", rate: AUDIO_SAMPLE_RATE },
|
|
83
|
+
outputAudioFormat: { type: "audio/pcm", rate: AUDIO_SAMPLE_RATE },
|
|
84
|
+
inputAudioTranscription: {},
|
|
85
|
+
outputAudioTranscription: {},
|
|
86
|
+
turnDetection: {
|
|
87
|
+
type: "server-vad",
|
|
88
|
+
prefixPaddingMs: 300,
|
|
89
|
+
silenceDurationMs: 500
|
|
90
|
+
},
|
|
91
|
+
tools: [
|
|
92
|
+
{
|
|
93
|
+
type: "function",
|
|
94
|
+
name: DELEGATION_TOOL,
|
|
95
|
+
description: "Pass execution work, actions, and session tasks to the underlying session agent. Include the complete standalone request and relevant conversational context.",
|
|
96
|
+
parameters: {
|
|
97
|
+
type: "object",
|
|
98
|
+
properties: {
|
|
99
|
+
request: {
|
|
100
|
+
type: "string",
|
|
101
|
+
description: "Complete standalone task for the session agent"
|
|
102
|
+
}
|
|
103
|
+
},
|
|
104
|
+
required: ["request"],
|
|
105
|
+
additionalProperties: false
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
]
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
for (const item of answer.initialItems) {
|
|
112
|
+
send(websocket, {
|
|
113
|
+
type: "conversation-item-create",
|
|
114
|
+
item: {
|
|
115
|
+
type: "text-message",
|
|
116
|
+
role: "user",
|
|
117
|
+
text: `<session_initial_item role="${item.role}">
|
|
118
|
+
${item.text}
|
|
119
|
+
</session_initial_item>`
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
await audio.startCapture(input.media);
|
|
124
|
+
input.onConnectionHealth("connected");
|
|
125
|
+
const stop = () => {
|
|
126
|
+
if (stopped) return;
|
|
127
|
+
stopped = true;
|
|
128
|
+
input.signal.removeEventListener("abort", stop);
|
|
129
|
+
websocket.removeEventListener("message", onMessage);
|
|
130
|
+
websocket.removeEventListener("close", onClose);
|
|
131
|
+
websocket.removeEventListener("error", onError);
|
|
132
|
+
for (const track of microphoneTracks) track.removeEventListener?.("ended", onMicrophoneEnded);
|
|
133
|
+
channel.close();
|
|
134
|
+
audio.dispose();
|
|
135
|
+
if (websocket.readyState === WebSocket.OPEN || websocket.readyState === WebSocket.CONNECTING) {
|
|
136
|
+
websocket.close(1e3, "OpenGeni realtime connection retired");
|
|
137
|
+
}
|
|
138
|
+
};
|
|
139
|
+
input.signal.addEventListener("abort", stop, { once: true });
|
|
140
|
+
return {
|
|
141
|
+
peerConnection: null,
|
|
142
|
+
events: channel.asRtcDataChannel(),
|
|
143
|
+
media: input.media,
|
|
144
|
+
operationId: input.operationId,
|
|
145
|
+
connectionId: answer.connectionId,
|
|
146
|
+
connectionEpoch: answer.connectionEpoch,
|
|
147
|
+
startupFenceSequence: answer.startupFenceSequence,
|
|
148
|
+
modeVersion: answer.modeVersion,
|
|
149
|
+
microphoneHealthy: () => microphoneTracks.length > 0 && microphoneTracks.every((track) => track.readyState !== "ended"),
|
|
150
|
+
audibleOutputState: () => audio.audibleOutputState(),
|
|
151
|
+
setOutputMuted: (muted) => audio.setMuted(muted),
|
|
152
|
+
activateRemoteAudio: () => void audio.resume(),
|
|
153
|
+
retryAudibleOutput: () => audio.resume(),
|
|
154
|
+
stop
|
|
155
|
+
};
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
function handleGatewayEvent(input) {
|
|
159
|
+
const event = input.event;
|
|
160
|
+
const eventId = providerEventId(event);
|
|
161
|
+
if (event.type === "session-created") {
|
|
162
|
+
const sessionId = stringValue(event.sessionId) ?? `gateway-${crypto.randomUUID()}`;
|
|
163
|
+
input.channel.providerEvent({
|
|
164
|
+
type: "session.started",
|
|
165
|
+
event_id: eventId,
|
|
166
|
+
session: { id: sessionId }
|
|
167
|
+
});
|
|
168
|
+
return;
|
|
169
|
+
}
|
|
170
|
+
if (event.type === "speech-started") {
|
|
171
|
+
const itemId = input.getCurrentOutputItemId();
|
|
172
|
+
if (itemId && input.audio.isPlaying()) {
|
|
173
|
+
const audioEndMs = input.audio.playbackOffsetMs();
|
|
174
|
+
input.audio.stopPlayback();
|
|
175
|
+
send(input.websocket, {
|
|
176
|
+
type: "conversation-item-truncate",
|
|
177
|
+
itemId,
|
|
178
|
+
contentIndex: 0,
|
|
179
|
+
audioEndMs: Math.max(0, Math.round(audioEndMs))
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
if (event.type === "input-transcription-completed") {
|
|
185
|
+
const transcript = stringValue(event.transcript) ?? "";
|
|
186
|
+
const itemId = stringValue(event.itemId) ?? crypto.randomUUID();
|
|
187
|
+
if (transcript.trim()) {
|
|
188
|
+
input.channel.providerEvent({
|
|
189
|
+
type: "turn.done",
|
|
190
|
+
event_id: eventId,
|
|
191
|
+
turn: { id: itemId, role: "user", transcript }
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
if (event.type === "audio-delta") {
|
|
197
|
+
const delta = stringValue(event.delta);
|
|
198
|
+
if (!delta) return;
|
|
199
|
+
const itemId = stringValue(event.itemId);
|
|
200
|
+
if (itemId) input.setCurrentOutputItemId(itemId);
|
|
201
|
+
input.audio.play(delta);
|
|
202
|
+
input.channel.providerEvent({ type: "output_audio.delta", event_id: eventId, audio: delta });
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
if (event.type === "audio-transcript-done" || event.type === "text-done") {
|
|
206
|
+
const itemId = stringValue(event.itemId) ?? crypto.randomUUID();
|
|
207
|
+
const transcript = stringValue(event.transcript) ?? stringValue(event.text) ?? "";
|
|
208
|
+
if (transcript.trim() && !input.finalizedAssistantItems.has(itemId)) {
|
|
209
|
+
input.finalizedAssistantItems.add(itemId);
|
|
210
|
+
input.channel.providerEvent({
|
|
211
|
+
type: "turn.done",
|
|
212
|
+
event_id: eventId,
|
|
213
|
+
turn: { id: itemId, role: "assistant", transcript }
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
if (event.type === "audio-done") return;
|
|
219
|
+
if (event.type === "function-call-arguments-done") {
|
|
220
|
+
const callId = stringValue(event.callId) ?? stringValue(event.itemId) ?? crypto.randomUUID();
|
|
221
|
+
const name = stringValue(event.name);
|
|
222
|
+
if (name !== DELEGATION_TOOL) {
|
|
223
|
+
send(input.websocket, {
|
|
224
|
+
type: "conversation-item-create",
|
|
225
|
+
item: {
|
|
226
|
+
type: "function-call-output",
|
|
227
|
+
callId,
|
|
228
|
+
name,
|
|
229
|
+
output: JSON.stringify({ error: "Unsupported realtime tool" })
|
|
230
|
+
}
|
|
231
|
+
});
|
|
232
|
+
send(input.websocket, { type: "response-create" });
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
const request = delegationRequest(stringValue(event.arguments) ?? "");
|
|
236
|
+
input.channel.providerEvent({
|
|
237
|
+
type: "delegation.created",
|
|
238
|
+
event_id: eventId,
|
|
239
|
+
item: {
|
|
240
|
+
id: callId,
|
|
241
|
+
type: "delegation",
|
|
242
|
+
target: "client",
|
|
243
|
+
content: [{ type: "input_text", text: request }]
|
|
244
|
+
}
|
|
245
|
+
});
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
if (event.type === "error") {
|
|
249
|
+
input.channel.providerEvent({
|
|
250
|
+
type: "error",
|
|
251
|
+
event_id: eventId,
|
|
252
|
+
message: stringValue(event.message) ?? "AI Gateway realtime provider error"
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
var GatewayRealtimeDataChannel = class extends EventTarget {
|
|
257
|
+
constructor(outbound) {
|
|
258
|
+
super();
|
|
259
|
+
this.outbound = outbound;
|
|
260
|
+
}
|
|
261
|
+
readyState = "connecting";
|
|
262
|
+
pendingOutbound = [];
|
|
263
|
+
outboundScheduled = false;
|
|
264
|
+
asRtcDataChannel() {
|
|
265
|
+
return this;
|
|
266
|
+
}
|
|
267
|
+
open() {
|
|
268
|
+
this.readyState = "open";
|
|
269
|
+
this.dispatchEvent(new Event("open"));
|
|
270
|
+
}
|
|
271
|
+
send(payload) {
|
|
272
|
+
if (this.readyState !== "open") throw new Error("Gateway realtime channel is not open");
|
|
273
|
+
let parsed;
|
|
274
|
+
try {
|
|
275
|
+
parsed = JSON.parse(payload);
|
|
276
|
+
} catch {
|
|
277
|
+
return;
|
|
278
|
+
}
|
|
279
|
+
if (!isRecord(parsed)) return;
|
|
280
|
+
this.pendingOutbound.push(parsed);
|
|
281
|
+
if (this.outboundScheduled) return;
|
|
282
|
+
this.outboundScheduled = true;
|
|
283
|
+
queueMicrotask(() => {
|
|
284
|
+
this.outboundScheduled = false;
|
|
285
|
+
const messages = this.pendingOutbound;
|
|
286
|
+
this.pendingOutbound = [];
|
|
287
|
+
if (messages.length > 0 && this.readyState === "open") this.outbound(messages);
|
|
288
|
+
});
|
|
289
|
+
}
|
|
290
|
+
providerEvent(event) {
|
|
291
|
+
if (this.readyState === "closed") return;
|
|
292
|
+
this.dispatchEvent(new MessageEvent("message", { data: JSON.stringify(event) }));
|
|
293
|
+
}
|
|
294
|
+
close() {
|
|
295
|
+
if (this.readyState === "closed") return;
|
|
296
|
+
this.readyState = "closed";
|
|
297
|
+
this.pendingOutbound = [];
|
|
298
|
+
this.dispatchEvent(new Event("close"));
|
|
299
|
+
}
|
|
300
|
+
};
|
|
301
|
+
function handleBridgeOutbound(websocket, messages) {
|
|
302
|
+
const groups = /* @__PURE__ */ new Map();
|
|
303
|
+
for (const message of messages) {
|
|
304
|
+
const type = stringValue(message.type);
|
|
305
|
+
if (type !== "session.context.append" && type !== "delegation.context.append") continue;
|
|
306
|
+
const id = stringValue(message.delegation_item_id) ?? null;
|
|
307
|
+
const channel = stringValue(message.channel) ?? "commentary";
|
|
308
|
+
const key = `${type}:${id ?? "session"}:${channel}`;
|
|
309
|
+
const content = Array.isArray(message.content) ? message.content : [];
|
|
310
|
+
const text = content.filter(isRecord).map((part) => stringValue(part.text) ?? "").join("");
|
|
311
|
+
const existing = groups.get(key);
|
|
312
|
+
groups.set(key, {
|
|
313
|
+
type,
|
|
314
|
+
id,
|
|
315
|
+
channel,
|
|
316
|
+
text: `${existing?.text ?? ""}${text}`
|
|
317
|
+
});
|
|
318
|
+
}
|
|
319
|
+
for (const group of groups.values()) {
|
|
320
|
+
if (group.type === "delegation.context.append" && group.id && group.channel === "speakable") {
|
|
321
|
+
send(websocket, {
|
|
322
|
+
type: "conversation-item-create",
|
|
323
|
+
item: {
|
|
324
|
+
type: "function-call-output",
|
|
325
|
+
callId: group.id,
|
|
326
|
+
name: DELEGATION_TOOL,
|
|
327
|
+
output: JSON.stringify({ result: group.text })
|
|
328
|
+
}
|
|
329
|
+
});
|
|
330
|
+
send(websocket, { type: "response-create" });
|
|
331
|
+
continue;
|
|
332
|
+
}
|
|
333
|
+
const wrapper = group.type === "delegation.context.append" ? group.channel === "commentary" ? "execution_progress" : "execution_result" : "session_update";
|
|
334
|
+
send(websocket, {
|
|
335
|
+
type: "conversation-item-create",
|
|
336
|
+
item: {
|
|
337
|
+
type: "text-message",
|
|
338
|
+
role: "user",
|
|
339
|
+
text: `<${wrapper}>
|
|
340
|
+
${group.text}
|
|
341
|
+
</${wrapper}>`
|
|
342
|
+
}
|
|
343
|
+
});
|
|
344
|
+
if (group.channel === "speakable") send(websocket, { type: "response-create" });
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
var GatewayRealtimeAudio = class {
|
|
348
|
+
constructor(options) {
|
|
349
|
+
this.options = options;
|
|
350
|
+
}
|
|
351
|
+
captureContext = null;
|
|
352
|
+
captureSource = null;
|
|
353
|
+
captureProcessor = null;
|
|
354
|
+
playbackContext = null;
|
|
355
|
+
playbackGain = null;
|
|
356
|
+
playbackTime = 0;
|
|
357
|
+
playbackStartedAt = 0;
|
|
358
|
+
activeSources = /* @__PURE__ */ new Set();
|
|
359
|
+
outputState = "inactive";
|
|
360
|
+
muted = false;
|
|
361
|
+
async startCapture(media) {
|
|
362
|
+
const context = new AudioContext({ sampleRate: AUDIO_SAMPLE_RATE });
|
|
363
|
+
this.captureContext = context;
|
|
364
|
+
const source = context.createMediaStreamSource(media);
|
|
365
|
+
const processor = context.createScriptProcessor(4096, 1, 1);
|
|
366
|
+
this.captureSource = source;
|
|
367
|
+
this.captureProcessor = processor;
|
|
368
|
+
processor.onaudioprocess = (event) => {
|
|
369
|
+
const samples = resample(
|
|
370
|
+
new Float32Array(event.inputBuffer.getChannelData(0)),
|
|
371
|
+
context.sampleRate,
|
|
372
|
+
AUDIO_SAMPLE_RATE
|
|
373
|
+
);
|
|
374
|
+
this.options.onAudio(encodePcm16(samples));
|
|
375
|
+
};
|
|
376
|
+
source.connect(processor);
|
|
377
|
+
processor.connect(context.destination);
|
|
378
|
+
await context.resume();
|
|
379
|
+
}
|
|
380
|
+
play(encoded) {
|
|
381
|
+
const context = this.playbackContext ??= new AudioContext({ sampleRate: AUDIO_SAMPLE_RATE });
|
|
382
|
+
if (context.state !== "running") {
|
|
383
|
+
void this.resume().then((resumed) => {
|
|
384
|
+
if (resumed) this.schedule(encoded);
|
|
385
|
+
});
|
|
386
|
+
return;
|
|
387
|
+
}
|
|
388
|
+
this.schedule(encoded);
|
|
389
|
+
}
|
|
390
|
+
schedule(encoded) {
|
|
391
|
+
const context = this.playbackContext;
|
|
392
|
+
if (!context) return;
|
|
393
|
+
const samples = decodePcm16(encoded);
|
|
394
|
+
const buffer = context.createBuffer(1, samples.length, AUDIO_SAMPLE_RATE);
|
|
395
|
+
buffer.getChannelData(0).set(samples);
|
|
396
|
+
const source = context.createBufferSource();
|
|
397
|
+
source.buffer = buffer;
|
|
398
|
+
source.connect(this.outputNode(context));
|
|
399
|
+
const startAt = Math.max(context.currentTime, this.playbackTime);
|
|
400
|
+
if (this.activeSources.size === 0) this.playbackStartedAt = startAt;
|
|
401
|
+
source.start(startAt);
|
|
402
|
+
this.playbackTime = startAt + buffer.duration;
|
|
403
|
+
this.activeSources.add(source);
|
|
404
|
+
this.publish("audible");
|
|
405
|
+
source.onended = () => {
|
|
406
|
+
this.activeSources.delete(source);
|
|
407
|
+
if (this.activeSources.size === 0) this.publish("inactive");
|
|
408
|
+
};
|
|
409
|
+
}
|
|
410
|
+
stopPlayback() {
|
|
411
|
+
for (const source of this.activeSources) {
|
|
412
|
+
try {
|
|
413
|
+
source.stop();
|
|
414
|
+
} catch {
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
this.activeSources.clear();
|
|
418
|
+
if (this.playbackContext) this.playbackTime = this.playbackContext.currentTime;
|
|
419
|
+
this.publish("inactive");
|
|
420
|
+
}
|
|
421
|
+
playbackOffsetMs() {
|
|
422
|
+
return this.playbackContext ? Math.max(0, (this.playbackContext.currentTime - this.playbackStartedAt) * 1e3) : 0;
|
|
423
|
+
}
|
|
424
|
+
audibleOutputState() {
|
|
425
|
+
return this.outputState;
|
|
426
|
+
}
|
|
427
|
+
setMuted(muted) {
|
|
428
|
+
this.muted = muted;
|
|
429
|
+
const context = this.playbackContext;
|
|
430
|
+
const gain = this.playbackGain;
|
|
431
|
+
if (context && gain) gain.gain.setValueAtTime(muted ? 0 : 1, context.currentTime);
|
|
432
|
+
}
|
|
433
|
+
isPlaying() {
|
|
434
|
+
return this.activeSources.size > 0;
|
|
435
|
+
}
|
|
436
|
+
async resume() {
|
|
437
|
+
const context = this.playbackContext ??= new AudioContext({ sampleRate: AUDIO_SAMPLE_RATE });
|
|
438
|
+
this.publish("pending");
|
|
439
|
+
try {
|
|
440
|
+
await context.resume();
|
|
441
|
+
this.publish(this.activeSources.size > 0 ? "audible" : "inactive");
|
|
442
|
+
return true;
|
|
443
|
+
} catch {
|
|
444
|
+
this.publish("blocked");
|
|
445
|
+
return false;
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
dispose() {
|
|
449
|
+
this.captureProcessor?.disconnect();
|
|
450
|
+
this.captureSource?.disconnect();
|
|
451
|
+
void this.captureContext?.close();
|
|
452
|
+
this.captureProcessor = null;
|
|
453
|
+
this.captureSource = null;
|
|
454
|
+
this.captureContext = null;
|
|
455
|
+
this.stopPlayback();
|
|
456
|
+
this.playbackGain?.disconnect();
|
|
457
|
+
void this.playbackContext?.close();
|
|
458
|
+
this.playbackGain = null;
|
|
459
|
+
this.playbackContext = null;
|
|
460
|
+
}
|
|
461
|
+
outputNode(context) {
|
|
462
|
+
if (!this.playbackGain) {
|
|
463
|
+
this.playbackGain = context.createGain();
|
|
464
|
+
this.playbackGain.gain.value = this.muted ? 0 : 1;
|
|
465
|
+
this.playbackGain.connect(context.destination);
|
|
466
|
+
}
|
|
467
|
+
return this.playbackGain;
|
|
468
|
+
}
|
|
469
|
+
publish(state) {
|
|
470
|
+
if (state === this.outputState) return;
|
|
471
|
+
this.outputState = state;
|
|
472
|
+
this.options.onAudibleOutputState(state);
|
|
473
|
+
}
|
|
474
|
+
};
|
|
475
|
+
function send(websocket, event) {
|
|
476
|
+
if (websocket.readyState !== WebSocket.OPEN) return;
|
|
477
|
+
websocket.send(JSON.stringify(event));
|
|
478
|
+
}
|
|
479
|
+
function delegationRequest(argumentsJson) {
|
|
480
|
+
try {
|
|
481
|
+
const parsed = JSON.parse(argumentsJson);
|
|
482
|
+
if (isRecord(parsed) && typeof parsed.request === "string" && parsed.request.trim()) {
|
|
483
|
+
return parsed.request.trim();
|
|
484
|
+
}
|
|
485
|
+
} catch {
|
|
486
|
+
}
|
|
487
|
+
return argumentsJson.trim() || "Continue the user's current request.";
|
|
488
|
+
}
|
|
489
|
+
function providerEventId(event) {
|
|
490
|
+
const raw = isRecord(event.raw) ? event.raw : null;
|
|
491
|
+
return stringValue(event.eventId) ?? stringValue(raw?.event_id) ?? stringValue(raw?.id) ?? crypto.randomUUID();
|
|
492
|
+
}
|
|
493
|
+
function encodePcm16(samples) {
|
|
494
|
+
const bytes = new Uint8Array(samples.length * 2);
|
|
495
|
+
const view = new DataView(bytes.buffer);
|
|
496
|
+
for (let index = 0; index < samples.length; index += 1) {
|
|
497
|
+
const sample = Math.max(-1, Math.min(1, samples[index] ?? 0));
|
|
498
|
+
view.setInt16(index * 2, sample < 0 ? sample * 32768 : sample * 32767, true);
|
|
499
|
+
}
|
|
500
|
+
let binary = "";
|
|
501
|
+
for (let start = 0; start < bytes.length; start += 32768) {
|
|
502
|
+
binary += String.fromCharCode(...bytes.subarray(start, start + 32768));
|
|
503
|
+
}
|
|
504
|
+
return btoa(binary);
|
|
505
|
+
}
|
|
506
|
+
function decodePcm16(encoded) {
|
|
507
|
+
const binary = atob(encoded);
|
|
508
|
+
const bytes = Uint8Array.from(binary, (character) => character.charCodeAt(0));
|
|
509
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
510
|
+
const samples = new Float32Array(Math.floor(bytes.byteLength / 2));
|
|
511
|
+
for (let index = 0; index < samples.length; index += 1) {
|
|
512
|
+
samples[index] = view.getInt16(index * 2, true) / 32768;
|
|
513
|
+
}
|
|
514
|
+
return samples;
|
|
515
|
+
}
|
|
516
|
+
function resample(input, inputRate, outputRate) {
|
|
517
|
+
if (inputRate === outputRate) return input;
|
|
518
|
+
const ratio = inputRate / outputRate;
|
|
519
|
+
const output = new Float32Array(Math.round(input.length / ratio));
|
|
520
|
+
for (let index = 0; index < output.length; index += 1) {
|
|
521
|
+
const source = index * ratio;
|
|
522
|
+
const floor = Math.floor(source);
|
|
523
|
+
const ceil = Math.min(floor + 1, input.length - 1);
|
|
524
|
+
const fraction = source - floor;
|
|
525
|
+
output[index] = (input[floor] ?? 0) * (1 - fraction) + (input[ceil] ?? 0) * fraction;
|
|
526
|
+
}
|
|
527
|
+
return output;
|
|
528
|
+
}
|
|
529
|
+
async function waitForWebSocketOpen(websocket, signal) {
|
|
530
|
+
if (websocket.readyState === WebSocket.OPEN) return;
|
|
531
|
+
await new Promise((resolve, reject) => {
|
|
532
|
+
const cleanup = () => {
|
|
533
|
+
websocket.removeEventListener("open", onOpen);
|
|
534
|
+
websocket.removeEventListener("error", onError);
|
|
535
|
+
websocket.removeEventListener("close", onClose);
|
|
536
|
+
signal.removeEventListener("abort", onAbort);
|
|
537
|
+
};
|
|
538
|
+
const onOpen = () => {
|
|
539
|
+
cleanup();
|
|
540
|
+
resolve();
|
|
541
|
+
};
|
|
542
|
+
const onError = () => {
|
|
543
|
+
cleanup();
|
|
544
|
+
reject(new Error("AI Gateway realtime WebSocket failed to open"));
|
|
545
|
+
};
|
|
546
|
+
const onClose = () => {
|
|
547
|
+
cleanup();
|
|
548
|
+
reject(new Error("AI Gateway realtime WebSocket closed before opening"));
|
|
549
|
+
};
|
|
550
|
+
const onAbort = () => {
|
|
551
|
+
cleanup();
|
|
552
|
+
reject(signal.reason ?? new DOMException("Aborted", "AbortError"));
|
|
553
|
+
};
|
|
554
|
+
websocket.addEventListener("open", onOpen, { once: true });
|
|
555
|
+
websocket.addEventListener("error", onError, { once: true });
|
|
556
|
+
websocket.addEventListener("close", onClose, { once: true });
|
|
557
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
558
|
+
});
|
|
559
|
+
}
|
|
560
|
+
function isRecord(value) {
|
|
561
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
562
|
+
}
|
|
563
|
+
function stringValue(value) {
|
|
564
|
+
return typeof value === "string" ? value : null;
|
|
565
|
+
}
|
|
566
|
+
function throwIfAborted(signal) {
|
|
567
|
+
if (signal.aborted) throw signal.reason ?? new DOMException("Aborted", "AbortError");
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
export {
|
|
571
|
+
createGatewayRealtimeTransportStarter
|
|
572
|
+
};
|
|
573
|
+
//# sourceMappingURL=chunk-DXDL7EEW.js.map
|