@opengeni/sdk 0.36.0 → 0.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/dist/chunk-DXDL7EEW.js +573 -0
- package/dist/chunk-DXDL7EEW.js.map +1 -0
- package/dist/chunk-OB5PGV6N.js +1988 -0
- package/dist/chunk-OB5PGV6N.js.map +1 -0
- package/dist/chunk-OINSI33U.js +97 -0
- package/dist/chunk-OINSI33U.js.map +1 -0
- package/dist/{chunk-I3KLDXFA.js → chunk-VOQ235WN.js} +136 -106
- package/dist/chunk-VOQ235WN.js.map +1 -0
- package/dist/client.d.ts +35 -2
- package/dist/codex-realtime-controller.d.ts +109 -0
- package/dist/codex-realtime-controller.js +12 -0
- package/dist/codex-realtime-controller.js.map +1 -0
- package/dist/codex-realtime-lifecycle.d.ts +18 -0
- package/dist/codex-realtime-v3-wire.d.ts +86 -0
- package/dist/codex-realtime-v3.d.ts +52 -0
- package/dist/codex-realtime.d.ts +68 -0
- package/dist/core.js +6 -4
- package/dist/gateway-realtime-transport.d.ts +2 -0
- package/dist/gateway-realtime-transport.js +7 -0
- package/dist/gateway-realtime-transport.js.map +1 -0
- package/dist/index.d.ts +10 -1
- package/dist/index.js +38 -6
- package/dist/index.js.map +1 -1
- package/dist/types.d.ts +197 -3
- package/package.json +10 -1
- package/src/client.ts +211 -17
- package/src/codex-realtime-controller.ts +1427 -0
- package/src/codex-realtime-lifecycle.ts +97 -0
- package/src/codex-realtime-v3-wire.ts +349 -0
- package/src/codex-realtime-v3.ts +575 -0
- package/src/codex-realtime.ts +411 -0
- package/src/gateway-realtime-transport.ts +650 -0
- package/src/index.ts +71 -0
- package/src/types.ts +244 -3
- package/dist/chunk-I3KLDXFA.js.map +0 -1
|
@@ -0,0 +1,1988 @@
|
|
|
1
|
+
import {
|
|
2
|
+
OpenGeniApiError
|
|
3
|
+
} from "./chunk-OINSI33U.js";
|
|
4
|
+
|
|
5
|
+
// src/codex-realtime-v3-wire.ts
|
|
6
|
+
var CODEX_REALTIME_CONTEXT_APPEND_MAX_BYTES = 500;
|
|
7
|
+
var CODEX_REALTIME_V3_MAX_EVENT_BYTES = 1048576;
|
|
8
|
+
var CODEX_REALTIME_V3_MAX_IDENTIFIER_BYTES = 1024;
|
|
9
|
+
var CODEX_REALTIME_V3_MAX_TEXT_BYTES = 131072;
|
|
10
|
+
function parseCodexRealtimeV3Event(payload) {
|
|
11
|
+
if (utf8ByteLength(payload) > CODEX_REALTIME_V3_MAX_EVENT_BYTES) {
|
|
12
|
+
return failure("oversized_event", null);
|
|
13
|
+
}
|
|
14
|
+
let value;
|
|
15
|
+
try {
|
|
16
|
+
value = JSON.parse(payload);
|
|
17
|
+
} catch {
|
|
18
|
+
return failure("invalid_json", null);
|
|
19
|
+
}
|
|
20
|
+
const event = record(value);
|
|
21
|
+
const typeField = boundedStringField(event, "type", CODEX_REALTIME_V3_MAX_IDENTIFIER_BYTES);
|
|
22
|
+
if (typeField.oversized) return failure("oversized_field", null);
|
|
23
|
+
const type = typeField.value;
|
|
24
|
+
if (!type) return failure("missing_type", null);
|
|
25
|
+
const providerEventIdField = firstBoundedStringField(
|
|
26
|
+
event,
|
|
27
|
+
["event_id", "id"],
|
|
28
|
+
CODEX_REALTIME_V3_MAX_IDENTIFIER_BYTES
|
|
29
|
+
);
|
|
30
|
+
if (providerEventIdField.oversized) return failure("oversized_field", type);
|
|
31
|
+
const providerEventId = providerEventIdField.value ?? null;
|
|
32
|
+
if (type === "session.started" || type === "session.updated") {
|
|
33
|
+
const session = record(event.session);
|
|
34
|
+
const sessionIdField = boundedStringField(
|
|
35
|
+
session,
|
|
36
|
+
"id",
|
|
37
|
+
CODEX_REALTIME_V3_MAX_IDENTIFIER_BYTES
|
|
38
|
+
);
|
|
39
|
+
const instructionsField = boundedStringField(
|
|
40
|
+
session,
|
|
41
|
+
"instructions",
|
|
42
|
+
CODEX_REALTIME_V3_MAX_TEXT_BYTES
|
|
43
|
+
);
|
|
44
|
+
if (sessionIdField.oversized || instructionsField.oversized) {
|
|
45
|
+
return failure("oversized_field", type);
|
|
46
|
+
}
|
|
47
|
+
const sessionId = sessionIdField.value;
|
|
48
|
+
if (!sessionId) return failure("invalid_shape", type);
|
|
49
|
+
return {
|
|
50
|
+
ok: true,
|
|
51
|
+
event: {
|
|
52
|
+
type,
|
|
53
|
+
providerEventId,
|
|
54
|
+
sessionId,
|
|
55
|
+
instructions: instructionsField.value ?? null
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
if (type === "input_transcript.added" || type === "output_transcript.added") {
|
|
60
|
+
const item = record(event.item);
|
|
61
|
+
const itemIdField = boundedStringField(item, "id", CODEX_REALTIME_V3_MAX_IDENTIFIER_BYTES);
|
|
62
|
+
const textField = boundedStringField(item, "text", CODEX_REALTIME_V3_MAX_TEXT_BYTES);
|
|
63
|
+
if (itemIdField.oversized || textField.oversized) {
|
|
64
|
+
return failure("oversized_field", type);
|
|
65
|
+
}
|
|
66
|
+
const text = textField.value;
|
|
67
|
+
if (text === void 0) return failure("invalid_shape", type);
|
|
68
|
+
return {
|
|
69
|
+
ok: true,
|
|
70
|
+
event: {
|
|
71
|
+
type,
|
|
72
|
+
providerEventId,
|
|
73
|
+
itemId: itemIdField.value ?? null,
|
|
74
|
+
text
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
if (type === "output_audio.delta") {
|
|
79
|
+
const audio = stringField(event, "audio");
|
|
80
|
+
if (audio === void 0) return failure("invalid_shape", type);
|
|
81
|
+
return {
|
|
82
|
+
ok: true,
|
|
83
|
+
event: {
|
|
84
|
+
type,
|
|
85
|
+
providerEventId,
|
|
86
|
+
audio,
|
|
87
|
+
startMs: finiteNumberField(event, "start_ms"),
|
|
88
|
+
endMs: finiteNumberField(event, "end_ms")
|
|
89
|
+
}
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
if (type === "turn.done") {
|
|
93
|
+
const turn = record(event.turn);
|
|
94
|
+
const turnIdField = boundedStringField(turn, "id", CODEX_REALTIME_V3_MAX_IDENTIFIER_BYTES);
|
|
95
|
+
const transcriptField = boundedStringField(
|
|
96
|
+
turn,
|
|
97
|
+
"transcript",
|
|
98
|
+
CODEX_REALTIME_V3_MAX_TEXT_BYTES
|
|
99
|
+
);
|
|
100
|
+
if (turnIdField.oversized || transcriptField.oversized) {
|
|
101
|
+
return failure("oversized_field", type);
|
|
102
|
+
}
|
|
103
|
+
const turnId = turnIdField.value;
|
|
104
|
+
const role = stringField(turn, "role");
|
|
105
|
+
const transcript = transcriptField.value;
|
|
106
|
+
if (!turnId || role !== "user" && role !== "assistant" || transcript === void 0) {
|
|
107
|
+
return failure("invalid_shape", type);
|
|
108
|
+
}
|
|
109
|
+
return {
|
|
110
|
+
ok: true,
|
|
111
|
+
event: { type, providerEventId, turnId, role, transcript }
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
if (type === "delegation.created") {
|
|
115
|
+
const item = record(event.item);
|
|
116
|
+
const delegationItemIdField = boundedStringField(
|
|
117
|
+
item,
|
|
118
|
+
"id",
|
|
119
|
+
CODEX_REALTIME_V3_MAX_IDENTIFIER_BYTES
|
|
120
|
+
);
|
|
121
|
+
if (delegationItemIdField.oversized) return failure("oversized_field", type);
|
|
122
|
+
const delegationItemId = delegationItemIdField.value;
|
|
123
|
+
if (!delegationItemId || stringField(item, "type") !== "delegation" || stringField(item, "target") !== "client" || !Array.isArray(item.content)) {
|
|
124
|
+
return failure("invalid_shape", type);
|
|
125
|
+
}
|
|
126
|
+
const inputParts = [];
|
|
127
|
+
let inputBytes = 0;
|
|
128
|
+
for (const rawContent of item.content) {
|
|
129
|
+
const content = record(rawContent);
|
|
130
|
+
if (stringField(content, "type") !== "input_text") continue;
|
|
131
|
+
const textField = boundedStringField(content, "text", CODEX_REALTIME_V3_MAX_TEXT_BYTES);
|
|
132
|
+
if (textField.oversized) return failure("oversized_field", type);
|
|
133
|
+
const text = textField.value ?? "";
|
|
134
|
+
inputBytes += utf8ByteLength(text);
|
|
135
|
+
if (inputBytes > CODEX_REALTIME_V3_MAX_TEXT_BYTES) {
|
|
136
|
+
return failure("oversized_field", type);
|
|
137
|
+
}
|
|
138
|
+
inputParts.push(text);
|
|
139
|
+
}
|
|
140
|
+
const inputTranscript = inputParts.join("");
|
|
141
|
+
return {
|
|
142
|
+
ok: true,
|
|
143
|
+
event: {
|
|
144
|
+
type,
|
|
145
|
+
providerEventId,
|
|
146
|
+
delegationItemId,
|
|
147
|
+
inputTranscript,
|
|
148
|
+
offsetMs: finiteNumberField(event, "offset_ms")
|
|
149
|
+
}
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
if (type === "error") {
|
|
153
|
+
const nested = record(event.error);
|
|
154
|
+
const message = stringField(event, "message") ?? stringField(nested, "message") ?? (event.error === void 0 ? void 0 : JSON.stringify(event.error));
|
|
155
|
+
if (message === void 0) return failure("invalid_shape", type);
|
|
156
|
+
if (utf8ByteLength(message) > CODEX_REALTIME_V3_MAX_TEXT_BYTES) {
|
|
157
|
+
return failure("oversized_field", type);
|
|
158
|
+
}
|
|
159
|
+
return { ok: true, event: { type, providerEventId, message } };
|
|
160
|
+
}
|
|
161
|
+
return failure("unsupported_type", type);
|
|
162
|
+
}
|
|
163
|
+
function encodeCodexRealtimeV3DelegationContextAppend(input) {
|
|
164
|
+
return contextAppendChunks(input.text).map((text) => ({
|
|
165
|
+
type: "delegation.context.append",
|
|
166
|
+
delegation_item_id: input.delegationItemId,
|
|
167
|
+
...input.channel ? { channel: input.channel } : {},
|
|
168
|
+
content: [{ type: "input_text", text }]
|
|
169
|
+
}));
|
|
170
|
+
}
|
|
171
|
+
function encodeCodexRealtimeV3SessionContextAppend(input) {
|
|
172
|
+
return contextAppendChunks(input.text).map((text) => ({
|
|
173
|
+
type: "session.context.append",
|
|
174
|
+
...input.channel ? { channel: input.channel } : {},
|
|
175
|
+
content: [{ type: "input_text", text }]
|
|
176
|
+
}));
|
|
177
|
+
}
|
|
178
|
+
function contextAppendChunks(text) {
|
|
179
|
+
if (utf8ByteLength(text) <= CODEX_REALTIME_CONTEXT_APPEND_MAX_BYTES) return [text];
|
|
180
|
+
const chunks = [];
|
|
181
|
+
let chunk = "";
|
|
182
|
+
let bytes = 0;
|
|
183
|
+
for (const character of text) {
|
|
184
|
+
const characterBytes = utf8ByteLength(character);
|
|
185
|
+
if (bytes + characterBytes > CODEX_REALTIME_CONTEXT_APPEND_MAX_BYTES && chunk) {
|
|
186
|
+
chunks.push(chunk);
|
|
187
|
+
chunk = "";
|
|
188
|
+
bytes = 0;
|
|
189
|
+
}
|
|
190
|
+
chunk += character;
|
|
191
|
+
bytes += characterBytes;
|
|
192
|
+
}
|
|
193
|
+
if (chunk || text.length === 0) chunks.push(chunk);
|
|
194
|
+
return chunks;
|
|
195
|
+
}
|
|
196
|
+
function utf8ByteLength(value) {
|
|
197
|
+
return new TextEncoder().encode(value).byteLength;
|
|
198
|
+
}
|
|
199
|
+
function failure(reason, eventType) {
|
|
200
|
+
return { ok: false, reason, eventType };
|
|
201
|
+
}
|
|
202
|
+
function record(value) {
|
|
203
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
204
|
+
}
|
|
205
|
+
function stringField(value, key) {
|
|
206
|
+
return typeof value[key] === "string" ? value[key] : void 0;
|
|
207
|
+
}
|
|
208
|
+
function boundedStringField(value, key, maxBytes) {
|
|
209
|
+
const candidate = stringField(value, key);
|
|
210
|
+
return {
|
|
211
|
+
value: candidate,
|
|
212
|
+
oversized: candidate !== void 0 && utf8ByteLength(candidate) > maxBytes
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
function firstBoundedStringField(value, keys, maxBytes) {
|
|
216
|
+
for (const key of keys) {
|
|
217
|
+
if (typeof value[key] === "string") return boundedStringField(value, key, maxBytes);
|
|
218
|
+
}
|
|
219
|
+
return { value: void 0, oversized: false };
|
|
220
|
+
}
|
|
221
|
+
function finiteNumberField(value, key) {
|
|
222
|
+
const candidate = value[key];
|
|
223
|
+
return typeof candidate === "number" && Number.isFinite(candidate) ? candidate : null;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// src/codex-realtime-v3.ts
|
|
227
|
+
var CODEX_REALTIME_V3_SYNC_MAX_ENTRIES = 64;
|
|
228
|
+
var CODEX_REALTIME_V3_PENDING_MAX_ENTRIES = 256;
|
|
229
|
+
var CODEX_REALTIME_V3_PENDING_MAX_BYTES = 16 * 1024 * 1024;
|
|
230
|
+
var REALTIME_DELEGATION_TRANSCRIPT_MAX_BYTES = 65536;
|
|
231
|
+
var REALTIME_DELEGATION_INPUT_MAX_BYTES = 65536;
|
|
232
|
+
function createCodexRealtimeV3Bridge(options) {
|
|
233
|
+
let closed = false;
|
|
234
|
+
let sealed = false;
|
|
235
|
+
let listening = false;
|
|
236
|
+
let speaking = false;
|
|
237
|
+
let activeDelegationId = null;
|
|
238
|
+
let lastError = null;
|
|
239
|
+
let ignoredEventCount = 0;
|
|
240
|
+
let lastIgnoredEventType = null;
|
|
241
|
+
let fatal = null;
|
|
242
|
+
let providerStarted;
|
|
243
|
+
let providerStartedAccepted = false;
|
|
244
|
+
let clientAckThroughSequence = null;
|
|
245
|
+
let pendingInbound = [];
|
|
246
|
+
let pendingInboundCount = 0;
|
|
247
|
+
let pendingInboundBytes = 0;
|
|
248
|
+
let flushing = null;
|
|
249
|
+
let flushRequestedWhileRunning = false;
|
|
250
|
+
let forceSync = false;
|
|
251
|
+
const clientReceivedSequences = /* @__PURE__ */ new Set();
|
|
252
|
+
const sentSequences = /* @__PURE__ */ new Set();
|
|
253
|
+
const finalizedTurnIds = /* @__PURE__ */ new Set();
|
|
254
|
+
let transcriptSinceDelegation = [];
|
|
255
|
+
let pendingDelegationUserTranscript = null;
|
|
256
|
+
const randomUUID = options.randomUUID ?? defaultRandomUUID;
|
|
257
|
+
const snapshot = () => ({
|
|
258
|
+
connectionId: options.connectionId,
|
|
259
|
+
connectionEpoch: options.connectionEpoch,
|
|
260
|
+
startupFenceSequence: options.startupFenceSequence,
|
|
261
|
+
modeVersion: options.modeVersion,
|
|
262
|
+
speaking,
|
|
263
|
+
activeDelegationId,
|
|
264
|
+
lastError,
|
|
265
|
+
ignoredEventCount,
|
|
266
|
+
lastIgnoredEventType,
|
|
267
|
+
pendingInbound: pendingInboundCount,
|
|
268
|
+
pendingInboundBytes,
|
|
269
|
+
clientAckThroughSequence,
|
|
270
|
+
providerAckSequences: [],
|
|
271
|
+
providerStarted: providerStartedAccepted,
|
|
272
|
+
fatal
|
|
273
|
+
});
|
|
274
|
+
const publish = () => options.onSnapshot?.(snapshot());
|
|
275
|
+
const triggerFatal = (message) => {
|
|
276
|
+
if (closed || fatal) return;
|
|
277
|
+
fatal = { code: "pending_overflow", message };
|
|
278
|
+
lastError = message;
|
|
279
|
+
publish();
|
|
280
|
+
try {
|
|
281
|
+
options.onFatal?.({ ...fatal });
|
|
282
|
+
} catch {
|
|
283
|
+
}
|
|
284
|
+
};
|
|
285
|
+
const enqueue = (entry) => {
|
|
286
|
+
if (closed || sealed || fatal) return false;
|
|
287
|
+
const bytes = utf8ByteLength2(JSON.stringify(entry));
|
|
288
|
+
if (pendingInboundCount + 1 > CODEX_REALTIME_V3_PENDING_MAX_ENTRIES || pendingInboundBytes + bytes > CODEX_REALTIME_V3_PENDING_MAX_BYTES) {
|
|
289
|
+
triggerFatal("Codex realtime durable event buffer exceeded its hard limit");
|
|
290
|
+
return false;
|
|
291
|
+
}
|
|
292
|
+
pendingInbound.push({ entry, bytes });
|
|
293
|
+
pendingInboundCount += 1;
|
|
294
|
+
pendingInboundBytes += bytes;
|
|
295
|
+
return true;
|
|
296
|
+
};
|
|
297
|
+
const hasWork = () => pendingInbound.length > 0 || !providerStartedAccepted && providerStarted !== void 0 || clientAckThroughSequence !== null || forceSync;
|
|
298
|
+
const runFlush = async () => {
|
|
299
|
+
while (true) {
|
|
300
|
+
if (closed || fatal) break;
|
|
301
|
+
const batch = pendingInbound.splice(0, CODEX_REALTIME_V3_SYNC_MAX_ENTRIES);
|
|
302
|
+
const startup = providerStartedAccepted ? void 0 : providerStarted;
|
|
303
|
+
const acknowledgedByClient = clientAckThroughSequence;
|
|
304
|
+
const poll = forceSync;
|
|
305
|
+
forceSync = false;
|
|
306
|
+
flushRequestedWhileRunning = false;
|
|
307
|
+
if (batch.length === 0 && !startup && acknowledgedByClient === null && !poll) break;
|
|
308
|
+
let result;
|
|
309
|
+
try {
|
|
310
|
+
result = await options.sync({
|
|
311
|
+
...options.owner,
|
|
312
|
+
connectionId: options.connectionId,
|
|
313
|
+
connectionEpoch: options.connectionEpoch,
|
|
314
|
+
...batch.length === 0 ? {} : { entries: batch.map(({ entry }) => entry) },
|
|
315
|
+
...startup ? { providerStarted: startup } : {},
|
|
316
|
+
...acknowledgedByClient === null ? {} : { clientAckThroughSequence: acknowledgedByClient }
|
|
317
|
+
});
|
|
318
|
+
} catch (error) {
|
|
319
|
+
pendingInbound = [...batch, ...pendingInbound];
|
|
320
|
+
throw error;
|
|
321
|
+
}
|
|
322
|
+
for (const item of batch) {
|
|
323
|
+
pendingInboundCount -= 1;
|
|
324
|
+
pendingInboundBytes -= item.bytes;
|
|
325
|
+
}
|
|
326
|
+
if (startup && providerStarted === startup) providerStartedAccepted = true;
|
|
327
|
+
if (acknowledgedByClient !== null && clientAckThroughSequence !== null && clientAckThroughSequence <= acknowledgedByClient) {
|
|
328
|
+
clientAckThroughSequence = null;
|
|
329
|
+
}
|
|
330
|
+
if (closed || fatal) break;
|
|
331
|
+
for (const entry of result.outbound) {
|
|
332
|
+
if (entry.clientAckedAt === null && !clientReceivedSequences.has(entry.sequence)) {
|
|
333
|
+
clientReceivedSequences.add(entry.sequence);
|
|
334
|
+
clientAckThroughSequence = Math.max(clientAckThroughSequence ?? 0, entry.sequence);
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
for (let index = 0; index < result.outbound.length; ) {
|
|
338
|
+
const entry = result.outbound[index];
|
|
339
|
+
if (sentSequences.has(entry.sequence)) {
|
|
340
|
+
index += 1;
|
|
341
|
+
continue;
|
|
342
|
+
}
|
|
343
|
+
if (entry.kind === "delegation_progress" && entry.delegationItemId) {
|
|
344
|
+
const progress = [];
|
|
345
|
+
while (index < result.outbound.length) {
|
|
346
|
+
const candidate = result.outbound[index];
|
|
347
|
+
if (candidate.kind !== "delegation_progress" || candidate.delegationItemId !== entry.delegationItemId) {
|
|
348
|
+
break;
|
|
349
|
+
}
|
|
350
|
+
if (!sentSequences.has(candidate.sequence)) progress.push(candidate);
|
|
351
|
+
index += 1;
|
|
352
|
+
}
|
|
353
|
+
sendDelegationProgress(options.events, entry.delegationItemId, progress);
|
|
354
|
+
for (const candidate of progress) sentSequences.add(candidate.sequence);
|
|
355
|
+
continue;
|
|
356
|
+
}
|
|
357
|
+
sendOutbound(options.events, entry);
|
|
358
|
+
sentSequences.add(entry.sequence);
|
|
359
|
+
if (entry.kind === "session_update" && entry.payload.source === "human_input" && entry.payload.delivery === "steer") {
|
|
360
|
+
activeDelegationId = null;
|
|
361
|
+
}
|
|
362
|
+
if ((entry.kind === "delegation_result" || entry.kind === "error") && entry.delegationItemId === activeDelegationId) {
|
|
363
|
+
activeDelegationId = null;
|
|
364
|
+
}
|
|
365
|
+
index += 1;
|
|
366
|
+
}
|
|
367
|
+
publish();
|
|
368
|
+
}
|
|
369
|
+
};
|
|
370
|
+
const requestFlush = (poll) => {
|
|
371
|
+
if (closed || fatal) return Promise.resolve();
|
|
372
|
+
if (poll) forceSync = true;
|
|
373
|
+
if (flushing) {
|
|
374
|
+
flushRequestedWhileRunning = true;
|
|
375
|
+
return flushing;
|
|
376
|
+
}
|
|
377
|
+
const task = Promise.resolve().then(runFlush).catch((error) => {
|
|
378
|
+
lastError = safeError(error);
|
|
379
|
+
publish();
|
|
380
|
+
throw error;
|
|
381
|
+
});
|
|
382
|
+
flushing = task;
|
|
383
|
+
void task.then(
|
|
384
|
+
() => {
|
|
385
|
+
if (flushing !== task) return;
|
|
386
|
+
flushing = null;
|
|
387
|
+
if (!closed && !fatal && (flushRequestedWhileRunning || hasWork())) {
|
|
388
|
+
flushRequestedWhileRunning = false;
|
|
389
|
+
void requestFlush(false).catch(() => void 0);
|
|
390
|
+
}
|
|
391
|
+
},
|
|
392
|
+
() => {
|
|
393
|
+
if (flushing === task) flushing = null;
|
|
394
|
+
}
|
|
395
|
+
);
|
|
396
|
+
return task;
|
|
397
|
+
};
|
|
398
|
+
const ingest = (payload) => {
|
|
399
|
+
if (closed || sealed || fatal) return Promise.resolve();
|
|
400
|
+
const parsed = parseCodexRealtimeV3Event(payload);
|
|
401
|
+
if (!parsed.ok) {
|
|
402
|
+
if (parsed.reason === "unsupported_type") {
|
|
403
|
+
ignoredEventCount += 1;
|
|
404
|
+
lastIgnoredEventType = parsed.eventType;
|
|
405
|
+
publish();
|
|
406
|
+
return Promise.resolve();
|
|
407
|
+
}
|
|
408
|
+
lastError = `Rejected Codex realtime V3 event: ${parsed.reason}`;
|
|
409
|
+
publish();
|
|
410
|
+
return Promise.resolve();
|
|
411
|
+
}
|
|
412
|
+
const event = parsed.event;
|
|
413
|
+
let durable = false;
|
|
414
|
+
if (event.type === "session.started") {
|
|
415
|
+
if (!providerStartedAccepted && providerStarted === void 0) {
|
|
416
|
+
providerStarted = {
|
|
417
|
+
providerSessionId: event.sessionId,
|
|
418
|
+
providerEventId: event.providerEventId
|
|
419
|
+
};
|
|
420
|
+
durable = true;
|
|
421
|
+
}
|
|
422
|
+
} else if (event.type === "input_transcript.added" || event.type === "output_transcript.added") {
|
|
423
|
+
} else if (event.type === "delegation.created") {
|
|
424
|
+
activeDelegationId = event.delegationItemId;
|
|
425
|
+
const transcript = delegationTranscript(transcriptSinceDelegation, event.inputTranscript);
|
|
426
|
+
const coveredTurnIds = transcriptSinceDelegation.map((entry) => entry.turnId);
|
|
427
|
+
durable = enqueue({
|
|
428
|
+
operationId: randomUUID(),
|
|
429
|
+
kind: "delegation_call",
|
|
430
|
+
providerEventId: event.providerEventId,
|
|
431
|
+
delegationItemId: event.delegationItemId,
|
|
432
|
+
text: renderRealtimeDelegationInput(event.inputTranscript, transcript),
|
|
433
|
+
payload: {
|
|
434
|
+
offsetMs: event.offsetMs,
|
|
435
|
+
inputTranscript: event.inputTranscript,
|
|
436
|
+
transcriptFenceTurnIds: coveredTurnIds
|
|
437
|
+
}
|
|
438
|
+
});
|
|
439
|
+
if (durable) {
|
|
440
|
+
const alreadyFinalized = transcriptSinceDelegation.some(
|
|
441
|
+
(entry) => entry.role === "user" && normalizedTranscript(entry.text) === normalizedTranscript(event.inputTranscript)
|
|
442
|
+
);
|
|
443
|
+
pendingDelegationUserTranscript = alreadyFinalized ? null : { delegationItemId: event.delegationItemId, text: event.inputTranscript };
|
|
444
|
+
transcriptSinceDelegation = [];
|
|
445
|
+
}
|
|
446
|
+
} else if (event.type === "output_audio.delta") {
|
|
447
|
+
speaking = true;
|
|
448
|
+
} else if (event.type === "turn.done") {
|
|
449
|
+
speaking = false;
|
|
450
|
+
if (event.transcript.length > 0 && !finalizedTurnIds.has(event.turnId)) {
|
|
451
|
+
const coveredByDelegationItemId = event.role === "user" && pendingDelegationUserTranscript !== null && normalizedTranscript(event.transcript) === normalizedTranscript(pendingDelegationUserTranscript.text) ? pendingDelegationUserTranscript.delegationItemId : null;
|
|
452
|
+
durable = enqueue(finalTranscript(randomUUID, event, coveredByDelegationItemId));
|
|
453
|
+
if (durable) {
|
|
454
|
+
finalizedTurnIds.add(event.turnId);
|
|
455
|
+
if (coveredByDelegationItemId) {
|
|
456
|
+
pendingDelegationUserTranscript = null;
|
|
457
|
+
} else {
|
|
458
|
+
transcriptSinceDelegation.push({
|
|
459
|
+
role: event.role,
|
|
460
|
+
text: event.transcript,
|
|
461
|
+
turnId: event.turnId
|
|
462
|
+
});
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
} else if (event.type === "error") {
|
|
467
|
+
lastError = event.message;
|
|
468
|
+
durable = enqueue({
|
|
469
|
+
operationId: randomUUID(),
|
|
470
|
+
kind: "error",
|
|
471
|
+
providerEventId: event.providerEventId,
|
|
472
|
+
text: event.message
|
|
473
|
+
});
|
|
474
|
+
}
|
|
475
|
+
publish();
|
|
476
|
+
return durable ? requestFlush(false) : Promise.resolve();
|
|
477
|
+
};
|
|
478
|
+
const onMessage = (message) => {
|
|
479
|
+
if (typeof message.data !== "string") return;
|
|
480
|
+
void ingest(message.data).catch(() => void 0);
|
|
481
|
+
};
|
|
482
|
+
const listen = () => {
|
|
483
|
+
if (closed || sealed || listening) return;
|
|
484
|
+
listening = true;
|
|
485
|
+
options.events.addEventListener("message", onMessage);
|
|
486
|
+
};
|
|
487
|
+
if (options.listen !== false) listen();
|
|
488
|
+
publish();
|
|
489
|
+
return {
|
|
490
|
+
snapshot,
|
|
491
|
+
ingest,
|
|
492
|
+
flush: () => requestFlush(true),
|
|
493
|
+
sealAndFlush: async () => {
|
|
494
|
+
if (closed) return;
|
|
495
|
+
sealed = true;
|
|
496
|
+
if (listening) options.events.removeEventListener("message", onMessage);
|
|
497
|
+
listening = false;
|
|
498
|
+
await requestFlush(true);
|
|
499
|
+
},
|
|
500
|
+
listen,
|
|
501
|
+
close: () => {
|
|
502
|
+
if (closed) return;
|
|
503
|
+
closed = true;
|
|
504
|
+
if (listening) options.events.removeEventListener("message", onMessage);
|
|
505
|
+
listening = false;
|
|
506
|
+
}
|
|
507
|
+
};
|
|
508
|
+
}
|
|
509
|
+
function finalTranscript(randomUUID, event, coveredByDelegationItemId) {
|
|
510
|
+
return {
|
|
511
|
+
operationId: randomUUID(),
|
|
512
|
+
kind: event.role === "user" ? "user_transcript" : "assistant_transcript",
|
|
513
|
+
providerEventId: event.providerEventId,
|
|
514
|
+
text: event.transcript,
|
|
515
|
+
payload: {
|
|
516
|
+
turnId: event.turnId,
|
|
517
|
+
...coveredByDelegationItemId ? { coveredByDelegationItemId } : {}
|
|
518
|
+
}
|
|
519
|
+
};
|
|
520
|
+
}
|
|
521
|
+
function delegationTranscript(entries, inputTranscript) {
|
|
522
|
+
const selected = [...entries];
|
|
523
|
+
const normalizedInput = normalizedTranscript(inputTranscript);
|
|
524
|
+
for (let index = selected.length - 1; index >= 0; index -= 1) {
|
|
525
|
+
const entry = selected[index];
|
|
526
|
+
if (entry?.role === "user" && normalizedInput.length > 0 && normalizedTranscript(entry.text) === normalizedInput) {
|
|
527
|
+
selected.splice(index, 1);
|
|
528
|
+
break;
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
const bounded = [];
|
|
532
|
+
let bytes = 0;
|
|
533
|
+
for (let index = selected.length - 1; index >= 0; index -= 1) {
|
|
534
|
+
const entry = selected[index];
|
|
535
|
+
const line = `${entry.role}: ${entry.text}`;
|
|
536
|
+
const lineBytes = utf8ByteLength2(line) + (bounded.length > 0 ? 1 : 0);
|
|
537
|
+
if (bytes + lineBytes > REALTIME_DELEGATION_TRANSCRIPT_MAX_BYTES) break;
|
|
538
|
+
bounded.unshift(entry);
|
|
539
|
+
bytes += lineBytes;
|
|
540
|
+
}
|
|
541
|
+
return bounded;
|
|
542
|
+
}
|
|
543
|
+
function renderRealtimeDelegationInput(inputTranscript, transcript) {
|
|
544
|
+
const input = takeUtf8Head(inputTranscript, REALTIME_DELEGATION_INPUT_MAX_BYTES);
|
|
545
|
+
const transcriptDelta = transcript.map((entry) => `${entry.role}: ${entry.text}`).join("\n");
|
|
546
|
+
return [
|
|
547
|
+
"<realtime_delegation>",
|
|
548
|
+
` <input>${escapeXmlText(input)}</input>`,
|
|
549
|
+
...transcriptDelta ? [` <transcript_delta>${escapeXmlText(transcriptDelta)}</transcript_delta>`] : [],
|
|
550
|
+
"</realtime_delegation>"
|
|
551
|
+
].join("\n");
|
|
552
|
+
}
|
|
553
|
+
function normalizedTranscript(value) {
|
|
554
|
+
return value.trim().replaceAll(/\s+/g, " ");
|
|
555
|
+
}
|
|
556
|
+
function takeUtf8Head(value, maximumBytes) {
|
|
557
|
+
const bytes = new TextEncoder().encode(value);
|
|
558
|
+
if (bytes.length <= maximumBytes) return value;
|
|
559
|
+
let end = maximumBytes;
|
|
560
|
+
while (end > 0 && (bytes[end] & 192) === 128) end -= 1;
|
|
561
|
+
return new TextDecoder().decode(bytes.subarray(0, end));
|
|
562
|
+
}
|
|
563
|
+
function escapeXmlText(value) {
|
|
564
|
+
return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">");
|
|
565
|
+
}
|
|
566
|
+
function sendOutbound(events, entry) {
|
|
567
|
+
if (events.readyState !== "open") throw new Error("Codex realtime data channel is not open");
|
|
568
|
+
const text = entry.text ?? JSON.stringify(entry.payload);
|
|
569
|
+
const payloadChannel = entry.payload.channel === "speakable" || entry.payload.channel === "commentary" ? entry.payload.channel : entry.payload.channel === null ? void 0 : entry.kind === "session_update" ? "commentary" : void 0;
|
|
570
|
+
const messages = (entry.kind === "delegation_result" || entry.kind === "error") && entry.delegationItemId ? encodeCodexRealtimeV3DelegationContextAppend({
|
|
571
|
+
delegationItemId: entry.delegationItemId,
|
|
572
|
+
text,
|
|
573
|
+
channel: payloadChannel ?? "speakable"
|
|
574
|
+
}) : encodeCodexRealtimeV3SessionContextAppend({ text, channel: payloadChannel });
|
|
575
|
+
for (const message of messages) events.send(JSON.stringify(message));
|
|
576
|
+
}
|
|
577
|
+
function sendDelegationProgress(events, delegationItemId, entries) {
|
|
578
|
+
if (entries.length === 0) return;
|
|
579
|
+
if (events.readyState !== "open") throw new Error("Codex realtime data channel is not open");
|
|
580
|
+
const text = entries.map((entry) => entry.text ?? "").join("");
|
|
581
|
+
if (text.length === 0) return;
|
|
582
|
+
for (const message of encodeCodexRealtimeV3DelegationContextAppend({
|
|
583
|
+
delegationItemId,
|
|
584
|
+
text,
|
|
585
|
+
channel: "commentary"
|
|
586
|
+
})) {
|
|
587
|
+
events.send(JSON.stringify(message));
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
function utf8ByteLength2(value) {
|
|
591
|
+
return new TextEncoder().encode(value).byteLength;
|
|
592
|
+
}
|
|
593
|
+
function defaultRandomUUID() {
|
|
594
|
+
if (!globalThis.crypto?.randomUUID) throw new Error("crypto.randomUUID is unavailable");
|
|
595
|
+
return globalThis.crypto.randomUUID();
|
|
596
|
+
}
|
|
597
|
+
function safeError(error) {
|
|
598
|
+
return error instanceof Error ? error.message : "Codex realtime bridge failed";
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
// src/codex-realtime.ts
|
|
602
|
+
var CodexRealtimeMicrophoneError = class extends Error {
|
|
603
|
+
constructor(code, message) {
|
|
604
|
+
super(message);
|
|
605
|
+
this.code = code;
|
|
606
|
+
this.name = "CodexRealtimeMicrophoneError";
|
|
607
|
+
}
|
|
608
|
+
};
|
|
609
|
+
async function startCodexRealtimeWebrtc(options) {
|
|
610
|
+
throwIfAborted(options.signal);
|
|
611
|
+
const peerConnection = (options.createPeerConnection ?? defaultPeerConnection)();
|
|
612
|
+
const events = peerConnection.createDataChannel("oai-events");
|
|
613
|
+
try {
|
|
614
|
+
options.onEventsCreated?.(events);
|
|
615
|
+
} catch (error) {
|
|
616
|
+
try {
|
|
617
|
+
events.close();
|
|
618
|
+
} finally {
|
|
619
|
+
peerConnection.close();
|
|
620
|
+
}
|
|
621
|
+
throw error;
|
|
622
|
+
}
|
|
623
|
+
let media = null;
|
|
624
|
+
let ownsMedia = false;
|
|
625
|
+
let stopped = false;
|
|
626
|
+
let remoteStream = null;
|
|
627
|
+
let remoteAudioActive = options.activateRemoteAudio ?? true;
|
|
628
|
+
let outputMuted = false;
|
|
629
|
+
let audibleOutput = "inactive";
|
|
630
|
+
let audioAttempt = 0;
|
|
631
|
+
const publishAudibleOutput = (next) => {
|
|
632
|
+
if (stopped && next !== "inactive") return;
|
|
633
|
+
audibleOutput = next;
|
|
634
|
+
options.onAudibleOutputState?.(next);
|
|
635
|
+
};
|
|
636
|
+
const playRemoteAudio = async () => {
|
|
637
|
+
const remoteAudio = options.remoteAudio;
|
|
638
|
+
const stream = remoteStream;
|
|
639
|
+
if (stopped || !remoteAudioActive || !remoteAudio || !stream) return false;
|
|
640
|
+
remoteAudio.autoplay = true;
|
|
641
|
+
remoteAudio.muted = outputMuted;
|
|
642
|
+
remoteAudio.srcObject = stream;
|
|
643
|
+
const attempt = ++audioAttempt;
|
|
644
|
+
publishAudibleOutput("pending");
|
|
645
|
+
try {
|
|
646
|
+
await remoteAudio.play();
|
|
647
|
+
if (stopped || attempt !== audioAttempt || remoteAudio.srcObject !== stream) return false;
|
|
648
|
+
publishAudibleOutput("audible");
|
|
649
|
+
return true;
|
|
650
|
+
} catch {
|
|
651
|
+
if (!stopped && attempt === audioAttempt && remoteAudio.srcObject === stream) {
|
|
652
|
+
publishAudibleOutput("blocked");
|
|
653
|
+
}
|
|
654
|
+
return false;
|
|
655
|
+
}
|
|
656
|
+
};
|
|
657
|
+
const onRemoteTrack = (event) => {
|
|
658
|
+
if (stopped || event.track.kind !== "audio") return;
|
|
659
|
+
const stream = event.streams[0];
|
|
660
|
+
if (!stream) return;
|
|
661
|
+
remoteStream = stream;
|
|
662
|
+
if (remoteAudioActive) void playRemoteAudio();
|
|
663
|
+
};
|
|
664
|
+
if (options.remoteAudio) peerConnection.addEventListener("track", onRemoteTrack);
|
|
665
|
+
const onConnectionState = () => {
|
|
666
|
+
if (stopped) return;
|
|
667
|
+
const peerState = peerConnection.connectionState;
|
|
668
|
+
const iceState = peerConnection.iceConnectionState;
|
|
669
|
+
if (peerState === "failed" || iceState === "failed") {
|
|
670
|
+
options.onConnectionHealth?.("failed");
|
|
671
|
+
} else if (peerState === "disconnected" || iceState === "disconnected") {
|
|
672
|
+
options.onConnectionHealth?.("disconnected");
|
|
673
|
+
} else if (peerState === "closed" || iceState === "closed") {
|
|
674
|
+
options.onConnectionHealth?.("closed");
|
|
675
|
+
} else if (peerState === "connected" || iceState === "connected" || iceState === "completed") {
|
|
676
|
+
options.onConnectionHealth?.("connected");
|
|
677
|
+
}
|
|
678
|
+
};
|
|
679
|
+
const onChannelClose = () => {
|
|
680
|
+
if (!stopped) options.onConnectionHealth?.("closed");
|
|
681
|
+
};
|
|
682
|
+
const onChannelError = () => {
|
|
683
|
+
if (!stopped) options.onConnectionHealth?.("failed");
|
|
684
|
+
};
|
|
685
|
+
peerConnection.addEventListener("connectionstatechange", onConnectionState);
|
|
686
|
+
peerConnection.addEventListener("iceconnectionstatechange", onConnectionState);
|
|
687
|
+
events.addEventListener("close", onChannelClose);
|
|
688
|
+
events.addEventListener("error", onChannelError);
|
|
689
|
+
let audioTracks = [];
|
|
690
|
+
const onMicrophoneEnded = () => {
|
|
691
|
+
if (!stopped) options.onMicrophoneEnded?.();
|
|
692
|
+
};
|
|
693
|
+
const stop = () => {
|
|
694
|
+
if (stopped) return;
|
|
695
|
+
stopped = true;
|
|
696
|
+
audioAttempt += 1;
|
|
697
|
+
options.signal?.removeEventListener("abort", stop);
|
|
698
|
+
if (options.remoteAudio) peerConnection.removeEventListener("track", onRemoteTrack);
|
|
699
|
+
peerConnection.removeEventListener("connectionstatechange", onConnectionState);
|
|
700
|
+
peerConnection.removeEventListener("iceconnectionstatechange", onConnectionState);
|
|
701
|
+
events.removeEventListener("close", onChannelClose);
|
|
702
|
+
events.removeEventListener("error", onChannelError);
|
|
703
|
+
for (const track of audioTracks) track.removeEventListener?.("ended", onMicrophoneEnded);
|
|
704
|
+
try {
|
|
705
|
+
events.close();
|
|
706
|
+
} finally {
|
|
707
|
+
try {
|
|
708
|
+
if (ownsMedia) media?.getTracks().forEach((track) => track.stop());
|
|
709
|
+
} finally {
|
|
710
|
+
if (options.remoteAudio && options.remoteAudio.srcObject === remoteStream) {
|
|
711
|
+
options.remoteAudio.pause();
|
|
712
|
+
options.remoteAudio.srcObject = null;
|
|
713
|
+
}
|
|
714
|
+
remoteStream = null;
|
|
715
|
+
publishAudibleOutput("inactive");
|
|
716
|
+
peerConnection.close();
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
};
|
|
720
|
+
options.signal?.addEventListener("abort", stop, { once: true });
|
|
721
|
+
try {
|
|
722
|
+
const getUserMedia = options.getUserMedia ?? defaultGetUserMedia;
|
|
723
|
+
if (options.media) {
|
|
724
|
+
media = options.media;
|
|
725
|
+
} else {
|
|
726
|
+
media = await acquireCodexRealtimeMicrophone({
|
|
727
|
+
getUserMedia,
|
|
728
|
+
signal: options.signal
|
|
729
|
+
});
|
|
730
|
+
ownsMedia = true;
|
|
731
|
+
}
|
|
732
|
+
throwIfAborted(options.signal);
|
|
733
|
+
audioTracks = media.getAudioTracks();
|
|
734
|
+
if (audioTracks.length === 0) {
|
|
735
|
+
throw new CodexRealtimeMicrophoneError(
|
|
736
|
+
"device_not_found",
|
|
737
|
+
"No microphone audio track is available"
|
|
738
|
+
);
|
|
739
|
+
}
|
|
740
|
+
if (!microphoneTracksHealthy(audioTracks)) {
|
|
741
|
+
throw new CodexRealtimeMicrophoneError(
|
|
742
|
+
"track_ended",
|
|
743
|
+
"The microphone audio track is no longer available"
|
|
744
|
+
);
|
|
745
|
+
}
|
|
746
|
+
for (const track of audioTracks) {
|
|
747
|
+
track.addEventListener?.("ended", onMicrophoneEnded);
|
|
748
|
+
peerConnection.addTrack(track, media);
|
|
749
|
+
}
|
|
750
|
+
const offer = await peerConnection.createOffer();
|
|
751
|
+
if (offer.type !== "offer" || !offer.sdp) {
|
|
752
|
+
throw new Error("Browser did not create a WebRTC SDP offer");
|
|
753
|
+
}
|
|
754
|
+
await peerConnection.setLocalDescription(offer);
|
|
755
|
+
throwIfAborted(options.signal);
|
|
756
|
+
const localSdp = peerConnection.localDescription?.sdp ?? offer.sdp;
|
|
757
|
+
const answer = await options.negotiate(
|
|
758
|
+
{
|
|
759
|
+
realtimeId: options.realtimeId,
|
|
760
|
+
operationId: options.operationId,
|
|
761
|
+
browserInstanceId: options.browserInstanceId,
|
|
762
|
+
ownerKey: options.ownerKey,
|
|
763
|
+
expectedVersion: options.expectedVersion,
|
|
764
|
+
expectedConnectionEpoch: options.expectedConnectionEpoch,
|
|
765
|
+
rotate: options.rotate,
|
|
766
|
+
browserActivation: "required",
|
|
767
|
+
sdp: localSdp,
|
|
768
|
+
version: "v3",
|
|
769
|
+
...options.instructions === void 0 ? {} : { instructions: options.instructions },
|
|
770
|
+
...options.voice === void 0 ? {} : { voice: options.voice }
|
|
771
|
+
},
|
|
772
|
+
{ signal: options.signal }
|
|
773
|
+
);
|
|
774
|
+
throwIfAborted(options.signal);
|
|
775
|
+
if (answer.version !== "v3" || answer.model !== "gpt-live-1-boulder-alpha" || !answer.sdp) {
|
|
776
|
+
throw new Error("OpenGeni returned an incompatible Codex realtime answer");
|
|
777
|
+
}
|
|
778
|
+
await peerConnection.setRemoteDescription({
|
|
779
|
+
type: "answer",
|
|
780
|
+
sdp: answer.sdp
|
|
781
|
+
});
|
|
782
|
+
throwIfAborted(options.signal);
|
|
783
|
+
return {
|
|
784
|
+
peerConnection,
|
|
785
|
+
events,
|
|
786
|
+
media,
|
|
787
|
+
operationId: options.operationId,
|
|
788
|
+
connectionId: answer.connectionId,
|
|
789
|
+
connectionEpoch: answer.connectionEpoch,
|
|
790
|
+
startupFenceSequence: answer.startupFenceSequence,
|
|
791
|
+
modeVersion: answer.modeVersion,
|
|
792
|
+
microphoneHealthy: () => microphoneTracksHealthy(audioTracks),
|
|
793
|
+
audibleOutputState: () => audibleOutput,
|
|
794
|
+
setOutputMuted: (muted) => {
|
|
795
|
+
outputMuted = muted;
|
|
796
|
+
if (options.remoteAudio) options.remoteAudio.muted = muted;
|
|
797
|
+
},
|
|
798
|
+
activateRemoteAudio: () => {
|
|
799
|
+
if (stopped || remoteAudioActive) return;
|
|
800
|
+
remoteAudioActive = true;
|
|
801
|
+
if (remoteStream) void playRemoteAudio();
|
|
802
|
+
},
|
|
803
|
+
retryAudibleOutput: playRemoteAudio,
|
|
804
|
+
stop
|
|
805
|
+
};
|
|
806
|
+
} catch (error) {
|
|
807
|
+
stop();
|
|
808
|
+
throw error;
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
async function acquireCodexRealtimeMicrophone(options = {}) {
|
|
812
|
+
const getUserMedia = options.getUserMedia ?? defaultGetUserMedia;
|
|
813
|
+
let media;
|
|
814
|
+
try {
|
|
815
|
+
media = await abortableMedia(getUserMedia({ audio: true }), options.signal);
|
|
816
|
+
} catch (error) {
|
|
817
|
+
throw microphoneAcquisitionError(error, options.signal);
|
|
818
|
+
}
|
|
819
|
+
const tracks = media.getAudioTracks();
|
|
820
|
+
if (tracks.length === 0) {
|
|
821
|
+
media.getTracks().forEach((track) => track.stop());
|
|
822
|
+
throw new CodexRealtimeMicrophoneError(
|
|
823
|
+
"device_not_found",
|
|
824
|
+
"No microphone audio track is available"
|
|
825
|
+
);
|
|
826
|
+
}
|
|
827
|
+
if (!microphoneTracksHealthy(tracks)) {
|
|
828
|
+
media.getTracks().forEach((track) => track.stop());
|
|
829
|
+
throw new CodexRealtimeMicrophoneError(
|
|
830
|
+
"track_ended",
|
|
831
|
+
"The microphone audio track is no longer available"
|
|
832
|
+
);
|
|
833
|
+
}
|
|
834
|
+
return media;
|
|
835
|
+
}
|
|
836
|
+
function codexRealtimeMicrophoneHealthy(media) {
|
|
837
|
+
return media !== null && microphoneTracksHealthy(media.getAudioTracks());
|
|
838
|
+
}
|
|
839
|
+
function microphoneTracksHealthy(tracks) {
|
|
840
|
+
return tracks.length > 0 && tracks.every((track) => track.readyState !== "ended");
|
|
841
|
+
}
|
|
842
|
+
function microphoneAcquisitionError(error, signal) {
|
|
843
|
+
if (signal?.aborted) {
|
|
844
|
+
return signal.reason instanceof Error ? signal.reason : new DOMException("Aborted", "AbortError");
|
|
845
|
+
}
|
|
846
|
+
const name = error instanceof Error ? error.name : "";
|
|
847
|
+
if (name === "NotAllowedError" || name === "SecurityError") {
|
|
848
|
+
return new CodexRealtimeMicrophoneError(
|
|
849
|
+
"permission_denied",
|
|
850
|
+
"Microphone permission was denied"
|
|
851
|
+
);
|
|
852
|
+
}
|
|
853
|
+
if (name === "NotFoundError" || name === "DevicesNotFoundError") {
|
|
854
|
+
return new CodexRealtimeMicrophoneError("device_not_found", "No microphone device was found");
|
|
855
|
+
}
|
|
856
|
+
if (name === "NotReadableError" || name === "TrackStartError" || name === "AbortError") {
|
|
857
|
+
return new CodexRealtimeMicrophoneError(
|
|
858
|
+
"device_unavailable",
|
|
859
|
+
"The microphone device is unavailable"
|
|
860
|
+
);
|
|
861
|
+
}
|
|
862
|
+
return new CodexRealtimeMicrophoneError(
|
|
863
|
+
"acquisition_failed",
|
|
864
|
+
"Microphone capture could not be started"
|
|
865
|
+
);
|
|
866
|
+
}
|
|
867
|
+
function defaultPeerConnection() {
|
|
868
|
+
if (typeof RTCPeerConnection === "undefined") {
|
|
869
|
+
throw new Error("WebRTC is not available in this environment");
|
|
870
|
+
}
|
|
871
|
+
return new RTCPeerConnection();
|
|
872
|
+
}
|
|
873
|
+
async function defaultGetUserMedia(constraints) {
|
|
874
|
+
if (typeof navigator === "undefined" || !navigator.mediaDevices?.getUserMedia) {
|
|
875
|
+
throw new Error("Microphone capture is not available in this environment");
|
|
876
|
+
}
|
|
877
|
+
return await navigator.mediaDevices.getUserMedia(constraints);
|
|
878
|
+
}
|
|
879
|
+
function throwIfAborted(signal) {
|
|
880
|
+
if (signal?.aborted) {
|
|
881
|
+
throw signal.reason ?? new DOMException("Aborted", "AbortError");
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
async function abortableMedia(pending, signal) {
|
|
885
|
+
if (!signal) return await pending;
|
|
886
|
+
throwIfAborted(signal);
|
|
887
|
+
let rejectAbort;
|
|
888
|
+
const aborted = new Promise((_resolve, reject) => {
|
|
889
|
+
rejectAbort = reject;
|
|
890
|
+
});
|
|
891
|
+
const onAbort = () => {
|
|
892
|
+
rejectAbort?.(signal.reason ?? new DOMException("Aborted", "AbortError"));
|
|
893
|
+
};
|
|
894
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
895
|
+
void pending.then(
|
|
896
|
+
(lateMedia) => {
|
|
897
|
+
if (signal.aborted) lateMedia.getTracks().forEach((track) => track.stop());
|
|
898
|
+
},
|
|
899
|
+
() => void 0
|
|
900
|
+
);
|
|
901
|
+
try {
|
|
902
|
+
return await Promise.race([pending, aborted]);
|
|
903
|
+
} finally {
|
|
904
|
+
signal.removeEventListener("abort", onAbort);
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
// src/codex-realtime-lifecycle.ts
|
|
909
|
+
function projectSessionRealtimeLifecycle(events) {
|
|
910
|
+
let projected = null;
|
|
911
|
+
for (const event of [...events].sort((left, right) => left.sequence - right.sequence)) {
|
|
912
|
+
const payload = recordValue(event.payload);
|
|
913
|
+
if (event.type === "session.realtime.started") {
|
|
914
|
+
const realtimeId = stringValue(payload?.realtimeId);
|
|
915
|
+
const operationId = stringValue(payload?.operationId);
|
|
916
|
+
const version = positiveInteger(payload?.version);
|
|
917
|
+
const connectionEpoch = positiveInteger(payload?.connectionEpoch);
|
|
918
|
+
const leaseExpiresAt = stringValue(payload?.leaseExpiresAt);
|
|
919
|
+
const model = realtimeModel(payload?.model);
|
|
920
|
+
if (realtimeId && operationId && version && connectionEpoch && leaseExpiresAt && model) {
|
|
921
|
+
projected = {
|
|
922
|
+
state: "active",
|
|
923
|
+
realtimeId,
|
|
924
|
+
operationId,
|
|
925
|
+
version,
|
|
926
|
+
connectionEpoch,
|
|
927
|
+
leaseExpiresAt,
|
|
928
|
+
model
|
|
929
|
+
};
|
|
930
|
+
}
|
|
931
|
+
} else if (event.type === "session.realtime.ended") {
|
|
932
|
+
const realtimeId = stringValue(payload?.realtimeId);
|
|
933
|
+
const operationId = stringValue(payload?.operationId);
|
|
934
|
+
const version = positiveInteger(payload?.version);
|
|
935
|
+
const connectionEpoch = positiveInteger(payload?.connectionEpoch);
|
|
936
|
+
const reason = endReason(payload?.reason);
|
|
937
|
+
if (realtimeId && operationId && version && connectionEpoch && reason) {
|
|
938
|
+
projected = {
|
|
939
|
+
state: "ended",
|
|
940
|
+
realtimeId,
|
|
941
|
+
operationId,
|
|
942
|
+
version,
|
|
943
|
+
connectionEpoch,
|
|
944
|
+
reason
|
|
945
|
+
};
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
return projected;
|
|
950
|
+
}
|
|
951
|
+
function realtimeModel(value) {
|
|
952
|
+
return value === "gpt-live-1-boulder-alpha" || value === "opengeni-gateway/openai/gpt-realtime-2.1" || value === "opengeni-gateway/openai/gpt-realtime-mini" || value === "opengeni-gateway/xai/grok-voice-think-fast-2.0" || value === "workspace-gateway/openai/gpt-realtime-2.1" || value === "workspace-gateway/openai/gpt-realtime-mini" || value === "workspace-gateway/xai/grok-voice-think-fast-2.0" ? value : null;
|
|
953
|
+
}
|
|
954
|
+
function recordValue(value) {
|
|
955
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
956
|
+
}
|
|
957
|
+
function stringValue(value) {
|
|
958
|
+
return typeof value === "string" && value.length > 0 ? value : null;
|
|
959
|
+
}
|
|
960
|
+
function positiveInteger(value) {
|
|
961
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value > 0 ? value : null;
|
|
962
|
+
}
|
|
963
|
+
function endReason(value) {
|
|
964
|
+
return value === "user_stop" || value === "browser_unload" || value === "lease_expired" ? value : null;
|
|
965
|
+
}
|
|
966
|
+
|
|
967
|
+
// src/codex-realtime-controller.ts
|
|
968
|
+
var HEARTBEAT_INTERVAL_MS = 1e4;
|
|
969
|
+
var OUTBOUND_SYNC_INTERVAL_MS = 1e3;
|
|
970
|
+
var CODEX_REALTIME_NEGOTIATION_TIMEOUT_MS = 2e4;
|
|
971
|
+
var DEFAULT_CONNECTION_ROTATION_INTERVAL_MS = 15 * 6e4;
|
|
972
|
+
var DEFAULT_RECONNECT_BACKOFF_MS = [250, 1e3, 2e3, 5e3];
|
|
973
|
+
var OWNER_RECORD_VERSION = 1;
|
|
974
|
+
function createCodexRealtimeController(options) {
|
|
975
|
+
const storage = options.storage ?? defaultStorage();
|
|
976
|
+
const storageKey = ownerStorageKey(
|
|
977
|
+
options.workspaceId,
|
|
978
|
+
options.sessionId,
|
|
979
|
+
options.ownerStorageNamespace
|
|
980
|
+
);
|
|
981
|
+
const model = options.model ?? "gpt-live-1-boulder-alpha";
|
|
982
|
+
const randomUUID = options.randomUUID ?? defaultRandomUUID2;
|
|
983
|
+
const scheduleInterval = options.setInterval ?? ((callback, delay) => globalThis.setInterval(callback, delay));
|
|
984
|
+
const unscheduleInterval = options.clearInterval ?? ((handle) => globalThis.clearInterval(handle));
|
|
985
|
+
const scheduleTimeout = options.setTimeout ?? ((callback, delay) => globalThis.setTimeout(callback, delay));
|
|
986
|
+
const unscheduleTimeout = options.clearTimeout ?? ((handle) => globalThis.clearTimeout(handle));
|
|
987
|
+
const rotationInterval = positiveDuration(
|
|
988
|
+
options.connectionRotationIntervalMs ?? DEFAULT_CONNECTION_ROTATION_INTERVAL_MS,
|
|
989
|
+
"connection rotation interval"
|
|
990
|
+
);
|
|
991
|
+
const negotiationTimeout = positiveDuration(
|
|
992
|
+
options.negotiationTimeoutMs ?? CODEX_REALTIME_NEGOTIATION_TIMEOUT_MS,
|
|
993
|
+
"negotiation timeout"
|
|
994
|
+
);
|
|
995
|
+
const reconnectBackoff = validateReconnectBackoff(
|
|
996
|
+
options.reconnectBackoffMs ?? DEFAULT_RECONNECT_BACKOFF_MS
|
|
997
|
+
);
|
|
998
|
+
const listeners = /* @__PURE__ */ new Set();
|
|
999
|
+
let owner = readOwnerRecord(storage, storageKey, options);
|
|
1000
|
+
let state = {
|
|
1001
|
+
status: owner ? "recovering" : "idle",
|
|
1002
|
+
realtimeId: null,
|
|
1003
|
+
mode: null,
|
|
1004
|
+
bridge: null,
|
|
1005
|
+
microphone: "inactive",
|
|
1006
|
+
inputMuted: false,
|
|
1007
|
+
audibleOutput: "inactive",
|
|
1008
|
+
outputMuted: false,
|
|
1009
|
+
connectionGeneration: 0,
|
|
1010
|
+
reconnectAttempt: 0,
|
|
1011
|
+
diagnostic: null,
|
|
1012
|
+
error: null
|
|
1013
|
+
};
|
|
1014
|
+
let active = null;
|
|
1015
|
+
let pendingAbort = null;
|
|
1016
|
+
let pendingGeneration = null;
|
|
1017
|
+
let microphone = null;
|
|
1018
|
+
let heartbeatTimer = null;
|
|
1019
|
+
let syncTimer = null;
|
|
1020
|
+
let rotationTimer = null;
|
|
1021
|
+
let reconnectTimer = null;
|
|
1022
|
+
let negotiationTimer = null;
|
|
1023
|
+
let closed = false;
|
|
1024
|
+
let stopping = false;
|
|
1025
|
+
let generation = 0;
|
|
1026
|
+
let reconnectAttempt = 0;
|
|
1027
|
+
let recoveryTerminal = false;
|
|
1028
|
+
let mutationTail = Promise.resolve();
|
|
1029
|
+
let connectionTask = null;
|
|
1030
|
+
const publish = (patch) => {
|
|
1031
|
+
state = { ...state, ...patch };
|
|
1032
|
+
for (const listener of listeners) listener({ ...state });
|
|
1033
|
+
};
|
|
1034
|
+
const diagnostic = (kind, message, recoverable, targetGeneration = state.connectionGeneration) => ({
|
|
1035
|
+
kind,
|
|
1036
|
+
message,
|
|
1037
|
+
recoverable,
|
|
1038
|
+
connectionGeneration: targetGeneration,
|
|
1039
|
+
attempt: reconnectAttempt
|
|
1040
|
+
});
|
|
1041
|
+
const exclusive = async (operation) => {
|
|
1042
|
+
const pending = mutationTail.then(operation, operation);
|
|
1043
|
+
mutationTail = pending.then(
|
|
1044
|
+
() => void 0,
|
|
1045
|
+
() => void 0
|
|
1046
|
+
);
|
|
1047
|
+
return await pending;
|
|
1048
|
+
};
|
|
1049
|
+
const stopTimers = () => {
|
|
1050
|
+
if (heartbeatTimer !== null) unscheduleInterval(heartbeatTimer);
|
|
1051
|
+
if (syncTimer !== null) unscheduleInterval(syncTimer);
|
|
1052
|
+
if (rotationTimer !== null) unscheduleTimeout(rotationTimer);
|
|
1053
|
+
if (reconnectTimer !== null) unscheduleTimeout(reconnectTimer);
|
|
1054
|
+
if (negotiationTimer !== null) unscheduleTimeout(negotiationTimer);
|
|
1055
|
+
heartbeatTimer = null;
|
|
1056
|
+
syncTimer = null;
|
|
1057
|
+
rotationTimer = null;
|
|
1058
|
+
reconnectTimer = null;
|
|
1059
|
+
negotiationTimer = null;
|
|
1060
|
+
};
|
|
1061
|
+
const stopNegotiationTimers = () => {
|
|
1062
|
+
if (heartbeatTimer !== null) unscheduleInterval(heartbeatTimer);
|
|
1063
|
+
if (rotationTimer !== null) unscheduleTimeout(rotationTimer);
|
|
1064
|
+
if (reconnectTimer !== null) unscheduleTimeout(reconnectTimer);
|
|
1065
|
+
if (negotiationTimer !== null) unscheduleTimeout(negotiationTimer);
|
|
1066
|
+
heartbeatTimer = null;
|
|
1067
|
+
rotationTimer = null;
|
|
1068
|
+
reconnectTimer = null;
|
|
1069
|
+
negotiationTimer = null;
|
|
1070
|
+
};
|
|
1071
|
+
const clearNegotiationTimer = () => {
|
|
1072
|
+
if (negotiationTimer !== null) unscheduleTimeout(negotiationTimer);
|
|
1073
|
+
negotiationTimer = null;
|
|
1074
|
+
};
|
|
1075
|
+
const releaseMicrophone = () => {
|
|
1076
|
+
const current = microphone;
|
|
1077
|
+
microphone = null;
|
|
1078
|
+
current?.getTracks().forEach((track) => track.stop());
|
|
1079
|
+
publish({ microphone: "inactive" });
|
|
1080
|
+
};
|
|
1081
|
+
const closeActive = () => {
|
|
1082
|
+
const current = active;
|
|
1083
|
+
active = null;
|
|
1084
|
+
current?.bridge.close();
|
|
1085
|
+
current?.transport.stop();
|
|
1086
|
+
publish({ bridge: null, audibleOutput: "inactive" });
|
|
1087
|
+
};
|
|
1088
|
+
const closeBrowserResources = (releaseMedia = true) => {
|
|
1089
|
+
stopTimers();
|
|
1090
|
+
pendingAbort?.abort(new DOMException("Codex realtime browser owner closed", "AbortError"));
|
|
1091
|
+
pendingAbort = null;
|
|
1092
|
+
pendingGeneration = null;
|
|
1093
|
+
closeActive();
|
|
1094
|
+
if (releaseMedia) releaseMicrophone();
|
|
1095
|
+
};
|
|
1096
|
+
const clearOwner = () => {
|
|
1097
|
+
owner = null;
|
|
1098
|
+
storage?.removeItem(storageKey);
|
|
1099
|
+
};
|
|
1100
|
+
const transitionEnded = (message = "Realtime mode ended") => {
|
|
1101
|
+
stopping = false;
|
|
1102
|
+
connectionTask = null;
|
|
1103
|
+
closeBrowserResources();
|
|
1104
|
+
clearOwner();
|
|
1105
|
+
reconnectAttempt = 0;
|
|
1106
|
+
recoveryTerminal = false;
|
|
1107
|
+
publish({
|
|
1108
|
+
status: "idle",
|
|
1109
|
+
realtimeId: null,
|
|
1110
|
+
mode: null,
|
|
1111
|
+
bridge: null,
|
|
1112
|
+
inputMuted: false,
|
|
1113
|
+
outputMuted: false,
|
|
1114
|
+
reconnectAttempt: 0,
|
|
1115
|
+
diagnostic: diagnostic("terminal_stop", message, false),
|
|
1116
|
+
error: null
|
|
1117
|
+
});
|
|
1118
|
+
};
|
|
1119
|
+
const ensureMicrophone = async (replace, signal) => {
|
|
1120
|
+
if (!replace && codexRealtimeMicrophoneHealthy(microphone)) return microphone;
|
|
1121
|
+
releaseMicrophone();
|
|
1122
|
+
publish({ microphone: "acquiring" });
|
|
1123
|
+
try {
|
|
1124
|
+
const acquired = await acquireCodexRealtimeMicrophone({
|
|
1125
|
+
signal,
|
|
1126
|
+
getUserMedia: options.getUserMedia
|
|
1127
|
+
});
|
|
1128
|
+
if (closed || stopping || signal.aborted) {
|
|
1129
|
+
acquired.getTracks().forEach((track) => track.stop());
|
|
1130
|
+
throw signal.reason ?? new DOMException("Aborted", "AbortError");
|
|
1131
|
+
}
|
|
1132
|
+
microphone = acquired;
|
|
1133
|
+
for (const track of acquired.getAudioTracks()) track.enabled = !state.inputMuted;
|
|
1134
|
+
publish({ microphone: "active" });
|
|
1135
|
+
return acquired;
|
|
1136
|
+
} catch (error) {
|
|
1137
|
+
if (error instanceof CodexRealtimeMicrophoneError) {
|
|
1138
|
+
const kind = error.code === "permission_denied" ? "permission_failure" : "device_failure";
|
|
1139
|
+
publish({
|
|
1140
|
+
microphone: error.code,
|
|
1141
|
+
status: state.mode?.state === "active" ? "recovering" : "error",
|
|
1142
|
+
diagnostic: diagnostic(kind, error.message, true),
|
|
1143
|
+
error: error.message
|
|
1144
|
+
});
|
|
1145
|
+
}
|
|
1146
|
+
throw error;
|
|
1147
|
+
}
|
|
1148
|
+
};
|
|
1149
|
+
const syncForGeneration = async (targetGeneration, realtimeId, request) => await exclusive(async () => {
|
|
1150
|
+
const current = state.mode;
|
|
1151
|
+
if (!owner || !current || current.id !== realtimeId || current.state !== "active" || active?.generation !== targetGeneration) {
|
|
1152
|
+
throw new Error("Codex realtime connection generation is no longer active");
|
|
1153
|
+
}
|
|
1154
|
+
return await options.client.syncSessionRealtimeLedger(
|
|
1155
|
+
options.workspaceId,
|
|
1156
|
+
options.sessionId,
|
|
1157
|
+
realtimeId,
|
|
1158
|
+
{ ...request, expectedVersion: current.version }
|
|
1159
|
+
);
|
|
1160
|
+
});
|
|
1161
|
+
const onAudibleOutput = (targetGeneration, next) => {
|
|
1162
|
+
if (active?.generation !== targetGeneration || closed || stopping) return;
|
|
1163
|
+
if (next === "blocked") {
|
|
1164
|
+
const message = "Browser blocked audible realtime output. Use Resume audio to continue listening.";
|
|
1165
|
+
publish({
|
|
1166
|
+
audibleOutput: next,
|
|
1167
|
+
diagnostic: diagnostic("autoplay_blocked", message, true),
|
|
1168
|
+
error: message
|
|
1169
|
+
});
|
|
1170
|
+
return;
|
|
1171
|
+
}
|
|
1172
|
+
publish({ audibleOutput: next, ...next === "audible" ? { error: null } : {} });
|
|
1173
|
+
};
|
|
1174
|
+
const startActiveIntervals = () => {
|
|
1175
|
+
if (heartbeatTimer === null) {
|
|
1176
|
+
heartbeatTimer = scheduleInterval(() => {
|
|
1177
|
+
void heartbeat().catch((error) => {
|
|
1178
|
+
if (!closed && !stopping) {
|
|
1179
|
+
publish({ error: safeError2(error) });
|
|
1180
|
+
scheduleRecovery("reconnect", false);
|
|
1181
|
+
}
|
|
1182
|
+
});
|
|
1183
|
+
}, HEARTBEAT_INTERVAL_MS);
|
|
1184
|
+
}
|
|
1185
|
+
if (syncTimer === null) {
|
|
1186
|
+
syncTimer = scheduleInterval(() => {
|
|
1187
|
+
void flush().catch((error) => {
|
|
1188
|
+
if (!closed && !stopping) publish({ error: safeError2(error) });
|
|
1189
|
+
});
|
|
1190
|
+
}, OUTBOUND_SYNC_INTERVAL_MS);
|
|
1191
|
+
}
|
|
1192
|
+
};
|
|
1193
|
+
const scheduleRecovery = (cause, replaceMicrophone, immediate = false) => {
|
|
1194
|
+
if (closed || stopping || recoveryTerminal || !owner || state.mode?.state !== "active") return;
|
|
1195
|
+
if (connectionTask || reconnectTimer !== null) return;
|
|
1196
|
+
if (rotationTimer !== null) unscheduleTimeout(rotationTimer);
|
|
1197
|
+
rotationTimer = null;
|
|
1198
|
+
startActiveIntervals();
|
|
1199
|
+
const kind = cause === "rotation" ? "rotation" : cause === "microphone" ? "device_failure" : "reconnect";
|
|
1200
|
+
const message = cause === "rotation" ? "Rotating the finite provider connection" : cause === "microphone" ? "Recovering microphone and realtime connection" : "Recovering the realtime provider connection";
|
|
1201
|
+
publish({
|
|
1202
|
+
status: "recovering",
|
|
1203
|
+
reconnectAttempt,
|
|
1204
|
+
diagnostic: diagnostic(kind, message, true),
|
|
1205
|
+
error: cause === "rotation" ? null : message
|
|
1206
|
+
});
|
|
1207
|
+
const delay = immediate ? 0 : reconnectBackoff[Math.min(reconnectAttempt, reconnectBackoff.length - 1)];
|
|
1208
|
+
reconnectTimer = scheduleTimeout(() => {
|
|
1209
|
+
reconnectTimer = null;
|
|
1210
|
+
const record2 = owner;
|
|
1211
|
+
const mode = state.mode;
|
|
1212
|
+
if (connectionTask || !record2 || !mode || closed || stopping || mode.state !== "active") {
|
|
1213
|
+
return;
|
|
1214
|
+
}
|
|
1215
|
+
connectionTask = establish(record2, mode, cause, true, replaceMicrophone).then(() => {
|
|
1216
|
+
reconnectAttempt = 0;
|
|
1217
|
+
publish({ reconnectAttempt: 0 });
|
|
1218
|
+
}).catch(async (error) => {
|
|
1219
|
+
connectionTask = null;
|
|
1220
|
+
await handleConnectionFailure(error, cause, replaceMicrophone);
|
|
1221
|
+
}).finally(() => {
|
|
1222
|
+
connectionTask = null;
|
|
1223
|
+
});
|
|
1224
|
+
}, delay);
|
|
1225
|
+
};
|
|
1226
|
+
const handleConnectionFailure = async (error, cause, replaceMicrophone) => {
|
|
1227
|
+
if (closed || stopping || isAbortError(error)) return;
|
|
1228
|
+
const message = safeError2(error);
|
|
1229
|
+
if (error instanceof CodexRealtimeMicrophoneError) {
|
|
1230
|
+
if (error.code === "track_ended" && state.mode?.state === "active") {
|
|
1231
|
+
reconnectAttempt += 1;
|
|
1232
|
+
publish({
|
|
1233
|
+
status: "recovering",
|
|
1234
|
+
microphone: error.code,
|
|
1235
|
+
reconnectAttempt,
|
|
1236
|
+
diagnostic: diagnostic("device_failure", error.message, true),
|
|
1237
|
+
error: error.message
|
|
1238
|
+
});
|
|
1239
|
+
scheduleRecovery("microphone", true);
|
|
1240
|
+
} else if (state.mode?.state === "active") {
|
|
1241
|
+
startActiveIntervals();
|
|
1242
|
+
}
|
|
1243
|
+
return;
|
|
1244
|
+
}
|
|
1245
|
+
if (error instanceof OpenGeniApiError && error.status === 404) {
|
|
1246
|
+
transitionEnded("Realtime owner no longer exists");
|
|
1247
|
+
return;
|
|
1248
|
+
}
|
|
1249
|
+
if (error instanceof OpenGeniApiError && error.status === 409 && error.retryable && owner) {
|
|
1250
|
+
try {
|
|
1251
|
+
const reconciled = await begin(owner, true, false);
|
|
1252
|
+
if (!reconciled || reconciled.state === "ended") return;
|
|
1253
|
+
} catch (reconcileError) {
|
|
1254
|
+
if (reconcileError instanceof OpenGeniApiError && !reconcileError.retryable) {
|
|
1255
|
+
stopTimers();
|
|
1256
|
+
if (!active) releaseMicrophone();
|
|
1257
|
+
recoveryTerminal = true;
|
|
1258
|
+
publish({
|
|
1259
|
+
status: "error",
|
|
1260
|
+
diagnostic: diagnostic("negotiation_failure", safeError2(reconcileError), false),
|
|
1261
|
+
error: safeError2(reconcileError)
|
|
1262
|
+
});
|
|
1263
|
+
return;
|
|
1264
|
+
}
|
|
1265
|
+
}
|
|
1266
|
+
} else if (error instanceof OpenGeniApiError && !error.retryable) {
|
|
1267
|
+
stopTimers();
|
|
1268
|
+
if (!active) releaseMicrophone();
|
|
1269
|
+
recoveryTerminal = true;
|
|
1270
|
+
publish({
|
|
1271
|
+
status: "error",
|
|
1272
|
+
diagnostic: diagnostic("negotiation_failure", message, false),
|
|
1273
|
+
error: message
|
|
1274
|
+
});
|
|
1275
|
+
return;
|
|
1276
|
+
}
|
|
1277
|
+
reconnectAttempt += 1;
|
|
1278
|
+
publish({
|
|
1279
|
+
status: "recovering",
|
|
1280
|
+
reconnectAttempt,
|
|
1281
|
+
diagnostic: diagnostic("negotiation_failure", message, true),
|
|
1282
|
+
error: message
|
|
1283
|
+
});
|
|
1284
|
+
scheduleRecovery(cause === "rotation" ? "reconnect" : cause, replaceMicrophone);
|
|
1285
|
+
};
|
|
1286
|
+
const onBridgeFatal = (targetGeneration, failedBridge, fatal) => {
|
|
1287
|
+
const current = active;
|
|
1288
|
+
if (closed || stopping || !current || current.generation !== targetGeneration) return;
|
|
1289
|
+
active = null;
|
|
1290
|
+
failedBridge.close();
|
|
1291
|
+
current.transport.stop();
|
|
1292
|
+
publish({
|
|
1293
|
+
status: "recovering",
|
|
1294
|
+
bridge: failedBridge.snapshot(),
|
|
1295
|
+
audibleOutput: "inactive",
|
|
1296
|
+
diagnostic: diagnostic("negotiation_failure", fatal.message, true, targetGeneration),
|
|
1297
|
+
error: fatal.message
|
|
1298
|
+
});
|
|
1299
|
+
scheduleRecovery("reconnect", false, true);
|
|
1300
|
+
};
|
|
1301
|
+
const onConnectionHealth = (targetGeneration, health) => {
|
|
1302
|
+
if (closed || stopping) return;
|
|
1303
|
+
if (pendingGeneration === targetGeneration && health !== "connected") {
|
|
1304
|
+
pendingAbort?.abort(new Error(`Codex realtime replacement ${health} before activation`));
|
|
1305
|
+
return;
|
|
1306
|
+
}
|
|
1307
|
+
if (active?.generation !== targetGeneration || health === "connected") return;
|
|
1308
|
+
scheduleRecovery("reconnect", false);
|
|
1309
|
+
};
|
|
1310
|
+
const onMicrophoneEnded = (targetGeneration) => {
|
|
1311
|
+
if (closed || stopping) return;
|
|
1312
|
+
const error = new CodexRealtimeMicrophoneError(
|
|
1313
|
+
"track_ended",
|
|
1314
|
+
"The microphone device was disconnected"
|
|
1315
|
+
);
|
|
1316
|
+
publish({
|
|
1317
|
+
microphone: "track_ended",
|
|
1318
|
+
diagnostic: diagnostic("device_failure", error.message, true),
|
|
1319
|
+
error: error.message
|
|
1320
|
+
});
|
|
1321
|
+
if (pendingGeneration === targetGeneration) {
|
|
1322
|
+
pendingAbort?.abort(error);
|
|
1323
|
+
return;
|
|
1324
|
+
}
|
|
1325
|
+
if (active?.generation !== targetGeneration) return;
|
|
1326
|
+
scheduleRecovery("microphone", true, true);
|
|
1327
|
+
};
|
|
1328
|
+
const startTimers = () => {
|
|
1329
|
+
stopTimers();
|
|
1330
|
+
startActiveIntervals();
|
|
1331
|
+
rotationTimer = scheduleTimeout(() => {
|
|
1332
|
+
rotationTimer = null;
|
|
1333
|
+
scheduleRecovery("rotation", false, true);
|
|
1334
|
+
}, rotationInterval);
|
|
1335
|
+
};
|
|
1336
|
+
const establish = async (record2, currentMode, cause, rotate, replaceMicrophone) => {
|
|
1337
|
+
if (currentMode.state !== "active") {
|
|
1338
|
+
transitionEnded();
|
|
1339
|
+
return;
|
|
1340
|
+
}
|
|
1341
|
+
if (rotate) {
|
|
1342
|
+
await heartbeat();
|
|
1343
|
+
const renewedMode = state.mode;
|
|
1344
|
+
if (!renewedMode || renewedMode.state !== "active") return;
|
|
1345
|
+
if (closed || stopping || owner !== record2) {
|
|
1346
|
+
throw new DOMException("Realtime connection generation was retired", "AbortError");
|
|
1347
|
+
}
|
|
1348
|
+
currentMode = renewedMode;
|
|
1349
|
+
}
|
|
1350
|
+
stopNegotiationTimers();
|
|
1351
|
+
const targetGeneration = ++generation;
|
|
1352
|
+
pendingGeneration = targetGeneration;
|
|
1353
|
+
const abort = new AbortController();
|
|
1354
|
+
pendingAbort = abort;
|
|
1355
|
+
const earlyEvents = createCodexRealtimeEarlyEventBuffer({
|
|
1356
|
+
onFatal: (error) => {
|
|
1357
|
+
if (pendingGeneration === targetGeneration && !abort.signal.aborted) abort.abort(error);
|
|
1358
|
+
}
|
|
1359
|
+
});
|
|
1360
|
+
negotiationTimer = scheduleTimeout(() => {
|
|
1361
|
+
if (pendingGeneration === targetGeneration && !abort.signal.aborted) {
|
|
1362
|
+
abort.abort(
|
|
1363
|
+
new CodexRealtimeGenerationError(
|
|
1364
|
+
"Codex realtime negotiation did not open within 20 seconds"
|
|
1365
|
+
)
|
|
1366
|
+
);
|
|
1367
|
+
}
|
|
1368
|
+
}, negotiationTimeout);
|
|
1369
|
+
publish({
|
|
1370
|
+
status: rotate ? "recovering" : "starting",
|
|
1371
|
+
realtimeId: currentMode.id,
|
|
1372
|
+
mode: currentMode,
|
|
1373
|
+
connectionGeneration: targetGeneration,
|
|
1374
|
+
diagnostic: cause === "rotation" ? diagnostic(
|
|
1375
|
+
"rotation",
|
|
1376
|
+
"Rotating the finite provider connection",
|
|
1377
|
+
true,
|
|
1378
|
+
targetGeneration
|
|
1379
|
+
) : cause === "reload" || cause === "reconnect" ? diagnostic("reconnect", "Reconnecting the same realtime mode", true, targetGeneration) : state.diagnostic,
|
|
1380
|
+
error: null
|
|
1381
|
+
});
|
|
1382
|
+
let connected;
|
|
1383
|
+
try {
|
|
1384
|
+
const media = await ensureMicrophone(replaceMicrophone, abort.signal);
|
|
1385
|
+
const operationId = randomUUID();
|
|
1386
|
+
const commonTransportInput = {
|
|
1387
|
+
client: options.client,
|
|
1388
|
+
workspaceId: options.workspaceId,
|
|
1389
|
+
sessionId: options.sessionId,
|
|
1390
|
+
realtimeId: currentMode.id,
|
|
1391
|
+
operationId,
|
|
1392
|
+
browserInstanceId: record2.browserInstanceId,
|
|
1393
|
+
ownerKey: record2.ownerKey,
|
|
1394
|
+
expectedVersion: currentMode.version,
|
|
1395
|
+
expectedConnectionEpoch: currentMode.connectionEpoch,
|
|
1396
|
+
rotate,
|
|
1397
|
+
signal: abort.signal,
|
|
1398
|
+
media,
|
|
1399
|
+
onEventsCreated: earlyEvents.attach,
|
|
1400
|
+
onAudibleOutputState: (next) => onAudibleOutput(targetGeneration, next),
|
|
1401
|
+
onMicrophoneEnded: () => onMicrophoneEnded(targetGeneration),
|
|
1402
|
+
onConnectionHealth: (health) => onConnectionHealth(targetGeneration, health)
|
|
1403
|
+
};
|
|
1404
|
+
connected = options.startTransport ? await options.startTransport(commonTransportInput) : await startCodexRealtimeWebrtc({
|
|
1405
|
+
...commonTransportInput,
|
|
1406
|
+
remoteAudio: options.remoteAudio,
|
|
1407
|
+
activateRemoteAudio: false,
|
|
1408
|
+
createPeerConnection: options.createPeerConnection,
|
|
1409
|
+
getUserMedia: options.getUserMedia,
|
|
1410
|
+
negotiate: async (request, requestOptions) => await options.client.negotiateCodexRealtimeWebrtc(
|
|
1411
|
+
options.workspaceId,
|
|
1412
|
+
options.sessionId,
|
|
1413
|
+
request,
|
|
1414
|
+
requestOptions
|
|
1415
|
+
)
|
|
1416
|
+
});
|
|
1417
|
+
} catch (error) {
|
|
1418
|
+
clearNegotiationTimer();
|
|
1419
|
+
earlyEvents.close();
|
|
1420
|
+
if (pendingGeneration === targetGeneration) {
|
|
1421
|
+
pendingGeneration = null;
|
|
1422
|
+
pendingAbort = null;
|
|
1423
|
+
}
|
|
1424
|
+
throw error;
|
|
1425
|
+
}
|
|
1426
|
+
try {
|
|
1427
|
+
await waitForDataChannelOpen(connected.events, abort.signal);
|
|
1428
|
+
if (closed || stopping || abort.signal.aborted || pendingGeneration !== targetGeneration || !connected.microphoneHealthy()) {
|
|
1429
|
+
if (!connected.microphoneHealthy() && !abort.signal.aborted) {
|
|
1430
|
+
abort.abort(
|
|
1431
|
+
new CodexRealtimeMicrophoneError(
|
|
1432
|
+
"track_ended",
|
|
1433
|
+
"The microphone audio track ended before realtime activation"
|
|
1434
|
+
)
|
|
1435
|
+
);
|
|
1436
|
+
}
|
|
1437
|
+
throw abort.signal.reason ?? new DOMException("Aborted", "AbortError");
|
|
1438
|
+
}
|
|
1439
|
+
const activated = await options.client.activateCodexRealtimeConnection(
|
|
1440
|
+
options.workspaceId,
|
|
1441
|
+
options.sessionId,
|
|
1442
|
+
currentMode.id,
|
|
1443
|
+
connected.connectionId,
|
|
1444
|
+
{
|
|
1445
|
+
operationId: connected.operationId,
|
|
1446
|
+
browserInstanceId: record2.browserInstanceId,
|
|
1447
|
+
ownerKey: record2.ownerKey,
|
|
1448
|
+
connectionEpoch: connected.connectionEpoch,
|
|
1449
|
+
expectedVersion: currentMode.version,
|
|
1450
|
+
expectedConnectionEpoch: currentMode.connectionEpoch
|
|
1451
|
+
},
|
|
1452
|
+
{ signal: abort.signal }
|
|
1453
|
+
);
|
|
1454
|
+
if (activated.mode.state !== "active") {
|
|
1455
|
+
clearNegotiationTimer();
|
|
1456
|
+
earlyEvents.close();
|
|
1457
|
+
connected.stop();
|
|
1458
|
+
transitionEnded();
|
|
1459
|
+
return;
|
|
1460
|
+
}
|
|
1461
|
+
if (closed || stopping || abort.signal.aborted || pendingGeneration !== targetGeneration || !connected.microphoneHealthy()) {
|
|
1462
|
+
if (!connected.microphoneHealthy() && !abort.signal.aborted) {
|
|
1463
|
+
abort.abort(
|
|
1464
|
+
new CodexRealtimeMicrophoneError(
|
|
1465
|
+
"track_ended",
|
|
1466
|
+
"The microphone audio track ended during realtime activation"
|
|
1467
|
+
)
|
|
1468
|
+
);
|
|
1469
|
+
}
|
|
1470
|
+
throw abort.signal.reason ?? new DOMException("Aborted", "AbortError");
|
|
1471
|
+
}
|
|
1472
|
+
clearNegotiationTimer();
|
|
1473
|
+
const previous = active;
|
|
1474
|
+
let bridge;
|
|
1475
|
+
bridge = createCodexRealtimeV3Bridge({
|
|
1476
|
+
events: connected.events,
|
|
1477
|
+
connectionId: connected.connectionId,
|
|
1478
|
+
connectionEpoch: connected.connectionEpoch,
|
|
1479
|
+
startupFenceSequence: connected.startupFenceSequence,
|
|
1480
|
+
modeVersion: activated.mode.version,
|
|
1481
|
+
owner: {
|
|
1482
|
+
browserInstanceId: record2.browserInstanceId,
|
|
1483
|
+
ownerKey: record2.ownerKey,
|
|
1484
|
+
expectedVersion: activated.mode.version
|
|
1485
|
+
},
|
|
1486
|
+
listen: false,
|
|
1487
|
+
sync: async (request) => await syncForGeneration(targetGeneration, activated.mode.id, request),
|
|
1488
|
+
randomUUID,
|
|
1489
|
+
onSnapshot: (nextBridge) => {
|
|
1490
|
+
if (active?.generation === targetGeneration) publish({ bridge: nextBridge });
|
|
1491
|
+
},
|
|
1492
|
+
onFatal: (fatal) => onBridgeFatal(targetGeneration, bridge, fatal)
|
|
1493
|
+
});
|
|
1494
|
+
active = { generation: targetGeneration, transport: connected, bridge };
|
|
1495
|
+
connected.setOutputMuted(state.outputMuted);
|
|
1496
|
+
recoveryTerminal = false;
|
|
1497
|
+
pendingAbort = null;
|
|
1498
|
+
pendingGeneration = null;
|
|
1499
|
+
publish({
|
|
1500
|
+
status: "active",
|
|
1501
|
+
realtimeId: activated.mode.id,
|
|
1502
|
+
mode: activated.mode,
|
|
1503
|
+
microphone: "active",
|
|
1504
|
+
audibleOutput: connected.audibleOutputState(),
|
|
1505
|
+
bridge: bridge.snapshot(),
|
|
1506
|
+
connectionGeneration: targetGeneration,
|
|
1507
|
+
reconnectAttempt: 0,
|
|
1508
|
+
diagnostic: cause === "rotation" ? diagnostic(
|
|
1509
|
+
"rotation",
|
|
1510
|
+
"Provider connection rotation completed",
|
|
1511
|
+
true,
|
|
1512
|
+
targetGeneration
|
|
1513
|
+
) : cause === "reload" || cause === "reconnect" || cause === "microphone" ? diagnostic("reconnect", "Realtime connection recovered", true, targetGeneration) : null,
|
|
1514
|
+
error: null
|
|
1515
|
+
});
|
|
1516
|
+
connected.activateRemoteAudio();
|
|
1517
|
+
previous?.bridge.close();
|
|
1518
|
+
previous?.transport.stop();
|
|
1519
|
+
earlyEvents.handoff(bridge);
|
|
1520
|
+
if (active?.generation !== targetGeneration) {
|
|
1521
|
+
throw new CodexRealtimeGenerationError(
|
|
1522
|
+
bridge.snapshot().fatal?.message ?? "Codex realtime connection generation was retired during activation"
|
|
1523
|
+
);
|
|
1524
|
+
}
|
|
1525
|
+
startTimers();
|
|
1526
|
+
} catch (error) {
|
|
1527
|
+
clearNegotiationTimer();
|
|
1528
|
+
earlyEvents.close();
|
|
1529
|
+
connected.stop();
|
|
1530
|
+
if (pendingGeneration === targetGeneration) {
|
|
1531
|
+
pendingGeneration = null;
|
|
1532
|
+
pendingAbort = null;
|
|
1533
|
+
}
|
|
1534
|
+
throw error;
|
|
1535
|
+
}
|
|
1536
|
+
};
|
|
1537
|
+
const begin = async (record2, recover, connectAfterBegin = true) => {
|
|
1538
|
+
const response = await options.client.beginSessionRealtime(
|
|
1539
|
+
options.workspaceId,
|
|
1540
|
+
options.sessionId,
|
|
1541
|
+
{
|
|
1542
|
+
operationId: record2.operationId,
|
|
1543
|
+
browserInstanceId: record2.browserInstanceId,
|
|
1544
|
+
ownerKey: record2.ownerKey,
|
|
1545
|
+
model
|
|
1546
|
+
}
|
|
1547
|
+
);
|
|
1548
|
+
if (response.mode.state !== "active") {
|
|
1549
|
+
transitionEnded();
|
|
1550
|
+
return null;
|
|
1551
|
+
}
|
|
1552
|
+
publish({ mode: response.mode, realtimeId: response.mode.id });
|
|
1553
|
+
if (connectAfterBegin) {
|
|
1554
|
+
await establish(
|
|
1555
|
+
record2,
|
|
1556
|
+
response.mode,
|
|
1557
|
+
recover ? "reload" : "manual",
|
|
1558
|
+
recover || response.replay,
|
|
1559
|
+
false
|
|
1560
|
+
);
|
|
1561
|
+
}
|
|
1562
|
+
return response.mode;
|
|
1563
|
+
};
|
|
1564
|
+
const heartbeat = async () => {
|
|
1565
|
+
await exclusive(async () => {
|
|
1566
|
+
const current = state.mode;
|
|
1567
|
+
if (!owner || !current || current.state !== "active") {
|
|
1568
|
+
throw new Error("Codex realtime owner is not active");
|
|
1569
|
+
}
|
|
1570
|
+
const result = await options.client.heartbeatSessionRealtime(
|
|
1571
|
+
options.workspaceId,
|
|
1572
|
+
options.sessionId,
|
|
1573
|
+
current.id,
|
|
1574
|
+
{
|
|
1575
|
+
browserInstanceId: owner.browserInstanceId,
|
|
1576
|
+
ownerKey: owner.ownerKey,
|
|
1577
|
+
expectedVersion: current.version
|
|
1578
|
+
}
|
|
1579
|
+
);
|
|
1580
|
+
if (result.mode.state === "ended") {
|
|
1581
|
+
transitionEnded("Realtime lease ended");
|
|
1582
|
+
return;
|
|
1583
|
+
}
|
|
1584
|
+
publish({ mode: result.mode, realtimeId: result.mode.id, error: null });
|
|
1585
|
+
});
|
|
1586
|
+
};
|
|
1587
|
+
const flush = async () => {
|
|
1588
|
+
await active?.bridge.flush();
|
|
1589
|
+
};
|
|
1590
|
+
const retry = async () => {
|
|
1591
|
+
if (!owner || state.mode?.state !== "active") {
|
|
1592
|
+
throw new Error("Codex realtime owner is not recoverable");
|
|
1593
|
+
}
|
|
1594
|
+
if (recoveryTerminal || state.diagnostic?.recoverable === false) {
|
|
1595
|
+
throw new Error("Codex realtime recovery is terminal; stop the mode before retrying");
|
|
1596
|
+
}
|
|
1597
|
+
if (reconnectTimer !== null) unscheduleTimeout(reconnectTimer);
|
|
1598
|
+
reconnectTimer = null;
|
|
1599
|
+
const replace = !codexRealtimeMicrophoneHealthy(microphone);
|
|
1600
|
+
scheduleRecovery(replace ? "microphone" : "manual", replace, true);
|
|
1601
|
+
};
|
|
1602
|
+
const retryAudibleOutput = async () => {
|
|
1603
|
+
if (!active || state.audibleOutput !== "blocked") return false;
|
|
1604
|
+
const target = active;
|
|
1605
|
+
const resumed = await target.transport.retryAudibleOutput();
|
|
1606
|
+
if (active?.generation !== target.generation) return false;
|
|
1607
|
+
return resumed;
|
|
1608
|
+
};
|
|
1609
|
+
const setInputMuted = (muted) => {
|
|
1610
|
+
if (state.inputMuted === muted) return;
|
|
1611
|
+
for (const track of microphone?.getAudioTracks() ?? []) track.enabled = !muted;
|
|
1612
|
+
publish({ inputMuted: muted });
|
|
1613
|
+
};
|
|
1614
|
+
const setOutputMuted = (muted) => {
|
|
1615
|
+
if (state.outputMuted === muted) return;
|
|
1616
|
+
active?.transport.setOutputMuted(muted);
|
|
1617
|
+
publish({ outputMuted: muted });
|
|
1618
|
+
};
|
|
1619
|
+
const controller = {
|
|
1620
|
+
snapshot: () => ({ ...state }),
|
|
1621
|
+
subscribe: (listener) => {
|
|
1622
|
+
listeners.add(listener);
|
|
1623
|
+
listener({ ...state });
|
|
1624
|
+
return () => listeners.delete(listener);
|
|
1625
|
+
},
|
|
1626
|
+
start: async () => {
|
|
1627
|
+
if (state.status === "lost_owner") {
|
|
1628
|
+
throw new Error("Realtime mode belongs to another browser owner");
|
|
1629
|
+
}
|
|
1630
|
+
if (!["idle", "error"].includes(state.status) || state.mode?.state === "active") return;
|
|
1631
|
+
closed = false;
|
|
1632
|
+
stopping = false;
|
|
1633
|
+
recoveryTerminal = false;
|
|
1634
|
+
const record2 = {
|
|
1635
|
+
version: OWNER_RECORD_VERSION,
|
|
1636
|
+
workspaceId: options.workspaceId,
|
|
1637
|
+
sessionId: options.sessionId,
|
|
1638
|
+
browserInstanceId: randomUUID(),
|
|
1639
|
+
ownerKey: `opengeni-realtime-owner:${randomUUID()}`,
|
|
1640
|
+
operationId: randomUUID()
|
|
1641
|
+
};
|
|
1642
|
+
owner = record2;
|
|
1643
|
+
storage?.setItem(storageKey, JSON.stringify(record2));
|
|
1644
|
+
publish({ status: "starting", error: null, diagnostic: null });
|
|
1645
|
+
try {
|
|
1646
|
+
await begin(record2, false);
|
|
1647
|
+
} catch (error) {
|
|
1648
|
+
closeBrowserResources(false);
|
|
1649
|
+
const failedMode = state.mode;
|
|
1650
|
+
if (failedMode?.state === "active") {
|
|
1651
|
+
await handleConnectionFailure(error, "reconnect", false);
|
|
1652
|
+
} else {
|
|
1653
|
+
clearOwner();
|
|
1654
|
+
publish({ status: "error", realtimeId: null, mode: null, error: safeError2(error) });
|
|
1655
|
+
}
|
|
1656
|
+
throw error;
|
|
1657
|
+
}
|
|
1658
|
+
},
|
|
1659
|
+
observeLifecycle: async (lifecycle) => {
|
|
1660
|
+
if (!lifecycle) {
|
|
1661
|
+
if (recoveryTerminal && state.mode?.state === "active") return;
|
|
1662
|
+
const record3 = readOwnerRecord(storage, storageKey, options);
|
|
1663
|
+
if (!record3) {
|
|
1664
|
+
transitionEnded();
|
|
1665
|
+
return;
|
|
1666
|
+
}
|
|
1667
|
+
if (connectionTask || state.status === "starting" || state.status === "active") return;
|
|
1668
|
+
closed = false;
|
|
1669
|
+
stopping = false;
|
|
1670
|
+
owner = record3;
|
|
1671
|
+
publish({ status: "recovering", error: null });
|
|
1672
|
+
connectionTask = begin(record3, true).then(() => void 0).catch(async (error) => {
|
|
1673
|
+
connectionTask = null;
|
|
1674
|
+
await handleConnectionFailure(error, "reload", false);
|
|
1675
|
+
}).finally(() => {
|
|
1676
|
+
connectionTask = null;
|
|
1677
|
+
});
|
|
1678
|
+
await connectionTask;
|
|
1679
|
+
return;
|
|
1680
|
+
}
|
|
1681
|
+
if (lifecycle.state === "ended") {
|
|
1682
|
+
const record3 = readOwnerRecord(storage, storageKey, options);
|
|
1683
|
+
if (record3 && record3.operationId !== lifecycle.operationId) {
|
|
1684
|
+
if (connectionTask || state.status === "active") return;
|
|
1685
|
+
closed = false;
|
|
1686
|
+
stopping = false;
|
|
1687
|
+
owner = record3;
|
|
1688
|
+
connectionTask = begin(record3, true).then(() => void 0).catch(async (error) => {
|
|
1689
|
+
connectionTask = null;
|
|
1690
|
+
await handleConnectionFailure(error, "reload", false);
|
|
1691
|
+
}).finally(() => {
|
|
1692
|
+
connectionTask = null;
|
|
1693
|
+
});
|
|
1694
|
+
await connectionTask;
|
|
1695
|
+
return;
|
|
1696
|
+
}
|
|
1697
|
+
if (state.realtimeId === null || lifecycle.realtimeId === state.realtimeId) {
|
|
1698
|
+
transitionEnded(`Realtime ended: ${lifecycle.reason}`);
|
|
1699
|
+
}
|
|
1700
|
+
return;
|
|
1701
|
+
}
|
|
1702
|
+
if (recoveryTerminal && state.realtimeId === lifecycle.realtimeId) return;
|
|
1703
|
+
if (state.status === "active" && state.realtimeId === lifecycle.realtimeId) return;
|
|
1704
|
+
const record2 = readOwnerRecord(storage, storageKey, options);
|
|
1705
|
+
if (!record2 || record2.operationId !== lifecycle.operationId) {
|
|
1706
|
+
closeBrowserResources();
|
|
1707
|
+
owner = null;
|
|
1708
|
+
const message = "Realtime is active in another browser owner. It can resume after that owner stops or its lease expires.";
|
|
1709
|
+
publish({
|
|
1710
|
+
status: "lost_owner",
|
|
1711
|
+
realtimeId: lifecycle.realtimeId,
|
|
1712
|
+
mode: null,
|
|
1713
|
+
bridge: null,
|
|
1714
|
+
diagnostic: diagnostic("lost_owner", message, false),
|
|
1715
|
+
error: message
|
|
1716
|
+
});
|
|
1717
|
+
return;
|
|
1718
|
+
}
|
|
1719
|
+
if (connectionTask || state.status === "starting") return;
|
|
1720
|
+
closed = false;
|
|
1721
|
+
stopping = false;
|
|
1722
|
+
owner = record2;
|
|
1723
|
+
publish({ status: "recovering", realtimeId: lifecycle.realtimeId, error: null });
|
|
1724
|
+
connectionTask = begin(record2, true).then(() => void 0).catch(async (error) => {
|
|
1725
|
+
connectionTask = null;
|
|
1726
|
+
await handleConnectionFailure(error, "reload", false);
|
|
1727
|
+
}).finally(() => {
|
|
1728
|
+
connectionTask = null;
|
|
1729
|
+
});
|
|
1730
|
+
await connectionTask;
|
|
1731
|
+
},
|
|
1732
|
+
heartbeat,
|
|
1733
|
+
flush,
|
|
1734
|
+
ingestProviderEvent: async (payload) => {
|
|
1735
|
+
if (!active) throw new Error("Codex realtime provider channel is not connected");
|
|
1736
|
+
await active.bridge.ingest(payload);
|
|
1737
|
+
},
|
|
1738
|
+
retry,
|
|
1739
|
+
retryAudibleOutput,
|
|
1740
|
+
setInputMuted,
|
|
1741
|
+
setOutputMuted,
|
|
1742
|
+
stop: async () => {
|
|
1743
|
+
const currentOwner = owner;
|
|
1744
|
+
if (!currentOwner || !state.mode) {
|
|
1745
|
+
if (state.status !== "lost_owner") transitionEnded();
|
|
1746
|
+
return;
|
|
1747
|
+
}
|
|
1748
|
+
stopping = true;
|
|
1749
|
+
publish({
|
|
1750
|
+
status: "stopping",
|
|
1751
|
+
diagnostic: diagnostic("terminal_stop", "Stopping realtime and releasing media", false),
|
|
1752
|
+
error: null
|
|
1753
|
+
});
|
|
1754
|
+
stopTimers();
|
|
1755
|
+
pendingAbort?.abort(new DOMException("Realtime stopped", "AbortError"));
|
|
1756
|
+
pendingGeneration = null;
|
|
1757
|
+
const retiredTask = connectionTask;
|
|
1758
|
+
connectionTask = null;
|
|
1759
|
+
void retiredTask?.catch(() => void 0);
|
|
1760
|
+
try {
|
|
1761
|
+
await active?.bridge.sealAndFlush();
|
|
1762
|
+
closeBrowserResources();
|
|
1763
|
+
let current = state.mode;
|
|
1764
|
+
if (!current) throw new Error("Codex realtime mode disappeared while stopping");
|
|
1765
|
+
let response;
|
|
1766
|
+
try {
|
|
1767
|
+
response = await exclusive(
|
|
1768
|
+
async () => await options.client.endSessionRealtime(
|
|
1769
|
+
options.workspaceId,
|
|
1770
|
+
options.sessionId,
|
|
1771
|
+
current.id,
|
|
1772
|
+
{
|
|
1773
|
+
browserInstanceId: currentOwner.browserInstanceId,
|
|
1774
|
+
ownerKey: currentOwner.ownerKey,
|
|
1775
|
+
expectedVersion: current.version,
|
|
1776
|
+
reason: "user_stop"
|
|
1777
|
+
}
|
|
1778
|
+
)
|
|
1779
|
+
);
|
|
1780
|
+
} catch (error) {
|
|
1781
|
+
if (!(error instanceof OpenGeniApiError) || error.status !== 409) throw error;
|
|
1782
|
+
const reconciled = await begin(currentOwner, true, false);
|
|
1783
|
+
if (!reconciled || reconciled.state === "ended") return;
|
|
1784
|
+
current = reconciled;
|
|
1785
|
+
response = await exclusive(
|
|
1786
|
+
async () => await options.client.endSessionRealtime(
|
|
1787
|
+
options.workspaceId,
|
|
1788
|
+
options.sessionId,
|
|
1789
|
+
current.id,
|
|
1790
|
+
{
|
|
1791
|
+
browserInstanceId: currentOwner.browserInstanceId,
|
|
1792
|
+
ownerKey: currentOwner.ownerKey,
|
|
1793
|
+
expectedVersion: current.version,
|
|
1794
|
+
reason: "user_stop"
|
|
1795
|
+
}
|
|
1796
|
+
)
|
|
1797
|
+
);
|
|
1798
|
+
}
|
|
1799
|
+
if (response.mode.state === "ended") transitionEnded("Realtime stopped by this browser");
|
|
1800
|
+
} catch (error) {
|
|
1801
|
+
stopping = false;
|
|
1802
|
+
owner = currentOwner;
|
|
1803
|
+
startActiveIntervals();
|
|
1804
|
+
publish({
|
|
1805
|
+
status: "recovering",
|
|
1806
|
+
diagnostic: diagnostic("terminal_stop", safeError2(error), true),
|
|
1807
|
+
error: safeError2(error)
|
|
1808
|
+
});
|
|
1809
|
+
throw error;
|
|
1810
|
+
}
|
|
1811
|
+
},
|
|
1812
|
+
close: () => {
|
|
1813
|
+
closed = true;
|
|
1814
|
+
stopping = false;
|
|
1815
|
+
closeBrowserResources();
|
|
1816
|
+
listeners.clear();
|
|
1817
|
+
}
|
|
1818
|
+
};
|
|
1819
|
+
return controller;
|
|
1820
|
+
}
|
|
1821
|
+
function createCodexRealtimeEarlyEventBuffer(input) {
|
|
1822
|
+
let events = null;
|
|
1823
|
+
let buffered = [];
|
|
1824
|
+
let bufferedBytes = 0;
|
|
1825
|
+
let fatal = false;
|
|
1826
|
+
const detach = () => {
|
|
1827
|
+
events?.removeEventListener("message", onMessage);
|
|
1828
|
+
events = null;
|
|
1829
|
+
};
|
|
1830
|
+
const fail = () => {
|
|
1831
|
+
if (fatal) return;
|
|
1832
|
+
fatal = true;
|
|
1833
|
+
detach();
|
|
1834
|
+
buffered = [];
|
|
1835
|
+
bufferedBytes = 0;
|
|
1836
|
+
input.onFatal(
|
|
1837
|
+
new CodexRealtimeGenerationError(
|
|
1838
|
+
"Codex realtime activation event buffer exceeded its hard limit"
|
|
1839
|
+
)
|
|
1840
|
+
);
|
|
1841
|
+
};
|
|
1842
|
+
const onMessage = (message) => {
|
|
1843
|
+
if (fatal || typeof message.data !== "string") return;
|
|
1844
|
+
const parsed = parseCodexRealtimeV3Event(message.data);
|
|
1845
|
+
if (!parsed.ok || parsed.event.type === "output_audio.delta") return;
|
|
1846
|
+
const bytes = new TextEncoder().encode(message.data).byteLength;
|
|
1847
|
+
if (bytes > CODEX_REALTIME_V3_MAX_EVENT_BYTES || buffered.length + 1 > CODEX_REALTIME_V3_PENDING_MAX_ENTRIES || bufferedBytes + bytes > CODEX_REALTIME_V3_PENDING_MAX_BYTES) {
|
|
1848
|
+
fail();
|
|
1849
|
+
return;
|
|
1850
|
+
}
|
|
1851
|
+
buffered.push(message.data);
|
|
1852
|
+
bufferedBytes += bytes;
|
|
1853
|
+
};
|
|
1854
|
+
return {
|
|
1855
|
+
attach: (channel) => {
|
|
1856
|
+
if (fatal || events === channel) return;
|
|
1857
|
+
if (events) throw new Error("Codex realtime activation buffer is already attached");
|
|
1858
|
+
events = channel;
|
|
1859
|
+
events.addEventListener("message", onMessage);
|
|
1860
|
+
},
|
|
1861
|
+
handoff: (bridge) => {
|
|
1862
|
+
if (fatal) {
|
|
1863
|
+
bridge.close();
|
|
1864
|
+
return;
|
|
1865
|
+
}
|
|
1866
|
+
for (let index = 0; index < buffered.length; index += 1) {
|
|
1867
|
+
void bridge.ingest(buffered[index]).catch(() => void 0);
|
|
1868
|
+
}
|
|
1869
|
+
buffered = [];
|
|
1870
|
+
bufferedBytes = 0;
|
|
1871
|
+
detach();
|
|
1872
|
+
bridge.listen();
|
|
1873
|
+
},
|
|
1874
|
+
close: () => {
|
|
1875
|
+
detach();
|
|
1876
|
+
buffered = [];
|
|
1877
|
+
bufferedBytes = 0;
|
|
1878
|
+
}
|
|
1879
|
+
};
|
|
1880
|
+
}
|
|
1881
|
+
var CodexRealtimeGenerationError = class extends Error {
|
|
1882
|
+
name = "CodexRealtimeGenerationError";
|
|
1883
|
+
};
|
|
1884
|
+
function ownerStorageKey(workspaceId, sessionId, namespace = "codex-realtime-owner") {
|
|
1885
|
+
return `opengeni:${namespace}:${workspaceId}:${sessionId}`;
|
|
1886
|
+
}
|
|
1887
|
+
function readOwnerRecord(storage, key, scope) {
|
|
1888
|
+
const raw = storage?.getItem(key);
|
|
1889
|
+
if (!raw) return null;
|
|
1890
|
+
try {
|
|
1891
|
+
const parsed = recordValue2(JSON.parse(raw));
|
|
1892
|
+
if (parsed?.version !== OWNER_RECORD_VERSION || parsed.workspaceId !== scope.workspaceId || parsed.sessionId !== scope.sessionId || !stringValue2(parsed.operationId) || !stringValue2(parsed.browserInstanceId) || !stringValue2(parsed.ownerKey) || String(parsed.ownerKey).length < 32) {
|
|
1893
|
+
storage?.removeItem(key);
|
|
1894
|
+
return null;
|
|
1895
|
+
}
|
|
1896
|
+
return parsed;
|
|
1897
|
+
} catch {
|
|
1898
|
+
storage?.removeItem(key);
|
|
1899
|
+
return null;
|
|
1900
|
+
}
|
|
1901
|
+
}
|
|
1902
|
+
function defaultStorage() {
|
|
1903
|
+
return typeof sessionStorage === "undefined" ? void 0 : sessionStorage;
|
|
1904
|
+
}
|
|
1905
|
+
function defaultRandomUUID2() {
|
|
1906
|
+
if (!globalThis.crypto?.randomUUID) throw new Error("crypto.randomUUID is unavailable");
|
|
1907
|
+
return globalThis.crypto.randomUUID();
|
|
1908
|
+
}
|
|
1909
|
+
function waitForDataChannelOpen(events, signal) {
|
|
1910
|
+
if (signal.aborted) {
|
|
1911
|
+
return Promise.reject(signal.reason ?? new DOMException("Aborted", "AbortError"));
|
|
1912
|
+
}
|
|
1913
|
+
if (events.readyState === "open") return Promise.resolve();
|
|
1914
|
+
if (events.readyState === "closing" || events.readyState === "closed") {
|
|
1915
|
+
return Promise.reject(new Error("Codex realtime data channel closed before opening"));
|
|
1916
|
+
}
|
|
1917
|
+
return new Promise((resolve, reject) => {
|
|
1918
|
+
const cleanup = () => {
|
|
1919
|
+
events.removeEventListener("open", onOpen);
|
|
1920
|
+
events.removeEventListener("close", onClose);
|
|
1921
|
+
events.removeEventListener("error", onError);
|
|
1922
|
+
signal.removeEventListener("abort", onAbort);
|
|
1923
|
+
};
|
|
1924
|
+
const onOpen = () => {
|
|
1925
|
+
cleanup();
|
|
1926
|
+
resolve();
|
|
1927
|
+
};
|
|
1928
|
+
const onClose = () => {
|
|
1929
|
+
cleanup();
|
|
1930
|
+
reject(new Error("Codex realtime data channel closed before opening"));
|
|
1931
|
+
};
|
|
1932
|
+
const onError = () => {
|
|
1933
|
+
cleanup();
|
|
1934
|
+
reject(new Error("Codex realtime data channel failed before opening"));
|
|
1935
|
+
};
|
|
1936
|
+
const onAbort = () => {
|
|
1937
|
+
cleanup();
|
|
1938
|
+
reject(signal.reason ?? new DOMException("Aborted", "AbortError"));
|
|
1939
|
+
};
|
|
1940
|
+
events.addEventListener("open", onOpen, { once: true });
|
|
1941
|
+
events.addEventListener("close", onClose, { once: true });
|
|
1942
|
+
events.addEventListener("error", onError, { once: true });
|
|
1943
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1944
|
+
if (signal.aborted) onAbort();
|
|
1945
|
+
else if (events.readyState === "open") onOpen();
|
|
1946
|
+
else if (events.readyState === "closing" || events.readyState === "closed") onClose();
|
|
1947
|
+
});
|
|
1948
|
+
}
|
|
1949
|
+
function positiveDuration(value, name) {
|
|
1950
|
+
if (!Number.isSafeInteger(value) || value <= 0)
|
|
1951
|
+
throw new Error(`Codex realtime ${name} is invalid`);
|
|
1952
|
+
return value;
|
|
1953
|
+
}
|
|
1954
|
+
function validateReconnectBackoff(values) {
|
|
1955
|
+
if (values.length === 0 || values.some((value) => !Number.isSafeInteger(value) || value < 0 || value > 6e4)) {
|
|
1956
|
+
throw new Error("Codex realtime reconnect backoff is invalid");
|
|
1957
|
+
}
|
|
1958
|
+
return [...values];
|
|
1959
|
+
}
|
|
1960
|
+
function recordValue2(value) {
|
|
1961
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
1962
|
+
}
|
|
1963
|
+
function stringValue2(value) {
|
|
1964
|
+
return typeof value === "string" && value.length > 0 ? value : null;
|
|
1965
|
+
}
|
|
1966
|
+
function isAbortError(error) {
|
|
1967
|
+
return error instanceof Error && error.name === "AbortError";
|
|
1968
|
+
}
|
|
1969
|
+
function safeError2(error) {
|
|
1970
|
+
return error instanceof Error ? error.message : "Codex realtime browser controller failed";
|
|
1971
|
+
}
|
|
1972
|
+
|
|
1973
|
+
export {
|
|
1974
|
+
CODEX_REALTIME_CONTEXT_APPEND_MAX_BYTES,
|
|
1975
|
+
parseCodexRealtimeV3Event,
|
|
1976
|
+
encodeCodexRealtimeV3DelegationContextAppend,
|
|
1977
|
+
encodeCodexRealtimeV3SessionContextAppend,
|
|
1978
|
+
contextAppendChunks,
|
|
1979
|
+
createCodexRealtimeV3Bridge,
|
|
1980
|
+
CodexRealtimeMicrophoneError,
|
|
1981
|
+
startCodexRealtimeWebrtc,
|
|
1982
|
+
acquireCodexRealtimeMicrophone,
|
|
1983
|
+
codexRealtimeMicrophoneHealthy,
|
|
1984
|
+
projectSessionRealtimeLifecycle,
|
|
1985
|
+
CODEX_REALTIME_NEGOTIATION_TIMEOUT_MS,
|
|
1986
|
+
createCodexRealtimeController
|
|
1987
|
+
};
|
|
1988
|
+
//# sourceMappingURL=chunk-OB5PGV6N.js.map
|