@kubohiroya/turbowarp-openai-realtime-api 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +373 -0
- package/README.ja.md +353 -0
- package/README.md +415 -0
- package/dist/composition.js +682 -0
- package/dist/extension-manifest.json +240 -0
- package/dist/turbowarp-openai-realtime-api.js +1581 -0
- package/dist/types/composition.d.ts +89 -0
- package/dist/types/config.d.ts +11 -0
- package/dist/types/extension-manifest.d.ts +30 -0
- package/dist/types/extension.d.ts +45 -0
- package/dist/types/index.d.ts +1 -0
- package/dist/types/realtime-session.d.ts +39 -0
- package/dist/types/relay-client.d.ts +16 -0
- package/dist/types/session-config.d.ts +27 -0
- package/dist/types/transport.d.ts +18 -0
- package/dist/types/usage.d.ts +34 -0
- package/dist/types/webrtc-transport.d.ts +26 -0
- package/package.json +79 -0
- package/schemas/extension-manifest.schema.json +49 -0
- package/src/block-definitions.json +347 -0
- package/src/composition.ts +318 -0
- package/src/config.ts +12 -0
- package/src/extension-manifest.ts +149 -0
- package/src/extension.ts +266 -0
- package/src/globals.d.ts +26 -0
- package/src/index.ts +14 -0
- package/src/realtime-session.ts +175 -0
- package/src/relay-client.ts +133 -0
- package/src/session-config.ts +71 -0
- package/src/transport.ts +18 -0
- package/src/usage.ts +127 -0
- package/src/webrtc-transport.ts +156 -0
|
@@ -0,0 +1,682 @@
|
|
|
1
|
+
import { createNamedFunctions } from "@kubohiroya/turbowarp-named-functions/composition";
|
|
2
|
+
//#region src/realtime-session.ts
|
|
3
|
+
/**
|
|
4
|
+
* Conversation logic over a transport: sending user text, collecting assistant output, and
|
|
5
|
+
* answering function calls with `function_call_output` followed by one `response.create`.
|
|
6
|
+
*/
|
|
7
|
+
var RealtimeSession = class {
|
|
8
|
+
constructor(hooks) {
|
|
9
|
+
this.hooks = hooks;
|
|
10
|
+
this.stateValue = "disconnected";
|
|
11
|
+
this.transport = null;
|
|
12
|
+
this.generation = 0;
|
|
13
|
+
}
|
|
14
|
+
get state() {
|
|
15
|
+
return this.stateValue;
|
|
16
|
+
}
|
|
17
|
+
async open(transport, clientSecret, microphone) {
|
|
18
|
+
this.close();
|
|
19
|
+
const generation = ++this.generation;
|
|
20
|
+
this.transport = transport;
|
|
21
|
+
this.setState("connecting");
|
|
22
|
+
try {
|
|
23
|
+
await transport.connect({
|
|
24
|
+
clientSecret,
|
|
25
|
+
microphone,
|
|
26
|
+
onEvent: (event) => {
|
|
27
|
+
if (generation === this.generation) this.handleEvent(event);
|
|
28
|
+
},
|
|
29
|
+
onClose: (reason) => {
|
|
30
|
+
if (generation !== this.generation) return;
|
|
31
|
+
this.transport = null;
|
|
32
|
+
this.hooks.onError(reason);
|
|
33
|
+
this.setState("failed");
|
|
34
|
+
}
|
|
35
|
+
});
|
|
36
|
+
} catch (error) {
|
|
37
|
+
if (generation === this.generation) {
|
|
38
|
+
this.transport = null;
|
|
39
|
+
this.setState("failed");
|
|
40
|
+
}
|
|
41
|
+
throw error;
|
|
42
|
+
}
|
|
43
|
+
if (generation === this.generation) this.setState("connected");
|
|
44
|
+
}
|
|
45
|
+
close() {
|
|
46
|
+
this.generation += 1;
|
|
47
|
+
const transport = this.transport;
|
|
48
|
+
this.transport = null;
|
|
49
|
+
transport?.close();
|
|
50
|
+
if (this.stateValue !== "disconnected") this.setState("disconnected");
|
|
51
|
+
}
|
|
52
|
+
sendText(text) {
|
|
53
|
+
this.send({
|
|
54
|
+
type: "conversation.item.create",
|
|
55
|
+
item: {
|
|
56
|
+
type: "message",
|
|
57
|
+
role: "user",
|
|
58
|
+
content: [{
|
|
59
|
+
type: "input_text",
|
|
60
|
+
text
|
|
61
|
+
}]
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
this.send({ type: "response.create" });
|
|
65
|
+
}
|
|
66
|
+
send(event) {
|
|
67
|
+
if (!this.transport || this.stateValue !== "connected") throw new Error("Not connected to Realtime.");
|
|
68
|
+
this.transport.send(event);
|
|
69
|
+
}
|
|
70
|
+
async handleEvent(event) {
|
|
71
|
+
if (event.type === "error") {
|
|
72
|
+
this.hooks.onError(describeApiError(event));
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
if (event.type !== "response.done") return;
|
|
76
|
+
const response = event.response;
|
|
77
|
+
if (typeof response === "object" && response !== null && "usage" in response) this.hooks.onUsage?.(response.usage);
|
|
78
|
+
const output = readResponseOutput(event);
|
|
79
|
+
if (output.text.length > 0) this.hooks.onResponseText(output.text);
|
|
80
|
+
if (output.calls.length === 0) return;
|
|
81
|
+
const generation = this.generation;
|
|
82
|
+
const results = await Promise.all(output.calls.map((call) => this.runCall(call)));
|
|
83
|
+
if (generation !== this.generation || !this.transport) return;
|
|
84
|
+
for (const result of results) this.transport.send({
|
|
85
|
+
type: "conversation.item.create",
|
|
86
|
+
item: {
|
|
87
|
+
type: "function_call_output",
|
|
88
|
+
call_id: result.callId,
|
|
89
|
+
output: result.output
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
this.transport.send({ type: "response.create" });
|
|
93
|
+
}
|
|
94
|
+
async runCall(call) {
|
|
95
|
+
let args;
|
|
96
|
+
try {
|
|
97
|
+
args = call.argumentsJson.trim().length === 0 ? {} : JSON.parse(call.argumentsJson);
|
|
98
|
+
} catch {
|
|
99
|
+
return {
|
|
100
|
+
callId: call.callId,
|
|
101
|
+
output: JSON.stringify({ error: "Arguments were not valid JSON." })
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
try {
|
|
105
|
+
const value = await this.hooks.callFunction(call.name, args);
|
|
106
|
+
return {
|
|
107
|
+
callId: call.callId,
|
|
108
|
+
output: JSON.stringify(value ?? null)
|
|
109
|
+
};
|
|
110
|
+
} catch (error) {
|
|
111
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
112
|
+
this.hooks.onError(message);
|
|
113
|
+
return {
|
|
114
|
+
callId: call.callId,
|
|
115
|
+
output: JSON.stringify({ error: message })
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
setState(state) {
|
|
120
|
+
this.stateValue = state;
|
|
121
|
+
this.hooks.onStateChange?.(state);
|
|
122
|
+
}
|
|
123
|
+
};
|
|
124
|
+
function readResponseOutput(event) {
|
|
125
|
+
const response = asRecord$1(event.response);
|
|
126
|
+
const items = Array.isArray(response?.output) ? response.output : [];
|
|
127
|
+
const texts = [];
|
|
128
|
+
const calls = [];
|
|
129
|
+
for (const rawItem of items) {
|
|
130
|
+
const item = asRecord$1(rawItem);
|
|
131
|
+
if (!item) continue;
|
|
132
|
+
if (item.type === "function_call" && typeof item.name === "string" && typeof item.call_id === "string") calls.push({
|
|
133
|
+
callId: item.call_id,
|
|
134
|
+
name: item.name,
|
|
135
|
+
argumentsJson: typeof item.arguments === "string" ? item.arguments : ""
|
|
136
|
+
});
|
|
137
|
+
else if (item.type === "message" && Array.isArray(item.content)) for (const rawPart of item.content) {
|
|
138
|
+
const part = asRecord$1(rawPart);
|
|
139
|
+
if (typeof part?.text === "string") texts.push(part.text);
|
|
140
|
+
else if (typeof part?.transcript === "string") texts.push(part.transcript);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
return {
|
|
144
|
+
text: texts.join(""),
|
|
145
|
+
calls
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
function describeApiError(event) {
|
|
149
|
+
const error = asRecord$1(event.error);
|
|
150
|
+
return `${typeof error?.message === "string" ? error.message : "Unknown Realtime API error."}${typeof error?.code === "string" ? ` (${error.code})` : ""}`;
|
|
151
|
+
}
|
|
152
|
+
function asRecord$1(value) {
|
|
153
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) ? value : null;
|
|
154
|
+
}
|
|
155
|
+
//#endregion
|
|
156
|
+
//#region src/relay-client.ts
|
|
157
|
+
var LOOPBACK_HOSTNAMES = /* @__PURE__ */ new Set([
|
|
158
|
+
"127.0.0.1",
|
|
159
|
+
"localhost",
|
|
160
|
+
"[::1]"
|
|
161
|
+
]);
|
|
162
|
+
var TOKEN_PATTERN = /^[A-Za-z0-9_-]{20,}$/u;
|
|
163
|
+
var CLIENT_SECRET_PATTERN = /^ek_[A-Za-z0-9_-]+$/u;
|
|
164
|
+
/** Accepts only a plain loopback HTTP origin so the relay token can never leave this machine. */
|
|
165
|
+
function normalizeRelayEndpoint(endpoint) {
|
|
166
|
+
let url;
|
|
167
|
+
try {
|
|
168
|
+
url = new URL(endpoint.trim());
|
|
169
|
+
} catch {
|
|
170
|
+
throw new TypeError("Relay endpoint must be a valid URL.");
|
|
171
|
+
}
|
|
172
|
+
if (url.protocol !== "http:" || !LOOPBACK_HOSTNAMES.has(url.hostname)) throw new TypeError("Relay endpoint must use HTTP on a loopback hostname.");
|
|
173
|
+
if (url.username.length > 0 || url.password.length > 0 || url.search.length > 0 || url.hash.length > 0 || url.pathname !== "/" && url.pathname !== "") throw new TypeError("Relay endpoint must contain only its loopback origin.");
|
|
174
|
+
return url.origin;
|
|
175
|
+
}
|
|
176
|
+
async function pairWithRelay(endpoint, code, fetcher = fetch, now = Date.now) {
|
|
177
|
+
const origin = normalizeRelayEndpoint(endpoint);
|
|
178
|
+
const trimmed = code.trim();
|
|
179
|
+
if (!/^\d{8}$/u.test(trimmed)) throw new TypeError("Relay pairing code must contain exactly eight digits.");
|
|
180
|
+
const record = requireRecord(await requestJson(fetcher, `${origin}/v1/pair`, {
|
|
181
|
+
method: "POST",
|
|
182
|
+
headers: { "content-type": "application/json" },
|
|
183
|
+
body: JSON.stringify({ code: trimmed }),
|
|
184
|
+
redirect: "error"
|
|
185
|
+
}), "Relay returned an invalid pairing response.");
|
|
186
|
+
const token = String(record.token ?? "");
|
|
187
|
+
if (!TOKEN_PATTERN.test(token)) throw new Error("Relay returned an invalid pairing token.");
|
|
188
|
+
if (typeof record.expiresAt !== "number" || !Number.isFinite(record.expiresAt) || record.expiresAt <= now()) throw new Error("Relay returned an invalid session expiration.");
|
|
189
|
+
return {
|
|
190
|
+
endpoint: origin,
|
|
191
|
+
token,
|
|
192
|
+
expiresAt: record.expiresAt
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
async function requestClientSecret(session, request, fetcher = fetch, now = Date.now) {
|
|
196
|
+
if (session.expiresAt <= now()) throw new Error("Relay session has expired. Pair with the local relay again.");
|
|
197
|
+
const data = requireRecord(requireRecord(await requestJson(fetcher, `${session.endpoint}/v1/openai/realtime/client-secrets`, {
|
|
198
|
+
method: "POST",
|
|
199
|
+
headers: {
|
|
200
|
+
"content-type": "application/json",
|
|
201
|
+
authorization: `Bearer ${session.token}`
|
|
202
|
+
},
|
|
203
|
+
body: JSON.stringify({ session: request }),
|
|
204
|
+
redirect: "error"
|
|
205
|
+
}), "Relay returned an invalid client secret response.").data, "Relay returned an invalid client secret response.");
|
|
206
|
+
const value = String(data.value ?? "");
|
|
207
|
+
if (!CLIENT_SECRET_PATTERN.test(value)) throw new Error("Relay returned an invalid client secret.");
|
|
208
|
+
if (typeof data.expiresAt !== "number" || !Number.isFinite(data.expiresAt) || data.expiresAt <= now()) throw new Error("Relay returned an expired client secret.");
|
|
209
|
+
return {
|
|
210
|
+
value,
|
|
211
|
+
expiresAt: data.expiresAt,
|
|
212
|
+
model: String(data.model ?? "")
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
async function requestJson(fetcher, url, init) {
|
|
216
|
+
const response = await fetcher(url, init);
|
|
217
|
+
const text = await response.text();
|
|
218
|
+
let result = null;
|
|
219
|
+
if (text.length > 0) try {
|
|
220
|
+
result = JSON.parse(text);
|
|
221
|
+
} catch {
|
|
222
|
+
throw new Error(`Relay returned a non-JSON response (${response.status}).`);
|
|
223
|
+
}
|
|
224
|
+
if (!response.ok) throw new Error(`Relay request failed (${response.status}): ${errorDetail(result)}`);
|
|
225
|
+
return result;
|
|
226
|
+
}
|
|
227
|
+
function requireRecord(value, message) {
|
|
228
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(message);
|
|
229
|
+
return value;
|
|
230
|
+
}
|
|
231
|
+
function errorDetail(value) {
|
|
232
|
+
if (typeof value === "object" && value !== null) {
|
|
233
|
+
const error = value.error;
|
|
234
|
+
if (typeof error === "object" && error !== null) {
|
|
235
|
+
const message = error.message;
|
|
236
|
+
if (typeof message === "string") return message;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
return "unknown error";
|
|
240
|
+
}
|
|
241
|
+
//#endregion
|
|
242
|
+
//#region src/session-config.ts
|
|
243
|
+
var MAX_INSTRUCTIONS_LENGTH = 16384;
|
|
244
|
+
var VOICE_PATTERN = /^[a-z0-9_-]{1,32}$/u;
|
|
245
|
+
var MODEL_PATTERN = /^[A-Za-z0-9._:-]{1,128}$/u;
|
|
246
|
+
var SUPPORTED_MODELS = ["gpt-realtime-2.1-mini", "gpt-realtime-2.1"];
|
|
247
|
+
function defaultSessionSettings() {
|
|
248
|
+
return {
|
|
249
|
+
model: "",
|
|
250
|
+
instructions: "",
|
|
251
|
+
voice: "marin",
|
|
252
|
+
outputMode: "audio"
|
|
253
|
+
};
|
|
254
|
+
}
|
|
255
|
+
/** Accepts an empty string (relay default) or a model identifier; the relay decides what is allowed. */
|
|
256
|
+
function normalizeModel(value) {
|
|
257
|
+
const model = value.trim();
|
|
258
|
+
if (model.length > 0 && !MODEL_PATTERN.test(model)) throw new TypeError("Model must be a model identifier.");
|
|
259
|
+
return model;
|
|
260
|
+
}
|
|
261
|
+
function normalizeVoice(value) {
|
|
262
|
+
const voice = value.trim().toLowerCase();
|
|
263
|
+
if (!VOICE_PATTERN.test(voice)) throw new TypeError("Voice must be a lowercase identifier.");
|
|
264
|
+
return voice;
|
|
265
|
+
}
|
|
266
|
+
function normalizeInstructions(value) {
|
|
267
|
+
if (value.length > 16384) throw new TypeError(`Instructions must be at most ${MAX_INSTRUCTIONS_LENGTH} characters.`);
|
|
268
|
+
return value;
|
|
269
|
+
}
|
|
270
|
+
function normalizeOutputMode(value) {
|
|
271
|
+
if (value === "audio" || value === "text") return value;
|
|
272
|
+
throw new TypeError("Output must be audio or text.");
|
|
273
|
+
}
|
|
274
|
+
function buildSessionRequest(settings, tools) {
|
|
275
|
+
if (tools.length > 32) throw new TypeError(`At most 32 functions can be exported as tools.`);
|
|
276
|
+
const request = {
|
|
277
|
+
voice: settings.voice,
|
|
278
|
+
outputModalities: [settings.outputMode]
|
|
279
|
+
};
|
|
280
|
+
if (settings.model.length > 0) request.model = settings.model;
|
|
281
|
+
if (settings.instructions.length > 0) request.instructions = settings.instructions;
|
|
282
|
+
if (tools.length > 0) request.tools = tools.map((tool) => ({ ...tool }));
|
|
283
|
+
return request;
|
|
284
|
+
}
|
|
285
|
+
//#endregion
|
|
286
|
+
//#region src/usage.ts
|
|
287
|
+
var MODEL_PRICES = {
|
|
288
|
+
"gpt-realtime-2.1": {
|
|
289
|
+
textInput: 4,
|
|
290
|
+
cachedTextInput: .4,
|
|
291
|
+
textOutput: 24,
|
|
292
|
+
audioInput: 32,
|
|
293
|
+
cachedAudioInput: .4,
|
|
294
|
+
audioOutput: 64
|
|
295
|
+
},
|
|
296
|
+
"gpt-realtime-2.1-mini": {
|
|
297
|
+
textInput: .6,
|
|
298
|
+
cachedTextInput: .06,
|
|
299
|
+
textOutput: 2.4,
|
|
300
|
+
audioInput: 10,
|
|
301
|
+
cachedAudioInput: .3,
|
|
302
|
+
audioOutput: 20
|
|
303
|
+
}
|
|
304
|
+
};
|
|
305
|
+
var PRICES_AS_OF = "2026-09-18";
|
|
306
|
+
function emptyUsage() {
|
|
307
|
+
return {
|
|
308
|
+
responses: 0,
|
|
309
|
+
inputTokens: 0,
|
|
310
|
+
outputTokens: 0,
|
|
311
|
+
cachedInputTokens: 0,
|
|
312
|
+
textInputTokens: 0,
|
|
313
|
+
audioInputTokens: 0,
|
|
314
|
+
cachedTextInputTokens: 0,
|
|
315
|
+
cachedAudioInputTokens: 0,
|
|
316
|
+
textOutputTokens: 0,
|
|
317
|
+
audioOutputTokens: 0,
|
|
318
|
+
estimatedCostUsd: 0,
|
|
319
|
+
unpricedResponses: 0
|
|
320
|
+
};
|
|
321
|
+
}
|
|
322
|
+
/** Adds one `response.usage` object to the totals. Unknown or malformed usage counts as zero tokens. */
|
|
323
|
+
function addUsage(totals, usage, model) {
|
|
324
|
+
const root = asRecord(usage);
|
|
325
|
+
if (!root) return totals;
|
|
326
|
+
const input = asRecord(root.input_token_details);
|
|
327
|
+
const output = asRecord(root.output_token_details);
|
|
328
|
+
const cachedDetails = asRecord(input?.cached_tokens_details);
|
|
329
|
+
const textInput = count(input?.text_tokens);
|
|
330
|
+
const audioInput = count(input?.audio_tokens);
|
|
331
|
+
const cached = count(input?.cached_tokens);
|
|
332
|
+
const cachedText = cachedDetails ? count(cachedDetails.text_tokens) : Math.min(cached, textInput);
|
|
333
|
+
const cachedAudio = cachedDetails ? count(cachedDetails.audio_tokens) : Math.max(0, cached - cachedText);
|
|
334
|
+
const textOutput = count(output?.text_tokens);
|
|
335
|
+
const audioOutput = count(output?.audio_tokens);
|
|
336
|
+
const next = {
|
|
337
|
+
responses: totals.responses + 1,
|
|
338
|
+
inputTokens: totals.inputTokens + count(root.input_tokens),
|
|
339
|
+
outputTokens: totals.outputTokens + count(root.output_tokens),
|
|
340
|
+
cachedInputTokens: totals.cachedInputTokens + cached,
|
|
341
|
+
textInputTokens: totals.textInputTokens + textInput,
|
|
342
|
+
audioInputTokens: totals.audioInputTokens + audioInput,
|
|
343
|
+
cachedTextInputTokens: totals.cachedTextInputTokens + cachedText,
|
|
344
|
+
cachedAudioInputTokens: totals.cachedAudioInputTokens + cachedAudio,
|
|
345
|
+
textOutputTokens: totals.textOutputTokens + textOutput,
|
|
346
|
+
audioOutputTokens: totals.audioOutputTokens + audioOutput,
|
|
347
|
+
estimatedCostUsd: totals.estimatedCostUsd,
|
|
348
|
+
unpricedResponses: totals.unpricedResponses
|
|
349
|
+
};
|
|
350
|
+
const price = MODEL_PRICES[model];
|
|
351
|
+
if (!price) {
|
|
352
|
+
next.unpricedResponses += 1;
|
|
353
|
+
return next;
|
|
354
|
+
}
|
|
355
|
+
next.estimatedCostUsd += (Math.max(0, textInput - cachedText) * price.textInput + cachedText * price.cachedTextInput + Math.max(0, audioInput - cachedAudio) * price.audioInput + cachedAudio * price.cachedAudioInput + textOutput * price.textOutput + audioOutput * price.audioOutput) / 1e6;
|
|
356
|
+
return next;
|
|
357
|
+
}
|
|
358
|
+
function count(value) {
|
|
359
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;
|
|
360
|
+
}
|
|
361
|
+
function asRecord(value) {
|
|
362
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) ? value : null;
|
|
363
|
+
}
|
|
364
|
+
//#endregion
|
|
365
|
+
//#region src/webrtc-transport.ts
|
|
366
|
+
var REALTIME_CALLS_URL = "https://api.openai.com/v1/realtime/calls";
|
|
367
|
+
var EVENTS_CHANNEL = "oai-events";
|
|
368
|
+
function browserWebRtcDependencies() {
|
|
369
|
+
return {
|
|
370
|
+
createPeerConnection: () => new RTCPeerConnection(),
|
|
371
|
+
getUserMedia: (constraints) => navigator.mediaDevices.getUserMedia(constraints),
|
|
372
|
+
createAudioElement: () => {
|
|
373
|
+
const element = document.createElement("audio");
|
|
374
|
+
element.autoplay = true;
|
|
375
|
+
return element;
|
|
376
|
+
},
|
|
377
|
+
fetch: (input, init) => fetch(input, init)
|
|
378
|
+
};
|
|
379
|
+
}
|
|
380
|
+
/** Browser-to-OpenAI WebRTC session authenticated with an ephemeral client secret. */
|
|
381
|
+
var WebRtcTransport = class {
|
|
382
|
+
constructor(deps = browserWebRtcDependencies()) {
|
|
383
|
+
this.deps = deps;
|
|
384
|
+
this.peer = null;
|
|
385
|
+
this.channel = null;
|
|
386
|
+
this.microphone = null;
|
|
387
|
+
this.audio = null;
|
|
388
|
+
this.closed = false;
|
|
389
|
+
}
|
|
390
|
+
async connect(options) {
|
|
391
|
+
this.closed = false;
|
|
392
|
+
try {
|
|
393
|
+
const peer = this.deps.createPeerConnection();
|
|
394
|
+
this.peer = peer;
|
|
395
|
+
const audio = this.deps.createAudioElement();
|
|
396
|
+
this.audio = audio;
|
|
397
|
+
peer.ontrack = (event) => {
|
|
398
|
+
audio.srcObject = event.streams[0] ?? null;
|
|
399
|
+
};
|
|
400
|
+
if (options.microphone) {
|
|
401
|
+
const stream = await this.deps.getUserMedia({ audio: true });
|
|
402
|
+
this.microphone = stream;
|
|
403
|
+
for (const track of stream.getAudioTracks()) peer.addTrack(track, stream);
|
|
404
|
+
} else peer.addTransceiver("audio", { direction: "recvonly" });
|
|
405
|
+
const channel = peer.createDataChannel(EVENTS_CHANNEL);
|
|
406
|
+
this.channel = channel;
|
|
407
|
+
channel.onmessage = (message) => {
|
|
408
|
+
const event = parseEvent(message.data);
|
|
409
|
+
if (event) options.onEvent(event);
|
|
410
|
+
};
|
|
411
|
+
channel.onclose = () => this.handleClose(options, "The Realtime data channel closed.");
|
|
412
|
+
peer.onconnectionstatechange = () => {
|
|
413
|
+
if (peer.connectionState === "failed" || peer.connectionState === "closed") this.handleClose(options, `The Realtime connection ${peer.connectionState}.`);
|
|
414
|
+
};
|
|
415
|
+
const opened = waitForOpen(channel, this.deps.channelOpenTimeoutMs ?? 15e3);
|
|
416
|
+
opened.catch(() => void 0);
|
|
417
|
+
const offer = await peer.createOffer();
|
|
418
|
+
await peer.setLocalDescription(offer);
|
|
419
|
+
const response = await this.deps.fetch(REALTIME_CALLS_URL, {
|
|
420
|
+
method: "POST",
|
|
421
|
+
body: offer.sdp ?? "",
|
|
422
|
+
headers: {
|
|
423
|
+
authorization: `Bearer ${options.clientSecret}`,
|
|
424
|
+
"content-type": "application/sdp"
|
|
425
|
+
},
|
|
426
|
+
redirect: "error"
|
|
427
|
+
});
|
|
428
|
+
const answer = await response.text();
|
|
429
|
+
if (!response.ok) throw new Error(`Realtime call setup failed (${response.status}).`);
|
|
430
|
+
await peer.setRemoteDescription({
|
|
431
|
+
type: "answer",
|
|
432
|
+
sdp: answer
|
|
433
|
+
});
|
|
434
|
+
await opened;
|
|
435
|
+
} catch (error) {
|
|
436
|
+
this.close();
|
|
437
|
+
throw error;
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
send(event) {
|
|
441
|
+
if (!this.channel || this.channel.readyState !== "open") throw new Error("Not connected to Realtime.");
|
|
442
|
+
this.channel.send(JSON.stringify(event));
|
|
443
|
+
}
|
|
444
|
+
close() {
|
|
445
|
+
this.closed = true;
|
|
446
|
+
const channel = this.channel;
|
|
447
|
+
const peer = this.peer;
|
|
448
|
+
this.channel = null;
|
|
449
|
+
this.peer = null;
|
|
450
|
+
if (channel) {
|
|
451
|
+
channel.onmessage = null;
|
|
452
|
+
channel.onclose = null;
|
|
453
|
+
channel.close();
|
|
454
|
+
}
|
|
455
|
+
if (peer) {
|
|
456
|
+
peer.onconnectionstatechange = null;
|
|
457
|
+
peer.ontrack = null;
|
|
458
|
+
peer.close();
|
|
459
|
+
}
|
|
460
|
+
for (const track of this.microphone?.getTracks() ?? []) track.stop();
|
|
461
|
+
this.microphone = null;
|
|
462
|
+
if (this.audio) {
|
|
463
|
+
this.audio.srcObject = null;
|
|
464
|
+
this.audio = null;
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
handleClose(options, reason) {
|
|
468
|
+
if (this.closed) return;
|
|
469
|
+
this.close();
|
|
470
|
+
options.onClose(reason);
|
|
471
|
+
}
|
|
472
|
+
};
|
|
473
|
+
function parseEvent(data) {
|
|
474
|
+
if (typeof data !== "string") return null;
|
|
475
|
+
try {
|
|
476
|
+
const value = JSON.parse(data);
|
|
477
|
+
if (typeof value === "object" && value !== null && typeof value.type === "string") return value;
|
|
478
|
+
} catch {}
|
|
479
|
+
return null;
|
|
480
|
+
}
|
|
481
|
+
function waitForOpen(channel, timeoutMs) {
|
|
482
|
+
if (channel.readyState === "open") return Promise.resolve();
|
|
483
|
+
return new Promise((resolve, reject) => {
|
|
484
|
+
const timer = setTimeout(() => reject(/* @__PURE__ */ new Error("Timed out opening the Realtime data channel.")), timeoutMs);
|
|
485
|
+
channel.addEventListener("open", () => {
|
|
486
|
+
clearTimeout(timer);
|
|
487
|
+
resolve();
|
|
488
|
+
}, { once: true });
|
|
489
|
+
});
|
|
490
|
+
}
|
|
491
|
+
//#endregion
|
|
492
|
+
//#region src/composition.ts
|
|
493
|
+
/**
|
|
494
|
+
* Composition API: the OpenAI Realtime capability without TurboWarp block definitions.
|
|
495
|
+
*
|
|
496
|
+
* A downstream extension (for example `@kubohiroya/turbowarp-voice-chat`) owns its own blocks and
|
|
497
|
+
* hats, and passes its own `define function` hat opcode. Importing this module does not register
|
|
498
|
+
* any TurboWarp extension and does not touch the `Scratch` global.
|
|
499
|
+
*/
|
|
500
|
+
var DEFAULT_RELAY_ENDPOINT = "http://127.0.0.1:8787";
|
|
501
|
+
function createRealtimeComposition(options) {
|
|
502
|
+
return new Composition(options);
|
|
503
|
+
}
|
|
504
|
+
var Composition = class {
|
|
505
|
+
constructor(options) {
|
|
506
|
+
this.options = options;
|
|
507
|
+
this.relayEndpoint = DEFAULT_RELAY_ENDPOINT;
|
|
508
|
+
this.relaySession = null;
|
|
509
|
+
this.settingsValue = defaultSessionSettings();
|
|
510
|
+
this.timeLimitSeconds = 0;
|
|
511
|
+
this.connectedAt = null;
|
|
512
|
+
this.timeLimitTimer = null;
|
|
513
|
+
this.activeModelValue = "";
|
|
514
|
+
this.lastResponse = "";
|
|
515
|
+
this.usageTotals = emptyUsage();
|
|
516
|
+
this.listeners = /* @__PURE__ */ new Set();
|
|
517
|
+
this.fetcher = options.fetch ?? ((input, init) => fetch(input, init));
|
|
518
|
+
this.createTransport = options.createTransport ?? (() => new WebRtcTransport());
|
|
519
|
+
this.now = options.now ?? Date.now;
|
|
520
|
+
this.setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
|
|
521
|
+
this.clearTimer = options.clearTimer ?? ((handle) => clearTimeout(handle));
|
|
522
|
+
this.ownsFunctions = options.functions === void 0;
|
|
523
|
+
this.namedFunctions = options.functions ?? createNamedFunctions({
|
|
524
|
+
runtime: options.runtime,
|
|
525
|
+
functionHatOpcode: options.functionHatOpcode
|
|
526
|
+
});
|
|
527
|
+
this.session = new RealtimeSession({
|
|
528
|
+
callFunction: (name, args) => this.namedFunctions.call(name, args, { exportedOnly: true }),
|
|
529
|
+
onResponseText: (text) => {
|
|
530
|
+
this.lastResponse = text;
|
|
531
|
+
this.emit({
|
|
532
|
+
type: "response",
|
|
533
|
+
text
|
|
534
|
+
});
|
|
535
|
+
},
|
|
536
|
+
onError: (message) => this.emit({
|
|
537
|
+
type: "error",
|
|
538
|
+
message
|
|
539
|
+
}),
|
|
540
|
+
onStateChange: (state) => this.handleState(state),
|
|
541
|
+
onUsage: (usage) => {
|
|
542
|
+
this.usageTotals = addUsage(this.usageTotals, usage, this.activeModelValue);
|
|
543
|
+
this.emit({
|
|
544
|
+
type: "usage",
|
|
545
|
+
usage: this.usage()
|
|
546
|
+
});
|
|
547
|
+
}
|
|
548
|
+
});
|
|
549
|
+
}
|
|
550
|
+
configureRelay(endpoint) {
|
|
551
|
+
this.relayEndpoint = normalizeRelayEndpoint(endpoint);
|
|
552
|
+
this.relaySession = null;
|
|
553
|
+
}
|
|
554
|
+
async pairRelay(code) {
|
|
555
|
+
this.relaySession = await pairWithRelay(this.relayEndpoint, code, this.fetcher, this.now);
|
|
556
|
+
}
|
|
557
|
+
isRelayPaired() {
|
|
558
|
+
return this.relaySession !== null && this.relaySession.expiresAt > this.now();
|
|
559
|
+
}
|
|
560
|
+
setModel(model) {
|
|
561
|
+
this.settingsValue = {
|
|
562
|
+
...this.settingsValue,
|
|
563
|
+
model: normalizeModel(model)
|
|
564
|
+
};
|
|
565
|
+
}
|
|
566
|
+
setInstructions(text) {
|
|
567
|
+
this.settingsValue = {
|
|
568
|
+
...this.settingsValue,
|
|
569
|
+
instructions: normalizeInstructions(text)
|
|
570
|
+
};
|
|
571
|
+
}
|
|
572
|
+
setVoice(voice) {
|
|
573
|
+
this.settingsValue = {
|
|
574
|
+
...this.settingsValue,
|
|
575
|
+
voice: normalizeVoice(voice)
|
|
576
|
+
};
|
|
577
|
+
}
|
|
578
|
+
setOutputMode(mode) {
|
|
579
|
+
this.settingsValue = {
|
|
580
|
+
...this.settingsValue,
|
|
581
|
+
outputMode: normalizeOutputMode(mode)
|
|
582
|
+
};
|
|
583
|
+
}
|
|
584
|
+
setSessionTimeLimit(seconds) {
|
|
585
|
+
if (!Number.isFinite(seconds) || seconds < 0) throw new TypeError("Session time limit must be zero or a positive number of seconds.");
|
|
586
|
+
this.timeLimitSeconds = seconds;
|
|
587
|
+
}
|
|
588
|
+
get settings() {
|
|
589
|
+
return { ...this.settingsValue };
|
|
590
|
+
}
|
|
591
|
+
get sessionTimeLimitSeconds() {
|
|
592
|
+
return this.timeLimitSeconds;
|
|
593
|
+
}
|
|
594
|
+
scanFunctions() {
|
|
595
|
+
return this.namedFunctions.scan();
|
|
596
|
+
}
|
|
597
|
+
get functions() {
|
|
598
|
+
return this.namedFunctions;
|
|
599
|
+
}
|
|
600
|
+
async connect(options) {
|
|
601
|
+
if (!this.relaySession || !this.isRelayPaired()) throw new Error("Pair with the local relay first.");
|
|
602
|
+
const request = buildSessionRequest(this.settingsValue, this.namedFunctions.tools());
|
|
603
|
+
const secret = await requestClientSecret(this.relaySession, request, this.fetcher, this.now);
|
|
604
|
+
this.activeModelValue = secret.model || this.settingsValue.model;
|
|
605
|
+
await this.session.open(this.createTransport(), secret.value, options.microphone);
|
|
606
|
+
}
|
|
607
|
+
disconnect() {
|
|
608
|
+
this.session.close();
|
|
609
|
+
}
|
|
610
|
+
get state() {
|
|
611
|
+
return this.session.state;
|
|
612
|
+
}
|
|
613
|
+
get activeModel() {
|
|
614
|
+
return this.activeModelValue;
|
|
615
|
+
}
|
|
616
|
+
sessionElapsedSeconds() {
|
|
617
|
+
return this.connectedAt === null ? 0 : Math.max(0, (this.now() - this.connectedAt) / 1e3);
|
|
618
|
+
}
|
|
619
|
+
sendText(text) {
|
|
620
|
+
this.session.sendText(text);
|
|
621
|
+
}
|
|
622
|
+
get lastResponseText() {
|
|
623
|
+
return this.lastResponse;
|
|
624
|
+
}
|
|
625
|
+
matchFunctionHat(name, thread) {
|
|
626
|
+
return this.namedFunctions.matchHat(name, thread);
|
|
627
|
+
}
|
|
628
|
+
functionArguments(thread) {
|
|
629
|
+
return this.namedFunctions.argumentsFor(thread);
|
|
630
|
+
}
|
|
631
|
+
returnFromFunction(thread, value) {
|
|
632
|
+
this.namedFunctions.returnFrom(thread, value);
|
|
633
|
+
}
|
|
634
|
+
usage() {
|
|
635
|
+
return { ...this.usageTotals };
|
|
636
|
+
}
|
|
637
|
+
resetUsage() {
|
|
638
|
+
this.usageTotals = emptyUsage();
|
|
639
|
+
this.emit({
|
|
640
|
+
type: "usage",
|
|
641
|
+
usage: this.usage()
|
|
642
|
+
});
|
|
643
|
+
}
|
|
644
|
+
subscribe(listener) {
|
|
645
|
+
this.listeners.add(listener);
|
|
646
|
+
return () => this.listeners.delete(listener);
|
|
647
|
+
}
|
|
648
|
+
release() {
|
|
649
|
+
this.disconnect();
|
|
650
|
+
this.listeners.clear();
|
|
651
|
+
if (this.ownsFunctions) this.namedFunctions.release();
|
|
652
|
+
}
|
|
653
|
+
handleState(state) {
|
|
654
|
+
if (state === "connected") {
|
|
655
|
+
this.connectedAt = this.now();
|
|
656
|
+
if (this.timeLimitSeconds > 0) this.timeLimitTimer = this.setTimer(() => this.reachTimeLimit(), this.timeLimitSeconds * 1e3);
|
|
657
|
+
} else if (state !== "connecting") {
|
|
658
|
+
this.connectedAt = null;
|
|
659
|
+
this.cancelTimeLimit();
|
|
660
|
+
this.namedFunctions.cancelAll("The Realtime session ended.");
|
|
661
|
+
}
|
|
662
|
+
this.emit({
|
|
663
|
+
type: "state",
|
|
664
|
+
state
|
|
665
|
+
});
|
|
666
|
+
}
|
|
667
|
+
reachTimeLimit() {
|
|
668
|
+
this.timeLimitTimer = null;
|
|
669
|
+
if (this.session.state !== "connected") return;
|
|
670
|
+
this.session.close();
|
|
671
|
+
this.emit({ type: "sessionTimeLimitReached" });
|
|
672
|
+
}
|
|
673
|
+
cancelTimeLimit() {
|
|
674
|
+
if (this.timeLimitTimer !== null) this.clearTimer(this.timeLimitTimer);
|
|
675
|
+
this.timeLimitTimer = null;
|
|
676
|
+
}
|
|
677
|
+
emit(event) {
|
|
678
|
+
for (const listener of [...this.listeners]) listener(event);
|
|
679
|
+
}
|
|
680
|
+
};
|
|
681
|
+
//#endregion
|
|
682
|
+
export { DEFAULT_RELAY_ENDPOINT, MODEL_PRICES, PRICES_AS_OF, SUPPORTED_MODELS, createRealtimeComposition };
|