@kubohiroya/turbowarp-openai-realtime-api 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1581 @@
1
+ // Name: TurboWarp-OpenAI-Realtime-API
2
+ // ID: kubohiroyaopenairealtime
3
+ // Description: Talk with the OpenAI Realtime API from TurboWarp through a localhost relay.
4
+ // By: Hiroya Kubo
5
+ // License: MPL-2.0
6
+
7
+ (function (Scratch) {
8
+ 'use strict';
9
+
10
+ //#region src/config.ts
11
+ var extensionConfig = {
12
+ id: "kubohiroyaopenairealtime",
13
+ slug: "turbowarp-openai-realtime-api",
14
+ name: "TurboWarp-OpenAI-Realtime-API",
15
+ description: "Talk with the OpenAI Realtime API from TurboWarp through a localhost relay.",
16
+ author: "Hiroya Kubo",
17
+ license: "MPL-2.0",
18
+ unsandboxed: true,
19
+ docsURI: "https://kubohiroya.github.io/turbowarp-openai-realtime-api/",
20
+ blockIconURI: "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0OCA0OCI+PHBhdGggZD0iTTggMTBoMzJhNCA0IDAgMCAxIDQgNHYxNmE0IDQgMCAwIDEtNCA0SDIybC05IDd2LTdIOGE0IDQgMCAwIDEtNC00VjE0YTQgNCAwIDAgMSA0LTR6IiBmaWxsPSIjMTBBMzdGIi8+PGcgc3Ryb2tlPSIjZmZmIiBzdHJva2Utd2lkdGg9IjMiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCI+PHBhdGggZD0iTTE0IDE5djYiLz48cGF0aCBkPSJNMjAgMTZ2MTIiLz48cGF0aCBkPSJNMjYgMTh2OCIvPjxwYXRoIGQ9Ik0zMiAxNXYxNCIvPjxwYXRoIGQ9Ik0zOCAyMHY0Ii8+PC9nPjwvc3ZnPg=="
21
+ };
22
+ var block_definitions_default = {
23
+ extensionName: "OpenAI Realtime",
24
+ blocks: [
25
+ {
26
+ "opcode": "configureRelay",
27
+ "blockType": "COMMAND",
28
+ "text": "configure local relay [ENDPOINT]",
29
+ "description": "Sets the loopback origin of the local capability-proxy relay. Clears any previous pairing.",
30
+ "descriptionJa": "localhostで動くcapability-proxy中継のループバックoriginを設定します。以前のペアリングは消去されます。",
31
+ "arguments": { "ENDPOINT": {
32
+ "type": "STRING",
33
+ "defaultValue": "http://127.0.0.1:8787"
34
+ } }
35
+ },
36
+ {
37
+ "opcode": "pairRelay",
38
+ "blockType": "COMMAND",
39
+ "text": "pair local relay with one-time code [CODE]",
40
+ "description": "Exchanges the eight-digit code printed by the relay for a session token kept only in memory.",
41
+ "descriptionJa": "中継が表示した8桁のコードを、メモリだけに保持するセッションtokenと交換します。",
42
+ "arguments": { "CODE": {
43
+ "type": "STRING",
44
+ "defaultValue": "00000000"
45
+ } }
46
+ },
47
+ {
48
+ "opcode": "isRelayPaired",
49
+ "blockType": "BOOLEAN",
50
+ "text": "local relay paired?",
51
+ "description": "Reports whether an unexpired relay session token is held in memory.",
52
+ "descriptionJa": "有効期限内の中継セッションtokenをメモリに保持しているかを返します。",
53
+ "arguments": {}
54
+ },
55
+ {
56
+ "opcode": "setInstructions",
57
+ "blockType": "COMMAND",
58
+ "text": "set instructions to [TEXT]",
59
+ "description": "Sets the system instructions used by the next connection.",
60
+ "descriptionJa": "次の接続で使うシステムへの指示(instructions)を設定します。",
61
+ "arguments": { "TEXT": {
62
+ "type": "STRING",
63
+ "defaultValue": "You are a friendly assistant. Answer briefly."
64
+ } }
65
+ },
66
+ {
67
+ "opcode": "setVoice",
68
+ "blockType": "COMMAND",
69
+ "text": "set voice to [VOICE]",
70
+ "description": "Sets the output voice used by the next connection.",
71
+ "descriptionJa": "次の接続で使う応答の声を設定します。",
72
+ "arguments": { "VOICE": {
73
+ "type": "STRING",
74
+ "menu": "voices",
75
+ "defaultValue": "marin"
76
+ } }
77
+ },
78
+ {
79
+ "opcode": "setOutputMode",
80
+ "blockType": "COMMAND",
81
+ "text": "set output to [MODE]",
82
+ "description": "Chooses spoken audio or text-only responses for the next connection.",
83
+ "descriptionJa": "次の接続で、音声で応答するかテキストだけで応答するかを選びます。",
84
+ "arguments": { "MODE": {
85
+ "type": "STRING",
86
+ "menu": "outputModes",
87
+ "defaultValue": "audio"
88
+ } }
89
+ },
90
+ {
91
+ "opcode": "setModel",
92
+ "blockType": "COMMAND",
93
+ "text": "set model to [MODEL]",
94
+ "description": "Chooses the Realtime model for the next connection. The relay only accepts models its operator allows; empty uses the relay default.",
95
+ "descriptionJa": "次の接続で使うRealtimeのモデルを選びます。中継の運用者が許可したモデルだけが使えます。空にすると中継の既定のモデルを使います。",
96
+ "arguments": { "MODEL": {
97
+ "type": "STRING",
98
+ "menu": "models",
99
+ "defaultValue": "gpt-realtime-2.1-mini"
100
+ } }
101
+ },
102
+ {
103
+ "opcode": "setSessionTimeLimit",
104
+ "blockType": "COMMAND",
105
+ "text": "set session time limit to [SECONDS] seconds",
106
+ "description": "Disconnects automatically after this many seconds from the next connection on. 0 disables the limit.",
107
+ "descriptionJa": "次の接続から、指定した秒数が経つと自動で切断します。0にすると制限しません。",
108
+ "arguments": { "SECONDS": {
109
+ "type": "NUMBER",
110
+ "defaultValue": "600"
111
+ } }
112
+ },
113
+ {
114
+ "opcode": "connect",
115
+ "blockType": "COMMAND",
116
+ "text": "connect to Realtime with microphone [MICROPHONE]",
117
+ "description": "Mints an ephemeral key through the relay and opens a WebRTC session. Exported functions become tools.",
118
+ "descriptionJa": "中継を通じて一時キーを発行し、WebRTCのセッションを開きます。ツールとして公開した関数はモデルから呼べるようになります。",
119
+ "arguments": { "MICROPHONE": {
120
+ "type": "STRING",
121
+ "menu": "onOff",
122
+ "defaultValue": "on"
123
+ } }
124
+ },
125
+ {
126
+ "opcode": "disconnect",
127
+ "blockType": "COMMAND",
128
+ "text": "disconnect from Realtime",
129
+ "description": "Closes the WebRTC session, stops the microphone, and fails pending function calls.",
130
+ "descriptionJa": "WebRTCのセッションを閉じてマイクを止め、実行中の関数呼び出しを失敗させます。",
131
+ "arguments": {}
132
+ },
133
+ {
134
+ "opcode": "isConnected",
135
+ "blockType": "BOOLEAN",
136
+ "text": "connected to Realtime?",
137
+ "description": "Reports whether the Realtime session is open.",
138
+ "descriptionJa": "Realtimeのセッションが開いているかを返します。",
139
+ "arguments": {}
140
+ },
141
+ {
142
+ "opcode": "connectionState",
143
+ "blockType": "REPORTER",
144
+ "text": "Realtime connection state",
145
+ "description": "Reports disconnected, connecting, connected, or failed.",
146
+ "descriptionJa": "disconnected、connecting、connected、failedのいずれかを返します。",
147
+ "arguments": {}
148
+ },
149
+ {
150
+ "opcode": "currentModel",
151
+ "blockType": "REPORTER",
152
+ "text": "Realtime model",
153
+ "description": "Reports the model the relay used for the current or last connection.",
154
+ "descriptionJa": "現在または直前の接続で中継が使ったモデルを返します。",
155
+ "arguments": {}
156
+ },
157
+ {
158
+ "opcode": "sessionElapsed",
159
+ "blockType": "REPORTER",
160
+ "text": "session elapsed seconds",
161
+ "description": "Reports seconds since the current session connected, or 0 when disconnected.",
162
+ "descriptionJa": "現在のセッションが接続してからの秒数を返します。切断中は0を返します。",
163
+ "arguments": {}
164
+ },
165
+ {
166
+ "opcode": "whenSessionTimeLimitReached",
167
+ "blockType": "HAT",
168
+ "text": "when session time limit is reached",
169
+ "description": "Starts after the session time limit disconnects the session.",
170
+ "descriptionJa": "セッションの時間上限によって切断されたときに起動します。",
171
+ "arguments": {}
172
+ },
173
+ {
174
+ "opcode": "sendText",
175
+ "blockType": "COMMAND",
176
+ "text": "send text [TEXT]",
177
+ "description": "Adds a user text message to the conversation and asks for a response.",
178
+ "descriptionJa": "利用者のテキストメッセージを会話に追加し、応答を要求します。",
179
+ "arguments": { "TEXT": {
180
+ "type": "STRING",
181
+ "defaultValue": "Hello!"
182
+ } }
183
+ },
184
+ {
185
+ "opcode": "whenResponseDone",
186
+ "blockType": "HAT",
187
+ "text": "when assistant finishes responding",
188
+ "description": "Starts when a response that contains assistant text or transcript completes.",
189
+ "descriptionJa": "アシスタントのテキストまたは音声の書き起こしを含む応答が完了したときに起動します。",
190
+ "arguments": {}
191
+ },
192
+ {
193
+ "opcode": "lastResponseText",
194
+ "blockType": "REPORTER",
195
+ "text": "last assistant response",
196
+ "description": "Reports the text or audio transcript of the most recent completed assistant response.",
197
+ "descriptionJa": "直近に完了したアシスタントの応答のテキスト、または音声の書き起こしを返します。",
198
+ "arguments": {}
199
+ },
200
+ {
201
+ "opcode": "defineFunction",
202
+ "blockType": "HAT",
203
+ "text": "define function [NAME] description [DESCRIPTION] args schema [SCHEMA] export as [EXPORT]",
204
+ "description": "Defines a function value. With export as tool, the model can call it; NAME, DESCRIPTION, and SCHEMA must be literal text.",
205
+ "descriptionJa": "関数を定義します。export asをtoolにすると、モデルから呼べるツールになります。NAME、DESCRIPTION、SCHEMAには文字列を直接書く必要があります。",
206
+ "arguments": {
207
+ "NAME": {
208
+ "type": "STRING",
209
+ "defaultValue": "get_score"
210
+ },
211
+ "DESCRIPTION": {
212
+ "type": "STRING",
213
+ "defaultValue": "Returns the player's current score."
214
+ },
215
+ "SCHEMA": {
216
+ "type": "STRING",
217
+ "defaultValue": "{\"type\":\"object\",\"properties\":{}}"
218
+ },
219
+ "EXPORT": {
220
+ "type": "STRING",
221
+ "menu": "exportModes",
222
+ "defaultValue": "tool"
223
+ }
224
+ }
225
+ },
226
+ {
227
+ "opcode": "functionArgument",
228
+ "blockType": "REPORTER",
229
+ "text": "function argument [PATH]",
230
+ "description": "Inside a function, reports the argument at a dotted path such as city or items.0.name.",
231
+ "descriptionJa": "関数の中で、cityやitems.0.nameのようなドット区切りのパスにある引数を返します。",
232
+ "arguments": { "PATH": {
233
+ "type": "STRING",
234
+ "defaultValue": "city"
235
+ } }
236
+ },
237
+ {
238
+ "opcode": "functionArgumentsJson",
239
+ "blockType": "REPORTER",
240
+ "text": "function arguments JSON",
241
+ "description": "Inside a function, reports all arguments as JSON text.",
242
+ "descriptionJa": "関数の中で、すべての引数をJSONテキストとして返します。",
243
+ "arguments": {}
244
+ },
245
+ {
246
+ "opcode": "returnValue",
247
+ "blockType": "COMMAND",
248
+ "text": "return [VALUE]",
249
+ "description": "Inside a function, returns a value and ends the script. JSON text is returned as JSON; other text is returned as a string.",
250
+ "descriptionJa": "関数の中で値を返し、スクリプトを終了します。JSONテキストはJSONとして、それ以外のテキストは文字列として返します。",
251
+ "arguments": { "VALUE": {
252
+ "type": "STRING",
253
+ "defaultValue": "{\"score\":10}"
254
+ } }
255
+ },
256
+ {
257
+ "opcode": "usageValue",
258
+ "blockType": "REPORTER",
259
+ "text": "Realtime usage [FIELD]",
260
+ "description": "Reports token usage totals since the last reset. costUSD is an estimate from published prices; the OpenAI dashboard is authoritative.",
261
+ "descriptionJa": "最後にリセットしてからの使用量の合計を返します。costUSDは公開価格からの概算で、正確な請求額はOpenAIの管理画面で確認してください。",
262
+ "arguments": { "FIELD": {
263
+ "type": "STRING",
264
+ "menu": "usageFields",
265
+ "defaultValue": "costUSD"
266
+ } }
267
+ },
268
+ {
269
+ "opcode": "resetUsage",
270
+ "blockType": "COMMAND",
271
+ "text": "reset Realtime usage",
272
+ "description": "Clears the usage totals.",
273
+ "descriptionJa": "使用量の合計を0に戻します。",
274
+ "arguments": {}
275
+ },
276
+ {
277
+ "opcode": "lastError",
278
+ "blockType": "REPORTER",
279
+ "text": "last Realtime error",
280
+ "description": "Reports the most recent relay, connection, or API error, or an empty string.",
281
+ "descriptionJa": "直近の中継、接続、APIのエラーを返します。エラーがなければ空文字列を返します。",
282
+ "arguments": {}
283
+ }
284
+ ],
285
+ menus: {
286
+ "voices": {
287
+ "acceptReporters": true,
288
+ "items": [
289
+ "alloy",
290
+ "ash",
291
+ "ballad",
292
+ "cedar",
293
+ "coral",
294
+ "echo",
295
+ "marin",
296
+ "sage",
297
+ "shimmer",
298
+ "verse"
299
+ ]
300
+ },
301
+ "outputModes": {
302
+ "acceptReporters": false,
303
+ "items": ["audio", "text"]
304
+ },
305
+ "onOff": {
306
+ "acceptReporters": false,
307
+ "items": ["on", "off"]
308
+ },
309
+ "exportModes": {
310
+ "acceptReporters": false,
311
+ "items": ["tool", "none"]
312
+ },
313
+ "models": {
314
+ "acceptReporters": true,
315
+ "items": ["gpt-realtime-2.1-mini", "gpt-realtime-2.1"]
316
+ },
317
+ "usageFields": {
318
+ "acceptReporters": false,
319
+ "items": [
320
+ "costUSD",
321
+ "responses",
322
+ "inputTokens",
323
+ "outputTokens",
324
+ "cachedInputTokens",
325
+ "textInputTokens",
326
+ "audioInputTokens",
327
+ "textOutputTokens",
328
+ "audioOutputTokens"
329
+ ]
330
+ }
331
+ }
332
+ };
333
+ //#endregion
334
+ //#region node_modules/.pnpm/@kubohiroya+turbowarp-named-functions@0.1.0/node_modules/@kubohiroya/turbowarp-named-functions/dist/composition.js
335
+ var FunctionDispatcher = class {
336
+ constructor(runtime, options) {
337
+ this.runtime = runtime;
338
+ this.options = options;
339
+ this.queue = [];
340
+ this.running = /* @__PURE__ */ new Map();
341
+ this.byThread = /* @__PURE__ */ new Map();
342
+ this.starting = null;
343
+ this.step = 0;
344
+ this.onAfterExecute = () => this.afterStep();
345
+ this.onStopAll = () => this.cancelAll("The project was stopped.");
346
+ this.timeoutMs = options.timeoutMs ?? 3e4;
347
+ this.setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
348
+ this.clearTimer = options.clearTimer ?? ((handle) => clearTimeout(handle));
349
+ runtime.on("AFTER_EXECUTE", this.onAfterExecute);
350
+ runtime.on("PROJECT_STOP_ALL", this.onStopAll);
351
+ }
352
+ /**
353
+ * Starts a named function. `caller` is the thread of the block making the call, if any; it is used
354
+ * to detect reentrant calls.
355
+ */
356
+ invoke(name, args, caller) {
357
+ if (!this.options.knownNames().has(name)) return Promise.reject(/* @__PURE__ */ new Error(`Unknown function: ${name}`));
358
+ const parentChain = caller ? this.byThread.get(caller)?.chain ?? [] : [];
359
+ if (parentChain.includes(name)) return Promise.reject(/* @__PURE__ */ new Error(`Reentrant call: ${[...parentChain, name].join(" -> ")}. Use a custom block for recursion.`));
360
+ return new Promise((resolve, reject) => {
361
+ const invocation = {
362
+ name,
363
+ args,
364
+ chain: [...parentChain, name],
365
+ resolve,
366
+ reject,
367
+ thread: null,
368
+ startedAtStep: -1,
369
+ timer: null,
370
+ settled: false
371
+ };
372
+ invocation.timer = this.setTimer(() => this.settle(invocation, /* @__PURE__ */ new Error(`Function ${name} timed out.`)), this.timeoutMs);
373
+ this.queue.push(invocation);
374
+ this.pump();
375
+ });
376
+ }
377
+ /** Hat predicate: true only for the script that should run the invocation being started. */
378
+ matchHat(name, thread) {
379
+ const invocation = this.starting;
380
+ if (!invocation || !thread || invocation.name !== name.trim()) return false;
381
+ invocation.thread = thread;
382
+ this.byThread.set(thread, invocation);
383
+ this.starting = null;
384
+ return true;
385
+ }
386
+ argumentsFor(thread) {
387
+ const invocation = thread ? this.byThread.get(thread) : void 0;
388
+ if (!invocation) throw new Error("This block can only be used inside a running function.");
389
+ return invocation.args;
390
+ }
391
+ returnFrom(thread, value) {
392
+ const invocation = thread ? this.byThread.get(thread) : void 0;
393
+ if (!invocation) throw new Error("return can only be used inside a running function.");
394
+ this.settle(invocation, null, value);
395
+ }
396
+ cancelAll(reason) {
397
+ for (const invocation of [...this.queue, ...this.running.values()]) this.settle(invocation, new Error(reason));
398
+ if (this.starting) this.settle(this.starting, new Error(reason));
399
+ }
400
+ get pendingCount() {
401
+ return this.queue.length + this.running.size;
402
+ }
403
+ release() {
404
+ this.cancelAll("Named functions were released.");
405
+ this.runtime.off?.("AFTER_EXECUTE", this.onAfterExecute);
406
+ this.runtime.off?.("PROJECT_STOP_ALL", this.onStopAll);
407
+ }
408
+ pump() {
409
+ if (this.starting) return;
410
+ const index = this.queue.findIndex((invocation) => !this.running.has(invocation.name));
411
+ if (index < 0) return;
412
+ const [invocation] = this.queue.splice(index, 1);
413
+ if (!invocation) return;
414
+ this.running.set(invocation.name, invocation);
415
+ this.starting = invocation;
416
+ invocation.startedAtStep = this.step;
417
+ this.runtime.startHats(this.options.hatOpcode);
418
+ }
419
+ afterStep() {
420
+ this.step += 1;
421
+ const starting = this.starting;
422
+ if (starting && this.step - starting.startedAtStep > 2) this.settle(starting, /* @__PURE__ */ new Error(`Function ${starting.name} did not start. Is its script already running?`));
423
+ for (const invocation of this.running.values()) if (invocation.thread && !this.runtime.threads.includes(invocation.thread)) this.settle(invocation, null, null);
424
+ this.pump();
425
+ }
426
+ settle(invocation, error, value) {
427
+ if (invocation.settled) return;
428
+ invocation.settled = true;
429
+ this.clearTimer(invocation.timer);
430
+ const queued = this.queue.indexOf(invocation);
431
+ if (queued >= 0) this.queue.splice(queued, 1);
432
+ if (this.running.get(invocation.name) === invocation) this.running.delete(invocation.name);
433
+ if (invocation.thread) this.byThread.delete(invocation.thread);
434
+ if (this.starting === invocation) this.starting = null;
435
+ if (error) invocation.reject(error);
436
+ else invocation.resolve(value);
437
+ }
438
+ };
439
+ /** Resolves a dotted path such as `items.0.name` inside parsed JSON arguments. */
440
+ function readArgumentPath(args, path) {
441
+ const trimmed = path.trim();
442
+ if (trimmed.length === 0) return args;
443
+ let current = args;
444
+ for (const segment of trimmed.split(".")) {
445
+ if (current === null || typeof current !== "object") return void 0;
446
+ current = current[segment];
447
+ }
448
+ return current;
449
+ }
450
+ /** Converts a value for a Scratch reporter: objects and arrays become JSON text. */
451
+ function toScratchValue(value) {
452
+ if (value === void 0 || value === null) return "";
453
+ if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value;
454
+ return JSON.stringify(value);
455
+ }
456
+ /** Parses block text: JSON text becomes JSON, anything else stays a string. */
457
+ function parseJsonOrText(text) {
458
+ const trimmed = text.trim();
459
+ if (trimmed.length === 0) return "";
460
+ try {
461
+ return JSON.parse(trimmed);
462
+ } catch {
463
+ return text;
464
+ }
465
+ }
466
+ var FUNCTION_NAME_PATTERN = /^[A-Za-z0-9_-]{1,64}$/u;
467
+ var MAX_DESCRIPTION_LENGTH = 1024;
468
+ /**
469
+ * Reads every `define function` hat in the project. NAME, DESCRIPTION, and SCHEMA are part of the
470
+ * tool contract sent to the model before any script runs, so they must be literal text.
471
+ */
472
+ function scanFunctionDefinitions(targets, hatOpcode) {
473
+ const functions = [];
474
+ const errors = [];
475
+ const seen = /* @__PURE__ */ new Map();
476
+ for (const target of targets) {
477
+ if (target.isOriginal === false) continue;
478
+ const targetName = target.getName?.() ?? (target.isStage ? "Stage" : "sprite");
479
+ const blocks = target.blocks._blocks;
480
+ for (const block of Object.values(blocks)) {
481
+ if (block.opcode !== hatOpcode || block.topLevel === false) continue;
482
+ const where = `${targetName} (block ${block.id})`;
483
+ try {
484
+ const definition = readDefinition(block, blocks, targetName);
485
+ const previous = seen.get(definition.name);
486
+ if (previous !== void 0) throw new Error(`function "${definition.name}" is already defined in ${previous}`);
487
+ seen.set(definition.name, where);
488
+ functions.push(definition);
489
+ } catch (error) {
490
+ errors.push(`${where}: ${error instanceof Error ? error.message : String(error)}`);
491
+ }
492
+ }
493
+ }
494
+ functions.sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0);
495
+ return {
496
+ functions,
497
+ errors
498
+ };
499
+ }
500
+ function toFunctionTools(functions) {
501
+ return functions.filter((definition) => definition.exportAs === "tool").map((definition) => ({
502
+ type: "function",
503
+ name: definition.name,
504
+ description: definition.description,
505
+ parameters: definition.parameters
506
+ }));
507
+ }
508
+ function readDefinition(block, blocks, targetName) {
509
+ const name = readLiteralInput(block, blocks, "NAME").trim();
510
+ if (!FUNCTION_NAME_PATTERN.test(name)) throw new Error("function name must be 1-64 letters, digits, \"_\" or \"-\"");
511
+ const description = readLiteralInput(block, blocks, "DESCRIPTION").trim();
512
+ if (description.length > 1024) throw new Error(`description must be at most ${MAX_DESCRIPTION_LENGTH} characters`);
513
+ const parameters = parseSchema(readLiteralInput(block, blocks, "SCHEMA"));
514
+ const exportAs = String(block.fields?.EXPORT?.value ?? "none") === "tool" ? "tool" : "none";
515
+ if (exportAs === "tool" && description.length === 0) throw new Error("a function exported as a tool needs a description");
516
+ return {
517
+ name,
518
+ description,
519
+ parameters,
520
+ exportAs,
521
+ targetName,
522
+ blockId: block.id
523
+ };
524
+ }
525
+ function readLiteralInput(block, blocks, inputName) {
526
+ const input = block.inputs?.[inputName];
527
+ if (!input) throw new Error(`${inputName} is missing`);
528
+ if (input.block && input.block !== input.shadow) throw new Error(`${inputName} must be literal text, not a reporter block`);
529
+ const shadowId = input.shadow ?? input.block;
530
+ const shadow = shadowId ? blocks[shadowId] : void 0;
531
+ const field = shadow?.fields ? Object.values(shadow.fields)[0] : void 0;
532
+ if (!field) throw new Error(`${inputName} is missing`);
533
+ return String(field.value ?? "");
534
+ }
535
+ function parseSchema(text) {
536
+ let value;
537
+ try {
538
+ value = JSON.parse(text);
539
+ } catch {
540
+ throw new Error("args schema must be valid JSON");
541
+ }
542
+ if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error("args schema must be a JSON object");
543
+ const schema = value;
544
+ if (schema.type !== "object") throw new Error("args schema must have \"type\": \"object\"");
545
+ return schema;
546
+ }
547
+ /**
548
+ * A small JSON Schema validator for function arguments. It interprets the schema at run time and
549
+ * never generates code, so it also works where `eval` is unavailable (for example Cloudflare Workers).
550
+ *
551
+ * Supported keywords: type, properties, required, additionalProperties (boolean), items, enum,
552
+ * const, minimum, maximum, minLength, maxLength, minItems, maxItems. Other keywords, such as
553
+ * description and title, are ignored.
554
+ */
555
+ function validateAgainstSchema(value, schema, path = "$") {
556
+ const rule = asRecord$2(schema);
557
+ if (!rule) return [];
558
+ const errors = [];
559
+ if (rule.type !== void 0) {
560
+ const types = Array.isArray(rule.type) ? rule.type : [rule.type];
561
+ if (!types.some((type) => matchesType(value, type))) return [`${path} must be ${types.join(" or ")}`];
562
+ }
563
+ if (Array.isArray(rule.enum) && !rule.enum.some((candidate) => deepEqual(candidate, value))) errors.push(`${path} must be one of ${rule.enum.map((item) => JSON.stringify(item)).join(", ")}`);
564
+ if ("const" in rule && !deepEqual(rule.const, value)) errors.push(`${path} must be ${JSON.stringify(rule.const)}`);
565
+ if (typeof value === "number") {
566
+ if (typeof rule.minimum === "number" && value < rule.minimum) errors.push(`${path} must be >= ${rule.minimum}`);
567
+ if (typeof rule.maximum === "number" && value > rule.maximum) errors.push(`${path} must be <= ${rule.maximum}`);
568
+ }
569
+ if (typeof value === "string") {
570
+ const length = [...value].length;
571
+ if (typeof rule.minLength === "number" && length < rule.minLength) errors.push(`${path} must have at least ${rule.minLength} characters`);
572
+ if (typeof rule.maxLength === "number" && length > rule.maxLength) errors.push(`${path} must have at most ${rule.maxLength} characters`);
573
+ }
574
+ if (Array.isArray(value)) {
575
+ if (typeof rule.minItems === "number" && value.length < rule.minItems) errors.push(`${path} must have at least ${rule.minItems} items`);
576
+ if (typeof rule.maxItems === "number" && value.length > rule.maxItems) errors.push(`${path} must have at most ${rule.maxItems} items`);
577
+ if (rule.items !== void 0) value.forEach((item, index) => errors.push(...validateAgainstSchema(item, rule.items, `${path}[${index}]`)));
578
+ }
579
+ const object = asRecord$2(value);
580
+ if (object) {
581
+ const properties = asRecord$2(rule.properties) ?? {};
582
+ if (Array.isArray(rule.required)) {
583
+ for (const name of rule.required) if (typeof name === "string" && !(name in object)) errors.push(`${path}.${name} is required`);
584
+ }
585
+ for (const [name, child] of Object.entries(object)) if (name in properties) errors.push(...validateAgainstSchema(child, properties[name], `${path}.${name}`));
586
+ else if (rule.additionalProperties === false) errors.push(`${path}.${name} is not allowed`);
587
+ }
588
+ return errors;
589
+ }
590
+ function matchesType(value, type) {
591
+ switch (type) {
592
+ case "object": return asRecord$2(value) !== null;
593
+ case "array": return Array.isArray(value);
594
+ case "string": return typeof value === "string";
595
+ case "number": return typeof value === "number" && Number.isFinite(value);
596
+ case "integer": return typeof value === "number" && Number.isInteger(value);
597
+ case "boolean": return typeof value === "boolean";
598
+ case "null": return value === null;
599
+ default: return false;
600
+ }
601
+ }
602
+ function deepEqual(left, right) {
603
+ return JSON.stringify(left) === JSON.stringify(right);
604
+ }
605
+ function asRecord$2(value) {
606
+ return typeof value === "object" && value !== null && !Array.isArray(value) ? value : null;
607
+ }
608
+ /**
609
+ * Composition API: named functions without TurboWarp block definitions.
610
+ *
611
+ * A consumer extension owns its `define function` hat (and its opcode) and forwards the hat
612
+ * predicate and the argument/return blocks here. Importing this module does not register a
613
+ * TurboWarp extension and does not touch the `Scratch` global.
614
+ */
615
+ function createNamedFunctions(options) {
616
+ return new NamedFunctionsImpl(options);
617
+ }
618
+ var NamedFunctionsImpl = class {
619
+ constructor(options) {
620
+ this.options = options;
621
+ this.promises = /* @__PURE__ */ new Map();
622
+ this.waiting = [];
623
+ this.inFlight = 0;
624
+ this.nextPromiseId = 1;
625
+ this.maxInFlight = Math.max(1, options.maxInFlight ?? 8);
626
+ this.maxRetained = Math.max(1, options.maxRetainedPromises ?? 256);
627
+ const dispatcherOptions = {
628
+ hatOpcode: options.functionHatOpcode,
629
+ knownNames: () => new Set(this.scan().functions.map((definition) => definition.name)),
630
+ ...options.timeoutMs === void 0 ? {} : { timeoutMs: options.timeoutMs },
631
+ ...options.setTimer ? { setTimer: options.setTimer } : {},
632
+ ...options.clearTimer ? { clearTimer: options.clearTimer } : {}
633
+ };
634
+ this.dispatcher = new FunctionDispatcher(options.runtime, dispatcherOptions);
635
+ options.runtime.on("PROJECT_STOP_ALL", () => this.promises.clear());
636
+ }
637
+ scan() {
638
+ return scanFunctionDefinitions(this.options.runtime.targets, this.options.functionHatOpcode);
639
+ }
640
+ tools() {
641
+ const scan = this.scan();
642
+ if (scan.errors.length > 0) throw new Error(`Invalid function definitions: ${scan.errors.join("; ")}`);
643
+ return toFunctionTools(scan.functions);
644
+ }
645
+ isDefined(name) {
646
+ return this.scan().functions.some((definition) => definition.name === name.trim());
647
+ }
648
+ call(name, args, options = {}) {
649
+ const trimmed = name.trim();
650
+ const definition = this.scan().functions.find((candidate) => candidate.name === trimmed);
651
+ if (!definition) return Promise.reject(/* @__PURE__ */ new Error(`Unknown function: ${trimmed}`));
652
+ if (options.exportedOnly && definition.exportAs !== "tool") return Promise.reject(/* @__PURE__ */ new Error(`Function ${trimmed} is not exported as a tool.`));
653
+ const errors = validateAgainstSchema(args, definition.parameters);
654
+ if (errors.length > 0) return Promise.reject(/* @__PURE__ */ new Error(`Invalid arguments for ${trimmed}: ${errors.join("; ")}`));
655
+ return this.dispatcher.invoke(trimmed, args, options.caller);
656
+ }
657
+ start(name, args, options = {}) {
658
+ const id = `p_${this.nextPromiseId++}`;
659
+ const promise = this.acquireSlot().then(() => this.call(name, args, options).finally(() => this.releaseSlot()));
660
+ const entry = {
661
+ promise,
662
+ settled: false
663
+ };
664
+ promise.then(() => entry.settled = true, () => entry.settled = true);
665
+ this.promises.set(id, entry);
666
+ this.evictSettled();
667
+ return { $promise: id };
668
+ }
669
+ await(ref) {
670
+ const entry = this.lookup(ref);
671
+ return entry ? entry.promise : Promise.reject(/* @__PURE__ */ new Error("Unknown or expired promise reference."));
672
+ }
673
+ async awaitAll(refs) {
674
+ if (!Array.isArray(refs)) throw new Error("await all needs a JSON array of promise references.");
675
+ return (await Promise.allSettled(refs.map((ref) => this.await(ref)))).map((result) => result.status === "fulfilled" ? result.value : { $error: messageOf$1(result.reason) });
676
+ }
677
+ isSettled(ref) {
678
+ return this.lookup(ref)?.settled ?? false;
679
+ }
680
+ matchHat(name, thread) {
681
+ return this.dispatcher.matchHat(name, thread);
682
+ }
683
+ argumentsFor(thread) {
684
+ return this.dispatcher.argumentsFor(thread);
685
+ }
686
+ returnFrom(thread, value) {
687
+ this.dispatcher.returnFrom(thread, value);
688
+ }
689
+ cancelAll(reason) {
690
+ this.dispatcher.cancelAll(reason);
691
+ }
692
+ release() {
693
+ this.dispatcher.release();
694
+ this.promises.clear();
695
+ }
696
+ lookup(ref) {
697
+ const id = typeof ref === "object" && ref !== null && typeof ref.$promise === "string" ? ref.$promise : void 0;
698
+ return id ? this.promises.get(id) : void 0;
699
+ }
700
+ acquireSlot() {
701
+ if (this.inFlight < this.maxInFlight) {
702
+ this.inFlight += 1;
703
+ return Promise.resolve();
704
+ }
705
+ return new Promise((resolve) => this.waiting.push(() => {
706
+ this.inFlight += 1;
707
+ resolve();
708
+ }));
709
+ }
710
+ releaseSlot() {
711
+ this.inFlight -= 1;
712
+ this.waiting.shift()?.();
713
+ }
714
+ evictSettled() {
715
+ if (this.promises.size <= this.maxRetained) return;
716
+ for (const [id, entry] of this.promises) {
717
+ if (this.promises.size <= this.maxRetained) break;
718
+ if (entry.settled) this.promises.delete(id);
719
+ }
720
+ }
721
+ };
722
+ function messageOf$1(error) {
723
+ return error instanceof Error ? error.message : String(error);
724
+ }
725
+ //#endregion
726
+ //#region src/realtime-session.ts
727
+ /**
728
+ * Conversation logic over a transport: sending user text, collecting assistant output, and
729
+ * answering function calls with `function_call_output` followed by one `response.create`.
730
+ */
731
+ var RealtimeSession = class {
732
+ constructor(hooks) {
733
+ this.hooks = hooks;
734
+ this.stateValue = "disconnected";
735
+ this.transport = null;
736
+ this.generation = 0;
737
+ }
738
+ get state() {
739
+ return this.stateValue;
740
+ }
741
+ async open(transport, clientSecret, microphone) {
742
+ this.close();
743
+ const generation = ++this.generation;
744
+ this.transport = transport;
745
+ this.setState("connecting");
746
+ try {
747
+ await transport.connect({
748
+ clientSecret,
749
+ microphone,
750
+ onEvent: (event) => {
751
+ if (generation === this.generation) this.handleEvent(event);
752
+ },
753
+ onClose: (reason) => {
754
+ if (generation !== this.generation) return;
755
+ this.transport = null;
756
+ this.hooks.onError(reason);
757
+ this.setState("failed");
758
+ }
759
+ });
760
+ } catch (error) {
761
+ if (generation === this.generation) {
762
+ this.transport = null;
763
+ this.setState("failed");
764
+ }
765
+ throw error;
766
+ }
767
+ if (generation === this.generation) this.setState("connected");
768
+ }
769
+ close() {
770
+ this.generation += 1;
771
+ const transport = this.transport;
772
+ this.transport = null;
773
+ transport?.close();
774
+ if (this.stateValue !== "disconnected") this.setState("disconnected");
775
+ }
776
+ sendText(text) {
777
+ this.send({
778
+ type: "conversation.item.create",
779
+ item: {
780
+ type: "message",
781
+ role: "user",
782
+ content: [{
783
+ type: "input_text",
784
+ text
785
+ }]
786
+ }
787
+ });
788
+ this.send({ type: "response.create" });
789
+ }
790
+ send(event) {
791
+ if (!this.transport || this.stateValue !== "connected") throw new Error("Not connected to Realtime.");
792
+ this.transport.send(event);
793
+ }
794
+ async handleEvent(event) {
795
+ if (event.type === "error") {
796
+ this.hooks.onError(describeApiError(event));
797
+ return;
798
+ }
799
+ if (event.type !== "response.done") return;
800
+ const response = event.response;
801
+ if (typeof response === "object" && response !== null && "usage" in response) this.hooks.onUsage?.(response.usage);
802
+ const output = readResponseOutput(event);
803
+ if (output.text.length > 0) this.hooks.onResponseText(output.text);
804
+ if (output.calls.length === 0) return;
805
+ const generation = this.generation;
806
+ const results = await Promise.all(output.calls.map((call) => this.runCall(call)));
807
+ if (generation !== this.generation || !this.transport) return;
808
+ for (const result of results) this.transport.send({
809
+ type: "conversation.item.create",
810
+ item: {
811
+ type: "function_call_output",
812
+ call_id: result.callId,
813
+ output: result.output
814
+ }
815
+ });
816
+ this.transport.send({ type: "response.create" });
817
+ }
818
+ async runCall(call) {
819
+ let args;
820
+ try {
821
+ args = call.argumentsJson.trim().length === 0 ? {} : JSON.parse(call.argumentsJson);
822
+ } catch {
823
+ return {
824
+ callId: call.callId,
825
+ output: JSON.stringify({ error: "Arguments were not valid JSON." })
826
+ };
827
+ }
828
+ try {
829
+ const value = await this.hooks.callFunction(call.name, args);
830
+ return {
831
+ callId: call.callId,
832
+ output: JSON.stringify(value ?? null)
833
+ };
834
+ } catch (error) {
835
+ const message = error instanceof Error ? error.message : String(error);
836
+ this.hooks.onError(message);
837
+ return {
838
+ callId: call.callId,
839
+ output: JSON.stringify({ error: message })
840
+ };
841
+ }
842
+ }
843
+ setState(state) {
844
+ this.stateValue = state;
845
+ this.hooks.onStateChange?.(state);
846
+ }
847
+ };
848
+ function readResponseOutput(event) {
849
+ const response = asRecord$1(event.response);
850
+ const items = Array.isArray(response?.output) ? response.output : [];
851
+ const texts = [];
852
+ const calls = [];
853
+ for (const rawItem of items) {
854
+ const item = asRecord$1(rawItem);
855
+ if (!item) continue;
856
+ if (item.type === "function_call" && typeof item.name === "string" && typeof item.call_id === "string") calls.push({
857
+ callId: item.call_id,
858
+ name: item.name,
859
+ argumentsJson: typeof item.arguments === "string" ? item.arguments : ""
860
+ });
861
+ else if (item.type === "message" && Array.isArray(item.content)) for (const rawPart of item.content) {
862
+ const part = asRecord$1(rawPart);
863
+ if (typeof part?.text === "string") texts.push(part.text);
864
+ else if (typeof part?.transcript === "string") texts.push(part.transcript);
865
+ }
866
+ }
867
+ return {
868
+ text: texts.join(""),
869
+ calls
870
+ };
871
+ }
872
+ function describeApiError(event) {
873
+ const error = asRecord$1(event.error);
874
+ return `${typeof error?.message === "string" ? error.message : "Unknown Realtime API error."}${typeof error?.code === "string" ? ` (${error.code})` : ""}`;
875
+ }
876
+ function asRecord$1(value) {
877
+ return typeof value === "object" && value !== null && !Array.isArray(value) ? value : null;
878
+ }
879
+ //#endregion
880
+ //#region src/relay-client.ts
881
+ var LOOPBACK_HOSTNAMES = /* @__PURE__ */ new Set([
882
+ "127.0.0.1",
883
+ "localhost",
884
+ "[::1]"
885
+ ]);
886
+ var TOKEN_PATTERN = /^[A-Za-z0-9_-]{20,}$/u;
887
+ var CLIENT_SECRET_PATTERN = /^ek_[A-Za-z0-9_-]+$/u;
888
+ /** Accepts only a plain loopback HTTP origin so the relay token can never leave this machine. */
889
+ function normalizeRelayEndpoint(endpoint) {
890
+ let url;
891
+ try {
892
+ url = new URL(endpoint.trim());
893
+ } catch {
894
+ throw new TypeError("Relay endpoint must be a valid URL.");
895
+ }
896
+ if (url.protocol !== "http:" || !LOOPBACK_HOSTNAMES.has(url.hostname)) throw new TypeError("Relay endpoint must use HTTP on a loopback hostname.");
897
+ if (url.username.length > 0 || url.password.length > 0 || url.search.length > 0 || url.hash.length > 0 || url.pathname !== "/" && url.pathname !== "") throw new TypeError("Relay endpoint must contain only its loopback origin.");
898
+ return url.origin;
899
+ }
900
+ async function pairWithRelay(endpoint, code, fetcher = fetch, now = Date.now) {
901
+ const origin = normalizeRelayEndpoint(endpoint);
902
+ const trimmed = code.trim();
903
+ if (!/^\d{8}$/u.test(trimmed)) throw new TypeError("Relay pairing code must contain exactly eight digits.");
904
+ const record = requireRecord(await requestJson(fetcher, `${origin}/v1/pair`, {
905
+ method: "POST",
906
+ headers: { "content-type": "application/json" },
907
+ body: JSON.stringify({ code: trimmed }),
908
+ redirect: "error"
909
+ }), "Relay returned an invalid pairing response.");
910
+ const token = String(record.token ?? "");
911
+ if (!TOKEN_PATTERN.test(token)) throw new Error("Relay returned an invalid pairing token.");
912
+ if (typeof record.expiresAt !== "number" || !Number.isFinite(record.expiresAt) || record.expiresAt <= now()) throw new Error("Relay returned an invalid session expiration.");
913
+ return {
914
+ endpoint: origin,
915
+ token,
916
+ expiresAt: record.expiresAt
917
+ };
918
+ }
919
+ async function requestClientSecret(session, request, fetcher = fetch, now = Date.now) {
920
+ if (session.expiresAt <= now()) throw new Error("Relay session has expired. Pair with the local relay again.");
921
+ const data = requireRecord(requireRecord(await requestJson(fetcher, `${session.endpoint}/v1/openai/realtime/client-secrets`, {
922
+ method: "POST",
923
+ headers: {
924
+ "content-type": "application/json",
925
+ authorization: `Bearer ${session.token}`
926
+ },
927
+ body: JSON.stringify({ session: request }),
928
+ redirect: "error"
929
+ }), "Relay returned an invalid client secret response.").data, "Relay returned an invalid client secret response.");
930
+ const value = String(data.value ?? "");
931
+ if (!CLIENT_SECRET_PATTERN.test(value)) throw new Error("Relay returned an invalid client secret.");
932
+ if (typeof data.expiresAt !== "number" || !Number.isFinite(data.expiresAt) || data.expiresAt <= now()) throw new Error("Relay returned an expired client secret.");
933
+ return {
934
+ value,
935
+ expiresAt: data.expiresAt,
936
+ model: String(data.model ?? "")
937
+ };
938
+ }
939
+ async function requestJson(fetcher, url, init) {
940
+ const response = await fetcher(url, init);
941
+ const text = await response.text();
942
+ let result = null;
943
+ if (text.length > 0) try {
944
+ result = JSON.parse(text);
945
+ } catch {
946
+ throw new Error(`Relay returned a non-JSON response (${response.status}).`);
947
+ }
948
+ if (!response.ok) throw new Error(`Relay request failed (${response.status}): ${errorDetail(result)}`);
949
+ return result;
950
+ }
951
+ function requireRecord(value, message) {
952
+ if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(message);
953
+ return value;
954
+ }
955
+ function errorDetail(value) {
956
+ if (typeof value === "object" && value !== null) {
957
+ const error = value.error;
958
+ if (typeof error === "object" && error !== null) {
959
+ const message = error.message;
960
+ if (typeof message === "string") return message;
961
+ }
962
+ }
963
+ return "unknown error";
964
+ }
965
+ //#endregion
966
+ //#region src/session-config.ts
967
+ var MAX_INSTRUCTIONS_LENGTH = 16384;
968
+ var VOICE_PATTERN = /^[a-z0-9_-]{1,32}$/u;
969
+ var MODEL_PATTERN = /^[A-Za-z0-9._:-]{1,128}$/u;
970
+ function defaultSessionSettings() {
971
+ return {
972
+ model: "",
973
+ instructions: "",
974
+ voice: "marin",
975
+ outputMode: "audio"
976
+ };
977
+ }
978
+ /** Accepts an empty string (relay default) or a model identifier; the relay decides what is allowed. */
979
+ function normalizeModel(value) {
980
+ const model = value.trim();
981
+ if (model.length > 0 && !MODEL_PATTERN.test(model)) throw new TypeError("Model must be a model identifier.");
982
+ return model;
983
+ }
984
+ function normalizeVoice(value) {
985
+ const voice = value.trim().toLowerCase();
986
+ if (!VOICE_PATTERN.test(voice)) throw new TypeError("Voice must be a lowercase identifier.");
987
+ return voice;
988
+ }
989
+ function normalizeInstructions(value) {
990
+ if (value.length > 16384) throw new TypeError(`Instructions must be at most ${MAX_INSTRUCTIONS_LENGTH} characters.`);
991
+ return value;
992
+ }
993
+ function normalizeOutputMode(value) {
994
+ if (value === "audio" || value === "text") return value;
995
+ throw new TypeError("Output must be audio or text.");
996
+ }
997
+ function buildSessionRequest(settings, tools) {
998
+ if (tools.length > 32) throw new TypeError(`At most 32 functions can be exported as tools.`);
999
+ const request = {
1000
+ voice: settings.voice,
1001
+ outputModalities: [settings.outputMode]
1002
+ };
1003
+ if (settings.model.length > 0) request.model = settings.model;
1004
+ if (settings.instructions.length > 0) request.instructions = settings.instructions;
1005
+ if (tools.length > 0) request.tools = tools.map((tool) => ({ ...tool }));
1006
+ return request;
1007
+ }
1008
+ //#endregion
1009
+ //#region src/usage.ts
1010
+ var MODEL_PRICES = {
1011
+ "gpt-realtime-2.1": {
1012
+ textInput: 4,
1013
+ cachedTextInput: .4,
1014
+ textOutput: 24,
1015
+ audioInput: 32,
1016
+ cachedAudioInput: .4,
1017
+ audioOutput: 64
1018
+ },
1019
+ "gpt-realtime-2.1-mini": {
1020
+ textInput: .6,
1021
+ cachedTextInput: .06,
1022
+ textOutput: 2.4,
1023
+ audioInput: 10,
1024
+ cachedAudioInput: .3,
1025
+ audioOutput: 20
1026
+ }
1027
+ };
1028
+ function emptyUsage() {
1029
+ return {
1030
+ responses: 0,
1031
+ inputTokens: 0,
1032
+ outputTokens: 0,
1033
+ cachedInputTokens: 0,
1034
+ textInputTokens: 0,
1035
+ audioInputTokens: 0,
1036
+ cachedTextInputTokens: 0,
1037
+ cachedAudioInputTokens: 0,
1038
+ textOutputTokens: 0,
1039
+ audioOutputTokens: 0,
1040
+ estimatedCostUsd: 0,
1041
+ unpricedResponses: 0
1042
+ };
1043
+ }
1044
+ /** Adds one `response.usage` object to the totals. Unknown or malformed usage counts as zero tokens. */
1045
+ function addUsage(totals, usage, model) {
1046
+ const root = asRecord(usage);
1047
+ if (!root) return totals;
1048
+ const input = asRecord(root.input_token_details);
1049
+ const output = asRecord(root.output_token_details);
1050
+ const cachedDetails = asRecord(input?.cached_tokens_details);
1051
+ const textInput = count(input?.text_tokens);
1052
+ const audioInput = count(input?.audio_tokens);
1053
+ const cached = count(input?.cached_tokens);
1054
+ const cachedText = cachedDetails ? count(cachedDetails.text_tokens) : Math.min(cached, textInput);
1055
+ const cachedAudio = cachedDetails ? count(cachedDetails.audio_tokens) : Math.max(0, cached - cachedText);
1056
+ const textOutput = count(output?.text_tokens);
1057
+ const audioOutput = count(output?.audio_tokens);
1058
+ const next = {
1059
+ responses: totals.responses + 1,
1060
+ inputTokens: totals.inputTokens + count(root.input_tokens),
1061
+ outputTokens: totals.outputTokens + count(root.output_tokens),
1062
+ cachedInputTokens: totals.cachedInputTokens + cached,
1063
+ textInputTokens: totals.textInputTokens + textInput,
1064
+ audioInputTokens: totals.audioInputTokens + audioInput,
1065
+ cachedTextInputTokens: totals.cachedTextInputTokens + cachedText,
1066
+ cachedAudioInputTokens: totals.cachedAudioInputTokens + cachedAudio,
1067
+ textOutputTokens: totals.textOutputTokens + textOutput,
1068
+ audioOutputTokens: totals.audioOutputTokens + audioOutput,
1069
+ estimatedCostUsd: totals.estimatedCostUsd,
1070
+ unpricedResponses: totals.unpricedResponses
1071
+ };
1072
+ const price = MODEL_PRICES[model];
1073
+ if (!price) {
1074
+ next.unpricedResponses += 1;
1075
+ return next;
1076
+ }
1077
+ next.estimatedCostUsd += (Math.max(0, textInput - cachedText) * price.textInput + cachedText * price.cachedTextInput + Math.max(0, audioInput - cachedAudio) * price.audioInput + cachedAudio * price.cachedAudioInput + textOutput * price.textOutput + audioOutput * price.audioOutput) / 1e6;
1078
+ return next;
1079
+ }
1080
+ function count(value) {
1081
+ return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;
1082
+ }
1083
+ function asRecord(value) {
1084
+ return typeof value === "object" && value !== null && !Array.isArray(value) ? value : null;
1085
+ }
1086
+ //#endregion
1087
+ //#region src/webrtc-transport.ts
1088
+ var REALTIME_CALLS_URL = "https://api.openai.com/v1/realtime/calls";
1089
+ var EVENTS_CHANNEL = "oai-events";
1090
+ function browserWebRtcDependencies() {
1091
+ return {
1092
+ createPeerConnection: () => new RTCPeerConnection(),
1093
+ getUserMedia: (constraints) => navigator.mediaDevices.getUserMedia(constraints),
1094
+ createAudioElement: () => {
1095
+ const element = document.createElement("audio");
1096
+ element.autoplay = true;
1097
+ return element;
1098
+ },
1099
+ fetch: (input, init) => fetch(input, init)
1100
+ };
1101
+ }
1102
+ /** Browser-to-OpenAI WebRTC session authenticated with an ephemeral client secret. */
1103
+ var WebRtcTransport = class {
1104
+ constructor(deps = browserWebRtcDependencies()) {
1105
+ this.deps = deps;
1106
+ this.peer = null;
1107
+ this.channel = null;
1108
+ this.microphone = null;
1109
+ this.audio = null;
1110
+ this.closed = false;
1111
+ }
1112
+ async connect(options) {
1113
+ this.closed = false;
1114
+ try {
1115
+ const peer = this.deps.createPeerConnection();
1116
+ this.peer = peer;
1117
+ const audio = this.deps.createAudioElement();
1118
+ this.audio = audio;
1119
+ peer.ontrack = (event) => {
1120
+ audio.srcObject = event.streams[0] ?? null;
1121
+ };
1122
+ if (options.microphone) {
1123
+ const stream = await this.deps.getUserMedia({ audio: true });
1124
+ this.microphone = stream;
1125
+ for (const track of stream.getAudioTracks()) peer.addTrack(track, stream);
1126
+ } else peer.addTransceiver("audio", { direction: "recvonly" });
1127
+ const channel = peer.createDataChannel(EVENTS_CHANNEL);
1128
+ this.channel = channel;
1129
+ channel.onmessage = (message) => {
1130
+ const event = parseEvent(message.data);
1131
+ if (event) options.onEvent(event);
1132
+ };
1133
+ channel.onclose = () => this.handleClose(options, "The Realtime data channel closed.");
1134
+ peer.onconnectionstatechange = () => {
1135
+ if (peer.connectionState === "failed" || peer.connectionState === "closed") this.handleClose(options, `The Realtime connection ${peer.connectionState}.`);
1136
+ };
1137
+ const opened = waitForOpen(channel, this.deps.channelOpenTimeoutMs ?? 15e3);
1138
+ opened.catch(() => void 0);
1139
+ const offer = await peer.createOffer();
1140
+ await peer.setLocalDescription(offer);
1141
+ const response = await this.deps.fetch(REALTIME_CALLS_URL, {
1142
+ method: "POST",
1143
+ body: offer.sdp ?? "",
1144
+ headers: {
1145
+ authorization: `Bearer ${options.clientSecret}`,
1146
+ "content-type": "application/sdp"
1147
+ },
1148
+ redirect: "error"
1149
+ });
1150
+ const answer = await response.text();
1151
+ if (!response.ok) throw new Error(`Realtime call setup failed (${response.status}).`);
1152
+ await peer.setRemoteDescription({
1153
+ type: "answer",
1154
+ sdp: answer
1155
+ });
1156
+ await opened;
1157
+ } catch (error) {
1158
+ this.close();
1159
+ throw error;
1160
+ }
1161
+ }
1162
+ send(event) {
1163
+ if (!this.channel || this.channel.readyState !== "open") throw new Error("Not connected to Realtime.");
1164
+ this.channel.send(JSON.stringify(event));
1165
+ }
1166
+ close() {
1167
+ this.closed = true;
1168
+ const channel = this.channel;
1169
+ const peer = this.peer;
1170
+ this.channel = null;
1171
+ this.peer = null;
1172
+ if (channel) {
1173
+ channel.onmessage = null;
1174
+ channel.onclose = null;
1175
+ channel.close();
1176
+ }
1177
+ if (peer) {
1178
+ peer.onconnectionstatechange = null;
1179
+ peer.ontrack = null;
1180
+ peer.close();
1181
+ }
1182
+ for (const track of this.microphone?.getTracks() ?? []) track.stop();
1183
+ this.microphone = null;
1184
+ if (this.audio) {
1185
+ this.audio.srcObject = null;
1186
+ this.audio = null;
1187
+ }
1188
+ }
1189
+ handleClose(options, reason) {
1190
+ if (this.closed) return;
1191
+ this.close();
1192
+ options.onClose(reason);
1193
+ }
1194
+ };
1195
+ function parseEvent(data) {
1196
+ if (typeof data !== "string") return null;
1197
+ try {
1198
+ const value = JSON.parse(data);
1199
+ if (typeof value === "object" && value !== null && typeof value.type === "string") return value;
1200
+ } catch {}
1201
+ return null;
1202
+ }
1203
+ function waitForOpen(channel, timeoutMs) {
1204
+ if (channel.readyState === "open") return Promise.resolve();
1205
+ return new Promise((resolve, reject) => {
1206
+ const timer = setTimeout(() => reject(/* @__PURE__ */ new Error("Timed out opening the Realtime data channel.")), timeoutMs);
1207
+ channel.addEventListener("open", () => {
1208
+ clearTimeout(timer);
1209
+ resolve();
1210
+ }, { once: true });
1211
+ });
1212
+ }
1213
+ //#endregion
1214
+ //#region src/composition.ts
1215
+ /**
1216
+ * Composition API: the OpenAI Realtime capability without TurboWarp block definitions.
1217
+ *
1218
+ * A downstream extension (for example `@kubohiroya/turbowarp-voice-chat`) owns its own blocks and
1219
+ * hats, and passes its own `define function` hat opcode. Importing this module does not register
1220
+ * any TurboWarp extension and does not touch the `Scratch` global.
1221
+ */
1222
+ var DEFAULT_RELAY_ENDPOINT = "http://127.0.0.1:8787";
1223
+ function createRealtimeComposition(options) {
1224
+ return new Composition(options);
1225
+ }
1226
+ var Composition = class {
1227
+ constructor(options) {
1228
+ this.options = options;
1229
+ this.relayEndpoint = DEFAULT_RELAY_ENDPOINT;
1230
+ this.relaySession = null;
1231
+ this.settingsValue = defaultSessionSettings();
1232
+ this.timeLimitSeconds = 0;
1233
+ this.connectedAt = null;
1234
+ this.timeLimitTimer = null;
1235
+ this.activeModelValue = "";
1236
+ this.lastResponse = "";
1237
+ this.usageTotals = emptyUsage();
1238
+ this.listeners = /* @__PURE__ */ new Set();
1239
+ this.fetcher = options.fetch ?? ((input, init) => fetch(input, init));
1240
+ this.createTransport = options.createTransport ?? (() => new WebRtcTransport());
1241
+ this.now = options.now ?? Date.now;
1242
+ this.setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
1243
+ this.clearTimer = options.clearTimer ?? ((handle) => clearTimeout(handle));
1244
+ this.ownsFunctions = options.functions === void 0;
1245
+ this.namedFunctions = options.functions ?? createNamedFunctions({
1246
+ runtime: options.runtime,
1247
+ functionHatOpcode: options.functionHatOpcode
1248
+ });
1249
+ this.session = new RealtimeSession({
1250
+ callFunction: (name, args) => this.namedFunctions.call(name, args, { exportedOnly: true }),
1251
+ onResponseText: (text) => {
1252
+ this.lastResponse = text;
1253
+ this.emit({
1254
+ type: "response",
1255
+ text
1256
+ });
1257
+ },
1258
+ onError: (message) => this.emit({
1259
+ type: "error",
1260
+ message
1261
+ }),
1262
+ onStateChange: (state) => this.handleState(state),
1263
+ onUsage: (usage) => {
1264
+ this.usageTotals = addUsage(this.usageTotals, usage, this.activeModelValue);
1265
+ this.emit({
1266
+ type: "usage",
1267
+ usage: this.usage()
1268
+ });
1269
+ }
1270
+ });
1271
+ }
1272
+ configureRelay(endpoint) {
1273
+ this.relayEndpoint = normalizeRelayEndpoint(endpoint);
1274
+ this.relaySession = null;
1275
+ }
1276
+ async pairRelay(code) {
1277
+ this.relaySession = await pairWithRelay(this.relayEndpoint, code, this.fetcher, this.now);
1278
+ }
1279
+ isRelayPaired() {
1280
+ return this.relaySession !== null && this.relaySession.expiresAt > this.now();
1281
+ }
1282
+ setModel(model) {
1283
+ this.settingsValue = {
1284
+ ...this.settingsValue,
1285
+ model: normalizeModel(model)
1286
+ };
1287
+ }
1288
+ setInstructions(text) {
1289
+ this.settingsValue = {
1290
+ ...this.settingsValue,
1291
+ instructions: normalizeInstructions(text)
1292
+ };
1293
+ }
1294
+ setVoice(voice) {
1295
+ this.settingsValue = {
1296
+ ...this.settingsValue,
1297
+ voice: normalizeVoice(voice)
1298
+ };
1299
+ }
1300
+ setOutputMode(mode) {
1301
+ this.settingsValue = {
1302
+ ...this.settingsValue,
1303
+ outputMode: normalizeOutputMode(mode)
1304
+ };
1305
+ }
1306
+ setSessionTimeLimit(seconds) {
1307
+ if (!Number.isFinite(seconds) || seconds < 0) throw new TypeError("Session time limit must be zero or a positive number of seconds.");
1308
+ this.timeLimitSeconds = seconds;
1309
+ }
1310
+ get settings() {
1311
+ return { ...this.settingsValue };
1312
+ }
1313
+ get sessionTimeLimitSeconds() {
1314
+ return this.timeLimitSeconds;
1315
+ }
1316
+ scanFunctions() {
1317
+ return this.namedFunctions.scan();
1318
+ }
1319
+ get functions() {
1320
+ return this.namedFunctions;
1321
+ }
1322
+ async connect(options) {
1323
+ if (!this.relaySession || !this.isRelayPaired()) throw new Error("Pair with the local relay first.");
1324
+ const request = buildSessionRequest(this.settingsValue, this.namedFunctions.tools());
1325
+ const secret = await requestClientSecret(this.relaySession, request, this.fetcher, this.now);
1326
+ this.activeModelValue = secret.model || this.settingsValue.model;
1327
+ await this.session.open(this.createTransport(), secret.value, options.microphone);
1328
+ }
1329
+ disconnect() {
1330
+ this.session.close();
1331
+ }
1332
+ get state() {
1333
+ return this.session.state;
1334
+ }
1335
+ get activeModel() {
1336
+ return this.activeModelValue;
1337
+ }
1338
+ sessionElapsedSeconds() {
1339
+ return this.connectedAt === null ? 0 : Math.max(0, (this.now() - this.connectedAt) / 1e3);
1340
+ }
1341
+ sendText(text) {
1342
+ this.session.sendText(text);
1343
+ }
1344
+ get lastResponseText() {
1345
+ return this.lastResponse;
1346
+ }
1347
+ matchFunctionHat(name, thread) {
1348
+ return this.namedFunctions.matchHat(name, thread);
1349
+ }
1350
+ functionArguments(thread) {
1351
+ return this.namedFunctions.argumentsFor(thread);
1352
+ }
1353
+ returnFromFunction(thread, value) {
1354
+ this.namedFunctions.returnFrom(thread, value);
1355
+ }
1356
+ usage() {
1357
+ return { ...this.usageTotals };
1358
+ }
1359
+ resetUsage() {
1360
+ this.usageTotals = emptyUsage();
1361
+ this.emit({
1362
+ type: "usage",
1363
+ usage: this.usage()
1364
+ });
1365
+ }
1366
+ subscribe(listener) {
1367
+ this.listeners.add(listener);
1368
+ return () => this.listeners.delete(listener);
1369
+ }
1370
+ release() {
1371
+ this.disconnect();
1372
+ this.listeners.clear();
1373
+ if (this.ownsFunctions) this.namedFunctions.release();
1374
+ }
1375
+ handleState(state) {
1376
+ if (state === "connected") {
1377
+ this.connectedAt = this.now();
1378
+ if (this.timeLimitSeconds > 0) this.timeLimitTimer = this.setTimer(() => this.reachTimeLimit(), this.timeLimitSeconds * 1e3);
1379
+ } else if (state !== "connecting") {
1380
+ this.connectedAt = null;
1381
+ this.cancelTimeLimit();
1382
+ this.namedFunctions.cancelAll("The Realtime session ended.");
1383
+ }
1384
+ this.emit({
1385
+ type: "state",
1386
+ state
1387
+ });
1388
+ }
1389
+ reachTimeLimit() {
1390
+ this.timeLimitTimer = null;
1391
+ if (this.session.state !== "connected") return;
1392
+ this.session.close();
1393
+ this.emit({ type: "sessionTimeLimitReached" });
1394
+ }
1395
+ cancelTimeLimit() {
1396
+ if (this.timeLimitTimer !== null) this.clearTimer(this.timeLimitTimer);
1397
+ this.timeLimitTimer = null;
1398
+ }
1399
+ emit(event) {
1400
+ for (const listener of [...this.listeners]) listener(event);
1401
+ }
1402
+ };
1403
+ //#endregion
1404
+ //#region src/extension.ts
1405
+ var blockDefinitions = block_definitions_default.blocks;
1406
+ var menuDefinitions = block_definitions_default.menus;
1407
+ var DEFINE_FUNCTION_OPCODE = `${extensionConfig.id}_defineFunction`;
1408
+ var RESPONSE_DONE_OPCODE = `${extensionConfig.id}_whenResponseDone`;
1409
+ var TIME_LIMIT_OPCODE = `${extensionConfig.id}_whenSessionTimeLimitReached`;
1410
+ var USAGE_FIELDS = {
1411
+ costUSD: (usage) => Math.round(usage.estimatedCostUsd * 1e6) / 1e6,
1412
+ responses: (usage) => usage.responses,
1413
+ inputTokens: (usage) => usage.inputTokens,
1414
+ outputTokens: (usage) => usage.outputTokens,
1415
+ cachedInputTokens: (usage) => usage.cachedInputTokens,
1416
+ textInputTokens: (usage) => usage.textInputTokens,
1417
+ audioInputTokens: (usage) => usage.audioInputTokens,
1418
+ textOutputTokens: (usage) => usage.textOutputTokens,
1419
+ audioOutputTokens: (usage) => usage.audioOutputTokens
1420
+ };
1421
+ /** Block surface over the Realtime composition. */
1422
+ var OpenAIRealtimeExtension = class {
1423
+ constructor(deps) {
1424
+ this.lastErrorMessage = "";
1425
+ this.realtime = createRealtimeComposition({
1426
+ ...deps,
1427
+ functionHatOpcode: DEFINE_FUNCTION_OPCODE
1428
+ });
1429
+ this.realtime.subscribe((event) => {
1430
+ if (event.type === "response") deps.runtime.startHats(RESPONSE_DONE_OPCODE);
1431
+ else if (event.type === "sessionTimeLimitReached") deps.runtime.startHats(TIME_LIMIT_OPCODE);
1432
+ else if (event.type === "error") this.lastErrorMessage = event.message;
1433
+ });
1434
+ }
1435
+ getInfo() {
1436
+ return {
1437
+ id: extensionConfig.id,
1438
+ name: Scratch.translate(block_definitions_default.extensionName),
1439
+ docsURI: extensionConfig.docsURI,
1440
+ blockIconURI: extensionConfig.blockIconURI,
1441
+ blocks: blockDefinitions.map((block) => this.toScratchBlock(block)),
1442
+ menus: Object.fromEntries(Object.entries(menuDefinitions).map(([id, menu]) => [id, {
1443
+ acceptReporters: menu.acceptReporters,
1444
+ items: [...menu.items]
1445
+ }]))
1446
+ };
1447
+ }
1448
+ configureRelay(args) {
1449
+ this.record(() => this.realtime.configureRelay(Scratch.Cast.toString(args.ENDPOINT)));
1450
+ }
1451
+ pairRelay(args) {
1452
+ return this.recordAsync(() => this.realtime.pairRelay(Scratch.Cast.toString(args.CODE)));
1453
+ }
1454
+ isRelayPaired() {
1455
+ return this.realtime.isRelayPaired();
1456
+ }
1457
+ setInstructions(args) {
1458
+ this.record(() => this.realtime.setInstructions(Scratch.Cast.toString(args.TEXT)));
1459
+ }
1460
+ setVoice(args) {
1461
+ this.record(() => this.realtime.setVoice(Scratch.Cast.toString(args.VOICE)));
1462
+ }
1463
+ setOutputMode(args) {
1464
+ this.record(() => this.realtime.setOutputMode(normalizeOutputMode(Scratch.Cast.toString(args.MODE))));
1465
+ }
1466
+ setModel(args) {
1467
+ this.record(() => this.realtime.setModel(Scratch.Cast.toString(args.MODEL)));
1468
+ }
1469
+ setSessionTimeLimit(args) {
1470
+ this.record(() => this.realtime.setSessionTimeLimit(Scratch.Cast.toNumber(args.SECONDS)));
1471
+ }
1472
+ connect(args) {
1473
+ return this.recordAsync(() => this.realtime.connect({ microphone: Scratch.Cast.toString(args.MICROPHONE) !== "off" }));
1474
+ }
1475
+ disconnect() {
1476
+ this.realtime.disconnect();
1477
+ }
1478
+ isConnected() {
1479
+ return this.realtime.state === "connected";
1480
+ }
1481
+ connectionState() {
1482
+ return this.realtime.state;
1483
+ }
1484
+ currentModel() {
1485
+ return this.realtime.activeModel;
1486
+ }
1487
+ sessionElapsed() {
1488
+ return Math.floor(this.realtime.sessionElapsedSeconds());
1489
+ }
1490
+ whenSessionTimeLimitReached() {
1491
+ return true;
1492
+ }
1493
+ sendText(args) {
1494
+ this.record(() => this.realtime.sendText(Scratch.Cast.toString(args.TEXT)));
1495
+ }
1496
+ whenResponseDone() {
1497
+ return true;
1498
+ }
1499
+ lastResponseText() {
1500
+ return this.realtime.lastResponseText;
1501
+ }
1502
+ defineFunction(args, util) {
1503
+ return this.realtime.matchFunctionHat(Scratch.Cast.toString(args.NAME), util?.thread);
1504
+ }
1505
+ functionArgument(args, util) {
1506
+ try {
1507
+ return toScratchValue(readArgumentPath(this.realtime.functionArguments(util?.thread), Scratch.Cast.toString(args.PATH)));
1508
+ } catch (error) {
1509
+ this.lastErrorMessage = messageOf(error);
1510
+ return "";
1511
+ }
1512
+ }
1513
+ functionArgumentsJson(_args, util) {
1514
+ try {
1515
+ return JSON.stringify(this.realtime.functionArguments(util?.thread) ?? null);
1516
+ } catch (error) {
1517
+ this.lastErrorMessage = messageOf(error);
1518
+ return "";
1519
+ }
1520
+ }
1521
+ returnValue(args, util) {
1522
+ try {
1523
+ this.realtime.returnFromFunction(util?.thread, parseJsonOrText(Scratch.Cast.toString(args.VALUE)));
1524
+ util?.stopThisScript?.();
1525
+ } catch (error) {
1526
+ this.lastErrorMessage = messageOf(error);
1527
+ }
1528
+ }
1529
+ usageValue(args) {
1530
+ const read = USAGE_FIELDS[Scratch.Cast.toString(args.FIELD)];
1531
+ return read ? read(this.realtime.usage()) : "";
1532
+ }
1533
+ resetUsage() {
1534
+ this.realtime.resetUsage();
1535
+ }
1536
+ lastError() {
1537
+ return this.lastErrorMessage;
1538
+ }
1539
+ record(action) {
1540
+ try {
1541
+ action();
1542
+ this.lastErrorMessage = "";
1543
+ } catch (error) {
1544
+ this.lastErrorMessage = messageOf(error);
1545
+ }
1546
+ }
1547
+ async recordAsync(action) {
1548
+ try {
1549
+ await action();
1550
+ this.lastErrorMessage = "";
1551
+ } catch (error) {
1552
+ this.lastErrorMessage = messageOf(error);
1553
+ }
1554
+ }
1555
+ toScratchBlock(block) {
1556
+ const scratchBlock = {
1557
+ opcode: block.opcode,
1558
+ blockType: Scratch.BlockType[block.blockType],
1559
+ text: Scratch.translate(block.text),
1560
+ arguments: Object.fromEntries(Object.entries(block.arguments).map(([name, argument]) => [name, {
1561
+ type: Scratch.ArgumentType[argument.type],
1562
+ defaultValue: argument.defaultValue,
1563
+ ...argument.menu ? { menu: argument.menu } : {}
1564
+ }]))
1565
+ };
1566
+ if (block.blockType === "HAT") scratchBlock.isEdgeActivated = false;
1567
+ return scratchBlock;
1568
+ }
1569
+ };
1570
+ function messageOf(error) {
1571
+ return error instanceof Error ? error.message : String(error);
1572
+ }
1573
+ //#endregion
1574
+ //#region src/index.ts
1575
+ if (extensionConfig.unsandboxed && !Scratch.extensions.unsandboxed) throw new Error(`${extensionConfig.name} must run unsandboxed.`);
1576
+ var runtime = Scratch.vm?.runtime;
1577
+ if (!runtime) throw new Error(`${extensionConfig.name} requires access to the TurboWarp VM runtime.`);
1578
+ Scratch.extensions.register(new OpenAIRealtimeExtension({ runtime }));
1579
+ //#endregion
1580
+
1581
+ })(Scratch);