@kubohiroya/turbowarp-openai-realtime-api 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +373 -0
- package/README.ja.md +353 -0
- package/README.md +415 -0
- package/dist/composition.js +682 -0
- package/dist/extension-manifest.json +240 -0
- package/dist/turbowarp-openai-realtime-api.js +1581 -0
- package/dist/types/composition.d.ts +89 -0
- package/dist/types/config.d.ts +11 -0
- package/dist/types/extension-manifest.d.ts +30 -0
- package/dist/types/extension.d.ts +45 -0
- package/dist/types/index.d.ts +1 -0
- package/dist/types/realtime-session.d.ts +39 -0
- package/dist/types/relay-client.d.ts +16 -0
- package/dist/types/session-config.d.ts +27 -0
- package/dist/types/transport.d.ts +18 -0
- package/dist/types/usage.d.ts +34 -0
- package/dist/types/webrtc-transport.d.ts +26 -0
- package/package.json +79 -0
- package/schemas/extension-manifest.schema.json +49 -0
- package/src/block-definitions.json +347 -0
- package/src/composition.ts +318 -0
- package/src/config.ts +12 -0
- package/src/extension-manifest.ts +149 -0
- package/src/extension.ts +266 -0
- package/src/globals.d.ts +26 -0
- package/src/index.ts +14 -0
- package/src/realtime-session.ts +175 -0
- package/src/relay-client.ts +133 -0
- package/src/session-config.ts +71 -0
- package/src/transport.ts +18 -0
- package/src/usage.ts +127 -0
- package/src/webrtc-transport.ts +156 -0
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
{
|
|
2
|
+
"extensionName": "OpenAI Realtime",
|
|
3
|
+
"blocks": [
|
|
4
|
+
{
|
|
5
|
+
"opcode": "configureRelay",
|
|
6
|
+
"blockType": "COMMAND",
|
|
7
|
+
"text": "configure local relay [ENDPOINT]",
|
|
8
|
+
"description": "Sets the loopback origin of the local capability-proxy relay. Clears any previous pairing.",
|
|
9
|
+
"descriptionJa": "localhostで動くcapability-proxy中継のループバックoriginを設定します。以前のペアリングは消去されます。",
|
|
10
|
+
"arguments": {
|
|
11
|
+
"ENDPOINT": {
|
|
12
|
+
"type": "STRING",
|
|
13
|
+
"defaultValue": "http://127.0.0.1:8787"
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
"opcode": "pairRelay",
|
|
19
|
+
"blockType": "COMMAND",
|
|
20
|
+
"text": "pair local relay with one-time code [CODE]",
|
|
21
|
+
"description": "Exchanges the eight-digit code printed by the relay for a session token kept only in memory.",
|
|
22
|
+
"descriptionJa": "中継が表示した8桁のコードを、メモリだけに保持するセッションtokenと交換します。",
|
|
23
|
+
"arguments": {
|
|
24
|
+
"CODE": {
|
|
25
|
+
"type": "STRING",
|
|
26
|
+
"defaultValue": "00000000"
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"opcode": "isRelayPaired",
|
|
32
|
+
"blockType": "BOOLEAN",
|
|
33
|
+
"text": "local relay paired?",
|
|
34
|
+
"description": "Reports whether an unexpired relay session token is held in memory.",
|
|
35
|
+
"descriptionJa": "有効期限内の中継セッションtokenをメモリに保持しているかを返します。",
|
|
36
|
+
"arguments": {}
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"opcode": "setInstructions",
|
|
40
|
+
"blockType": "COMMAND",
|
|
41
|
+
"text": "set instructions to [TEXT]",
|
|
42
|
+
"description": "Sets the system instructions used by the next connection.",
|
|
43
|
+
"descriptionJa": "次の接続で使うシステムへの指示(instructions)を設定します。",
|
|
44
|
+
"arguments": {
|
|
45
|
+
"TEXT": {
|
|
46
|
+
"type": "STRING",
|
|
47
|
+
"defaultValue": "You are a friendly assistant. Answer briefly."
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
"opcode": "setVoice",
|
|
53
|
+
"blockType": "COMMAND",
|
|
54
|
+
"text": "set voice to [VOICE]",
|
|
55
|
+
"description": "Sets the output voice used by the next connection.",
|
|
56
|
+
"descriptionJa": "次の接続で使う応答の声を設定します。",
|
|
57
|
+
"arguments": {
|
|
58
|
+
"VOICE": {
|
|
59
|
+
"type": "STRING",
|
|
60
|
+
"menu": "voices",
|
|
61
|
+
"defaultValue": "marin"
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
"opcode": "setOutputMode",
|
|
67
|
+
"blockType": "COMMAND",
|
|
68
|
+
"text": "set output to [MODE]",
|
|
69
|
+
"description": "Chooses spoken audio or text-only responses for the next connection.",
|
|
70
|
+
"descriptionJa": "次の接続で、音声で応答するかテキストだけで応答するかを選びます。",
|
|
71
|
+
"arguments": {
|
|
72
|
+
"MODE": {
|
|
73
|
+
"type": "STRING",
|
|
74
|
+
"menu": "outputModes",
|
|
75
|
+
"defaultValue": "audio"
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
"opcode": "setModel",
|
|
81
|
+
"blockType": "COMMAND",
|
|
82
|
+
"text": "set model to [MODEL]",
|
|
83
|
+
"description": "Chooses the Realtime model for the next connection. The relay only accepts models its operator allows; empty uses the relay default.",
|
|
84
|
+
"descriptionJa": "次の接続で使うRealtimeのモデルを選びます。中継の運用者が許可したモデルだけが使えます。空にすると中継の既定のモデルを使います。",
|
|
85
|
+
"arguments": {
|
|
86
|
+
"MODEL": {
|
|
87
|
+
"type": "STRING",
|
|
88
|
+
"menu": "models",
|
|
89
|
+
"defaultValue": "gpt-realtime-2.1-mini"
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
"opcode": "setSessionTimeLimit",
|
|
95
|
+
"blockType": "COMMAND",
|
|
96
|
+
"text": "set session time limit to [SECONDS] seconds",
|
|
97
|
+
"description": "Disconnects automatically after this many seconds from the next connection on. 0 disables the limit.",
|
|
98
|
+
"descriptionJa": "次の接続から、指定した秒数が経つと自動で切断します。0にすると制限しません。",
|
|
99
|
+
"arguments": {
|
|
100
|
+
"SECONDS": {
|
|
101
|
+
"type": "NUMBER",
|
|
102
|
+
"defaultValue": "600"
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
},
|
|
106
|
+
{
|
|
107
|
+
"opcode": "connect",
|
|
108
|
+
"blockType": "COMMAND",
|
|
109
|
+
"text": "connect to Realtime with microphone [MICROPHONE]",
|
|
110
|
+
"description": "Mints an ephemeral key through the relay and opens a WebRTC session. Exported functions become tools.",
|
|
111
|
+
"descriptionJa": "中継を通じて一時キーを発行し、WebRTCのセッションを開きます。ツールとして公開した関数はモデルから呼べるようになります。",
|
|
112
|
+
"arguments": {
|
|
113
|
+
"MICROPHONE": {
|
|
114
|
+
"type": "STRING",
|
|
115
|
+
"menu": "onOff",
|
|
116
|
+
"defaultValue": "on"
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
},
|
|
120
|
+
{
|
|
121
|
+
"opcode": "disconnect",
|
|
122
|
+
"blockType": "COMMAND",
|
|
123
|
+
"text": "disconnect from Realtime",
|
|
124
|
+
"description": "Closes the WebRTC session, stops the microphone, and fails pending function calls.",
|
|
125
|
+
"descriptionJa": "WebRTCのセッションを閉じてマイクを止め、実行中の関数呼び出しを失敗させます。",
|
|
126
|
+
"arguments": {}
|
|
127
|
+
},
|
|
128
|
+
{
|
|
129
|
+
"opcode": "isConnected",
|
|
130
|
+
"blockType": "BOOLEAN",
|
|
131
|
+
"text": "connected to Realtime?",
|
|
132
|
+
"description": "Reports whether the Realtime session is open.",
|
|
133
|
+
"descriptionJa": "Realtimeのセッションが開いているかを返します。",
|
|
134
|
+
"arguments": {}
|
|
135
|
+
},
|
|
136
|
+
{
|
|
137
|
+
"opcode": "connectionState",
|
|
138
|
+
"blockType": "REPORTER",
|
|
139
|
+
"text": "Realtime connection state",
|
|
140
|
+
"description": "Reports disconnected, connecting, connected, or failed.",
|
|
141
|
+
"descriptionJa": "disconnected、connecting、connected、failedのいずれかを返します。",
|
|
142
|
+
"arguments": {}
|
|
143
|
+
},
|
|
144
|
+
{
|
|
145
|
+
"opcode": "currentModel",
|
|
146
|
+
"blockType": "REPORTER",
|
|
147
|
+
"text": "Realtime model",
|
|
148
|
+
"description": "Reports the model the relay used for the current or last connection.",
|
|
149
|
+
"descriptionJa": "現在または直前の接続で中継が使ったモデルを返します。",
|
|
150
|
+
"arguments": {}
|
|
151
|
+
},
|
|
152
|
+
{
|
|
153
|
+
"opcode": "sessionElapsed",
|
|
154
|
+
"blockType": "REPORTER",
|
|
155
|
+
"text": "session elapsed seconds",
|
|
156
|
+
"description": "Reports seconds since the current session connected, or 0 when disconnected.",
|
|
157
|
+
"descriptionJa": "現在のセッションが接続してからの秒数を返します。切断中は0を返します。",
|
|
158
|
+
"arguments": {}
|
|
159
|
+
},
|
|
160
|
+
{
|
|
161
|
+
"opcode": "whenSessionTimeLimitReached",
|
|
162
|
+
"blockType": "HAT",
|
|
163
|
+
"text": "when session time limit is reached",
|
|
164
|
+
"description": "Starts after the session time limit disconnects the session.",
|
|
165
|
+
"descriptionJa": "セッションの時間上限によって切断されたときに起動します。",
|
|
166
|
+
"arguments": {}
|
|
167
|
+
},
|
|
168
|
+
{
|
|
169
|
+
"opcode": "sendText",
|
|
170
|
+
"blockType": "COMMAND",
|
|
171
|
+
"text": "send text [TEXT]",
|
|
172
|
+
"description": "Adds a user text message to the conversation and asks for a response.",
|
|
173
|
+
"descriptionJa": "利用者のテキストメッセージを会話に追加し、応答を要求します。",
|
|
174
|
+
"arguments": {
|
|
175
|
+
"TEXT": {
|
|
176
|
+
"type": "STRING",
|
|
177
|
+
"defaultValue": "Hello!"
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
},
|
|
181
|
+
{
|
|
182
|
+
"opcode": "whenResponseDone",
|
|
183
|
+
"blockType": "HAT",
|
|
184
|
+
"text": "when assistant finishes responding",
|
|
185
|
+
"description": "Starts when a response that contains assistant text or transcript completes.",
|
|
186
|
+
"descriptionJa": "アシスタントのテキストまたは音声の書き起こしを含む応答が完了したときに起動します。",
|
|
187
|
+
"arguments": {}
|
|
188
|
+
},
|
|
189
|
+
{
|
|
190
|
+
"opcode": "lastResponseText",
|
|
191
|
+
"blockType": "REPORTER",
|
|
192
|
+
"text": "last assistant response",
|
|
193
|
+
"description": "Reports the text or audio transcript of the most recent completed assistant response.",
|
|
194
|
+
"descriptionJa": "直近に完了したアシスタントの応答のテキスト、または音声の書き起こしを返します。",
|
|
195
|
+
"arguments": {}
|
|
196
|
+
},
|
|
197
|
+
{
|
|
198
|
+
"opcode": "defineFunction",
|
|
199
|
+
"blockType": "HAT",
|
|
200
|
+
"text": "define function [NAME] description [DESCRIPTION] args schema [SCHEMA] export as [EXPORT]",
|
|
201
|
+
"description": "Defines a function value. With export as tool, the model can call it; NAME, DESCRIPTION, and SCHEMA must be literal text.",
|
|
202
|
+
"descriptionJa": "関数を定義します。export asをtoolにすると、モデルから呼べるツールになります。NAME、DESCRIPTION、SCHEMAには文字列を直接書く必要があります。",
|
|
203
|
+
"arguments": {
|
|
204
|
+
"NAME": {
|
|
205
|
+
"type": "STRING",
|
|
206
|
+
"defaultValue": "get_score"
|
|
207
|
+
},
|
|
208
|
+
"DESCRIPTION": {
|
|
209
|
+
"type": "STRING",
|
|
210
|
+
"defaultValue": "Returns the player's current score."
|
|
211
|
+
},
|
|
212
|
+
"SCHEMA": {
|
|
213
|
+
"type": "STRING",
|
|
214
|
+
"defaultValue": "{\"type\":\"object\",\"properties\":{}}"
|
|
215
|
+
},
|
|
216
|
+
"EXPORT": {
|
|
217
|
+
"type": "STRING",
|
|
218
|
+
"menu": "exportModes",
|
|
219
|
+
"defaultValue": "tool"
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
},
|
|
223
|
+
{
|
|
224
|
+
"opcode": "functionArgument",
|
|
225
|
+
"blockType": "REPORTER",
|
|
226
|
+
"text": "function argument [PATH]",
|
|
227
|
+
"description": "Inside a function, reports the argument at a dotted path such as city or items.0.name.",
|
|
228
|
+
"descriptionJa": "関数の中で、cityやitems.0.nameのようなドット区切りのパスにある引数を返します。",
|
|
229
|
+
"arguments": {
|
|
230
|
+
"PATH": {
|
|
231
|
+
"type": "STRING",
|
|
232
|
+
"defaultValue": "city"
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
},
|
|
236
|
+
{
|
|
237
|
+
"opcode": "functionArgumentsJson",
|
|
238
|
+
"blockType": "REPORTER",
|
|
239
|
+
"text": "function arguments JSON",
|
|
240
|
+
"description": "Inside a function, reports all arguments as JSON text.",
|
|
241
|
+
"descriptionJa": "関数の中で、すべての引数をJSONテキストとして返します。",
|
|
242
|
+
"arguments": {}
|
|
243
|
+
},
|
|
244
|
+
{
|
|
245
|
+
"opcode": "returnValue",
|
|
246
|
+
"blockType": "COMMAND",
|
|
247
|
+
"text": "return [VALUE]",
|
|
248
|
+
"description": "Inside a function, returns a value and ends the script. JSON text is returned as JSON; other text is returned as a string.",
|
|
249
|
+
"descriptionJa": "関数の中で値を返し、スクリプトを終了します。JSONテキストはJSONとして、それ以外のテキストは文字列として返します。",
|
|
250
|
+
"arguments": {
|
|
251
|
+
"VALUE": {
|
|
252
|
+
"type": "STRING",
|
|
253
|
+
"defaultValue": "{\"score\":10}"
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
},
|
|
257
|
+
{
|
|
258
|
+
"opcode": "usageValue",
|
|
259
|
+
"blockType": "REPORTER",
|
|
260
|
+
"text": "Realtime usage [FIELD]",
|
|
261
|
+
"description": "Reports token usage totals since the last reset. costUSD is an estimate from published prices; the OpenAI dashboard is authoritative.",
|
|
262
|
+
"descriptionJa": "最後にリセットしてからの使用量の合計を返します。costUSDは公開価格からの概算で、正確な請求額はOpenAIの管理画面で確認してください。",
|
|
263
|
+
"arguments": {
|
|
264
|
+
"FIELD": {
|
|
265
|
+
"type": "STRING",
|
|
266
|
+
"menu": "usageFields",
|
|
267
|
+
"defaultValue": "costUSD"
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
},
|
|
271
|
+
{
|
|
272
|
+
"opcode": "resetUsage",
|
|
273
|
+
"blockType": "COMMAND",
|
|
274
|
+
"text": "reset Realtime usage",
|
|
275
|
+
"description": "Clears the usage totals.",
|
|
276
|
+
"descriptionJa": "使用量の合計を0に戻します。",
|
|
277
|
+
"arguments": {}
|
|
278
|
+
},
|
|
279
|
+
{
|
|
280
|
+
"opcode": "lastError",
|
|
281
|
+
"blockType": "REPORTER",
|
|
282
|
+
"text": "last Realtime error",
|
|
283
|
+
"description": "Reports the most recent relay, connection, or API error, or an empty string.",
|
|
284
|
+
"descriptionJa": "直近の中継、接続、APIのエラーを返します。エラーがなければ空文字列を返します。",
|
|
285
|
+
"arguments": {}
|
|
286
|
+
}
|
|
287
|
+
],
|
|
288
|
+
"menus": {
|
|
289
|
+
"voices": {
|
|
290
|
+
"acceptReporters": true,
|
|
291
|
+
"items": [
|
|
292
|
+
"alloy",
|
|
293
|
+
"ash",
|
|
294
|
+
"ballad",
|
|
295
|
+
"cedar",
|
|
296
|
+
"coral",
|
|
297
|
+
"echo",
|
|
298
|
+
"marin",
|
|
299
|
+
"sage",
|
|
300
|
+
"shimmer",
|
|
301
|
+
"verse"
|
|
302
|
+
]
|
|
303
|
+
},
|
|
304
|
+
"outputModes": {
|
|
305
|
+
"acceptReporters": false,
|
|
306
|
+
"items": [
|
|
307
|
+
"audio",
|
|
308
|
+
"text"
|
|
309
|
+
]
|
|
310
|
+
},
|
|
311
|
+
"onOff": {
|
|
312
|
+
"acceptReporters": false,
|
|
313
|
+
"items": [
|
|
314
|
+
"on",
|
|
315
|
+
"off"
|
|
316
|
+
]
|
|
317
|
+
},
|
|
318
|
+
"exportModes": {
|
|
319
|
+
"acceptReporters": false,
|
|
320
|
+
"items": [
|
|
321
|
+
"tool",
|
|
322
|
+
"none"
|
|
323
|
+
]
|
|
324
|
+
},
|
|
325
|
+
"models": {
|
|
326
|
+
"acceptReporters": true,
|
|
327
|
+
"items": [
|
|
328
|
+
"gpt-realtime-2.1-mini",
|
|
329
|
+
"gpt-realtime-2.1"
|
|
330
|
+
]
|
|
331
|
+
},
|
|
332
|
+
"usageFields": {
|
|
333
|
+
"acceptReporters": false,
|
|
334
|
+
"items": [
|
|
335
|
+
"costUSD",
|
|
336
|
+
"responses",
|
|
337
|
+
"inputTokens",
|
|
338
|
+
"outputTokens",
|
|
339
|
+
"cachedInputTokens",
|
|
340
|
+
"textInputTokens",
|
|
341
|
+
"audioInputTokens",
|
|
342
|
+
"textOutputTokens",
|
|
343
|
+
"audioOutputTokens"
|
|
344
|
+
]
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
}
|
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Composition API: the OpenAI Realtime capability without TurboWarp block definitions.
|
|
3
|
+
*
|
|
4
|
+
* A downstream extension (for example `@kubohiroya/turbowarp-voice-chat`) owns its own blocks and
|
|
5
|
+
* hats, and passes its own `define function` hat opcode. Importing this module does not register
|
|
6
|
+
* any TurboWarp extension and does not touch the `Scratch` global.
|
|
7
|
+
*/
|
|
8
|
+
import {
|
|
9
|
+
createNamedFunctions,
|
|
10
|
+
type FunctionScan,
|
|
11
|
+
type NamedFunctions,
|
|
12
|
+
type RuntimeLike,
|
|
13
|
+
type RuntimeThread
|
|
14
|
+
} from '@kubohiroya/turbowarp-named-functions/composition';
|
|
15
|
+
import {RealtimeSession, type ConnectionState} from './realtime-session.js';
|
|
16
|
+
import {
|
|
17
|
+
normalizeRelayEndpoint,
|
|
18
|
+
pairWithRelay,
|
|
19
|
+
requestClientSecret,
|
|
20
|
+
type FetchLike,
|
|
21
|
+
type RelaySession
|
|
22
|
+
} from './relay-client.js';
|
|
23
|
+
import {
|
|
24
|
+
buildSessionRequest,
|
|
25
|
+
defaultSessionSettings,
|
|
26
|
+
normalizeInstructions,
|
|
27
|
+
normalizeModel,
|
|
28
|
+
normalizeOutputMode,
|
|
29
|
+
normalizeVoice,
|
|
30
|
+
type OutputMode,
|
|
31
|
+
type SessionSettings
|
|
32
|
+
} from './session-config.js';
|
|
33
|
+
import type {RealtimeTransport} from './transport.js';
|
|
34
|
+
import {addUsage, emptyUsage, type UsageTotals} from './usage.js';
|
|
35
|
+
import {WebRtcTransport} from './webrtc-transport.js';
|
|
36
|
+
|
|
37
|
+
export type {ConnectionState} from './realtime-session.js';
|
|
38
|
+
export type {OutputMode, SessionSettings} from './session-config.js';
|
|
39
|
+
export {SUPPORTED_MODELS} from './session-config.js';
|
|
40
|
+
export type {UsageTotals} from './usage.js';
|
|
41
|
+
export {MODEL_PRICES, PRICES_AS_OF} from './usage.js';
|
|
42
|
+
export type {RealtimeTransport, RealtimeEvent, TransportConnectOptions} from './transport.js';
|
|
43
|
+
export type {NamedFunctions, RuntimeLike, RuntimeThread} from '@kubohiroya/turbowarp-named-functions/composition';
|
|
44
|
+
|
|
45
|
+
export const DEFAULT_RELAY_ENDPOINT = 'http://127.0.0.1:8787';
|
|
46
|
+
|
|
47
|
+
export type RealtimeCompositionEvent =
|
|
48
|
+
| {type: 'state'; state: ConnectionState}
|
|
49
|
+
| {type: 'response'; text: string}
|
|
50
|
+
| {type: 'usage'; usage: Readonly<UsageTotals>}
|
|
51
|
+
| {type: 'error'; message: string}
|
|
52
|
+
| {type: 'sessionTimeLimitReached'};
|
|
53
|
+
|
|
54
|
+
export type RealtimeCompositionListener = (event: RealtimeCompositionEvent) => void;
|
|
55
|
+
|
|
56
|
+
export interface RealtimeCompositionOptions {
|
|
57
|
+
runtime: RuntimeLike;
|
|
58
|
+
/** Opcode of the downstream extension's `define function` hat, e.g. `myextension_defineFunction`. */
|
|
59
|
+
functionHatOpcode: string;
|
|
60
|
+
/**
|
|
61
|
+
* Named functions to expose as tools. Pass one shared instance when another composition also uses
|
|
62
|
+
* the same `define function` hats; otherwise one is created from `functionHatOpcode`.
|
|
63
|
+
*/
|
|
64
|
+
functions?: NamedFunctions;
|
|
65
|
+
fetch?: FetchLike;
|
|
66
|
+
createTransport?: () => RealtimeTransport;
|
|
67
|
+
now?: () => number;
|
|
68
|
+
setTimer?: (callback: () => void, ms: number) => unknown;
|
|
69
|
+
clearTimer?: (handle: unknown) => void;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export interface RealtimeComposition {
|
|
73
|
+
configureRelay(endpoint: string): void;
|
|
74
|
+
pairRelay(code: string): Promise<void>;
|
|
75
|
+
isRelayPaired(): boolean;
|
|
76
|
+
|
|
77
|
+
setModel(model: string): void;
|
|
78
|
+
setInstructions(text: string): void;
|
|
79
|
+
setVoice(voice: string): void;
|
|
80
|
+
setOutputMode(mode: OutputMode): void;
|
|
81
|
+
/** Seconds; 0 disables the limit. Applies from the next connection. */
|
|
82
|
+
setSessionTimeLimit(seconds: number): void;
|
|
83
|
+
readonly settings: Readonly<SessionSettings>;
|
|
84
|
+
readonly sessionTimeLimitSeconds: number;
|
|
85
|
+
|
|
86
|
+
/** Validates `define function` hats without connecting. */
|
|
87
|
+
scanFunctions(): FunctionScan;
|
|
88
|
+
/** The named functions used for tools. */
|
|
89
|
+
readonly functions: NamedFunctions;
|
|
90
|
+
connect(options: {microphone: boolean}): Promise<void>;
|
|
91
|
+
disconnect(): void;
|
|
92
|
+
readonly state: ConnectionState;
|
|
93
|
+
/** Model reported by the relay for the current or last connection; empty before connecting. */
|
|
94
|
+
readonly activeModel: string;
|
|
95
|
+
sessionElapsedSeconds(): number;
|
|
96
|
+
sendText(text: string): void;
|
|
97
|
+
readonly lastResponseText: string;
|
|
98
|
+
|
|
99
|
+
/** Call from the `define function` hat predicate. */
|
|
100
|
+
matchFunctionHat(name: string, thread: RuntimeThread | undefined): boolean;
|
|
101
|
+
functionArguments(thread: RuntimeThread | undefined): unknown;
|
|
102
|
+
returnFromFunction(thread: RuntimeThread | undefined, value: unknown): void;
|
|
103
|
+
|
|
104
|
+
usage(): Readonly<UsageTotals>;
|
|
105
|
+
resetUsage(): void;
|
|
106
|
+
|
|
107
|
+
subscribe(listener: RealtimeCompositionListener): () => void;
|
|
108
|
+
/** Disconnects and detaches listeners. */
|
|
109
|
+
release(): void;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export function createRealtimeComposition(options: RealtimeCompositionOptions): RealtimeComposition {
|
|
113
|
+
return new Composition(options);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
class Composition implements RealtimeComposition {
|
|
117
|
+
private relayEndpoint = DEFAULT_RELAY_ENDPOINT;
|
|
118
|
+
private relaySession: RelaySession | null = null;
|
|
119
|
+
private settingsValue: SessionSettings = defaultSessionSettings();
|
|
120
|
+
private timeLimitSeconds = 0;
|
|
121
|
+
private connectedAt: number | null = null;
|
|
122
|
+
private timeLimitTimer: unknown = null;
|
|
123
|
+
private activeModelValue = '';
|
|
124
|
+
private lastResponse = '';
|
|
125
|
+
private usageTotals: UsageTotals = emptyUsage();
|
|
126
|
+
private readonly listeners = new Set<RealtimeCompositionListener>();
|
|
127
|
+
private readonly fetcher: FetchLike;
|
|
128
|
+
private readonly createTransport: () => RealtimeTransport;
|
|
129
|
+
private readonly now: () => number;
|
|
130
|
+
private readonly setTimer: (callback: () => void, ms: number) => unknown;
|
|
131
|
+
private readonly clearTimer: (handle: unknown) => void;
|
|
132
|
+
private readonly namedFunctions: NamedFunctions;
|
|
133
|
+
private readonly ownsFunctions: boolean;
|
|
134
|
+
private readonly session: RealtimeSession;
|
|
135
|
+
|
|
136
|
+
public constructor(private readonly options: RealtimeCompositionOptions) {
|
|
137
|
+
this.fetcher = options.fetch ?? ((input, init) => fetch(input, init));
|
|
138
|
+
this.createTransport = options.createTransport ?? (() => new WebRtcTransport());
|
|
139
|
+
this.now = options.now ?? Date.now;
|
|
140
|
+
this.setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
|
|
141
|
+
this.clearTimer = options.clearTimer ?? ((handle) => clearTimeout(handle as ReturnType<typeof setTimeout>));
|
|
142
|
+
this.ownsFunctions = options.functions === undefined;
|
|
143
|
+
this.namedFunctions =
|
|
144
|
+
options.functions ??
|
|
145
|
+
createNamedFunctions({runtime: options.runtime, functionHatOpcode: options.functionHatOpcode});
|
|
146
|
+
this.session = new RealtimeSession({
|
|
147
|
+
callFunction: (name, args) => this.namedFunctions.call(name, args, {exportedOnly: true}),
|
|
148
|
+
onResponseText: (text) => {
|
|
149
|
+
this.lastResponse = text;
|
|
150
|
+
this.emit({type: 'response', text});
|
|
151
|
+
},
|
|
152
|
+
onError: (message) => this.emit({type: 'error', message}),
|
|
153
|
+
onStateChange: (state) => this.handleState(state),
|
|
154
|
+
onUsage: (usage) => {
|
|
155
|
+
this.usageTotals = addUsage(this.usageTotals, usage, this.activeModelValue);
|
|
156
|
+
this.emit({type: 'usage', usage: this.usage()});
|
|
157
|
+
}
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// ---- relay --------------------------------------------------------------------------------
|
|
162
|
+
|
|
163
|
+
public configureRelay(endpoint: string): void {
|
|
164
|
+
this.relayEndpoint = normalizeRelayEndpoint(endpoint);
|
|
165
|
+
this.relaySession = null;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
public async pairRelay(code: string): Promise<void> {
|
|
169
|
+
this.relaySession = await pairWithRelay(this.relayEndpoint, code, this.fetcher, this.now);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
public isRelayPaired(): boolean {
|
|
173
|
+
return this.relaySession !== null && this.relaySession.expiresAt > this.now();
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// ---- settings -----------------------------------------------------------------------------
|
|
177
|
+
|
|
178
|
+
public setModel(model: string): void {
|
|
179
|
+
this.settingsValue = {...this.settingsValue, model: normalizeModel(model)};
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
public setInstructions(text: string): void {
|
|
183
|
+
this.settingsValue = {...this.settingsValue, instructions: normalizeInstructions(text)};
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
public setVoice(voice: string): void {
|
|
187
|
+
this.settingsValue = {...this.settingsValue, voice: normalizeVoice(voice)};
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
public setOutputMode(mode: OutputMode): void {
|
|
191
|
+
this.settingsValue = {...this.settingsValue, outputMode: normalizeOutputMode(mode)};
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
public setSessionTimeLimit(seconds: number): void {
|
|
195
|
+
if (!Number.isFinite(seconds) || seconds < 0) {
|
|
196
|
+
throw new TypeError('Session time limit must be zero or a positive number of seconds.');
|
|
197
|
+
}
|
|
198
|
+
this.timeLimitSeconds = seconds;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
public get settings(): Readonly<SessionSettings> {
|
|
202
|
+
return {...this.settingsValue};
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
public get sessionTimeLimitSeconds(): number {
|
|
206
|
+
return this.timeLimitSeconds;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// ---- connection ---------------------------------------------------------------------------
|
|
210
|
+
|
|
211
|
+
public scanFunctions(): FunctionScan {
|
|
212
|
+
return this.namedFunctions.scan();
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
public get functions(): NamedFunctions {
|
|
216
|
+
return this.namedFunctions;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
public async connect(options: {microphone: boolean}): Promise<void> {
|
|
220
|
+
if (!this.relaySession || !this.isRelayPaired()) throw new Error('Pair with the local relay first.');
|
|
221
|
+
const request = buildSessionRequest(this.settingsValue, this.namedFunctions.tools());
|
|
222
|
+
const secret = await requestClientSecret(this.relaySession, request, this.fetcher, this.now);
|
|
223
|
+
this.activeModelValue = secret.model || this.settingsValue.model;
|
|
224
|
+
await this.session.open(this.createTransport(), secret.value, options.microphone);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
public disconnect(): void {
|
|
228
|
+
this.session.close();
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
public get state(): ConnectionState {
|
|
232
|
+
return this.session.state;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
public get activeModel(): string {
|
|
236
|
+
return this.activeModelValue;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
public sessionElapsedSeconds(): number {
|
|
240
|
+
return this.connectedAt === null ? 0 : Math.max(0, (this.now() - this.connectedAt) / 1000);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
public sendText(text: string): void {
|
|
244
|
+
this.session.sendText(text);
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
public get lastResponseText(): string {
|
|
248
|
+
return this.lastResponse;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// ---- function values ----------------------------------------------------------------------
|
|
252
|
+
|
|
253
|
+
public matchFunctionHat(name: string, thread: RuntimeThread | undefined): boolean {
|
|
254
|
+
return this.namedFunctions.matchHat(name, thread);
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
public functionArguments(thread: RuntimeThread | undefined): unknown {
|
|
258
|
+
return this.namedFunctions.argumentsFor(thread);
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
public returnFromFunction(thread: RuntimeThread | undefined, value: unknown): void {
|
|
262
|
+
this.namedFunctions.returnFrom(thread, value);
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// ---- usage --------------------------------------------------------------------------------
|
|
266
|
+
|
|
267
|
+
public usage(): Readonly<UsageTotals> {
|
|
268
|
+
return {...this.usageTotals};
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
public resetUsage(): void {
|
|
272
|
+
this.usageTotals = emptyUsage();
|
|
273
|
+
this.emit({type: 'usage', usage: this.usage()});
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
// ---- events -------------------------------------------------------------------------------
|
|
277
|
+
|
|
278
|
+
public subscribe(listener: RealtimeCompositionListener): () => void {
|
|
279
|
+
this.listeners.add(listener);
|
|
280
|
+
return () => this.listeners.delete(listener);
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
public release(): void {
|
|
284
|
+
this.disconnect();
|
|
285
|
+
this.listeners.clear();
|
|
286
|
+
if (this.ownsFunctions) this.namedFunctions.release();
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
private handleState(state: ConnectionState): void {
|
|
290
|
+
if (state === 'connected') {
|
|
291
|
+
this.connectedAt = this.now();
|
|
292
|
+
if (this.timeLimitSeconds > 0) {
|
|
293
|
+
this.timeLimitTimer = this.setTimer(() => this.reachTimeLimit(), this.timeLimitSeconds * 1000);
|
|
294
|
+
}
|
|
295
|
+
} else if (state !== 'connecting') {
|
|
296
|
+
this.connectedAt = null;
|
|
297
|
+
this.cancelTimeLimit();
|
|
298
|
+
this.namedFunctions.cancelAll('The Realtime session ended.');
|
|
299
|
+
}
|
|
300
|
+
this.emit({type: 'state', state});
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
private reachTimeLimit(): void {
|
|
304
|
+
this.timeLimitTimer = null;
|
|
305
|
+
if (this.session.state !== 'connected') return;
|
|
306
|
+
this.session.close();
|
|
307
|
+
this.emit({type: 'sessionTimeLimitReached'});
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
private cancelTimeLimit(): void {
|
|
311
|
+
if (this.timeLimitTimer !== null) this.clearTimer(this.timeLimitTimer);
|
|
312
|
+
this.timeLimitTimer = null;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
private emit(event: RealtimeCompositionEvent): void {
|
|
316
|
+
for (const listener of [...this.listeners]) listener(event);
|
|
317
|
+
}
|
|
318
|
+
}
|
package/src/config.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export const extensionConfig = {
|
|
2
|
+
id: 'kubohiroyaopenairealtime',
|
|
3
|
+
slug: 'turbowarp-openai-realtime-api',
|
|
4
|
+
name: 'TurboWarp-OpenAI-Realtime-API',
|
|
5
|
+
description: 'Talk with the OpenAI Realtime API from TurboWarp through a localhost relay.',
|
|
6
|
+
author: 'Hiroya Kubo',
|
|
7
|
+
license: 'MPL-2.0',
|
|
8
|
+
unsandboxed: true,
|
|
9
|
+
docsURI: 'https://kubohiroya.github.io/turbowarp-openai-realtime-api/',
|
|
10
|
+
blockIconURI:
|
|
11
|
+
'data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0OCA0OCI+PHBhdGggZD0iTTggMTBoMzJhNCA0IDAgMCAxIDQgNHYxNmE0IDQgMCAwIDEtNCA0SDIybC05IDd2LTdIOGE0IDQgMCAwIDEtNC00VjE0YTQgNCAwIDAgMSA0LTR6IiBmaWxsPSIjMTBBMzdGIi8+PGcgc3Ryb2tlPSIjZmZmIiBzdHJva2Utd2lkdGg9IjMiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCI+PHBhdGggZD0iTTE0IDE5djYiLz48cGF0aCBkPSJNMjAgMTZ2MTIiLz48cGF0aCBkPSJNMjYgMTh2OCIvPjxwYXRoIGQ9Ik0zMiAxNXYxNCIvPjxwYXRoIGQ9Ik0zOCAyMHY0Ii8+PC9nPjwvc3ZnPg=='
|
|
12
|
+
} as const;
|