@kubohiroya/turbowarp-openai-realtime-api 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +373 -0
- package/README.ja.md +353 -0
- package/README.md +415 -0
- package/dist/composition.js +682 -0
- package/dist/extension-manifest.json +240 -0
- package/dist/turbowarp-openai-realtime-api.js +1581 -0
- package/dist/types/composition.d.ts +89 -0
- package/dist/types/config.d.ts +11 -0
- package/dist/types/extension-manifest.d.ts +30 -0
- package/dist/types/extension.d.ts +45 -0
- package/dist/types/index.d.ts +1 -0
- package/dist/types/realtime-session.d.ts +39 -0
- package/dist/types/relay-client.d.ts +16 -0
- package/dist/types/session-config.d.ts +27 -0
- package/dist/types/transport.d.ts +18 -0
- package/dist/types/usage.d.ts +34 -0
- package/dist/types/webrtc-transport.d.ts +26 -0
- package/package.json +79 -0
- package/schemas/extension-manifest.schema.json +49 -0
- package/src/block-definitions.json +347 -0
- package/src/composition.ts +318 -0
- package/src/config.ts +12 -0
- package/src/extension-manifest.ts +149 -0
- package/src/extension.ts +266 -0
- package/src/globals.d.ts +26 -0
- package/src/index.ts +14 -0
- package/src/realtime-session.ts +175 -0
- package/src/relay-client.ts +133 -0
- package/src/session-config.ts +71 -0
- package/src/transport.ts +18 -0
- package/src/usage.ts +127 -0
- package/src/webrtc-transport.ts +156 -0
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import type {Plugin} from 'vite';
|
|
2
|
+
|
|
3
|
+
export const EXTENSION_MANIFEST_FORMAT_VERSION = 1 as const;
|
|
4
|
+
|
|
5
|
+
export interface ExtensionManifestArgument {
|
|
6
|
+
id: string;
|
|
7
|
+
type: string;
|
|
8
|
+
menu?: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface ExtensionManifestBlock {
|
|
12
|
+
opcode: string;
|
|
13
|
+
blockType: string;
|
|
14
|
+
arguments: ExtensionManifestArgument[];
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface ExtensionManifestMenu {
|
|
18
|
+
id: string;
|
|
19
|
+
acceptReporters: boolean;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface ExtensionManifest {
|
|
23
|
+
formatVersion: typeof EXTENSION_MANIFEST_FORMAT_VERSION;
|
|
24
|
+
id: string;
|
|
25
|
+
blocks: ExtensionManifestBlock[];
|
|
26
|
+
menus: ExtensionManifestMenu[];
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface ExtensionManifestPluginOptions {
|
|
30
|
+
id: string;
|
|
31
|
+
definitions: unknown;
|
|
32
|
+
fileName?: string;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function createExtensionManifest(id: string, definitions: unknown): ExtensionManifest {
|
|
36
|
+
if (!/^[a-z0-9]+$/.test(id)) {
|
|
37
|
+
throw new TypeError('Extension manifest ID must contain only lowercase letters and numbers.');
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const source = requireRecord(definitions, 'Block definitions');
|
|
41
|
+
const sourceBlocks = source.blocks;
|
|
42
|
+
if (!Array.isArray(sourceBlocks)) {
|
|
43
|
+
throw new TypeError('Block definitions must contain a blocks array.');
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const menus = normalizeMenus(source.menus);
|
|
47
|
+
const menuIds = new Set(menus.map((menu) => menu.id));
|
|
48
|
+
const seenOpcodes = new Set<string>();
|
|
49
|
+
const blocks = sourceBlocks.map((block, index) => {
|
|
50
|
+
const normalized = normalizeBlock(block, index, menuIds);
|
|
51
|
+
if (seenOpcodes.has(normalized.opcode)) {
|
|
52
|
+
throw new TypeError(`Duplicate block opcode: ${normalized.opcode}`);
|
|
53
|
+
}
|
|
54
|
+
seenOpcodes.add(normalized.opcode);
|
|
55
|
+
return normalized;
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
return {
|
|
59
|
+
formatVersion: EXTENSION_MANIFEST_FORMAT_VERSION,
|
|
60
|
+
id,
|
|
61
|
+
blocks: blocks.sort((left, right) => compareIds(left.opcode, right.opcode)),
|
|
62
|
+
menus
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function serializeExtensionManifest(id: string, definitions: unknown): string {
|
|
67
|
+
return `${JSON.stringify(createExtensionManifest(id, definitions), null, 2)}\n`;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function extensionManifestPlugin(options: ExtensionManifestPluginOptions): Plugin {
|
|
71
|
+
return {
|
|
72
|
+
name: 'extension-api-manifest',
|
|
73
|
+
apply: 'build',
|
|
74
|
+
enforce: 'post',
|
|
75
|
+
generateBundle() {
|
|
76
|
+
this.emitFile({
|
|
77
|
+
type: 'asset',
|
|
78
|
+
fileName: options.fileName ?? 'extension-manifest.json',
|
|
79
|
+
source: serializeExtensionManifest(options.id, options.definitions)
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function normalizeBlock(
|
|
86
|
+
value: unknown,
|
|
87
|
+
index: number,
|
|
88
|
+
menuIds: ReadonlySet<string>
|
|
89
|
+
): ExtensionManifestBlock {
|
|
90
|
+
const block = requireRecord(value, `Block at index ${index}`);
|
|
91
|
+
const opcode = requireNonEmptyString(block.opcode, `Block at index ${index} opcode`);
|
|
92
|
+
const blockType = requireNonEmptyString(block.blockType, `Block ${opcode} blockType`);
|
|
93
|
+
const sourceArguments = block.arguments ?? {};
|
|
94
|
+
const argumentRecord = requireRecord(sourceArguments, `Block ${opcode} arguments`);
|
|
95
|
+
const argumentsList = Object.entries(argumentRecord).map(([argumentId, argument]) => {
|
|
96
|
+
requireNonEmptyString(argumentId, `Block ${opcode} argument ID`);
|
|
97
|
+
const definition = requireRecord(argument, `Block ${opcode} argument ${argumentId}`);
|
|
98
|
+
const type = requireNonEmptyString(
|
|
99
|
+
definition.type,
|
|
100
|
+
`Block ${opcode} argument ${argumentId} type`
|
|
101
|
+
);
|
|
102
|
+
const menu = definition.menu;
|
|
103
|
+
if (menu !== undefined && (typeof menu !== 'string' || !menuIds.has(menu))) {
|
|
104
|
+
throw new TypeError(`Block ${opcode} argument ${argumentId} references unknown menu: ${menu}`);
|
|
105
|
+
}
|
|
106
|
+
return menu === undefined ? {id: argumentId, type} : {id: argumentId, type, menu};
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
return {
|
|
110
|
+
opcode,
|
|
111
|
+
blockType,
|
|
112
|
+
arguments: argumentsList.sort((left, right) => compareIds(left.id, right.id))
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function normalizeMenus(value: unknown): ExtensionManifestMenu[] {
|
|
117
|
+
const menuRecord = requireRecord(value ?? {}, 'Block definition menus');
|
|
118
|
+
return Object.entries(menuRecord)
|
|
119
|
+
.map(([id, menu]) => {
|
|
120
|
+
requireNonEmptyString(id, 'Menu ID');
|
|
121
|
+
const definition = requireRecord(menu, `Menu ${id}`);
|
|
122
|
+
const acceptReporters = definition.acceptReporters ?? false;
|
|
123
|
+
if (typeof acceptReporters !== 'boolean') {
|
|
124
|
+
throw new TypeError(`Menu ${id} acceptReporters must be a boolean.`);
|
|
125
|
+
}
|
|
126
|
+
return {id, acceptReporters};
|
|
127
|
+
})
|
|
128
|
+
.sort((left, right) => compareIds(left.id, right.id));
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function requireRecord(value: unknown, label: string): Record<string, unknown> {
|
|
132
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
|
|
133
|
+
throw new TypeError(`${label} must be an object.`);
|
|
134
|
+
}
|
|
135
|
+
return value as Record<string, unknown>;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function requireNonEmptyString(value: unknown, label: string): string {
|
|
139
|
+
if (typeof value !== 'string' || value.length === 0) {
|
|
140
|
+
throw new TypeError(`${label} must be a non-empty string.`);
|
|
141
|
+
}
|
|
142
|
+
return value;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function compareIds(left: string, right: string): number {
|
|
146
|
+
if (left < right) return -1;
|
|
147
|
+
if (left > right) return 1;
|
|
148
|
+
return 0;
|
|
149
|
+
}
|
package/src/extension.ts
ADDED
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
import {extensionConfig} from './config';
|
|
2
|
+
import definitions from './block-definitions.json';
|
|
3
|
+
import {
|
|
4
|
+
createRealtimeComposition,
|
|
5
|
+
type RealtimeComposition,
|
|
6
|
+
type RealtimeCompositionOptions,
|
|
7
|
+
type UsageTotals
|
|
8
|
+
} from './composition.js';
|
|
9
|
+
import {
|
|
10
|
+
parseJsonOrText,
|
|
11
|
+
readArgumentPath,
|
|
12
|
+
toScratchValue,
|
|
13
|
+
type BlockUtilityLike
|
|
14
|
+
} from '@kubohiroya/turbowarp-named-functions/composition';
|
|
15
|
+
import {normalizeOutputMode} from './session-config.js';
|
|
16
|
+
|
|
17
|
+
type BlockTypeName = 'COMMAND' | 'REPORTER' | 'BOOLEAN' | 'HAT';
|
|
18
|
+
type ArgumentTypeName = 'STRING' | 'NUMBER' | 'BOOLEAN';
|
|
19
|
+
|
|
20
|
+
interface DefinitionArgument {
|
|
21
|
+
type: ArgumentTypeName;
|
|
22
|
+
defaultValue: string;
|
|
23
|
+
menu?: string;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
interface BlockDefinition {
|
|
27
|
+
opcode: string;
|
|
28
|
+
blockType: BlockTypeName;
|
|
29
|
+
text: string;
|
|
30
|
+
description: string;
|
|
31
|
+
arguments: Record<string, DefinitionArgument>;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
interface MenuDefinition {
|
|
35
|
+
acceptReporters: boolean;
|
|
36
|
+
items: string[];
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const blockDefinitions = definitions.blocks as readonly BlockDefinition[];
|
|
40
|
+
const menuDefinitions = definitions.menus as Record<string, MenuDefinition>;
|
|
41
|
+
|
|
42
|
+
export const DEFINE_FUNCTION_OPCODE = `${extensionConfig.id}_defineFunction`;
|
|
43
|
+
export const RESPONSE_DONE_OPCODE = `${extensionConfig.id}_whenResponseDone`;
|
|
44
|
+
export const TIME_LIMIT_OPCODE = `${extensionConfig.id}_whenSessionTimeLimitReached`;
|
|
45
|
+
|
|
46
|
+
export type RealtimeExtensionDependencies = Omit<RealtimeCompositionOptions, 'functionHatOpcode'>;
|
|
47
|
+
|
|
48
|
+
type Args = Record<string, unknown>;
|
|
49
|
+
|
|
50
|
+
const USAGE_FIELDS: Record<string, (usage: UsageTotals) => number> = {
|
|
51
|
+
costUSD: (usage) => Math.round(usage.estimatedCostUsd * 1_000_000) / 1_000_000,
|
|
52
|
+
responses: (usage) => usage.responses,
|
|
53
|
+
inputTokens: (usage) => usage.inputTokens,
|
|
54
|
+
outputTokens: (usage) => usage.outputTokens,
|
|
55
|
+
cachedInputTokens: (usage) => usage.cachedInputTokens,
|
|
56
|
+
textInputTokens: (usage) => usage.textInputTokens,
|
|
57
|
+
audioInputTokens: (usage) => usage.audioInputTokens,
|
|
58
|
+
textOutputTokens: (usage) => usage.textOutputTokens,
|
|
59
|
+
audioOutputTokens: (usage) => usage.audioOutputTokens
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
/** Block surface over the Realtime composition. */
|
|
63
|
+
export class OpenAIRealtimeExtension implements TurboWarpExtension {
|
|
64
|
+
private lastErrorMessage = '';
|
|
65
|
+
private readonly realtime: RealtimeComposition;
|
|
66
|
+
|
|
67
|
+
public constructor(deps: RealtimeExtensionDependencies) {
|
|
68
|
+
this.realtime = createRealtimeComposition({...deps, functionHatOpcode: DEFINE_FUNCTION_OPCODE});
|
|
69
|
+
this.realtime.subscribe((event) => {
|
|
70
|
+
if (event.type === 'response') deps.runtime.startHats(RESPONSE_DONE_OPCODE);
|
|
71
|
+
else if (event.type === 'sessionTimeLimitReached') deps.runtime.startHats(TIME_LIMIT_OPCODE);
|
|
72
|
+
else if (event.type === 'error') this.lastErrorMessage = event.message;
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
public getInfo(): Record<string, unknown> {
|
|
77
|
+
return {
|
|
78
|
+
id: extensionConfig.id,
|
|
79
|
+
name: Scratch.translate(definitions.extensionName),
|
|
80
|
+
docsURI: extensionConfig.docsURI,
|
|
81
|
+
blockIconURI: extensionConfig.blockIconURI,
|
|
82
|
+
blocks: blockDefinitions.map((block) => this.toScratchBlock(block)),
|
|
83
|
+
menus: Object.fromEntries(
|
|
84
|
+
Object.entries(menuDefinitions).map(([id, menu]) => [
|
|
85
|
+
id,
|
|
86
|
+
{acceptReporters: menu.acceptReporters, items: [...menu.items]}
|
|
87
|
+
])
|
|
88
|
+
)
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// ---- relay ------------------------------------------------------------------------------
|
|
93
|
+
|
|
94
|
+
public configureRelay(args: Args): void {
|
|
95
|
+
this.record(() => this.realtime.configureRelay(Scratch.Cast.toString(args.ENDPOINT)));
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
public pairRelay(args: Args): Promise<void> {
|
|
99
|
+
return this.recordAsync(() => this.realtime.pairRelay(Scratch.Cast.toString(args.CODE)));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
public isRelayPaired(): boolean {
|
|
103
|
+
return this.realtime.isRelayPaired();
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// ---- session settings ---------------------------------------------------------------------
|
|
107
|
+
|
|
108
|
+
public setInstructions(args: Args): void {
|
|
109
|
+
this.record(() => this.realtime.setInstructions(Scratch.Cast.toString(args.TEXT)));
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
public setVoice(args: Args): void {
|
|
113
|
+
this.record(() => this.realtime.setVoice(Scratch.Cast.toString(args.VOICE)));
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
public setOutputMode(args: Args): void {
|
|
117
|
+
this.record(() => this.realtime.setOutputMode(normalizeOutputMode(Scratch.Cast.toString(args.MODE))));
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
public setModel(args: Args): void {
|
|
121
|
+
this.record(() => this.realtime.setModel(Scratch.Cast.toString(args.MODEL)));
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
public setSessionTimeLimit(args: Args): void {
|
|
125
|
+
this.record(() => this.realtime.setSessionTimeLimit(Scratch.Cast.toNumber(args.SECONDS)));
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// ---- connection ---------------------------------------------------------------------------
|
|
129
|
+
|
|
130
|
+
public connect(args: Args): Promise<void> {
|
|
131
|
+
return this.recordAsync(() =>
|
|
132
|
+
this.realtime.connect({microphone: Scratch.Cast.toString(args.MICROPHONE) !== 'off'})
|
|
133
|
+
);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
public disconnect(): void {
|
|
137
|
+
this.realtime.disconnect();
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
public isConnected(): boolean {
|
|
141
|
+
return this.realtime.state === 'connected';
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
public connectionState(): string {
|
|
145
|
+
return this.realtime.state;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
public currentModel(): string {
|
|
149
|
+
return this.realtime.activeModel;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
public sessionElapsed(): number {
|
|
153
|
+
return Math.floor(this.realtime.sessionElapsedSeconds());
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
public whenSessionTimeLimitReached(): boolean {
|
|
157
|
+
return true;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// ---- conversation -------------------------------------------------------------------------
|
|
161
|
+
|
|
162
|
+
public sendText(args: Args): void {
|
|
163
|
+
this.record(() => this.realtime.sendText(Scratch.Cast.toString(args.TEXT)));
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
public whenResponseDone(): boolean {
|
|
167
|
+
return true;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
public lastResponseText(): string {
|
|
171
|
+
return this.realtime.lastResponseText;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// ---- function values ----------------------------------------------------------------------
|
|
175
|
+
|
|
176
|
+
public defineFunction(args: Args, util?: BlockUtilityLike): boolean {
|
|
177
|
+
return this.realtime.matchFunctionHat(Scratch.Cast.toString(args.NAME), util?.thread);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
public functionArgument(args: Args, util?: BlockUtilityLike): string | number | boolean {
|
|
181
|
+
try {
|
|
182
|
+
const value = readArgumentPath(this.realtime.functionArguments(util?.thread), Scratch.Cast.toString(args.PATH));
|
|
183
|
+
return toScratchValue(value);
|
|
184
|
+
} catch (error) {
|
|
185
|
+
this.lastErrorMessage = messageOf(error);
|
|
186
|
+
return '';
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
public functionArgumentsJson(_args: Args, util?: BlockUtilityLike): string {
|
|
191
|
+
try {
|
|
192
|
+
return JSON.stringify(this.realtime.functionArguments(util?.thread) ?? null);
|
|
193
|
+
} catch (error) {
|
|
194
|
+
this.lastErrorMessage = messageOf(error);
|
|
195
|
+
return '';
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
public returnValue(args: Args, util?: BlockUtilityLike & {stopThisScript?: () => void}): void {
|
|
200
|
+
try {
|
|
201
|
+
this.realtime.returnFromFunction(util?.thread, parseJsonOrText(Scratch.Cast.toString(args.VALUE)));
|
|
202
|
+
util?.stopThisScript?.();
|
|
203
|
+
} catch (error) {
|
|
204
|
+
this.lastErrorMessage = messageOf(error);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// ---- usage and diagnostics ----------------------------------------------------------------
|
|
209
|
+
|
|
210
|
+
public usageValue(args: Args): number | string {
|
|
211
|
+
const read = USAGE_FIELDS[Scratch.Cast.toString(args.FIELD)];
|
|
212
|
+
return read ? read(this.realtime.usage()) : '';
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
public resetUsage(): void {
|
|
216
|
+
this.realtime.resetUsage();
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
public lastError(): string {
|
|
220
|
+
return this.lastErrorMessage;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// ---- internals ----------------------------------------------------------------------------
|
|
224
|
+
|
|
225
|
+
private record(action: () => void): void {
|
|
226
|
+
try {
|
|
227
|
+
action();
|
|
228
|
+
this.lastErrorMessage = '';
|
|
229
|
+
} catch (error) {
|
|
230
|
+
this.lastErrorMessage = messageOf(error);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
private async recordAsync(action: () => Promise<void>): Promise<void> {
|
|
235
|
+
try {
|
|
236
|
+
await action();
|
|
237
|
+
this.lastErrorMessage = '';
|
|
238
|
+
} catch (error) {
|
|
239
|
+
this.lastErrorMessage = messageOf(error);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
private toScratchBlock(block: BlockDefinition): Record<string, unknown> {
|
|
244
|
+
const scratchBlock: Record<string, unknown> = {
|
|
245
|
+
opcode: block.opcode,
|
|
246
|
+
blockType: Scratch.BlockType[block.blockType],
|
|
247
|
+
text: Scratch.translate(block.text),
|
|
248
|
+
arguments: Object.fromEntries(
|
|
249
|
+
Object.entries(block.arguments).map(([name, argument]) => [
|
|
250
|
+
name,
|
|
251
|
+
{
|
|
252
|
+
type: Scratch.ArgumentType[argument.type],
|
|
253
|
+
defaultValue: argument.defaultValue,
|
|
254
|
+
...(argument.menu ? {menu: argument.menu} : {})
|
|
255
|
+
}
|
|
256
|
+
])
|
|
257
|
+
)
|
|
258
|
+
};
|
|
259
|
+
if (block.blockType === 'HAT') scratchBlock.isEdgeActivated = false;
|
|
260
|
+
return scratchBlock;
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
function messageOf(error: unknown): string {
|
|
265
|
+
return error instanceof Error ? error.message : String(error);
|
|
266
|
+
}
|
package/src/globals.d.ts
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
interface TurboWarpExtension {
|
|
2
|
+
getInfo(): Record<string, unknown>;
|
|
3
|
+
}
|
|
4
|
+
|
|
5
|
+
interface ScratchTranslate {
|
|
6
|
+
(text: string): string;
|
|
7
|
+
(message: {default: string; description?: string}, placeholders?: Record<string, string | number>): string;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
interface ScratchApi {
|
|
11
|
+
extensions: {
|
|
12
|
+
unsandboxed: boolean;
|
|
13
|
+
register(extension: TurboWarpExtension): void;
|
|
14
|
+
};
|
|
15
|
+
vm?: {runtime: unknown};
|
|
16
|
+
BlockType: Record<'COMMAND' | 'REPORTER' | 'BOOLEAN' | 'HAT', string>;
|
|
17
|
+
ArgumentType: Record<'STRING' | 'NUMBER' | 'BOOLEAN', string>;
|
|
18
|
+
Cast: {
|
|
19
|
+
toString(value: unknown): string;
|
|
20
|
+
toNumber(value: unknown): number;
|
|
21
|
+
toBoolean(value: unknown): boolean;
|
|
22
|
+
};
|
|
23
|
+
translate: ScratchTranslate;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
declare const Scratch: ScratchApi;
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import {extensionConfig} from './config.js';
|
|
2
|
+
import {OpenAIRealtimeExtension} from './extension.js';
|
|
3
|
+
import type {RuntimeLike} from '@kubohiroya/turbowarp-named-functions/composition';
|
|
4
|
+
|
|
5
|
+
if (extensionConfig.unsandboxed && !Scratch.extensions.unsandboxed) {
|
|
6
|
+
throw new Error(`${extensionConfig.name} must run unsandboxed.`);
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
const runtime = Scratch.vm?.runtime as RuntimeLike | undefined;
|
|
10
|
+
if (!runtime) {
|
|
11
|
+
throw new Error(`${extensionConfig.name} requires access to the TurboWarp VM runtime.`);
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
Scratch.extensions.register(new OpenAIRealtimeExtension({runtime}));
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import type {RealtimeEvent, RealtimeTransport} from './transport.js';
|
|
2
|
+
|
|
3
|
+
export type ConnectionState = 'disconnected' | 'connecting' | 'connected' | 'failed';
|
|
4
|
+
|
|
5
|
+
export interface FunctionCall {
|
|
6
|
+
callId: string;
|
|
7
|
+
name: string;
|
|
8
|
+
argumentsJson: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface RealtimeSessionHooks {
|
|
12
|
+
/** Runs a function exported as a tool and resolves with its return value. */
|
|
13
|
+
callFunction: (name: string, args: unknown) => Promise<unknown>;
|
|
14
|
+
onResponseText: (text: string) => void;
|
|
15
|
+
onError: (message: string) => void;
|
|
16
|
+
onStateChange?: (state: ConnectionState) => void;
|
|
17
|
+
/** Receives `response.usage` from every `response.done`, including function-call rounds. */
|
|
18
|
+
onUsage?: (usage: unknown) => void;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Conversation logic over a transport: sending user text, collecting assistant output, and
|
|
23
|
+
* answering function calls with `function_call_output` followed by one `response.create`.
|
|
24
|
+
*/
|
|
25
|
+
export class RealtimeSession {
|
|
26
|
+
private stateValue: ConnectionState = 'disconnected';
|
|
27
|
+
private transport: RealtimeTransport | null = null;
|
|
28
|
+
private generation = 0;
|
|
29
|
+
|
|
30
|
+
public constructor(private readonly hooks: RealtimeSessionHooks) {}
|
|
31
|
+
|
|
32
|
+
public get state(): ConnectionState {
|
|
33
|
+
return this.stateValue;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
public async open(
|
|
37
|
+
transport: RealtimeTransport,
|
|
38
|
+
clientSecret: string,
|
|
39
|
+
microphone: boolean
|
|
40
|
+
): Promise<void> {
|
|
41
|
+
this.close();
|
|
42
|
+
const generation = ++this.generation;
|
|
43
|
+
this.transport = transport;
|
|
44
|
+
this.setState('connecting');
|
|
45
|
+
try {
|
|
46
|
+
await transport.connect({
|
|
47
|
+
clientSecret,
|
|
48
|
+
microphone,
|
|
49
|
+
onEvent: (event) => {
|
|
50
|
+
if (generation === this.generation) void this.handleEvent(event);
|
|
51
|
+
},
|
|
52
|
+
onClose: (reason) => {
|
|
53
|
+
if (generation !== this.generation) return;
|
|
54
|
+
this.transport = null;
|
|
55
|
+
this.hooks.onError(reason);
|
|
56
|
+
this.setState('failed');
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
} catch (error) {
|
|
60
|
+
if (generation === this.generation) {
|
|
61
|
+
this.transport = null;
|
|
62
|
+
this.setState('failed');
|
|
63
|
+
}
|
|
64
|
+
throw error;
|
|
65
|
+
}
|
|
66
|
+
if (generation === this.generation) this.setState('connected');
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
public close(): void {
|
|
70
|
+
this.generation += 1;
|
|
71
|
+
const transport = this.transport;
|
|
72
|
+
this.transport = null;
|
|
73
|
+
transport?.close();
|
|
74
|
+
if (this.stateValue !== 'disconnected') this.setState('disconnected');
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
public sendText(text: string): void {
|
|
78
|
+
this.send({
|
|
79
|
+
type: 'conversation.item.create',
|
|
80
|
+
item: {type: 'message', role: 'user', content: [{type: 'input_text', text}]}
|
|
81
|
+
});
|
|
82
|
+
this.send({type: 'response.create'});
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
private send(event: RealtimeEvent): void {
|
|
86
|
+
if (!this.transport || this.stateValue !== 'connected') throw new Error('Not connected to Realtime.');
|
|
87
|
+
this.transport.send(event);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
private async handleEvent(event: RealtimeEvent): Promise<void> {
|
|
91
|
+
if (event.type === 'error') {
|
|
92
|
+
this.hooks.onError(describeApiError(event));
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
if (event.type !== 'response.done') return;
|
|
96
|
+
const response = event.response;
|
|
97
|
+
if (typeof response === 'object' && response !== null && 'usage' in response) {
|
|
98
|
+
this.hooks.onUsage?.((response as {usage: unknown}).usage);
|
|
99
|
+
}
|
|
100
|
+
const output = readResponseOutput(event);
|
|
101
|
+
if (output.text.length > 0) this.hooks.onResponseText(output.text);
|
|
102
|
+
if (output.calls.length === 0) return;
|
|
103
|
+
|
|
104
|
+
const generation = this.generation;
|
|
105
|
+
const results = await Promise.all(output.calls.map((call) => this.runCall(call)));
|
|
106
|
+
if (generation !== this.generation || !this.transport) return;
|
|
107
|
+
for (const result of results) {
|
|
108
|
+
this.transport.send({
|
|
109
|
+
type: 'conversation.item.create',
|
|
110
|
+
item: {type: 'function_call_output', call_id: result.callId, output: result.output}
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
this.transport.send({type: 'response.create'});
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
private async runCall(call: FunctionCall): Promise<{callId: string; output: string}> {
|
|
117
|
+
let args: unknown;
|
|
118
|
+
try {
|
|
119
|
+
args = call.argumentsJson.trim().length === 0 ? {} : (JSON.parse(call.argumentsJson) as unknown);
|
|
120
|
+
} catch {
|
|
121
|
+
return {callId: call.callId, output: JSON.stringify({error: 'Arguments were not valid JSON.'})};
|
|
122
|
+
}
|
|
123
|
+
try {
|
|
124
|
+
const value = await this.hooks.callFunction(call.name, args);
|
|
125
|
+
return {callId: call.callId, output: JSON.stringify(value ?? null)};
|
|
126
|
+
} catch (error) {
|
|
127
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
128
|
+
this.hooks.onError(message);
|
|
129
|
+
return {callId: call.callId, output: JSON.stringify({error: message})};
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
private setState(state: ConnectionState): void {
|
|
134
|
+
this.stateValue = state;
|
|
135
|
+
this.hooks.onStateChange?.(state);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
export function readResponseOutput(event: RealtimeEvent): {text: string; calls: FunctionCall[]} {
|
|
140
|
+
const response = asRecord(event.response);
|
|
141
|
+
const items = Array.isArray(response?.output) ? response.output : [];
|
|
142
|
+
const texts: string[] = [];
|
|
143
|
+
const calls: FunctionCall[] = [];
|
|
144
|
+
for (const rawItem of items) {
|
|
145
|
+
const item = asRecord(rawItem);
|
|
146
|
+
if (!item) continue;
|
|
147
|
+
if (item.type === 'function_call' && typeof item.name === 'string' && typeof item.call_id === 'string') {
|
|
148
|
+
calls.push({
|
|
149
|
+
callId: item.call_id,
|
|
150
|
+
name: item.name,
|
|
151
|
+
argumentsJson: typeof item.arguments === 'string' ? item.arguments : ''
|
|
152
|
+
});
|
|
153
|
+
} else if (item.type === 'message' && Array.isArray(item.content)) {
|
|
154
|
+
for (const rawPart of item.content) {
|
|
155
|
+
const part = asRecord(rawPart);
|
|
156
|
+
if (typeof part?.text === 'string') texts.push(part.text);
|
|
157
|
+
else if (typeof part?.transcript === 'string') texts.push(part.transcript);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
return {text: texts.join(''), calls};
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function describeApiError(event: RealtimeEvent): string {
|
|
165
|
+
const error = asRecord(event.error);
|
|
166
|
+
const message = typeof error?.message === 'string' ? error.message : 'Unknown Realtime API error.';
|
|
167
|
+
const code = typeof error?.code === 'string' ? ` (${error.code})` : '';
|
|
168
|
+
return `${message}${code}`;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function asRecord(value: unknown): Record<string, unknown> | null {
|
|
172
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
|
173
|
+
? (value as Record<string, unknown>)
|
|
174
|
+
: null;
|
|
175
|
+
}
|