@kubohiroya/turbowarp-openai-realtime-api 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,89 @@
1
+ /**
2
+ * Composition API: the OpenAI Realtime capability without TurboWarp block definitions.
3
+ *
4
+ * A downstream extension (for example `@kubohiroya/turbowarp-voice-chat`) owns its own blocks and
5
+ * hats, and passes its own `define function` hat opcode. Importing this module does not register
6
+ * any TurboWarp extension and does not touch the `Scratch` global.
7
+ */
8
+ import { type FunctionScan, type NamedFunctions, type RuntimeLike, type RuntimeThread } from '@kubohiroya/turbowarp-named-functions/composition';
9
+ import { type ConnectionState } from './realtime-session.js';
10
+ import { type FetchLike } from './relay-client.js';
11
+ import { type OutputMode, type SessionSettings } from './session-config.js';
12
+ import type { RealtimeTransport } from './transport.js';
13
+ import { type UsageTotals } from './usage.js';
14
+ export type { ConnectionState } from './realtime-session.js';
15
+ export type { OutputMode, SessionSettings } from './session-config.js';
16
+ export { SUPPORTED_MODELS } from './session-config.js';
17
+ export type { UsageTotals } from './usage.js';
18
+ export { MODEL_PRICES, PRICES_AS_OF } from './usage.js';
19
+ export type { RealtimeTransport, RealtimeEvent, TransportConnectOptions } from './transport.js';
20
+ export type { NamedFunctions, RuntimeLike, RuntimeThread } from '@kubohiroya/turbowarp-named-functions/composition';
21
+ export declare const DEFAULT_RELAY_ENDPOINT = "http://127.0.0.1:8787";
22
+ export type RealtimeCompositionEvent = {
23
+ type: 'state';
24
+ state: ConnectionState;
25
+ } | {
26
+ type: 'response';
27
+ text: string;
28
+ } | {
29
+ type: 'usage';
30
+ usage: Readonly<UsageTotals>;
31
+ } | {
32
+ type: 'error';
33
+ message: string;
34
+ } | {
35
+ type: 'sessionTimeLimitReached';
36
+ };
37
+ export type RealtimeCompositionListener = (event: RealtimeCompositionEvent) => void;
38
+ export interface RealtimeCompositionOptions {
39
+ runtime: RuntimeLike;
40
+ /** Opcode of the downstream extension's `define function` hat, e.g. `myextension_defineFunction`. */
41
+ functionHatOpcode: string;
42
+ /**
43
+ * Named functions to expose as tools. Pass one shared instance when another composition also uses
44
+ * the same `define function` hats; otherwise one is created from `functionHatOpcode`.
45
+ */
46
+ functions?: NamedFunctions;
47
+ fetch?: FetchLike;
48
+ createTransport?: () => RealtimeTransport;
49
+ now?: () => number;
50
+ setTimer?: (callback: () => void, ms: number) => unknown;
51
+ clearTimer?: (handle: unknown) => void;
52
+ }
53
+ export interface RealtimeComposition {
54
+ configureRelay(endpoint: string): void;
55
+ pairRelay(code: string): Promise<void>;
56
+ isRelayPaired(): boolean;
57
+ setModel(model: string): void;
58
+ setInstructions(text: string): void;
59
+ setVoice(voice: string): void;
60
+ setOutputMode(mode: OutputMode): void;
61
+ /** Seconds; 0 disables the limit. Applies from the next connection. */
62
+ setSessionTimeLimit(seconds: number): void;
63
+ readonly settings: Readonly<SessionSettings>;
64
+ readonly sessionTimeLimitSeconds: number;
65
+ /** Validates `define function` hats without connecting. */
66
+ scanFunctions(): FunctionScan;
67
+ /** The named functions used for tools. */
68
+ readonly functions: NamedFunctions;
69
+ connect(options: {
70
+ microphone: boolean;
71
+ }): Promise<void>;
72
+ disconnect(): void;
73
+ readonly state: ConnectionState;
74
+ /** Model reported by the relay for the current or last connection; empty before connecting. */
75
+ readonly activeModel: string;
76
+ sessionElapsedSeconds(): number;
77
+ sendText(text: string): void;
78
+ readonly lastResponseText: string;
79
+ /** Call from the `define function` hat predicate. */
80
+ matchFunctionHat(name: string, thread: RuntimeThread | undefined): boolean;
81
+ functionArguments(thread: RuntimeThread | undefined): unknown;
82
+ returnFromFunction(thread: RuntimeThread | undefined, value: unknown): void;
83
+ usage(): Readonly<UsageTotals>;
84
+ resetUsage(): void;
85
+ subscribe(listener: RealtimeCompositionListener): () => void;
86
+ /** Disconnects and detaches listeners. */
87
+ release(): void;
88
+ }
89
+ export declare function createRealtimeComposition(options: RealtimeCompositionOptions): RealtimeComposition;
@@ -0,0 +1,11 @@
1
+ export declare const extensionConfig: {
2
+ readonly id: "kubohiroyaopenairealtime";
3
+ readonly slug: "turbowarp-openai-realtime-api";
4
+ readonly name: "TurboWarp-OpenAI-Realtime-API";
5
+ readonly description: "Talk with the OpenAI Realtime API from TurboWarp through a localhost relay.";
6
+ readonly author: "Hiroya Kubo";
7
+ readonly license: "MPL-2.0";
8
+ readonly unsandboxed: true;
9
+ readonly docsURI: "https://kubohiroya.github.io/turbowarp-openai-realtime-api/";
10
+ readonly blockIconURI: "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0OCA0OCI+PHBhdGggZD0iTTggMTBoMzJhNCA0IDAgMCAxIDQgNHYxNmE0IDQgMCAwIDEtNCA0SDIybC05IDd2LTdIOGE0IDQgMCAwIDEtNC00VjE0YTQgNCAwIDAgMSA0LTR6IiBmaWxsPSIjMTBBMzdGIi8+PGcgc3Ryb2tlPSIjZmZmIiBzdHJva2Utd2lkdGg9IjMiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCI+PHBhdGggZD0iTTE0IDE5djYiLz48cGF0aCBkPSJNMjAgMTZ2MTIiLz48cGF0aCBkPSJNMjYgMTh2OCIvPjxwYXRoIGQ9Ik0zMiAxNXYxNCIvPjxwYXRoIGQ9Ik0zOCAyMHY0Ii8+PC9nPjwvc3ZnPg==";
11
+ };
@@ -0,0 +1,30 @@
1
+ import type { Plugin } from 'vite';
2
+ export declare const EXTENSION_MANIFEST_FORMAT_VERSION: 1;
3
+ export interface ExtensionManifestArgument {
4
+ id: string;
5
+ type: string;
6
+ menu?: string;
7
+ }
8
+ export interface ExtensionManifestBlock {
9
+ opcode: string;
10
+ blockType: string;
11
+ arguments: ExtensionManifestArgument[];
12
+ }
13
+ export interface ExtensionManifestMenu {
14
+ id: string;
15
+ acceptReporters: boolean;
16
+ }
17
+ export interface ExtensionManifest {
18
+ formatVersion: typeof EXTENSION_MANIFEST_FORMAT_VERSION;
19
+ id: string;
20
+ blocks: ExtensionManifestBlock[];
21
+ menus: ExtensionManifestMenu[];
22
+ }
23
+ export interface ExtensionManifestPluginOptions {
24
+ id: string;
25
+ definitions: unknown;
26
+ fileName?: string;
27
+ }
28
+ export declare function createExtensionManifest(id: string, definitions: unknown): ExtensionManifest;
29
+ export declare function serializeExtensionManifest(id: string, definitions: unknown): string;
30
+ export declare function extensionManifestPlugin(options: ExtensionManifestPluginOptions): Plugin;
@@ -0,0 +1,45 @@
1
+ import { type RealtimeCompositionOptions } from './composition.js';
2
+ import { type BlockUtilityLike } from '@kubohiroya/turbowarp-named-functions/composition';
3
+ export declare const DEFINE_FUNCTION_OPCODE: string;
4
+ export declare const RESPONSE_DONE_OPCODE: string;
5
+ export declare const TIME_LIMIT_OPCODE: string;
6
+ export type RealtimeExtensionDependencies = Omit<RealtimeCompositionOptions, 'functionHatOpcode'>;
7
+ type Args = Record<string, unknown>;
8
+ /** Block surface over the Realtime composition. */
9
+ export declare class OpenAIRealtimeExtension implements TurboWarpExtension {
10
+ private lastErrorMessage;
11
+ private readonly realtime;
12
+ constructor(deps: RealtimeExtensionDependencies);
13
+ getInfo(): Record<string, unknown>;
14
+ configureRelay(args: Args): void;
15
+ pairRelay(args: Args): Promise<void>;
16
+ isRelayPaired(): boolean;
17
+ setInstructions(args: Args): void;
18
+ setVoice(args: Args): void;
19
+ setOutputMode(args: Args): void;
20
+ setModel(args: Args): void;
21
+ setSessionTimeLimit(args: Args): void;
22
+ connect(args: Args): Promise<void>;
23
+ disconnect(): void;
24
+ isConnected(): boolean;
25
+ connectionState(): string;
26
+ currentModel(): string;
27
+ sessionElapsed(): number;
28
+ whenSessionTimeLimitReached(): boolean;
29
+ sendText(args: Args): void;
30
+ whenResponseDone(): boolean;
31
+ lastResponseText(): string;
32
+ defineFunction(args: Args, util?: BlockUtilityLike): boolean;
33
+ functionArgument(args: Args, util?: BlockUtilityLike): string | number | boolean;
34
+ functionArgumentsJson(_args: Args, util?: BlockUtilityLike): string;
35
+ returnValue(args: Args, util?: BlockUtilityLike & {
36
+ stopThisScript?: () => void;
37
+ }): void;
38
+ usageValue(args: Args): number | string;
39
+ resetUsage(): void;
40
+ lastError(): string;
41
+ private record;
42
+ private recordAsync;
43
+ private toScratchBlock;
44
+ }
45
+ export {};
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,39 @@
1
+ import type { RealtimeEvent, RealtimeTransport } from './transport.js';
2
+ export type ConnectionState = 'disconnected' | 'connecting' | 'connected' | 'failed';
3
+ export interface FunctionCall {
4
+ callId: string;
5
+ name: string;
6
+ argumentsJson: string;
7
+ }
8
+ export interface RealtimeSessionHooks {
9
+ /** Runs a function exported as a tool and resolves with its return value. */
10
+ callFunction: (name: string, args: unknown) => Promise<unknown>;
11
+ onResponseText: (text: string) => void;
12
+ onError: (message: string) => void;
13
+ onStateChange?: (state: ConnectionState) => void;
14
+ /** Receives `response.usage` from every `response.done`, including function-call rounds. */
15
+ onUsage?: (usage: unknown) => void;
16
+ }
17
+ /**
18
+ * Conversation logic over a transport: sending user text, collecting assistant output, and
19
+ * answering function calls with `function_call_output` followed by one `response.create`.
20
+ */
21
+ export declare class RealtimeSession {
22
+ private readonly hooks;
23
+ private stateValue;
24
+ private transport;
25
+ private generation;
26
+ constructor(hooks: RealtimeSessionHooks);
27
+ get state(): ConnectionState;
28
+ open(transport: RealtimeTransport, clientSecret: string, microphone: boolean): Promise<void>;
29
+ close(): void;
30
+ sendText(text: string): void;
31
+ private send;
32
+ private handleEvent;
33
+ private runCall;
34
+ private setState;
35
+ }
36
+ export declare function readResponseOutput(event: RealtimeEvent): {
37
+ text: string;
38
+ calls: FunctionCall[];
39
+ };
@@ -0,0 +1,16 @@
1
+ import type { RealtimeSessionRequest } from './session-config.js';
2
+ export type FetchLike = (input: string, init?: RequestInit) => Promise<Response>;
3
+ export interface RelaySession {
4
+ endpoint: string;
5
+ token: string;
6
+ expiresAt: number;
7
+ }
8
+ export interface ClientSecret {
9
+ value: string;
10
+ expiresAt: number;
11
+ model: string;
12
+ }
13
+ /** Accepts only a plain loopback HTTP origin so the relay token can never leave this machine. */
14
+ export declare function normalizeRelayEndpoint(endpoint: string): string;
15
+ export declare function pairWithRelay(endpoint: string, code: string, fetcher?: FetchLike, now?: () => number): Promise<RelaySession>;
16
+ export declare function requestClientSecret(session: RelaySession, request: RealtimeSessionRequest, fetcher?: FetchLike, now?: () => number): Promise<ClientSecret>;
@@ -0,0 +1,27 @@
1
+ import type { FunctionTool } from '@kubohiroya/turbowarp-named-functions/composition';
2
+ export type OutputMode = 'audio' | 'text';
3
+ /** Request body sent to the relay's client-secret route (see capability-proxy). */
4
+ export interface RealtimeSessionRequest {
5
+ model?: string;
6
+ instructions?: string;
7
+ voice?: string;
8
+ outputModalities?: [OutputMode];
9
+ tools?: FunctionTool[];
10
+ }
11
+ export interface SessionSettings {
12
+ /** Empty means the relay's default model. */
13
+ model: string;
14
+ instructions: string;
15
+ voice: string;
16
+ outputMode: OutputMode;
17
+ }
18
+ export declare const MAX_INSTRUCTIONS_LENGTH = 16384;
19
+ export declare const MAX_TOOLS = 32;
20
+ export declare const SUPPORTED_MODELS: readonly ["gpt-realtime-2.1-mini", "gpt-realtime-2.1"];
21
+ export declare function defaultSessionSettings(): SessionSettings;
22
+ /** Accepts an empty string (relay default) or a model identifier; the relay decides what is allowed. */
23
+ export declare function normalizeModel(value: string): string;
24
+ export declare function normalizeVoice(value: string): string;
25
+ export declare function normalizeInstructions(value: string): string;
26
+ export declare function normalizeOutputMode(value: string): OutputMode;
27
+ export declare function buildSessionRequest(settings: SessionSettings, tools: readonly FunctionTool[]): RealtimeSessionRequest;
@@ -0,0 +1,18 @@
1
+ export type RealtimeEvent = Record<string, unknown> & {
2
+ type: string;
3
+ };
4
+ export interface TransportConnectOptions {
5
+ clientSecret: string;
6
+ microphone: boolean;
7
+ onEvent: (event: RealtimeEvent) => void;
8
+ onClose: (reason: string) => void;
9
+ }
10
+ /**
11
+ * A connection to the Realtime API. WebRTC is the primary implementation; a WebSocket transport can
12
+ * implement the same interface later because both carry the same JSON events.
13
+ */
14
+ export interface RealtimeTransport {
15
+ connect(options: TransportConnectOptions): Promise<void>;
16
+ send(event: RealtimeEvent): void;
17
+ close(): void;
18
+ }
@@ -0,0 +1,34 @@
1
+ /**
2
+ * Token usage totals from `response.done` events and an estimated cost.
3
+ *
4
+ * Prices are USD per 1M tokens from the OpenAI pricing page as of 2026-09-18. They are an estimate
5
+ * for learners; the OpenAI dashboard is the source of truth for billing.
6
+ */
7
+ export interface ModelPrice {
8
+ textInput: number;
9
+ cachedTextInput: number;
10
+ textOutput: number;
11
+ audioInput: number;
12
+ cachedAudioInput: number;
13
+ audioOutput: number;
14
+ }
15
+ export declare const MODEL_PRICES: Readonly<Record<string, ModelPrice>>;
16
+ export declare const PRICES_AS_OF = "2026-09-18";
17
+ export interface UsageTotals {
18
+ responses: number;
19
+ inputTokens: number;
20
+ outputTokens: number;
21
+ cachedInputTokens: number;
22
+ textInputTokens: number;
23
+ audioInputTokens: number;
24
+ cachedTextInputTokens: number;
25
+ cachedAudioInputTokens: number;
26
+ textOutputTokens: number;
27
+ audioOutputTokens: number;
28
+ estimatedCostUsd: number;
29
+ /** Responses whose model has no known price; their cost is not included. */
30
+ unpricedResponses: number;
31
+ }
32
+ export declare function emptyUsage(): UsageTotals;
33
+ /** Adds one `response.usage` object to the totals. Unknown or malformed usage counts as zero tokens. */
34
+ export declare function addUsage(totals: UsageTotals, usage: unknown, model: string): UsageTotals;
@@ -0,0 +1,26 @@
1
+ import type { FetchLike } from './relay-client.js';
2
+ import type { RealtimeEvent, RealtimeTransport, TransportConnectOptions } from './transport.js';
3
+ export declare const REALTIME_CALLS_URL = "https://api.openai.com/v1/realtime/calls";
4
+ export declare const EVENTS_CHANNEL = "oai-events";
5
+ export interface WebRtcDependencies {
6
+ createPeerConnection: () => RTCPeerConnection;
7
+ getUserMedia: (constraints: MediaStreamConstraints) => Promise<MediaStream>;
8
+ createAudioElement: () => HTMLAudioElement;
9
+ fetch: FetchLike;
10
+ channelOpenTimeoutMs?: number;
11
+ }
12
+ export declare function browserWebRtcDependencies(): WebRtcDependencies;
13
+ /** Browser-to-OpenAI WebRTC session authenticated with an ephemeral client secret. */
14
+ export declare class WebRtcTransport implements RealtimeTransport {
15
+ private readonly deps;
16
+ private peer;
17
+ private channel;
18
+ private microphone;
19
+ private audio;
20
+ private closed;
21
+ constructor(deps?: WebRtcDependencies);
22
+ connect(options: TransportConnectOptions): Promise<void>;
23
+ send(event: RealtimeEvent): void;
24
+ close(): void;
25
+ private handleClose;
26
+ }
package/package.json ADDED
@@ -0,0 +1,79 @@
1
+ {
2
+ "name": "@kubohiroya/turbowarp-openai-realtime-api",
3
+ "version": "0.1.0",
4
+ "description": "A TurboWarp extension for the OpenAI Realtime API, using a localhost relay for credentials.",
5
+ "author": "Hiroya Kubo <hiroya@cuc.ac.jp>",
6
+ "license": "MPL-2.0",
7
+ "homepage": "https://github.com/kubohiroya/turbowarp-openai-realtime-api#readme",
8
+ "repository": {
9
+ "type": "git",
10
+ "url": "git+https://github.com/kubohiroya/turbowarp-openai-realtime-api.git"
11
+ },
12
+ "bugs": {
13
+ "url": "https://github.com/kubohiroya/turbowarp-openai-realtime-api/issues"
14
+ },
15
+ "keywords": [
16
+ "turbowarp",
17
+ "scratch",
18
+ "extension",
19
+ "openai",
20
+ "realtime",
21
+ "webrtc",
22
+ "voice",
23
+ "vite",
24
+ "typescript"
25
+ ],
26
+ "type": "module",
27
+ "files": [
28
+ "dist/",
29
+ "schemas/",
30
+ "src/",
31
+ "README.md",
32
+ "README.ja.md",
33
+ "LICENSE"
34
+ ],
35
+ "publishConfig": {
36
+ "access": "public",
37
+ "registry": "https://registry.npmjs.org/"
38
+ },
39
+ "packageManager": "pnpm@11.11.0",
40
+ "engines": {
41
+ "node": ">=22.18.0"
42
+ },
43
+ "scripts": {
44
+ "dev": "vite build --watch",
45
+ "typecheck": "tsc --noEmit",
46
+ "build": "vite build && vite build --config vite.composition.config.ts && tsc --project tsconfig.build.json",
47
+ "lint": "eslint src tests scripts vite.config.ts vite.composition.config.ts eslint.config.ts",
48
+ "test": "vitest run",
49
+ "docs": "node scripts/generate-readme.ts",
50
+ "docs:check": "node scripts/generate-readme.ts --check",
51
+ "repo:check": "node scripts/check-repo.ts",
52
+ "pack:check": "npm pack --dry-run --ignore-scripts",
53
+ "check": "pnpm run typecheck && pnpm run lint && pnpm run test && pnpm run docs:check && pnpm run check:dist && pnpm run types:composition:check && pnpm run repo:check && pnpm run pack:check",
54
+ "check:dist": "pnpm run build && node scripts/check-dist.ts",
55
+ "types:composition:check": "tsc --project tsconfig.composition-consumer.json"
56
+ },
57
+ "devDependencies": {
58
+ "@eslint/js": "^9.39.5",
59
+ "@kubohiroya/vite-plugin-turbowarp-extension": "0.3.0",
60
+ "@types/node": "^24.13.3",
61
+ "eslint": "^9.39.5",
62
+ "jiti": "^2.7.0",
63
+ "typescript": "^5.9.3",
64
+ "typescript-eslint": "^8.66.0",
65
+ "vite": "^8.2.2",
66
+ "vitest": "^5.0.0"
67
+ },
68
+ "exports": {
69
+ "./composition": {
70
+ "types": "./dist/types/composition.d.ts",
71
+ "import": "./dist/composition.js"
72
+ },
73
+ "./turbowarp-openai-realtime-api.js": "./dist/turbowarp-openai-realtime-api.js",
74
+ "./package.json": "./package.json"
75
+ },
76
+ "dependencies": {
77
+ "@kubohiroya/turbowarp-named-functions": "0.1.0"
78
+ }
79
+ }
@@ -0,0 +1,49 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "https://raw.githubusercontent.com/kubohiroya/turbowarp-openai-realtime-api/main/schemas/extension-manifest.schema.json",
4
+ "title": "TurboWarp Extension API Manifest",
5
+ "type": "object",
6
+ "additionalProperties": false,
7
+ "required": ["formatVersion", "id", "blocks", "menus"],
8
+ "properties": {
9
+ "formatVersion": {"const": 1},
10
+ "id": {"type": "string", "pattern": "^[a-z0-9]+$"},
11
+ "blocks": {
12
+ "type": "array",
13
+ "items": {
14
+ "type": "object",
15
+ "additionalProperties": false,
16
+ "required": ["opcode", "blockType", "arguments"],
17
+ "properties": {
18
+ "opcode": {"type": "string", "minLength": 1},
19
+ "blockType": {"type": "string", "minLength": 1},
20
+ "arguments": {
21
+ "type": "array",
22
+ "items": {
23
+ "type": "object",
24
+ "additionalProperties": false,
25
+ "required": ["id", "type"],
26
+ "properties": {
27
+ "id": {"type": "string", "minLength": 1},
28
+ "type": {"type": "string", "minLength": 1},
29
+ "menu": {"type": "string", "minLength": 1}
30
+ }
31
+ }
32
+ }
33
+ }
34
+ }
35
+ },
36
+ "menus": {
37
+ "type": "array",
38
+ "items": {
39
+ "type": "object",
40
+ "additionalProperties": false,
41
+ "required": ["id", "acceptReporters"],
42
+ "properties": {
43
+ "id": {"type": "string", "minLength": 1},
44
+ "acceptReporters": {"type": "boolean"}
45
+ }
46
+ }
47
+ }
48
+ }
49
+ }