@gravity-platform/openai-realtime 1.0.1

package/dist/Realtime/service/core/processing/TextAccumulator.js

@@ -0,0 +1,75 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TextAccumulator = void 0;
+const plugin_base_1 = require("@gravity-platform/plugin-base");
+// Module-level platform calls cause startup freezes (docs-starter/nodes/CLAUDE.md
+// rule 5) — resolve the logger lazily instead
+function getLogger() {
+    return (0, plugin_base_1.getPlatformDependencies)().createLogger("RealtimeTextAccumulator");
+}
+class TextAccumulator {
+    constructor(sessionId, emit) {
+        this.sessionId = sessionId;
+        this.emit = emit;
+        this.transcription = "";
+        this.assistantResponse = "";
+        this.allTurns = [];
+        this.progressLog = "";
+        this.logger = getLogger();
+    }
+    appendAssistant(text) {
+        this.assistantResponse += text;
+    }
+    setAssistantText(text) {
+        this.assistantResponse = text;
+    }
+    setUserTranscript(transcript) {
+        this.transcription = transcript;
+    }
+    emitProgress(text) {
+        this.progressLog += text;
+        this.emit?.({ __outputs: { progress: this.progressLog } });
+    }
+    emitConversation() {
+        this.logger.debug("Turn complete", {
+            sessionId: this.sessionId,
+            queryLength: this.transcription.length,
+            responseLength: this.assistantResponse.length,
+        });
+        if (this.transcription || this.assistantResponse) {
+            this.allTurns.push({ query: this.transcription, response: this.assistantResponse });
+            const turnNum = this.allTurns.length;
+            const q = this.transcription ? `Q${turnNum}: ${this.transcription}\n` : "";
+            const a = this.assistantResponse ? `A${turnNum}: ${this.assistantResponse}\n` : "";
+            this.emitProgress(q + a);
+        }
+    }
+    emitFinal() {
+        if (!this.emit || this.allTurns.length === 0)
+            return;
+        const lastTurn = this.allTurns[this.allTurns.length - 1];
+        this.logger.info("Emitting final conversation", {
+            sessionId: this.sessionId,
+            turns: this.allTurns.length,
+        });
+        this.emit({
+            __outputs: {
+                text: {
+                    query: lastTurn.query,
+                    response: lastTurn.response,
+                },
+            },
+        });
+    }
+    resetTurn() {
+        this.transcription = "";
+        this.assistantResponse = "";
+    }
+    getTranscription() {
+        return this.transcription;
+    }
+    getAssistantResponse() {
+        return this.assistantResponse;
+    }
+}
+exports.TextAccumulator = TextAccumulator;

package/dist/Realtime/service/core/processing/UsageStatsCollector.d.ts

@@ -0,0 +1,14 @@
+import { StreamUsageStats } from "../../../../util/types";
+export declare class UsageStatsCollector {
+    private stats;
+    incrementChunkCount(): void;
+    /**
+     * Accumulate per-response usage. OpenAI bills each response with its own
+     * input tokens (including context), so summing across the session's turns is
+     * the accurate total — overwriting would record only the final turn.
+     */
+    addUsage(inputTokens: number, outputTokens: number): void;
+    setTextResults(transcription: string, assistantResponse: string): void;
+    getUsageStats(): StreamUsageStats;
+    reset(): void;
+}

package/dist/Realtime/service/core/processing/UsageStatsCollector.js

@@ -0,0 +1,52 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UsageStatsCollector = void 0;
+class UsageStatsCollector {
+    constructor() {
+        this.stats = {
+            estimated: false,
+            total_tokens: 0,
+            inputTokens: 0,
+            outputTokens: 0,
+            chunk_count: 0,
+            textOutput: "",
+            transcription: "",
+            assistantResponse: "",
+        };
+    }
+    incrementChunkCount() {
+        this.stats.chunk_count++;
+    }
+    /**
+     * Accumulate per-response usage. OpenAI bills each response with its own
+     * input tokens (including context), so summing across the session's turns is
+     * the accurate total — overwriting would record only the final turn.
+     */
+    addUsage(inputTokens, outputTokens) {
+        this.stats.inputTokens += inputTokens;
+        this.stats.outputTokens += outputTokens;
+        this.stats.total_tokens = this.stats.inputTokens + this.stats.outputTokens;
+        this.stats.estimated = false;
+    }
+    setTextResults(transcription, assistantResponse) {
+        this.stats.transcription = transcription;
+        this.stats.assistantResponse = assistantResponse;
+        this.stats.textOutput = transcription + assistantResponse;
+    }
+    getUsageStats() {
+        return { ...this.stats };
+    }
+    reset() {
+        this.stats = {
+            estimated: false,
+            total_tokens: 0,
+            inputTokens: 0,
+            outputTokens: 0,
+            chunk_count: 0,
+            textOutput: "",
+            transcription: "",
+            assistantResponse: "",
+        };
+    }
+}
+exports.UsageStatsCollector = UsageStatsCollector;

package/dist/Realtime/service/core/streaming/RealtimeSessionRegistry.d.ts

@@ -0,0 +1,10 @@
+import { WsClient } from "./WsClient";
+declare class RealtimeSessionRegistry {
+    private sessions;
+    register(conversationId: string, wsClient: WsClient): void;
+    get(conversationId: string): WsClient | undefined;
+    remove(conversationId: string): void;
+    clear(): void;
+}
+export declare const realtimeSessionRegistry: RealtimeSessionRegistry;
+export {};

package/dist/Realtime/service/core/streaming/RealtimeSessionRegistry.js

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.realtimeSessionRegistry = void 0;
+class RealtimeSessionRegistry {
+    constructor() {
+        this.sessions = new Map();
+    }
+    register(conversationId, wsClient) {
+        // A duplicate START_CALL must not orphan the previous session: the old
+        // client would become unreachable by END_CALL and stay open (and billed)
+        const existing = this.sessions.get(conversationId);
+        if (existing && existing !== wsClient) {
+            try {
+                existing.close();
+            }
+            catch {
+                /* already closed */
+            }
+        }
+        this.sessions.set(conversationId, wsClient);
+    }
+    get(conversationId) {
+        return this.sessions.get(conversationId);
+    }
+    remove(conversationId) {
+        this.sessions.delete(conversationId);
+    }
+    clear() {
+        this.sessions.clear();
+    }
+}
+exports.realtimeSessionRegistry = new RealtimeSessionRegistry();

package/dist/Realtime/service/core/streaming/SessionManager.d.ts

@@ -0,0 +1,14 @@
+import { OpenAIRealtimeConfig, StreamingMetadata } from "../../../../util/types";
+import { WsClient } from "./WsClient";
+import { RealtimeResponseProcessor } from "../processing/ResponseProcessor";
+export interface RealtimeSession {
+    sessionId: string;
+    isActive: boolean;
+    wsClient: WsClient;
+    responseProcessor: RealtimeResponseProcessor;
+}
+export declare class SessionManager {
+    private sessions;
+    createSession(_config: OpenAIRealtimeConfig, metadata: StreamingMetadata, responseProcessor: RealtimeResponseProcessor, wsClient: WsClient): RealtimeSession;
+    endSession(sessionId: string): void;
+}

package/dist/Realtime/service/core/streaming/SessionManager.js

@@ -0,0 +1,33 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SessionManager = void 0;
+const plugin_base_1 = require("@gravity-platform/plugin-base");
+function getLogger() {
+    return (0, plugin_base_1.getPlatformDependencies)().createLogger("RealtimeSessionManager");
+}
+class SessionManager {
+    constructor() {
+        this.sessions = new Map();
+    }
+    createSession(_config, metadata, responseProcessor, wsClient) {
+        const sessionId = metadata.workflowId || `session_${Date.now()}`;
+        const session = {
+            sessionId,
+            isActive: true,
+            wsClient,
+            responseProcessor,
+        };
+        this.sessions.set(sessionId, session);
+        return session;
+    }
+    endSession(sessionId) {
+        const session = this.sessions.get(sessionId);
+        if (session) {
+            session.isActive = false;
+            session.responseProcessor.cleanup();
+            this.sessions.delete(sessionId);
+            getLogger().info("Session ended", { sessionId });
+        }
+    }
+}
+exports.SessionManager = SessionManager;

package/dist/Realtime/service/core/streaming/WsClient.d.ts

@@ -0,0 +1,11 @@
+export declare class WsClient {
+    private ws;
+    private logger;
+    private messageHandler;
+    connect(apiKey: string): Promise<void>;
+    send(event: any): void;
+    onMessage(handler: (event: any) => void): void;
+    close(): void;
+    waitForClose(): Promise<void>;
+    get isOpen(): boolean;
+}

package/dist/Realtime/service/core/streaming/WsClient.js

@@ -0,0 +1,93 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WsClient = void 0;
+const ws_1 = __importDefault(require("ws"));
+const plugin_base_1 = require("@gravity-platform/plugin-base");
+const constants_1 = require("../../../constants");
+function getLogger() {
+    return (0, plugin_base_1.getPlatformDependencies)().createLogger("RealtimeWsClient");
+}
+class WsClient {
+    constructor() {
+        this.ws = null;
+        this.logger = getLogger();
+        this.messageHandler = null;
+    }
+    async connect(apiKey) {
+        const url = `${constants_1.REALTIME_WS_URL}?model=${constants_1.REALTIME_MODEL_ID}`;
+        this.ws = new ws_1.default(url, {
+            headers: {
+                Authorization: `Bearer ${apiKey}`,
+            },
+        });
+        return new Promise((resolve, reject) => {
+            if (!this.ws)
+                return reject(new Error("WebSocket not initialized"));
+            let settled = false;
+            this.ws.once("open", () => {
+                settled = true;
+                this.logger.info("OpenAI Realtime WebSocket connected");
+                resolve();
+            });
+            // Persistent handler: an unhandled 'error' event on the socket would crash
+            // the process. Rejects the connect promise only during the connection phase.
+            this.ws.on("error", (err) => {
+                this.logger.error("WebSocket error", { error: err.message });
+                if (!settled) {
+                    settled = true;
+                    reject(err);
+                }
+            });
+            this.ws.on("message", (data) => {
+                try {
+                    const event = JSON.parse(data.toString());
+                    if (this.messageHandler) {
+                        this.messageHandler(event);
+                    }
+                }
+                catch (err) {
+                    this.logger.error("Failed to parse WebSocket message", { error: err.message });
+                }
+            });
+            this.ws.on("close", (code, reason) => {
+                this.logger.info("OpenAI Realtime WebSocket closed", { code, reason: reason.toString() });
+            });
+        });
+    }
+    send(event) {
+        if (!this.ws || this.ws.readyState !== ws_1.default.OPEN) {
+            this.logger.warn("Cannot send — WebSocket not open");
+            return;
+        }
+        this.ws.send(JSON.stringify(event));
+    }
+    onMessage(handler) {
+        this.messageHandler = handler;
+    }
+    close() {
+        if (this.ws) {
+            this.ws.close();
+            this.ws = null;
+        }
+    }
+    waitForClose() {
+        return new Promise((resolve) => {
+            if (!this.ws) {
+                resolve();
+                return;
+            }
+            if (!this.isOpen) {
+                resolve();
+                return;
+            }
+            this.ws.once("close", () => resolve());
+        });
+    }
+    get isOpen() {
+        return this.ws?.readyState === ws_1.default.OPEN;
+    }
+}
+exports.WsClient = WsClient;

package/dist/Realtime/service/index.d.ts

@@ -0,0 +1,6 @@
+import { OpenAIRealtimeConfig, StreamUsageStats, StreamingMetadata } from "../../util/types";
+export declare class RealtimeVoiceService {
+    private orchestrator;
+    generateVoiceStream(config: OpenAIRealtimeConfig, metadata: StreamingMetadata, context: any, emit?: (output: any) => void): Promise<StreamUsageStats>;
+}
+export type { OpenAIRealtimeConfig, StreamUsageStats, StreamingMetadata };

package/dist/Realtime/service/index.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RealtimeVoiceService = void 0;
+const SessionOrchestrator_1 = require("./core/orchestration/SessionOrchestrator");
+class RealtimeVoiceService {
+    constructor() {
+        this.orchestrator = new SessionOrchestrator_1.SessionOrchestrator();
+    }
+    async generateVoiceStream(config, metadata, context, emit) {
+        return this.orchestrator.orchestrateSession(config, metadata, context, emit);
+    }
+}
+exports.RealtimeVoiceService = RealtimeVoiceService;

package/dist/Realtime/service/io/events/incoming/builders/AudioAppendBuilder.d.ts

@@ -0,0 +1,4 @@
+export declare class AudioAppendBuilder {
+    static build(base64Audio: string): Record<string, unknown>;
+    static buildCommit(): Record<string, unknown>;
+}

package/dist/Realtime/service/io/events/incoming/builders/AudioAppendBuilder.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AudioAppendBuilder = void 0;
+class AudioAppendBuilder {
+    static build(base64Audio) {
+        return {
+            type: "input_audio_buffer.append",
+            audio: base64Audio,
+        };
+    }
+    static buildCommit() {
+        return { type: "input_audio_buffer.commit" };
+    }
+}
+exports.AudioAppendBuilder = AudioAppendBuilder;

package/dist/Realtime/service/io/events/incoming/builders/ConversationItemBuilder.d.ts

@@ -0,0 +1,5 @@
+export declare class ConversationItemBuilder {
+    static buildUserMessage(text: string): Record<string, unknown>;
+    static buildAssistantMessage(text: string): Record<string, unknown>;
+    static buildFunctionCallOutput(callId: string, output: string): Record<string, unknown>;
+}

package/dist/Realtime/service/io/events/incoming/builders/ConversationItemBuilder.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConversationItemBuilder = void 0;
+class ConversationItemBuilder {
+    static buildUserMessage(text) {
+        return {
+            type: "conversation.item.create",
+            item: {
+                type: "message",
+                role: "user",
+                content: [{ type: "input_text", text }],
+            },
+        };
+    }
+    static buildAssistantMessage(text) {
+        return {
+            type: "conversation.item.create",
+            item: {
+                type: "message",
+                role: "assistant",
+                content: [{ type: "text", text }],
+            },
+        };
+    }
+    static buildFunctionCallOutput(callId, output) {
+        return {
+            type: "conversation.item.create",
+            item: {
+                type: "function_call_output",
+                call_id: callId,
+                output,
+            },
+        };
+    }
+}
+exports.ConversationItemBuilder = ConversationItemBuilder;

package/dist/Realtime/service/io/events/incoming/builders/ResponseCreateBuilder.d.ts

@@ -0,0 +1,3 @@
+export declare class ResponseCreateBuilder {
+    static build(): Record<string, unknown>;
+}

package/dist/Realtime/service/io/events/incoming/builders/ResponseCreateBuilder.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ResponseCreateBuilder = void 0;
+class ResponseCreateBuilder {
+    static build() {
+        return { type: "response.create" };
+    }
+}
+exports.ResponseCreateBuilder = ResponseCreateBuilder;

package/dist/Realtime/service/io/events/incoming/builders/SessionUpdateBuilder.d.ts

@@ -0,0 +1,4 @@
+import { OpenAIRealtimeConfig } from "../../../../../../util/types";
+export declare class SessionUpdateBuilder {
+    static build(config: OpenAIRealtimeConfig): Record<string, unknown>;
+}

package/dist/Realtime/service/io/events/incoming/builders/SessionUpdateBuilder.js

@@ -0,0 +1,61 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SessionUpdateBuilder = void 0;
+const constants_1 = require("../../../../../constants");
+const VALID_VOICES = new Set(["alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse", "marin", "cedar"]);
+function normalizeVoice(input) {
+    if (!input)
+        return "alloy";
+    const lower = input.toLowerCase();
+    return VALID_VOICES.has(lower) ? lower : "alloy";
+}
+class SessionUpdateBuilder {
+    static build(config) {
+        const turnDetection = config.turnDetection === "disabled"
+            ? null
+            : config.turnDetection === "server_vad"
+                ? {
+                    type: "server_vad",
+                    create_response: true,
+                    interrupt_response: true,
+                    silence_duration_ms: 700,
+                    prefix_padding_ms: 300,
+                    threshold: 0.7,
+                }
+                : {
+                    type: "semantic_vad",
+                    create_response: true,
+                    interrupt_response: true,
+                    eagerness: "high",
+                };
+        const session = {
+            type: "realtime",
+            model: constants_1.REALTIME_MODEL_ID,
+            output_modalities: ["audio"],
+            audio: {
+                input: {
+                    format: { type: "audio/pcm", rate: 24000 },
+                    noise_reduction: { type: "near_field" },
+                    transcription: { model: "gpt-4o-mini-transcribe" },
+                    turn_detection: turnDetection,
+                },
+                output: {
+                    format: { type: "audio/pcm", rate: 24000 },
+                    voice: normalizeVoice(config.voice),
+                },
+            },
+        };
+        if (config.systemPrompt) {
+            session.instructions = config.systemPrompt;
+        }
+        if (config.maxResponseOutputTokens !== undefined) {
+            session.max_output_tokens = config.maxResponseOutputTokens;
+        }
+        if (config.tools && config.tools.length > 0) {
+            session.tools = config.tools;
+            session.tool_choice = "auto";
+        }
+        return { type: "session.update", session };
+    }
+}
+exports.SessionUpdateBuilder = SessionUpdateBuilder;

package/dist/Realtime/service/io/publishers/WebSocketAudioPublisher.d.ts

@@ -0,0 +1,28 @@
+import { StreamingMetadata, AudioState } from "../../../../util/types";
+export declare class WebSocketAudioPublisher {
+    private chunkBuffers;
+    publishAudio(config: {
+        audioData: string;
+        format: string;
+        sourceType: string;
+        conversationId: string;
+        metadata: StreamingMetadata;
+        audioState: AudioState;
+        index: number;
+    }): Promise<void>;
+    private addToBuffer;
+    private flushBuffer;
+    publishState(config: {
+        state: AudioState | string;
+        conversationId?: string;
+        metadata?: StreamingMetadata;
+        message?: string;
+        additionalMetadata?: Record<string, unknown>;
+    }): Promise<void>;
+    /**
+     * Drop any buffered audio without sending it — used on user barge-in so the
+     * client doesn't hear a trailing fragment of the interrupted response
+     */
+    discardBuffer(conversationId: string): void;
+    cleanup(conversationId: string): Promise<void>;
+}

package/dist/Realtime/service/io/publishers/WebSocketAudioPublisher.js

@@ -0,0 +1,101 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebSocketAudioPublisher = void 0;
+const plugin_base_1 = require("@gravity-platform/plugin-base");
+function getLogger() {
+    return (0, plugin_base_1.getPlatformDependencies)().createLogger("RealtimeWebSocketAudioPublisher");
+}
+function getAudioWSManager() {
+    return (0, plugin_base_1.getPlatformDependencies)().getAudioWebSocketManager?.();
+}
+const TARGET_CHUNK_SIZE = 32768;
+const MAX_BUFFER_DELAY = 50;
+class WebSocketAudioPublisher {
+    constructor() {
+        this.chunkBuffers = new Map();
+    }
+    async publishAudio(config) {
+        const audioWSManager = getAudioWSManager();
+        if (!audioWSManager)
+            return;
+        if (config.audioState === "SPEECH_STARTED" || config.audioState === "SPEECH_ENDED") {
+            audioWSManager.sendControl(config.conversationId, {
+                type: "audioState",
+                state: config.audioState,
+                metadata: config.metadata,
+            });
+            if (config.audioState === "SPEECH_ENDED") {
+                await this.flushBuffer(config.conversationId);
+            }
+            return;
+        }
+        const audioBuffer = Buffer.from(config.audioData, "base64");
+        this.addToBuffer(config.conversationId, audioBuffer);
+    }
+    addToBuffer(conversationId, chunk) {
+        let buffer = this.chunkBuffers.get(conversationId);
+        if (!buffer) {
+            buffer = { chunks: [], totalSize: 0 };
+            this.chunkBuffers.set(conversationId, buffer);
+        }
+        buffer.chunks.push(chunk);
+        buffer.totalSize += chunk.length;
+        if (buffer.timer)
+            clearTimeout(buffer.timer);
+        if (buffer.totalSize >= TARGET_CHUNK_SIZE) {
+            this.flushBuffer(conversationId);
+        }
+        else {
+            buffer.timer = setTimeout(() => this.flushBuffer(conversationId), MAX_BUFFER_DELAY);
+        }
+    }
+    async flushBuffer(conversationId) {
+        const buffer = this.chunkBuffers.get(conversationId);
+        if (!buffer || buffer.chunks.length === 0)
+            return;
+        const audioWSManager = getAudioWSManager();
+        if (!audioWSManager)
+            return;
+        const combined = Buffer.concat(buffer.chunks);
+        audioWSManager.sendAudio(conversationId, combined);
+        buffer.chunks = [];
+        buffer.totalSize = 0;
+        if (buffer.timer) {
+            clearTimeout(buffer.timer);
+            buffer.timer = undefined;
+        }
+    }
+    async publishState(config) {
+        const audioWSManager = getAudioWSManager();
+        if (!audioWSManager || !config.conversationId)
+            return;
+        if (config.state === "SPEECH_ENDED") {
+            await this.flushBuffer(config.conversationId);
+        }
+        const mergedMetadata = { ...(config.metadata || {}), ...(config.additionalMetadata || {}) };
+        audioWSManager.sendControl(config.conversationId, {
+            type: "AUDIO_STATE",
+            state: config.state,
+            message: config.message,
+            metadata: mergedMetadata,
+        });
+    }
+    /**
+     * Drop any buffered audio without sending it — used on user barge-in so the
+     * client doesn't hear a trailing fragment of the interrupted response
+     */
+    discardBuffer(conversationId) {
+        const buffer = this.chunkBuffers.get(conversationId);
+        if (buffer?.timer)
+            clearTimeout(buffer.timer);
+        this.chunkBuffers.delete(conversationId);
+    }
+    async cleanup(conversationId) {
+        await this.flushBuffer(conversationId);
+        const buffer = this.chunkBuffers.get(conversationId);
+        if (buffer?.timer)
+            clearTimeout(buffer.timer);
+        this.chunkBuffers.delete(conversationId);
+    }
+}
+exports.WebSocketAudioPublisher = WebSocketAudioPublisher;

package/dist/Realtime/service/io/websocket/RealtimeWebSocketAudioSubscriber.d.ts

@@ -0,0 +1,13 @@
+import { WsClient } from "../../core/streaming/WsClient";
+export declare class RealtimeWebSocketAudioSubscriber {
+    private sessions;
+    private static instance;
+    private logger;
+    private constructor();
+    static getInstance(): RealtimeWebSocketAudioSubscriber;
+    private setupWebSocketHandlers;
+    registerSession(wsSessionId: string, chatId: string, wsClient: WsClient): void;
+    handleAudioData(sessionId: string, audioData: ArrayBuffer): Promise<void>;
+    handleControlMessage(sessionId: string, message: any): Promise<void>;
+    unregisterSession(sessionId: string): void;
+}