@gravity-platform/openai-realtime 1.0.1

package/dist/Realtime/service/core/orchestration/SessionOrchestrator.js

@@ -0,0 +1,333 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SessionOrchestrator = void 0;
+const plugin_base_1 = require("@gravity-platform/plugin-base");
+const WsClient_1 = require("../streaming/WsClient");
+const SessionManager_1 = require("../streaming/SessionManager");
+const ResponseProcessor_1 = require("../processing/ResponseProcessor");
+const RealtimeSessionRegistry_1 = require("../streaming/RealtimeSessionRegistry");
+const SessionUpdateBuilder_1 = require("../../io/events/incoming/builders/SessionUpdateBuilder");
+const ConversationItemBuilder_1 = require("../../io/events/incoming/builders/ConversationItemBuilder");
+const ResponseCreateBuilder_1 = require("../../io/events/incoming/builders/ResponseCreateBuilder");
+const RealtimeWebSocketAudioSubscriber_1 = require("../../io/websocket/RealtimeWebSocketAudioSubscriber");
+const WebSocketAudioPublisher_1 = require("../../io/publishers/WebSocketAudioPublisher");
+const constants_1 = require("../../../constants");
+function getLogger() {
+    return (0, plugin_base_1.getPlatformDependencies)().createLogger("RealtimeSessionOrchestrator");
+}
+const EMPTY_STATS = {
+    estimated: true,
+    total_tokens: 0,
+    inputTokens: 0,
+    outputTokens: 0,
+    chunk_count: 0,
+    textOutput: "",
+    transcription: "",
+    assistantResponse: "",
+};
+class SessionOrchestrator {
+    constructor() {
+        this.logger = getLogger();
+        this.sessionManager = new SessionManager_1.SessionManager();
+    }
+    async orchestrateSession(config, metadata, context, emit) {
+        const controlSignal = config.controlSignal ?? "START_CALL";
+        const conversationId = metadata.conversationId || metadata.workflowId || "unknown";
+        if (controlSignal === "END_CALL") {
+            return this.handleEndCall(conversationId);
+        }
+        return this.handleStartCall(config, metadata, context, emit, conversationId);
+    }
+    async handleStartCall(config, metadata, context, emit, conversationId) {
+        // MCP discovery (timeout after 3s to avoid blocking session startup)
+        const api = context?.api;
+        if (api?.callService) {
+            let discoveryTimer;
+            try {
+                const discovery = api.callService("getSchema", {}, context);
+                // If the timeout wins the race, a later rejection from the discovery call
+                // must not surface as an unhandled rejection
+                discovery.catch(() => { });
+                const mcpSchema = await Promise.race([
+                    discovery,
+                    new Promise((_, reject) => {
+                        discoveryTimer = setTimeout(() => reject(new Error("MCP discovery timeout")), 3000);
+                    }),
+                ]);
+                if (mcpSchema?.methods) {
+                    config.tools = Object.entries(mcpSchema.methods).map(([name, m]) => ({
+                        type: "function",
+                        name,
+                        description: m.description ?? `Execute ${name}`,
+                        parameters: m.input ?? { type: "object", properties: {} },
+                    }));
+                    config.mcpService = {};
+                    for (const [methodName] of Object.entries(mcpSchema.methods)) {
+                        config.mcpService[methodName] = (input) => api.callService(methodName, input, context);
+                    }
+                    this.logger.info(`MCP tools configured: ${config.tools.length}`);
+                }
+            }
+            catch {
+                // No MCP connected — continue
+            }
+            finally {
+                if (discoveryTimer)
+                    clearTimeout(discoveryTimer);
+            }
+        }
+        // Credentials - using field signature pattern
+        const credentials = this.getCredentials(context);
+        const wsClient = new WsClient_1.WsClient();
+        const responseProcessor = new ResponseProcessor_1.RealtimeResponseProcessor(metadata.workflowId || "unknown", metadata, emit, wsClient);
+        // Track parallel tool calls — request the next response only when ALL
+        // dispatched tools have completed AND the model's response.done has arrived
+        // (a fast tool can finish before the response that requested it settles).
+        let pendingToolCount = 0;
+        let awaitingResponseDone = false;
+        let needsSessionUpdate = false;
+        const TOOL_TIMEOUT_MS = parseInt(process.env.REALTIME_TOOL_TIMEOUT_MS || "30000", 10);
+        const withTimeout = (promise, ms, label) => new Promise((resolve, reject) => {
+            const timer = setTimeout(() => reject(new Error(`Tool ${label} timed out after ${ms}ms`)), ms);
+            promise.then((v) => {
+                clearTimeout(timer);
+                resolve(v);
+            }, (e) => {
+                clearTimeout(timer);
+                reject(e);
+            });
+        });
+        const saveTrace = (args) => {
+            if (!context?.api?.saveMCPTrace || !metadata.executionId || !metadata.nodeId)
+                return;
+            const endTime = Date.now();
+            context.api
+                .saveMCPTrace({
+                executionId: metadata.executionId,
+                parentNodeId: metadata.nodeId,
+                toolName: args.toolName,
+                arguments: args.toolInput,
+                result: args.result,
+                startTime: args.startTime,
+                endTime,
+                duration: endTime - args.startTime,
+                success: args.success,
+                ...(args.error ? { error: args.error } : {}),
+            })
+                .catch((err) => this.logger.warn("Failed to save MCP trace", { error: err?.message }));
+        };
+        const maybeRequestNextResponse = () => {
+            if (pendingToolCount > 0 || awaitingResponseDone)
+                return;
+            if (needsSessionUpdate) {
+                wsClient.send(SessionUpdateBuilder_1.SessionUpdateBuilder.build(config));
+                this.logger.info("Registered discovered MCPs as tools");
+                needsSessionUpdate = false;
+            }
+            wsClient.send(ResponseCreateBuilder_1.ResponseCreateBuilder.build());
+        };
+        const executeToolCall = async (toolName, toolInput, callId) => {
+            // The entire body is guarded: this runs fire-and-forget, so any escape
+            // would be an unhandled rejection and would skip the counter decrement
+            try {
+                this.logger.info("Tool use", { toolName, callId });
+                const startTime = Date.now();
+                let result;
+                let toolError;
+                try {
+                    if (config.mcpService?.[toolName]) {
+                        result = await withTimeout(config.mcpService[toolName](toolInput), TOOL_TIMEOUT_MS, toolName);
+                        emit?.({ __outputs: { mcpResult: { name: toolName, arguments: toolInput, result } } });
+                        saveTrace({ toolName, toolInput, result, startTime, success: true });
+                    }
+                    else {
+                        toolError = `No handler for tool: ${toolName}`;
+                        result = { error: toolError };
+                        saveTrace({ toolName, toolInput, result: null, startTime, success: false, error: toolError });
+                    }
+                }
+                catch (err) {
+                    toolError = `Tool execution failed: ${err.message}`;
+                    result = { error: toolError };
+                    saveTrace({ toolName, toolInput, result: null, startTime, success: false, error: toolError });
+                }
+                // Notify client that tool call finished
+                try {
+                    const completionPublisher = new WebSocketAudioPublisher_1.WebSocketAudioPublisher();
+                    await completionPublisher.publishState({
+                        state: "TOOL_USE_COMPLETED",
+                        conversationId,
+                        metadata,
+                        message: toolError ? `Tool failed: ${toolName}` : `Tool completed: ${toolName}`,
+                        additionalMetadata: { toolName, callId, error: toolError },
+                    });
+                }
+                catch (err) {
+                    this.logger.warn("Failed to publish TOOL_USE_COMPLETED", { error: err?.message });
+                }
+                // Send function output back to OpenAI
+                let serialized;
+                try {
+                    serialized = JSON.stringify(result);
+                }
+                catch {
+                    serialized = JSON.stringify({ error: "Tool result was not serializable" });
+                }
+                wsClient.send(ConversationItemBuilder_1.ConversationItemBuilder.buildFunctionCallOutput(callId, serialized));
+                // Register dynamically discovered MCP tools from findIntent/discoverRelated results
+                if ((toolName === "findIntent" || toolName === "discoverRelated") && Array.isArray(result) && api?.callService) {
+                    config.tools = config.tools || [];
+                    config.mcpService = config.mcpService || {};
+                    for (const item of result) {
+                        if (item?.object_type === "mcp" && item?.metadata?.schema?.methods) {
+                            const methodName = Object.keys(item.metadata.schema.methods)[0];
+                            if (!methodName || config.mcpService[methodName])
+                                continue;
+                            const methodDef = item.metadata.schema.methods[methodName];
+                            config.tools.push({
+                                type: "function",
+                                name: methodName,
+                                description: methodDef?.description || item.description || item.title,
+                                parameters: methodDef?.input || { type: "object", properties: { message: { type: "string" } } },
+                            });
+                            config.mcpService[methodName] = (input) => api.callService(methodName, input, context);
+                            needsSessionUpdate = true;
+                        }
+                    }
+                }
+            }
+            catch (err) {
+                this.logger.error("Tool call handling failed", { toolName, callId, error: err?.message });
+                // Best effort: the model must not be left waiting on this call_id
+                try {
+                    wsClient.send(ConversationItemBuilder_1.ConversationItemBuilder.buildFunctionCallOutput(callId, JSON.stringify({ error: "Tool call failed" })));
+                }
+                catch {
+                    /* socket may be closed */
+                }
+            }
+            finally {
+                pendingToolCount--;
+                maybeRequestNextResponse();
+            }
+        };
+        // Tool use handler — called SYNCHRONOUSLY per function-call event so the
+        // pending count is accurate before any tool can complete
+        responseProcessor.onToolUse = ({ toolName, toolInput, callId }) => {
+            pendingToolCount++;
+            awaitingResponseDone = true;
+            void executeToolCall(toolName, toolInput, callId);
+        };
+        // The response that requested the tools has settled — safe to continue
+        // once all dispatched tools have finished
+        responseProcessor.onToolResponseDone = () => {
+            awaitingResponseDone = false;
+            maybeRequestNextResponse();
+        };
+        const session = this.sessionManager.createSession(config, metadata, responseProcessor, wsClient);
+        // Connect
+        await wsClient.connect(credentials.apiKey);
+        RealtimeSessionRegistry_1.realtimeSessionRegistry.register(conversationId, wsClient);
+        const wsSubscriber = RealtimeWebSocketAudioSubscriber_1.RealtimeWebSocketAudioSubscriber.getInstance();
+        // From here on the OpenAI session is live and billed — any exit path MUST
+        // close the socket and deregister, or the session is orphaned with no way
+        // to end it
+        try {
+            wsClient.onMessage((event) => {
+                responseProcessor.processEvent(event).catch((err) => {
+                    this.logger.error("Event processing error", { error: err.message });
+                });
+            });
+            // Configure session
+            wsClient.send(SessionUpdateBuilder_1.SessionUpdateBuilder.build(config));
+            // Replay history
+            if (config.conversationHistory?.length) {
+                for (const msg of config.conversationHistory) {
+                    wsClient.send(msg.role === "user"
+                        ? ConversationItemBuilder_1.ConversationItemBuilder.buildUserMessage(msg.content)
+                        : ConversationItemBuilder_1.ConversationItemBuilder.buildAssistantMessage(msg.content));
+                }
+            }
+            // Send initial request first — model processing time provides a natural
+            // buffer while the frontend initialises audio playback after SESSION_READY.
+            if (config.initialRequest) {
+                wsClient.send(ConversationItemBuilder_1.ConversationItemBuilder.buildUserMessage(config.initialRequest));
+                wsClient.send(ResponseCreateBuilder_1.ResponseCreateBuilder.build());
+            }
+            // Register mic audio subscriber
+            wsSubscriber.registerSession(conversationId, metadata.chatId || "", wsClient);
+            // Publish SESSION_READY — frontend begins audio playback setup on receipt
+            const publisher = new WebSocketAudioPublisher_1.WebSocketAudioPublisher();
+            await publisher.publishState({
+                state: "SESSION_READY",
+                conversationId,
+                metadata,
+                message: "OpenAI Realtime audio session ready",
+                additionalMetadata: { nodeId: context?.nodeId || "openairealtimevoice1", serverVad: true },
+            });
+            // Hold until WS closes (triggered by END_CALL)
+            this.logger.info("⏳ [SESSION] Waiting for WebSocket close...", { conversationId });
+            await wsClient.waitForClose();
+            this.logger.info("🔚 [SESSION] WebSocket closed — session ending", { conversationId });
+        }
+        finally {
+            wsClient.close(); // no-op if already closed
+            this.sessionManager.endSession(session.sessionId);
+            RealtimeSessionRegistry_1.realtimeSessionRegistry.remove(conversationId);
+            wsSubscriber.unregisterSession(conversationId);
+        }
+        responseProcessor.emitFinal();
+        const result = responseProcessor.getUsageStats();
+        // Save token usage
+        if (result.total_tokens > 0) {
+            try {
+                await (0, plugin_base_1.getPlatformDependencies)().saveTokenUsage({
+                    workflowId: metadata.workflowId,
+                    executionId: metadata.executionId,
+                    nodeId: metadata.nodeId,
+                    nodeType: "OpenAIRealtimeVoice",
+                    model: constants_1.REALTIME_MODEL_ID,
+                    usage: {
+                        total_tokens: result.total_tokens,
+                        input_tokens: result.inputTokens,
+                        output_tokens: result.outputTokens,
+                    },
+                    timestamp: new Date(),
+                });
+            }
+            catch (err) {
+                this.logger.error("Failed to save token usage", { error: err.message });
+            }
+        }
+        return result;
+    }
+    handleEndCall(conversationId) {
+        const wsClient = RealtimeSessionRegistry_1.realtimeSessionRegistry.get(conversationId);
+        if (wsClient) {
+            this.logger.info("END_CALL — closing Realtime WebSocket", { conversationId });
+            wsClient.close();
+        }
+        else {
+            this.logger.warn("END_CALL — no active session found", { conversationId });
+        }
+        return { ...EMPTY_STATS };
+    }
+    getCredentials(context) {
+        const available = context.credentials || {};
+        // Prefer the credential declared on the node definition: the platform passes
+        // ALL workflow credentials here, and others (e.g. xAI) also carry an apiKey
+        // field — the generic signature scan below could pick the wrong provider.
+        const preferred = available.openaiCredential || available.openAICredential;
+        if (preferred?.apiKey) {
+            return { apiKey: preferred.apiKey };
+        }
+        // Fallback: field signature pattern (docs-starter/nodes/04-credentials.md)
+        for (const val of Object.values(available)) {
+            if (val?.apiKey) {
+                return { apiKey: val.apiKey };
+            }
+        }
+        throw new Error("OpenAI credentials not configured");
+    }
+}
+exports.SessionOrchestrator = SessionOrchestrator;

package/dist/Realtime/service/core/processing/AudioHandler.d.ts

@@ -0,0 +1,17 @@
+import { StreamingMetadata } from "../../../../util/types";
+export declare class AudioHandler {
+    private conversationId;
+    private metadata;
+    private chunkIndex;
+    private publisher;
+    constructor(conversationId: string, metadata: StreamingMetadata);
+    handleAudioStart(): Promise<void>;
+    bufferAudioChunk(base64Audio: string): Promise<void>;
+    handleAudioEnd(): Promise<void>;
+    /**
+     * User barge-in: drop buffered assistant audio instead of flushing it, but
+     * still send SPEECH_ENDED — the client relies on it to unmute the mic
+     */
+    handleInterruption(): Promise<void>;
+    cleanup(): void;
+}

package/dist/Realtime/service/core/processing/AudioHandler.js

@@ -0,0 +1,57 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AudioHandler = void 0;
+const WebSocketAudioPublisher_1 = require("../../io/publishers/WebSocketAudioPublisher");
+class AudioHandler {
+    constructor(conversationId, metadata) {
+        this.conversationId = conversationId;
+        this.metadata = metadata;
+        this.chunkIndex = 0;
+        this.publisher = new WebSocketAudioPublisher_1.WebSocketAudioPublisher();
+    }
+    async handleAudioStart() {
+        await this.publisher.publishState({
+            state: "SPEECH_STARTED",
+            conversationId: this.conversationId,
+            metadata: this.metadata,
+            message: "Assistant started speaking",
+        });
+    }
+    async bufferAudioChunk(base64Audio) {
+        await this.publisher.publishAudio({
+            audioData: base64Audio,
+            format: "pcm16",
+            sourceType: "OpenAIRealtime",
+            conversationId: this.conversationId,
+            metadata: this.metadata,
+            audioState: "SPEECH_STREAMING",
+            index: this.chunkIndex++,
+        });
+    }
+    async handleAudioEnd() {
+        await this.publisher.publishState({
+            state: "SPEECH_ENDED",
+            conversationId: this.conversationId,
+            metadata: this.metadata,
+            message: "Assistant finished speaking",
+        });
+        await this.publisher.cleanup(this.conversationId);
+    }
+    /**
+     * User barge-in: drop buffered assistant audio instead of flushing it, but
+     * still send SPEECH_ENDED — the client relies on it to unmute the mic
+     */
+    async handleInterruption() {
+        this.publisher.discardBuffer(this.conversationId);
+        await this.publisher.publishState({
+            state: "SPEECH_ENDED",
+            conversationId: this.conversationId,
+            metadata: this.metadata,
+            message: "Assistant interrupted by user",
+        });
+    }
+    cleanup() {
+        this.chunkIndex = 0;
+    }
+}
+exports.AudioHandler = AudioHandler;

package/dist/Realtime/service/core/processing/ResponseProcessor.d.ts

@@ -0,0 +1,34 @@
+import { StreamingMetadata, StreamUsageStats } from "../../../../util/types";
+import type { WsClient } from "../streaming/WsClient";
+export declare class RealtimeResponseProcessor {
+    private sessionId;
+    private metadata;
+    private wsClient?;
+    private audioHandler;
+    private textAccumulator;
+    private usageStatsCollector;
+    private audioStarted;
+    private readonly logger;
+    private hasToolCallsInCurrentResponse;
+    /** Called synchronously per function-call event — must not be awaited so the
+     *  orchestrator's pending count is accurate before any tool can complete */
+    onToolUse?: (toolUse: {
+        toolName: string;
+        toolInput: any;
+        callId: string;
+    }) => void;
+    /** Called when a response that dispatched tool calls reaches response.done */
+    onToolResponseDone?: () => void;
+    emitProgress(text: string): void;
+    constructor(sessionId: string, metadata: StreamingMetadata, emit?: (output: any) => void, wsClient?: WsClient | undefined);
+    processEvent(event: any): Promise<void>;
+    private routeEvent;
+    private handleResponseDone;
+    private publishUserSpeechState;
+    private handleFunctionCall;
+    private finalizeUsageStats;
+    emitFinal(): void;
+    getUsageStats(): StreamUsageStats;
+    handleError(error: any): Promise<void>;
+    cleanup(): void;
+}

package/dist/Realtime/service/core/processing/ResponseProcessor.js

@@ -0,0 +1,199 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RealtimeResponseProcessor = void 0;
+const plugin_base_1 = require("@gravity-platform/plugin-base");
+const AudioHandler_1 = require("./AudioHandler");
+const TextAccumulator_1 = require("./TextAccumulator");
+const UsageStatsCollector_1 = require("./UsageStatsCollector");
+const WebSocketAudioPublisher_1 = require("../../io/publishers/WebSocketAudioPublisher");
+// Module-level platform calls cause startup freezes (docs-starter/nodes/CLAUDE.md
+// rule 5) — resolve the logger lazily instead
+function getLogger() {
+    return (0, plugin_base_1.getPlatformDependencies)().createLogger("RealtimeResponseProcessor");
+}
+class RealtimeResponseProcessor {
+    emitProgress(text) {
+        this.textAccumulator.emitProgress(text);
+    }
+    constructor(sessionId, metadata, emit, wsClient) {
+        this.sessionId = sessionId;
+        this.metadata = metadata;
+        this.wsClient = wsClient;
+        this.usageStatsCollector = new UsageStatsCollector_1.UsageStatsCollector();
+        this.audioStarted = false;
+        this.logger = getLogger();
+        this.hasToolCallsInCurrentResponse = false;
+        const conversationId = metadata.conversationId || sessionId;
+        this.audioHandler = new AudioHandler_1.AudioHandler(conversationId, metadata);
+        this.textAccumulator = new TextAccumulator_1.TextAccumulator(sessionId, emit);
+    }
+    async processEvent(event) {
+        try {
+            await this.routeEvent(event);
+        }
+        catch (err) {
+            this.logger.error("Error processing event", {
+                sessionId: this.sessionId,
+                type: event?.type,
+                error: err instanceof Error ? err.message : err,
+            });
+        }
+    }
+    async routeEvent(event) {
+        switch (event.type) {
+            case "response.output_audio.delta":
+                if (!this.audioStarted) {
+                    this.audioStarted = true;
+                    await this.audioHandler.handleAudioStart();
+                }
+                this.usageStatsCollector.incrementChunkCount();
+                await this.audioHandler.bufferAudioChunk(event.delta);
+                break;
+            case "response.output_audio.done":
+                await this.audioHandler.handleAudioEnd();
+                this.audioStarted = false;
+                break;
+            case "response.output_audio_transcript.delta":
+                this.textAccumulator.appendAssistant(event.delta ?? "");
+                break;
+            case "response.output_audio_transcript.done":
+                if (event.transcript) {
+                    this.textAccumulator.setAssistantText(event.transcript);
+                }
+                break;
+            case "conversation.item.input_audio_transcription.completed":
+                this.textAccumulator.setUserTranscript(event.transcript ?? "");
+                break;
+            case "response.function_call_arguments.done":
+                await this.handleFunctionCall(event);
+                break;
+            case "session.created":
+                this.logger.info("Session created", { session: event.session });
+                break;
+            case "session.updated":
+                this.logger.debug("Session config accepted", {
+                    voice: event.session?.audio?.output?.voice,
+                    turn_detection: event.session?.audio?.input?.turn_detection,
+                });
+                break;
+            case "input_audio_buffer.speech_started":
+                this.logger.debug("VAD: speech started", { sessionId: this.sessionId });
+                if (this.audioStarted) {
+                    // Barge-in: drop buffered assistant audio rather than flushing it
+                    await this.audioHandler.handleInterruption();
+                    this.audioStarted = false;
+                }
+                this.textAccumulator.resetTurn();
+                await this.publishUserSpeechState("USER_SPEECH_STARTED");
+                break;
+            case "input_audio_buffer.speech_stopped":
+                this.logger.debug("VAD: speech stopped", { sessionId: this.sessionId });
+                await this.publishUserSpeechState("USER_SPEECH_ENDED");
+                break;
+            case "input_audio_buffer.committed":
+                this.logger.debug("Audio buffer committed", { sessionId: this.sessionId });
+                break;
+            case "response.done":
+                this.handleResponseDone(event);
+                break;
+            case "response.function_call_arguments.delta":
+            case "response.output_item.added":
+            case "response.output_item.done":
+            case "response.content_part.added":
+            case "response.content_part.done":
+            case "response.created":
+            case "conversation.item.created":
+            case "conversation.item.done":
+            case "conversation.item.added":
+            case "rate_limits.updated":
+                break;
+            case "error":
+                this.logger.error("Realtime API error", { error: event.error, sessionId: this.sessionId });
+                break;
+            default:
+                this.logger.debug("Unhandled event", { type: event.type });
+                break;
+        }
+    }
+    handleResponseDone(event) {
+        this.finalizeUsageStats(event);
+        if (this.hasToolCallsInCurrentResponse) {
+            // Tool calls were dispatched — the orchestrator sends response.create once
+            // all tool outputs are submitted AND this response has settled
+            this.hasToolCallsInCurrentResponse = false;
+            this.onToolResponseDone?.();
+            return;
+        }
+        // Normal text/audio response — emit conversation turn
+        this.textAccumulator.emitConversation();
+        this.textAccumulator.resetTurn();
+    }
+    async publishUserSpeechState(state) {
+        const conversationId = this.metadata.conversationId || this.sessionId;
+        const publisher = new WebSocketAudioPublisher_1.WebSocketAudioPublisher();
+        try {
+            await publisher.publishState({
+                state,
+                conversationId,
+                metadata: this.metadata,
+                message: state === "USER_SPEECH_STARTED" ? "User started speaking" : "User finished speaking",
+            });
+        }
+        catch (err) {
+            this.logger.warn("Failed to publish user speech state", { state, error: err?.message });
+        }
+    }
+    async handleFunctionCall(event) {
+        const toolName = event.name;
+        const callId = event.call_id;
+        let toolInput = {};
+        try {
+            toolInput = JSON.parse(event.arguments || "{}");
+        }
+        catch {
+            this.logger.warn("Failed to parse tool arguments", { arguments: event.arguments });
+        }
+        this.hasToolCallsInCurrentResponse = true;
+        // Dispatch FIRST, synchronously — awaiting the state publish before counting
+        // opens a race where a fast parallel tool hits zero pending and requests the
+        // next response early
+        if (this.onToolUse) {
+            this.onToolUse({ toolName, toolInput, callId });
+        }
+        const conversationId = this.metadata.conversationId || this.sessionId;
+        const publisher = new WebSocketAudioPublisher_1.WebSocketAudioPublisher();
+        await publisher.publishState({
+            state: "TOOL_USE",
+            conversationId,
+            metadata: this.metadata,
+            message: `Using tool: ${toolName}`,
+            additionalMetadata: { toolName, callId, toolInput },
+        });
+    }
+    finalizeUsageStats(event) {
+        const usage = event.response?.usage;
+        if (usage) {
+            this.usageStatsCollector.addUsage(usage.input_tokens ?? 0, usage.output_tokens ?? 0);
+        }
+        this.usageStatsCollector.setTextResults(this.textAccumulator.getTranscription(), this.textAccumulator.getAssistantResponse());
+    }
+    emitFinal() {
+        this.textAccumulator.emitFinal();
+    }
+    getUsageStats() {
+        return this.usageStatsCollector.getUsageStats();
+    }
+    async handleError(error) {
+        this.logger.error("Stream error", { sessionId: this.sessionId, error: error.message || error });
+        if (this.audioStarted) {
+            await this.audioHandler.handleAudioEnd();
+            this.audioStarted = false;
+        }
+    }
+    cleanup() {
+        this.audioHandler.cleanup();
+        this.textAccumulator.resetTurn();
+        this.usageStatsCollector.reset();
+    }
+}
+exports.RealtimeResponseProcessor = RealtimeResponseProcessor;

package/dist/Realtime/service/core/processing/TextAccumulator.d.ts

@@ -0,0 +1,19 @@
+export declare class TextAccumulator {
+    private sessionId;
+    private emit?;
+    private transcription;
+    private assistantResponse;
+    private allTurns;
+    private progressLog;
+    private readonly logger;
+    constructor(sessionId: string, emit?: ((output: any) => void) | undefined);
+    appendAssistant(text: string): void;
+    setAssistantText(text: string): void;
+    setUserTranscript(transcript: string): void;
+    emitProgress(text: string): void;
+    emitConversation(): void;
+    emitFinal(): void;
+    resetTurn(): void;
+    getTranscription(): string;
+    getAssistantResponse(): string;
+}