@gravity-platform/openai-realtime 1.0.1

package/dist/Realtime/service/io/websocket/RealtimeWebSocketAudioSubscriber.js

@@ -0,0 +1,94 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RealtimeWebSocketAudioSubscriber = void 0;
+const plugin_base_1 = require("@gravity-platform/plugin-base");
+const AudioAppendBuilder_1 = require("../events/incoming/builders/AudioAppendBuilder");
+function getLogger() {
+    return (0, plugin_base_1.getPlatformDependencies)().createLogger("RealtimeWebSocketAudioSubscriber");
+}
+function getAudioWSManager() {
+    return (0, plugin_base_1.getPlatformDependencies)().getAudioWebSocketManager?.();
+}
+function resample16kTo24k(input) {
+    const srcSamples = input.length / 2;
+    const dstSamples = Math.round((srcSamples * 24000) / 16000);
+    const output = Buffer.alloc(dstSamples * 2);
+    const ratio = srcSamples / dstSamples;
+    for (let i = 0; i < dstSamples; i++) {
+        const srcPos = i * ratio;
+        const srcIndex = Math.floor(srcPos);
+        const frac = srcPos - srcIndex;
+        const s0 = input.readInt16LE(srcIndex * 2);
+        const s1 = srcIndex + 1 < srcSamples ? input.readInt16LE((srcIndex + 1) * 2) : s0;
+        const sample = Math.round(s0 + frac * (s1 - s0));
+        output.writeInt16LE(Math.max(-32768, Math.min(32767, sample)), i * 2);
+    }
+    return output;
+}
+class RealtimeWebSocketAudioSubscriber {
+    constructor() {
+        this.sessions = new Map();
+        this.logger = getLogger();
+        this.setupWebSocketHandlers();
+    }
+    static getInstance() {
+        if (!RealtimeWebSocketAudioSubscriber.instance) {
+            RealtimeWebSocketAudioSubscriber.instance = new RealtimeWebSocketAudioSubscriber();
+        }
+        return RealtimeWebSocketAudioSubscriber.instance;
+    }
+    setupWebSocketHandlers() {
+        const audioWSManager = getAudioWSManager();
+        if (audioWSManager?.setAudioDataHandler) {
+            audioWSManager.setAudioDataHandler(this.handleAudioData.bind(this));
+            audioWSManager.setControlMessageHandler(this.handleControlMessage.bind(this));
+            this.logger.info("Realtime WebSocket audio subscriber registered");
+        }
+        else {
+            this.logger.warn("AudioWebSocketManager not available");
+        }
+    }
+    registerSession(wsSessionId, chatId, wsClient) {
+        this.sessions.set(wsSessionId, { wsClient, chatId, isActive: true });
+        const audioWSManager = getAudioWSManager();
+        audioWSManager?.startAudioSession?.(wsSessionId, wsSessionId);
+        this.logger.info("Realtime audio session registered", { wsSessionId, chatId });
+    }
+    async handleAudioData(sessionId, audioData) {
+        const session = this.sessions.get(sessionId);
+        if (!session || !session.isActive)
+            return;
+        if (!session.wsClient.isOpen) {
+            this.logger.warn("Mic audio but WsClient is closed", { sessionId });
+            return;
+        }
+        const resampled = resample16kTo24k(Buffer.from(audioData));
+        const base64 = resampled.toString("base64");
+        session.wsClient.send(AudioAppendBuilder_1.AudioAppendBuilder.build(base64));
+    }
+    async handleControlMessage(sessionId, message) {
+        const type = message?.type;
+        if (type === "stop" || type === "END_CALL") {
+            const session = this.sessions.get(sessionId);
+            if (!session)
+                return;
+            this.logger.info("Control 'stop' — closing Realtime WebSocket", { sessionId });
+            session.isActive = false;
+            try {
+                session.wsClient.close();
+            }
+            catch (err) {
+                this.logger.warn("Error closing WS on stop", { sessionId, error: err?.message });
+            }
+        }
+    }
+    unregisterSession(sessionId) {
+        const session = this.sessions.get(sessionId);
+        if (session) {
+            session.isActive = false;
+            this.sessions.delete(sessionId);
+            this.logger.info("Realtime audio session unregistered", { sessionId });
+        }
+    }
+}
+exports.RealtimeWebSocketAudioSubscriber = RealtimeWebSocketAudioSubscriber;

package/dist/credentials/index.d.ts

@@ -0,0 +1,14 @@
+export declare const OpenAICredential: {
+    name: string;
+    displayName: string;
+    description: string;
+    properties: {
+        name: string;
+        displayName: string;
+        type: "string";
+        required: boolean;
+        secret: boolean;
+        description: string;
+        placeholder: string;
+    }[];
+};

package/dist/credentials/index.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenAICredential = void 0;
+exports.OpenAICredential = {
+    name: "openaiCredential",
+    displayName: "OpenAI",
+    description: "Credentials for OpenAI Realtime API",
+    properties: [
+        {
+            name: "apiKey",
+            displayName: "API Key",
+            type: "string",
+            required: true,
+            secret: true,
+            description: "Your OpenAI API key from platform.openai.com",
+            placeholder: "sk-...",
+        },
+    ],
+};

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+declare const plugin: import("@gravity-platform/plugin-base").GravityPlugin;
+export default plugin;

package/dist/index.js

@@ -0,0 +1,54 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const plugin_base_1 = require("@gravity-platform/plugin-base");
+const package_json_1 = __importDefault(require("../package.json"));
+const plugin = (0, plugin_base_1.createPlugin)({
+    name: package_json_1.default.name,
+    version: package_json_1.default.version,
+    description: package_json_1.default.description,
+    async setup(api) {
+        const { initializePlatformFromAPI } = await Promise.resolve().then(() => __importStar(require("@gravity-platform/plugin-base")));
+        initializePlatformFromAPI(api);
+        const { RealtimeVoiceNode } = await Promise.resolve().then(() => __importStar(require("./Realtime/node")));
+        api.registerNode(RealtimeVoiceNode);
+        const { OpenAICredential } = await Promise.resolve().then(() => __importStar(require("./credentials")));
+        api.registerCredential(OpenAICredential);
+    },
+});
+exports.default = plugin;

package/dist/util/types.d.ts

@@ -0,0 +1,40 @@
+export interface OpenAIRealtimeConfig {
+    systemPrompt?: string;
+    conversationHistory?: Array<{
+        role: "user" | "assistant";
+        content: string;
+    }>;
+    initialRequest?: string;
+    voice?: "alloy" | "ash" | "ballad" | "coral" | "echo" | "sage" | "shimmer" | "verse" | "marin" | "cedar";
+    turnDetection?: "semantic_vad" | "server_vad" | "disabled";
+    redisChannel?: string;
+    maxResponseOutputTokens?: number;
+    tools?: Array<{
+        type: "function";
+        name: string;
+        description: string;
+        parameters: any;
+    }>;
+    mcpService?: Record<string, (input: any) => Promise<any>>;
+    controlSignal?: "START_CALL" | "END_CALL";
+}
+export interface StreamUsageStats {
+    estimated: boolean;
+    total_tokens: number;
+    inputTokens: number;
+    outputTokens: number;
+    chunk_count: number;
+    textOutput: string;
+    transcription: string;
+    assistantResponse: string;
+}
+export interface StreamingMetadata {
+    workflowId?: string;
+    executionId?: string;
+    nodeId?: string;
+    chatId?: string;
+    conversationId?: string;
+    userId?: string;
+    providerId?: string;
+}
+export type AudioState = "SPEECH_STARTED" | "SPEECH_STREAMING" | "SPEECH_ENDED" | "SESSION_READY" | "TOOL_USE" | "TOOL_USE_COMPLETED" | "USER_SPEECH_STARTED" | "USER_SPEECH_ENDED";

package/dist/util/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "@gravity-platform/openai-realtime",
+  "version": "1.0.1",
+  "description": "OpenAI Realtime API (gpt-realtime-2) integration for Gravity workflow system",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "clean": "rm -rf dist"
+  },
+  "dependencies": {
+    "pino": "^10.1.0",
+    "ws": "^8.18.0"
+  },
+  "devDependencies": {
+    "@types/ws": "^8.5.14",
+    "typescript": "^5.0.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "gravity": {
+    "displayName": "OpenAI Realtime",
+    "category": "ai",
+    "logoUrl": "https://res.cloudinary.com/sonik/image/upload/v1749262616/gravity/icons/ChatGPT-Logo.svg.webp",
+    "nodes": [
+      {
+        "name": "OpenAI Realtime Voice",
+        "type": "CallbackNode",
+        "description": "Real-time voice conversation with gpt-realtime-2 via WebSocket",
+        "category": "AI",
+        "mcp": false
+      }
+    ],
+    "features": [
+      "Real-time voice conversation",
+      "WebSocket streaming",
+      "Function calling support",
+      "Low-latency responses",
+      "Server VAD (Voice Activity Detection)",
+      "Audio input/output"
+    ],
+    "credentials": [
+      {
+        "name": "OpenAI API Key",
+        "type": "openaiApiKey",
+        "required": true,
+        "description": "API key from platform.openai.com"
+      }
+    ]
+  },
+  "peerDependencies": {
+    "@gravity-platform/plugin-base": ">=1.0.0"
+  }
+}