opc-agent 2.0.2 → 3.0.0

package/dist/channels/voice.d.ts

@@ -22,15 +22,74 @@ export interface VoiceChannelConfig {
     sampleRate?: number;
     language?: string;
 }
+export interface VoiceConfig {
+    sttProvider: 'whisper' | 'web-speech' | 'deepgram';
+    ttsProvider: 'edge-tts' | 'openai-tts' | 'elevenlabs';
+    sttApiKey?: string;
+    ttsApiKey?: string;
+    voice?: string;
+    language?: string;
+}
+export declare class WhisperSTTProvider implements STTProvider {
+    name: string;
+    private apiKey;
+    constructor(apiKey: string);
+    transcribe(audio: Buffer, options?: STTOptions): Promise<string>;
+}
+export declare class DeepgramSTTProvider implements STTProvider {
+    name: string;
+    private apiKey;
+    constructor(apiKey: string);
+    transcribe(audio: Buffer, options?: STTOptions): Promise<string>;
+}
+export declare class EdgeTTSProvider implements TTSProvider {
+    name: string;
+    private defaultVoice;
+    constructor(voice?: string);
+    synthesize(text: string, options?: TTSOptions): Promise<Buffer>;
+}
+export declare class OpenAITTSProvider implements TTSProvider {
+    name: string;
+    private apiKey;
+    private defaultVoice;
+    constructor(apiKey: string, voice?: string);
+    synthesize(text: string, options?: TTSOptions): Promise<Buffer>;
+}
+export declare class ElevenLabsTTSProvider implements TTSProvider {
+    name: string;
+    private apiKey;
+    private defaultVoice;
+    constructor(apiKey: string, voice?: string);
+    synthesize(text: string, options?: TTSOptions): Promise<Buffer>;
+}
+export declare function createVoiceProviders(config: VoiceConfig): {
+    stt?: STTProvider;
+    tts?: TTSProvider;
+};
 export declare class VoiceChannel extends BaseChannel {
     type: string;
     private config;
     private logger;
     private running;
+    private conversationActive;
     constructor(config?: VoiceChannelConfig);
     start(): Promise<void>;
     stop(): Promise<void>;
     isRunning(): boolean;
+    /** Transcribe audio to text */
+    transcribe(audio: Buffer, format?: string): Promise<string>;
+    /** Synthesize text to audio */
+    synthesize(text: string, voice?: string): Promise<Buffer>;
+    /** Start real-time conversation mode */
+    startConversation(onMessage: (text: string) => Promise<string>): Promise<void>;
+    /** Process a single turn in conversation mode */
+    processConversationTurn(audio: Buffer, onMessage: (text: string) => Promise<string>): Promise<{
+        text: string;
+        response: string;
+        audioResponse?: Buffer;
+    }>;
+    stopConversation(): void;
+    isConversationActive(): boolean;
     /** Process audio input: STT → Agent → TTS */
     processAudio(audio: Buffer): Promise<{
         text: string;

package/dist/channels/voice.js

@@ -1,14 +1,324 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.VoiceChannel = void 0;
+exports.VoiceChannel = exports.ElevenLabsTTSProvider = exports.OpenAITTSProvider = exports.EdgeTTSProvider = exports.DeepgramSTTProvider = exports.WhisperSTTProvider = void 0;
+exports.createVoiceProviders = createVoiceProviders;
 const index_1 = require("./index");
 const logger_1 = require("../core/logger");
+const https = __importStar(require("https"));
+// ── Whisper STT Provider ────────────────────────────────────
+class WhisperSTTProvider {
+    name = 'whisper';
+    apiKey;
+    constructor(apiKey) {
+        this.apiKey = apiKey;
+    }
+    async transcribe(audio, options) {
+        const FormData = (await Promise.resolve(`${'form-data'}`).then(s => __importStar(require(s))).catch(() => null));
+        // Build multipart form data manually
+        const boundary = '----OPCBoundary' + Date.now().toString(36);
+        const parts = [];
+        // file field
+        parts.push(Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="audio.wav"\r\nContent-Type: audio/wav\r\n\r\n`));
+        parts.push(audio);
+        parts.push(Buffer.from('\r\n'));
+        // model field
+        parts.push(Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="model"\r\n\r\nwhisper-1\r\n`));
+        // language field
+        if (options?.language) {
+            parts.push(Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="language"\r\n\r\n${options.language}\r\n`));
+        }
+        parts.push(Buffer.from(`--${boundary}--\r\n`));
+        const body = Buffer.concat(parts);
+        return new Promise((resolve, reject) => {
+            const req = https.request({
+                hostname: 'api.openai.com',
+                path: '/v1/audio/transcriptions',
+                method: 'POST',
+                headers: {
+                    'Authorization': `Bearer ${this.apiKey}`,
+                    'Content-Type': `multipart/form-data; boundary=${boundary}`,
+                    'Content-Length': body.length,
+                },
+            }, (res) => {
+                const chunks = [];
+                res.on('data', (c) => chunks.push(c));
+                res.on('end', () => {
+                    try {
+                        const data = JSON.parse(Buffer.concat(chunks).toString());
+                        resolve(data.text ?? '');
+                    }
+                    catch (e) {
+                        reject(new Error('Failed to parse Whisper response'));
+                    }
+                });
+            });
+            req.on('error', reject);
+            req.write(body);
+            req.end();
+        });
+    }
+}
+exports.WhisperSTTProvider = WhisperSTTProvider;
+// ── Deepgram STT Provider ───────────────────────────────────
+class DeepgramSTTProvider {
+    name = 'deepgram';
+    apiKey;
+    constructor(apiKey) {
+        this.apiKey = apiKey;
+    }
+    async transcribe(audio, options) {
+        const lang = options?.language ?? 'en';
+        return new Promise((resolve, reject) => {
+            const req = https.request({
+                hostname: 'api.deepgram.com',
+                path: `/v1/listen?language=${lang}&model=nova-2`,
+                method: 'POST',
+                headers: {
+                    'Authorization': `Token ${this.apiKey}`,
+                    'Content-Type': 'audio/wav',
+                    'Content-Length': audio.length,
+                },
+            }, (res) => {
+                const chunks = [];
+                res.on('data', (c) => chunks.push(c));
+                res.on('end', () => {
+                    try {
+                        const data = JSON.parse(Buffer.concat(chunks).toString());
+                        const transcript = data?.results?.channels?.[0]?.alternatives?.[0]?.transcript ?? '';
+                        resolve(transcript);
+                    }
+                    catch {
+                        reject(new Error('Failed to parse Deepgram response'));
+                    }
+                });
+            });
+            req.on('error', reject);
+            req.write(audio);
+            req.end();
+        });
+    }
+}
+exports.DeepgramSTTProvider = DeepgramSTTProvider;
+// ── Edge TTS Provider (free, no API key) ────────────────────
+class EdgeTTSProvider {
+    name = 'edge-tts';
+    defaultVoice;
+    constructor(voice) {
+        this.defaultVoice = voice ?? 'en-US-AriaNeural';
+    }
+    async synthesize(text, options) {
+        const WebSocket = (await Promise.resolve(`${'ws'}`).then(s => __importStar(require(s))).catch(() => null))?.default;
+        if (!WebSocket) {
+            throw new Error('ws package required for Edge TTS. Install with: npm i ws');
+        }
+        const voice = options?.voice ?? this.defaultVoice;
+        const requestId = [...Array(32)].map(() => Math.random().toString(16)[2]).join('');
+        const timestamp = new Date().toISOString();
+        const endpoint = `wss://speech.platform.bing.com/consumer/speech/synthesize/readaloud/edge/v1?TrustedClientToken=6A5AA1D4EAFF4E9FB37E23D68491D6F4&ConnectionId=${requestId}`;
+        return new Promise((resolve, reject) => {
+            const ws = new WebSocket(endpoint, {
+                headers: {
+                    'Origin': 'chrome-extension://jdiccldimpdaibmpdkjnbmckianbfold',
+                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+                },
+            });
+            const audioChunks = [];
+            let headerSent = false;
+            ws.on('open', () => {
+                // Send config
+                ws.send(`Content-Type:application/json; charset=utf-8\r\nPath:speech.config\r\n\r\n{"context":{"synthesis":{"audio":{"metadataoptions":{"sentenceBoundaryEnabled":"false","wordBoundaryEnabled":"false"},"outputFormat":"audio-24khz-48kbitrate-mono-mp3"}}}}`);
+                // Send SSML
+                const ssml = `<speak version='1.0' xmlns='http://www.w3.org/2001/10/synthesis' xml:lang='en-US'><voice name='${voice}'>${escapeXml(text)}</voice></speak>`;
+                ws.send(`X-RequestId:${requestId}\r\nContent-Type:application/ssml+xml\r\nX-Timestamp:${timestamp}\r\nPath:ssml\r\n\r\n${ssml}`);
+            });
+            ws.on('message', (data) => {
+                if (typeof data === 'string' || (Buffer.isBuffer(data) && data.toString().includes('Path:turn.end'))) {
+                    if (typeof data === 'string' && data.includes('Path:turn.end')) {
+                        ws.close();
+                        resolve(Buffer.concat(audioChunks));
+                    }
+                }
+                else if (Buffer.isBuffer(data)) {
+                    // Binary message — extract audio after header
+                    const headerEnd = data.indexOf(Buffer.from('\r\n\r\n'));
+                    if (headerEnd !== -1) {
+                        audioChunks.push(data.slice(headerEnd + 4));
+                    }
+                }
+            });
+            ws.on('error', (err) => {
+                reject(new Error(`Edge TTS WebSocket error: ${err.message}`));
+            });
+            ws.on('close', () => {
+                if (audioChunks.length > 0) {
+                    resolve(Buffer.concat(audioChunks));
+                }
+            });
+            // Timeout
+            setTimeout(() => {
+                ws.close();
+                if (audioChunks.length > 0) {
+                    resolve(Buffer.concat(audioChunks));
+                }
+                else {
+                    reject(new Error('Edge TTS timeout'));
+                }
+            }, 30000);
+        });
+    }
+}
+exports.EdgeTTSProvider = EdgeTTSProvider;
+function escapeXml(text) {
+    return text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;').replace(/'/g, '&apos;');
+}
+// ── OpenAI TTS Provider ─────────────────────────────────────
+class OpenAITTSProvider {
+    name = 'openai-tts';
+    apiKey;
+    defaultVoice;
+    constructor(apiKey, voice) {
+        this.apiKey = apiKey;
+        this.defaultVoice = voice ?? 'alloy';
+    }
+    async synthesize(text, options) {
+        const voice = options?.voice ?? this.defaultVoice;
+        const body = JSON.stringify({
+            model: 'tts-1',
+            input: text,
+            voice,
+            speed: options?.speed ?? 1.0,
+        });
+        return new Promise((resolve, reject) => {
+            const req = https.request({
+                hostname: 'api.openai.com',
+                path: '/v1/audio/speech',
+                method: 'POST',
+                headers: {
+                    'Authorization': `Bearer ${this.apiKey}`,
+                    'Content-Type': 'application/json',
+                    'Content-Length': Buffer.byteLength(body),
+                },
+            }, (res) => {
+                const chunks = [];
+                res.on('data', (c) => chunks.push(c));
+                res.on('end', () => resolve(Buffer.concat(chunks)));
+            });
+            req.on('error', reject);
+            req.write(body);
+            req.end();
+        });
+    }
+}
+exports.OpenAITTSProvider = OpenAITTSProvider;
+// ── ElevenLabs TTS Provider ─────────────────────────────────
+class ElevenLabsTTSProvider {
+    name = 'elevenlabs';
+    apiKey;
+    defaultVoice;
+    constructor(apiKey, voice) {
+        this.apiKey = apiKey;
+        this.defaultVoice = voice ?? '21m00Tcm4TlvDq8ikWAM'; // Rachel
+    }
+    async synthesize(text, options) {
+        const voiceId = options?.voice ?? this.defaultVoice;
+        const body = JSON.stringify({
+            text,
+            model_id: 'eleven_monolingual_v1',
+        });
+        return new Promise((resolve, reject) => {
+            const req = https.request({
+                hostname: 'api.elevenlabs.io',
+                path: `/v1/text-to-speech/${voiceId}`,
+                method: 'POST',
+                headers: {
+                    'xi-api-key': this.apiKey,
+                    'Content-Type': 'application/json',
+                    'Content-Length': Buffer.byteLength(body),
+                },
+            }, (res) => {
+                const chunks = [];
+                res.on('data', (c) => chunks.push(c));
+                res.on('end', () => resolve(Buffer.concat(chunks)));
+            });
+            req.on('error', reject);
+            req.write(body);
+            req.end();
+        });
+    }
+}
+exports.ElevenLabsTTSProvider = ElevenLabsTTSProvider;
+// ── Voice Config Factory ────────────────────────────────────
+function createVoiceProviders(config) {
+    let stt;
+    let tts;
+    switch (config.sttProvider) {
+        case 'whisper':
+            if (config.sttApiKey)
+                stt = new WhisperSTTProvider(config.sttApiKey);
+            break;
+        case 'deepgram':
+            if (config.sttApiKey)
+                stt = new DeepgramSTTProvider(config.sttApiKey);
+            break;
+        case 'web-speech':
+            // Browser only — not available in Node.js
+            break;
+    }
+    switch (config.ttsProvider) {
+        case 'edge-tts':
+            tts = new EdgeTTSProvider(config.voice);
+            break;
+        case 'openai-tts':
+            if (config.ttsApiKey)
+                tts = new OpenAITTSProvider(config.ttsApiKey, config.voice);
+            break;
+        case 'elevenlabs':
+            if (config.ttsApiKey)
+                tts = new ElevenLabsTTSProvider(config.ttsApiKey, config.voice);
+            break;
+    }
+    return { stt, tts };
+}
 // ── Voice Channel ───────────────────────────────────────────
 class VoiceChannel extends index_1.BaseChannel {
     type = 'voice';
     config;
     logger = new logger_1.Logger('voice-channel');
     running = false;
+    conversationActive = false;
     constructor(config) {
         super();
         this.config = config ?? {};
@@ -22,11 +332,51 @@ class VoiceChannel extends index_1.BaseChannel {
     }
     async stop() {
         this.running = false;
+        this.conversationActive = false;
         this.logger.info('Voice channel stopped');
     }
     isRunning() {
         return this.running;
     }
+    /** Transcribe audio to text */
+    async transcribe(audio, format) {
+        if (!this.config.sttProvider) {
+            throw new Error('No STT provider configured');
+        }
+        return this.config.sttProvider.transcribe(audio, { language: this.config.language });
+    }
+    /** Synthesize text to audio */
+    async synthesize(text, voice) {
+        if (!this.config.ttsProvider) {
+            throw new Error('No TTS provider configured');
+        }
+        return this.config.ttsProvider.synthesize(text, { voice });
+    }
+    /** Start real-time conversation mode */
+    async startConversation(onMessage) {
+        if (!this.running)
+            await this.start();
+        this.conversationActive = true;
+        this.logger.info('Conversation mode started');
+        // In a real implementation, this would set up a microphone stream.
+        // For now, expose the conversation loop for programmatic use.
+    }
+    /** Process a single turn in conversation mode */
+    async processConversationTurn(audio, onMessage) {
+        const text = await this.transcribe(audio);
+        const response = await onMessage(text);
+        let audioResponse;
+        if (this.config.ttsProvider) {
+            audioResponse = await this.synthesize(response);
+        }
+        return { text, response, audioResponse };
+    }
+    stopConversation() {
+        this.conversationActive = false;
+    }
+    isConversationActive() {
+        return this.conversationActive;
+    }
     /** Process audio input: STT → Agent → TTS */
     async processAudio(audio) {
         if (!this.handler)