opc-agent 4.1.6 → 4.1.8

package/dist/channels/voice.js

@@ -1,4 +1,20 @@
 "use strict";
+/**
+ * Voice processing module — STT (Speech-to-Text) and TTS (Text-to-Speech)
+ *
+ * STT providers:
+ * 1. Edge STT (free, via edge-stt or Whisper local)
+ * 2. Volcano Engine / Doubao STT (best Chinese, ~¥0.01/req)
+ * 3. OpenAI Whisper API ($0.006/min)
+ * 4. Local Whisper (free, needs model download)
+ *
+ * TTS providers:
+ * 1. edge-tts (free, Microsoft voices, excellent Chinese)
+ * 2. Volcano Engine TTS (Doubao voices)
+ * 3. OpenAI TTS ($0.015/1K chars)
+ *
+ * Default: Whisper API for STT + edge-tts for TTS (best quality/cost ratio)
+ */
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
     var desc = Object.getOwnPropertyDescriptor(m, k);
@@ -33,385 +49,244 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.VoiceChannel = exports.ElevenLabsTTSProvider = exports.OpenAITTSProvider = exports.EdgeTTSProvider = exports.DeepgramSTTProvider = exports.WhisperSTTProvider = void 0;
-exports.createVoiceProviders = createVoiceProviders;
-const index_1 = require("./index");
-const logger_1 = require("../core/logger");
+exports.VoiceProcessor = void 0;
+exports.createVoiceProcessor = createVoiceProcessor;
+const child_process_1 = require("child_process");
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
 const https = __importStar(require("https"));
-// ── Whisper STT Provider ────────────────────────────────────
-class WhisperSTTProvider {
-    name = 'whisper';
-    apiKey;
-    constructor(apiKey) {
-        this.apiKey = apiKey;
+const http = __importStar(require("http"));
+const DEFAULT_CONFIG = {
+    sttProvider: 'whisper-api',
+    ttsProvider: 'edge-tts',
+    ttsVoice: 'zh-CN-XiaoxiaoNeural',
+    ttsLang: 'zh-CN',
+    tempDir: '.opc/voice-tmp',
+};
+class VoiceProcessor {
+    config;
+    constructor(config) {
+        this.config = { ...DEFAULT_CONFIG, ...config };
+        const dir = this.config.tempDir || '.opc/voice-tmp';
+        if (!fs.existsSync(dir))
+            fs.mkdirSync(dir, { recursive: true });
+    }
+    /**
+     * Convert speech audio to text (STT)
+     * @param audioPath Path to audio file (.ogg, .mp3, .wav, .m4a)
+     * @returns Transcribed text
+     */
+    async speechToText(audioPath) {
+        switch (this.config.sttProvider) {
+            case 'whisper-api':
+                return this.whisperApiSTT(audioPath);
+            case 'whisper-local':
+                return this.whisperLocalSTT(audioPath);
+            case 'volcano':
+                return this.volcanoSTT(audioPath);
+            default:
+                throw new Error(`STT not configured. Set voice.sttProvider in config.`);
+        }
+    }
+    /**
+     * Convert text to speech audio (TTS)
+     * @param text Text to convert
+     * @returns Path to generated audio file (.mp3)
+     */
+    async textToSpeech(text) {
+        switch (this.config.ttsProvider) {
+            case 'edge-tts':
+                return this.edgeTTS(text);
+            case 'openai-tts':
+                return this.openaiTTS(text);
+            case 'volcano':
+                return this.volcanoTTS(text);
+            default:
+                throw new Error(`TTS not configured. Set voice.ttsProvider in config.`);
+        }
+    }
+    /** Check if voice processing is available */
+    isSTTAvailable() {
+        if (this.config.sttProvider === 'none')
+            return false;
+        if (this.config.sttProvider === 'whisper-api') {
+            return !!(this.config.openaiApiKey || process.env.OPENAI_API_KEY);
+        }
+        if (this.config.sttProvider === 'whisper-local') {
+            return this.checkOllamaWhisper();
+        }
+        return true;
     }
-    async transcribe(audio, options) {
-        const FormData = (await Promise.resolve(`${'form-data'}`).then(s => __importStar(require(s))).catch(() => null));
-        // Build multipart form data manually
-        const boundary = '----OPCBoundary' + Date.now().toString(36);
+    isTTSAvailable() {
+        if (this.config.ttsProvider === 'none')
+            return false;
+        if (this.config.ttsProvider === 'edge-tts')
+            return this.checkEdgeTTS();
+        return true;
+    }
+    // ─── STT Providers ───
+    async whisperApiSTT(audioPath) {
+        const apiKey = this.config.openaiApiKey || process.env.OPENAI_API_KEY;
+        const baseUrl = this.config.openaiBaseUrl || process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1';
+        if (!apiKey)
+            throw new Error('OpenAI API key required for Whisper STT');
+        // Use multipart/form-data with fetch
+        const fileBuffer = fs.readFileSync(audioPath);
+        const fileName = path.basename(audioPath);
+        // Build multipart body manually
+        const boundary = '----OPCVoice' + Date.now();
         const parts = [];
-        // file field
-        parts.push(Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="audio.wav"\r\nContent-Type: audio/wav\r\n\r\n`));
-        parts.push(audio);
+        // file part
+        parts.push(Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${fileName}"\r\nContent-Type: audio/ogg\r\n\r\n`));
+        parts.push(fileBuffer);
         parts.push(Buffer.from('\r\n'));
-        // model field
+        // model part
         parts.push(Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="model"\r\n\r\nwhisper-1\r\n`));
-        // language field
-        if (options?.language) {
-            parts.push(Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="language"\r\n\r\n${options.language}\r\n`));
-        }
+        // language part (optimize for Chinese)
+        parts.push(Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="language"\r\n\r\nzh\r\n`));
         parts.push(Buffer.from(`--${boundary}--\r\n`));
         const body = Buffer.concat(parts);
-        return new Promise((resolve, reject) => {
-            const req = https.request({
-                hostname: 'api.openai.com',
-                path: '/v1/audio/transcriptions',
-                method: 'POST',
-                headers: {
-                    'Authorization': `Bearer ${this.apiKey}`,
-                    'Content-Type': `multipart/form-data; boundary=${boundary}`,
-                    'Content-Length': body.length,
-                },
-            }, (res) => {
-                const chunks = [];
-                res.on('data', (c) => chunks.push(c));
-                res.on('end', () => {
-                    try {
-                        const data = JSON.parse(Buffer.concat(chunks).toString());
-                        resolve(data.text ?? '');
-                    }
-                    catch (e) {
-                        reject(new Error('Failed to parse Whisper response'));
-                    }
-                });
-            });
-            req.on('error', reject);
-            req.write(body);
-            req.end();
+        const url = `${baseUrl}/audio/transcriptions`;
+        const response = await fetch(url, {
+            method: 'POST',
+            headers: {
+                'Authorization': `Bearer ${apiKey}`,
+                'Content-Type': `multipart/form-data; boundary=${boundary}`,
+            },
+            body,
         });
+        if (!response.ok) {
+            const err = await response.text();
+            throw new Error(`Whisper API error (${response.status}): ${err}`);
+        }
+        const result = await response.json();
+        return result.text?.trim() || '';
     }
-}
-exports.WhisperSTTProvider = WhisperSTTProvider;
-// ── Deepgram STT Provider ───────────────────────────────────
-class DeepgramSTTProvider {
-    name = 'deepgram';
-    apiKey;
-    constructor(apiKey) {
-        this.apiKey = apiKey;
-    }
-    async transcribe(audio, options) {
-        const lang = options?.language ?? 'en';
-        return new Promise((resolve, reject) => {
-            const req = https.request({
-                hostname: 'api.deepgram.com',
-                path: `/v1/listen?language=${lang}&model=nova-2`,
-                method: 'POST',
-                headers: {
-                    'Authorization': `Token ${this.apiKey}`,
-                    'Content-Type': 'audio/wav',
-                    'Content-Length': audio.length,
-                },
-            }, (res) => {
-                const chunks = [];
-                res.on('data', (c) => chunks.push(c));
-                res.on('end', () => {
-                    try {
-                        const data = JSON.parse(Buffer.concat(chunks).toString());
-                        const transcript = data?.results?.channels?.[0]?.alternatives?.[0]?.transcript ?? '';
-                        resolve(transcript);
-                    }
-                    catch {
-                        reject(new Error('Failed to parse Deepgram response'));
-                    }
-                });
+    async whisperLocalSTT(audioPath) {
+        // Use Ollama's audio models or local whisper.cpp
+        const ollamaUrl = this.config.ollamaUrl || 'http://localhost:11434';
+        try {
+            // Try whisper via Ollama (if audio model available)
+            // Fallback: use whisper.cpp CLI
+            const result = (0, child_process_1.execSync)(`whisper "${audioPath}" --language zh --output_format txt`, {
+                encoding: 'utf-8',
+                timeout: 30000,
             });
-            req.on('error', reject);
-            req.write(audio);
-            req.end();
-        });
+            return result.trim();
+        }
+        catch {
+            throw new Error('Local Whisper not available. Install whisper.cpp or use whisper-api provider.');
+        }
     }
-}
-exports.DeepgramSTTProvider = DeepgramSTTProvider;
-// ── Edge TTS Provider (free, no API key) ────────────────────
-class EdgeTTSProvider {
-    name = 'edge-tts';
-    defaultVoice;
-    constructor(voice) {
-        this.defaultVoice = voice ?? 'en-US-AriaNeural';
+    async volcanoSTT(_audioPath) {
+        // Volcano Engine STT (豆包同源)
+        // TODO: Implement when credentials provided
+        throw new Error('Volcano STT not yet implemented. Use whisper-api for now.');
     }
-    async synthesize(text, options) {
-        const WebSocket = (await Promise.resolve(`${'ws'}`).then(s => __importStar(require(s))).catch(() => null))?.default;
-        if (!WebSocket) {
-            throw new Error('ws package required for Edge TTS. Install with: npm i ws');
-        }
-        const voice = options?.voice ?? this.defaultVoice;
-        const requestId = [...Array(32)].map(() => Math.random().toString(16)[2]).join('');
-        const timestamp = new Date().toISOString();
-        const endpoint = `wss://speech.platform.bing.com/consumer/speech/synthesize/readaloud/edge/v1?TrustedClientToken=6A5AA1D4EAFF4E9FB37E23D68491D6F4&ConnectionId=${requestId}`;
+    // ─── TTS Providers ───
+    async edgeTTS(text) {
+        const voice = this.config.ttsVoice || 'zh-CN-XiaoxiaoNeural';
+        const outPath = path.join(this.config.tempDir || '.opc/voice-tmp', `tts-${Date.now()}.mp3`);
+        // edge-tts is a Python package: pip install edge-tts
         return new Promise((resolve, reject) => {
-            const ws = new WebSocket(endpoint, {
-                headers: {
-                    'Origin': 'chrome-extension://jdiccldimpdaibmpdkjnbmckianbfold',
-                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
-                },
-            });
-            const audioChunks = [];
-            let headerSent = false;
-            ws.on('open', () => {
-                // Send config
-                ws.send(`Content-Type:application/json; charset=utf-8\r\nPath:speech.config\r\n\r\n{"context":{"synthesis":{"audio":{"metadataoptions":{"sentenceBoundaryEnabled":"false","wordBoundaryEnabled":"false"},"outputFormat":"audio-24khz-48kbitrate-mono-mp3"}}}}`);
-                // Send SSML
-                const ssml = `<speak version='1.0' xmlns='http://www.w3.org/2001/10/synthesis' xml:lang='en-US'><voice name='${voice}'>${escapeXml(text)}</voice></speak>`;
-                ws.send(`X-RequestId:${requestId}\r\nContent-Type:application/ssml+xml\r\nX-Timestamp:${timestamp}\r\nPath:ssml\r\n\r\n${ssml}`);
-            });
-            ws.on('message', (data) => {
-                if (typeof data === 'string' || (Buffer.isBuffer(data) && data.toString().includes('Path:turn.end'))) {
-                    if (typeof data === 'string' && data.includes('Path:turn.end')) {
-                        ws.close();
-                        resolve(Buffer.concat(audioChunks));
-                    }
-                }
-                else if (Buffer.isBuffer(data)) {
-                    // Binary message — extract audio after header
-                    const headerEnd = data.indexOf(Buffer.from('\r\n\r\n'));
-                    if (headerEnd !== -1) {
-                        audioChunks.push(data.slice(headerEnd + 4));
-                    }
-                }
-            });
-            ws.on('error', (err) => {
-                reject(new Error(`Edge TTS WebSocket error: ${err.message}`));
-            });
-            ws.on('close', () => {
-                if (audioChunks.length > 0) {
-                    resolve(Buffer.concat(audioChunks));
-                }
-            });
-            // Timeout
-            setTimeout(() => {
-                ws.close();
-                if (audioChunks.length > 0) {
-                    resolve(Buffer.concat(audioChunks));
+            const escaped = text.replace(/"/g, '\\"').replace(/\n/g, ' ');
+            (0, child_process_1.exec)(`edge-tts --voice "${voice}" --text "${escaped}" --write-media "${outPath}"`, {
+                timeout: 30000,
+            }, (err) => {
+                if (err) {
+                    reject(new Error(`edge-tts failed: ${err.message}. Install with: pip install edge-tts`));
                 }
                 else {
-                    reject(new Error('Edge TTS timeout'));
+                    resolve(outPath);
                 }
-            }, 30000);
-        });
-    }
-}
-exports.EdgeTTSProvider = EdgeTTSProvider;
-function escapeXml(text) {
-    return text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;').replace(/'/g, '&apos;');
-}
-// ── OpenAI TTS Provider ─────────────────────────────────────
-class OpenAITTSProvider {
-    name = 'openai-tts';
-    apiKey;
-    defaultVoice;
-    constructor(apiKey, voice) {
-        this.apiKey = apiKey;
-        this.defaultVoice = voice ?? 'alloy';
-    }
-    async synthesize(text, options) {
-        const voice = options?.voice ?? this.defaultVoice;
-        const body = JSON.stringify({
-            model: 'tts-1',
-            input: text,
-            voice,
-            speed: options?.speed ?? 1.0,
-        });
-        return new Promise((resolve, reject) => {
-            const req = https.request({
-                hostname: 'api.openai.com',
-                path: '/v1/audio/speech',
-                method: 'POST',
-                headers: {
-                    'Authorization': `Bearer ${this.apiKey}`,
-                    'Content-Type': 'application/json',
-                    'Content-Length': Buffer.byteLength(body),
-                },
-            }, (res) => {
-                const chunks = [];
-                res.on('data', (c) => chunks.push(c));
-                res.on('end', () => resolve(Buffer.concat(chunks)));
-            });
-            req.on('error', reject);
-            req.write(body);
-            req.end();
-        });
-    }
-}
-exports.OpenAITTSProvider = OpenAITTSProvider;
-// ── ElevenLabs TTS Provider ─────────────────────────────────
-class ElevenLabsTTSProvider {
-    name = 'elevenlabs';
-    apiKey;
-    defaultVoice;
-    constructor(apiKey, voice) {
-        this.apiKey = apiKey;
-        this.defaultVoice = voice ?? '21m00Tcm4TlvDq8ikWAM'; // Rachel
-    }
-    async synthesize(text, options) {
-        const voiceId = options?.voice ?? this.defaultVoice;
-        const body = JSON.stringify({
-            text,
-            model_id: 'eleven_monolingual_v1',
-        });
-        return new Promise((resolve, reject) => {
-            const req = https.request({
-                hostname: 'api.elevenlabs.io',
-                path: `/v1/text-to-speech/${voiceId}`,
-                method: 'POST',
-                headers: {
-                    'xi-api-key': this.apiKey,
-                    'Content-Type': 'application/json',
-                    'Content-Length': Buffer.byteLength(body),
-                },
-            }, (res) => {
-                const chunks = [];
-                res.on('data', (c) => chunks.push(c));
-                res.on('end', () => resolve(Buffer.concat(chunks)));
             });
-            req.on('error', reject);
-            req.write(body);
-            req.end();
         });
     }
-}
-exports.ElevenLabsTTSProvider = ElevenLabsTTSProvider;
-// ── Voice Config Factory ────────────────────────────────────
-function createVoiceProviders(config) {
-    let stt;
-    let tts;
-    switch (config.sttProvider) {
-        case 'whisper':
-            if (config.sttApiKey)
-                stt = new WhisperSTTProvider(config.sttApiKey);
-            break;
-        case 'deepgram':
-            if (config.sttApiKey)
-                stt = new DeepgramSTTProvider(config.sttApiKey);
-            break;
-        case 'web-speech':
-            // Browser only — not available in Node.js
-            break;
-    }
-    switch (config.ttsProvider) {
-        case 'edge-tts':
-            tts = new EdgeTTSProvider(config.voice);
-            break;
-        case 'openai-tts':
-            if (config.ttsApiKey)
-                tts = new OpenAITTSProvider(config.ttsApiKey, config.voice);
-            break;
-        case 'elevenlabs':
-            if (config.ttsApiKey)
-                tts = new ElevenLabsTTSProvider(config.ttsApiKey, config.voice);
-            break;
-    }
-    return { stt, tts };
-}
-// ── Voice Channel ───────────────────────────────────────────
-class VoiceChannel extends index_1.BaseChannel {
-    type = 'voice';
-    config;
-    logger = new logger_1.Logger('voice-channel');
-    running = false;
-    conversationActive = false;
-    constructor(config) {
-        super();
-        this.config = config ?? {};
-    }
-    async start() {
-        this.running = true;
-        this.logger.info('Voice channel started', {
-            stt: this.config.sttProvider?.name ?? 'none',
-            tts: this.config.ttsProvider?.name ?? 'none',
+    async openaiTTS(text) {
+        const apiKey = this.config.openaiApiKey || process.env.OPENAI_API_KEY;
+        const baseUrl = this.config.openaiBaseUrl || process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1';
+        if (!apiKey)
+            throw new Error('OpenAI API key required for TTS');
+        const outPath = path.join(this.config.tempDir || '.opc/voice-tmp', `tts-${Date.now()}.mp3`);
+        const response = await fetch(`${baseUrl}/audio/speech`, {
+            method: 'POST',
+            headers: {
+                'Authorization': `Bearer ${apiKey}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                model: 'tts-1',
+                voice: 'nova',
+                input: text,
+            }),
         });
+        if (!response.ok)
+            throw new Error(`OpenAI TTS error: ${response.status}`);
+        const buffer = Buffer.from(await response.arrayBuffer());
+        fs.writeFileSync(outPath, buffer);
+        return outPath;
     }
-    async stop() {
-        this.running = false;
-        this.conversationActive = false;
-        this.logger.info('Voice channel stopped');
-    }
-    isRunning() {
-        return this.running;
+    async volcanoTTS(_text) {
+        throw new Error('Volcano TTS not yet implemented.');
     }
-    /** Transcribe audio to text */
-    async transcribe(audio, format) {
-        if (!this.config.sttProvider) {
-            throw new Error('No STT provider configured');
+    // ─── Helpers ───
+    checkEdgeTTS() {
+        try {
+            (0, child_process_1.execSync)('edge-tts --version', { stdio: 'pipe', timeout: 5000 });
+            return true;
         }
-        return this.config.sttProvider.transcribe(audio, { language: this.config.language });
-    }
-    /** Synthesize text to audio */
-    async synthesize(text, voice) {
-        if (!this.config.ttsProvider) {
-            throw new Error('No TTS provider configured');
+        catch {
+            return false;
         }
-        return this.config.ttsProvider.synthesize(text, { voice });
     }
-    /** Start real-time conversation mode */
-    async startConversation(onMessage) {
-        if (!this.running)
-            await this.start();
-        this.conversationActive = true;
-        this.logger.info('Conversation mode started');
-        // In a real implementation, this would set up a microphone stream.
-        // For now, expose the conversation loop for programmatic use.
-    }
-    /** Process a single turn in conversation mode */
-    async processConversationTurn(audio, onMessage) {
-        const text = await this.transcribe(audio);
-        const response = await onMessage(text);
-        let audioResponse;
-        if (this.config.ttsProvider) {
-            audioResponse = await this.synthesize(response);
-        }
-        return { text, response, audioResponse };
-    }
-    stopConversation() {
-        this.conversationActive = false;
-    }
-    isConversationActive() {
-        return this.conversationActive;
-    }
-    /** Process audio input: STT → Agent → TTS */
-    async processAudio(audio) {
-        if (!this.handler)
-            throw new Error('No message handler set');
-        // STT
-        let text;
-        if (this.config.sttProvider) {
-            text = await this.config.sttProvider.transcribe(audio, { language: this.config.language });
-        }
-        else {
-            text = audio.toString('utf-8'); // Fallback: treat as text
+    checkOllamaWhisper() {
+        try {
+            (0, child_process_1.execSync)('whisper --help', { stdio: 'pipe', timeout: 5000 });
+            return true;
         }
-        this.logger.debug('STT result', { text });
-        // Create message and send to agent
-        const message = {
-            id: `voice_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
-            role: 'user',
-            content: text,
-            timestamp: Date.now(),
-            metadata: { channel: 'voice' },
-        };
-        const response = await this.handler(message);
-        // TTS
-        let audioResponse;
-        if (this.config.ttsProvider) {
-            audioResponse = await this.config.ttsProvider.synthesize(response.content, { language: this.config.language });
+        catch {
+            return false;
         }
-        return { text, response: response.content, audioResponse };
     }
-    setSTTProvider(provider) {
-        this.config.sttProvider = provider;
+    /** Download a file from URL to local path */
+    async downloadFile(url, destPath) {
+        return new Promise((resolve, reject) => {
+            const client = url.startsWith('https') ? https : http;
+            const req = client.get(url, (res) => {
+                if (res.statusCode === 301 || res.statusCode === 302) {
+                    // Follow redirect
+                    this.downloadFile(res.headers.location, destPath).then(resolve).catch(reject);
+                    return;
+                }
+                const ws = fs.createWriteStream(destPath);
+                res.pipe(ws);
+                ws.on('finish', () => { ws.close(); resolve(); });
+                ws.on('error', reject);
+            });
+            req.on('error', reject);
+            req.setTimeout(30000, () => { req.destroy(); reject(new Error('Download timeout')); });
+        });
     }
-    setTTSProvider(provider) {
-        this.config.ttsProvider = provider;
+    /** Cleanup temp files */
+    cleanup() {
+        const dir = this.config.tempDir || '.opc/voice-tmp';
+        try {
+            const files = fs.readdirSync(dir);
+            const now = Date.now();
+            for (const f of files) {
+                const fp = path.join(dir, f);
+                const stat = fs.statSync(fp);
+                // Remove files older than 1 hour
+                if (now - stat.mtimeMs > 3600000) {
+                    fs.unlinkSync(fp);
+                }
+            }
+        }
+        catch { /* ignore */ }
     }
 }
-exports.VoiceChannel = VoiceChannel;
+exports.VoiceProcessor = VoiceProcessor;
+function createVoiceProcessor(config) {
+    return new VoiceProcessor(config);
+}
 //# sourceMappingURL=voice.js.map