@framers/agentos-ext-voice-synthesis 1.0.0 → 2.0.1

package/dist/tools/textToSpeech.js

@@ -1,7 +1,11 @@
 /**
- * ElevenLabs TTS Tool — text-to-speech synthesis.
+ * Multi-provider TTS Tool — text-to-speech synthesis.
+ *
+ * Supports: OpenAI TTS, ElevenLabs, Ollama (local), any OpenAI-compatible TTS API.
+ * Auto-detects available provider from API keys in environment.
  */
-const VOICES = {
+// ── ElevenLabs voice name → ID mapping ──
+const ELEVENLABS_VOICES = {
     rachel: '21m00Tcm4TlvDq8ikWAM',
     domi: 'AZnzlk1XvdvUeBnXmlld',
     bella: 'EXAVITQu4vr4xnSDxMaL',
@@ -11,62 +15,208 @@ const VOICES = {
     adam: 'pNInz6obpgDQGcFmaJgB',
     sam: 'yoZ06aMxZJJ28mfd3POQ',
 };
+// ── OpenAI voice options ──
+const OPENAI_VOICES = ['alloy', 'echo', 'fable', 'onyx', 'nova', 'shimmer'];
 export class TextToSpeechTool {
-    id = 'elevenlabs-tts-v1';
+    id = 'tts-multi-provider-v1';
     name = 'text_to_speech';
     displayName = 'Text to Speech';
-    description = 'Convert text to speech using ElevenLabs. Returns base64-encoded MP3 audio. ' +
-        'Voices: rachel, domi, bella, antoni, josh, arnold, adam, sam.';
+    description = 'Convert text to speech audio. Supports multiple providers: OpenAI TTS (alloy/echo/fable/onyx/nova/shimmer), ' +
+        'ElevenLabs (rachel/domi/bella/antoni/josh/arnold/adam/sam), or local Ollama TTS. ' +
+        'Auto-detects available provider from API keys. Returns base64-encoded audio.';
     category = 'media';
-    version = '1.0.0';
+    version = '2.0.0';
     hasSideEffects = false;
     inputSchema = {
         type: 'object',
         properties: {
-            text: { type: 'string', description: 'Text to convert. Max 5000 chars.' },
-            voice: { type: 'string', default: 'rachel', description: 'Voice name or ID.' },
-            model: { type: 'string', default: 'eleven_monolingual_v1' },
-            stability: { type: 'number', minimum: 0, maximum: 1, default: 0.5 },
-            similarity_boost: { type: 'number', minimum: 0, maximum: 1, default: 0.75 },
+            text: { type: 'string', description: 'Text to convert to speech. Max 5000 chars.' },
+            voice: {
+                type: 'string',
+                description: 'Voice name. OpenAI: alloy, echo, fable, onyx, nova (default), shimmer. ' +
+                    'ElevenLabs: rachel (default), domi, bella, antoni, josh, arnold, adam, sam. ' +
+                    'Or a custom voice ID.',
+            },
+            model: {
+                type: 'string',
+                description: 'TTS model. OpenAI: tts-1 (default), tts-1-hd. ElevenLabs: eleven_monolingual_v1 (default), eleven_multilingual_v2.',
+            },
+            provider: {
+                type: 'string',
+                enum: ['openai', 'elevenlabs', 'ollama', 'auto'],
+                description: 'TTS provider. Default: auto (detects from available API keys).',
+            },
+            speed: { type: 'number', minimum: 0.25, maximum: 4.0, description: 'OpenAI speed multiplier (0.25-4.0).' },
+            stability: { type: 'number', minimum: 0, maximum: 1, description: 'ElevenLabs voice stability (0-1).' },
+            similarity_boost: { type: 'number', minimum: 0, maximum: 1, description: 'ElevenLabs similarity boost (0-1).' },
+            format: { type: 'string', enum: ['mp3', 'opus', 'aac', 'flac', 'wav'], description: 'Output audio format.' },
         },
         required: ['text'],
     };
     requiredCapabilities = ['capability:tts'];
-    apiKey;
-    constructor(apiKey) {
-        this.apiKey = apiKey || process.env.ELEVENLABS_API_KEY || '';
+    config;
+    constructor(config) {
+        this.config = {
+            openaiApiKey: config?.openaiApiKey || process.env.OPENAI_API_KEY || '',
+            openaiBaseUrl: config?.openaiBaseUrl || process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1',
+            elevenLabsApiKey: config?.elevenLabsApiKey || process.env.ELEVENLABS_API_KEY || '',
+            ollamaBaseUrl: config?.ollamaBaseUrl || process.env.OLLAMA_BASE_URL || 'http://localhost:11434',
+            defaultProvider: config?.defaultProvider || process.env.TTS_PROVIDER || 'auto',
+        };
+    }
+    resolveProvider(requested) {
+        const pref = requested || this.config.defaultProvider || 'auto';
+        if (pref !== 'auto') {
+            // Verify the requested provider has credentials
+            if (pref === 'openai' && this.config.openaiApiKey)
+                return 'openai';
+            if (pref === 'elevenlabs' && this.config.elevenLabsApiKey)
+                return 'elevenlabs';
+            if (pref === 'ollama')
+                return 'ollama';
+            // Fall through to auto if requested provider isn't configured
+        }
+        // Auto-detect: prefer OpenAI (cheaper, faster), then ElevenLabs, then Ollama
+        if (this.config.openaiApiKey)
+            return 'openai';
+        if (this.config.elevenLabsApiKey)
+            return 'elevenlabs';
+        return 'ollama'; // Local fallback — may or may not have TTS model
     }
     async execute(args, _context) {
-        if (!this.apiKey)
-            return { success: false, error: 'ELEVENLABS_API_KEY not configured.' };
         const text = args.text.slice(0, 5000);
-        const voiceId = VOICES[(args.voice || 'rachel').toLowerCase()] || args.voice || VOICES.rachel;
+        const provider = this.resolveProvider(args.provider);
+        if (!provider) {
+            return {
+                success: false,
+                error: 'No TTS provider available. Set one of: OPENAI_API_KEY, ELEVENLABS_API_KEY, or configure Ollama with a TTS model. ' +
+                    'Get an OpenAI key at https://platform.openai.com/api-keys or ElevenLabs at https://elevenlabs.io',
+            };
+        }
+        try {
+            switch (provider) {
+                case 'openai':
+                    return await this.synthesizeOpenAI(text, args);
+                case 'elevenlabs':
+                    return await this.synthesizeElevenLabs(text, args);
+                case 'ollama':
+                    return await this.synthesizeOllama(text, args);
+                default:
+                    return { success: false, error: `Unknown TTS provider: ${provider}` };
+            }
+        }
+        catch (err) {
+            return { success: false, error: `TTS failed (${provider}): ${err.message}` };
+        }
+    }
+    // ── OpenAI TTS ──
+    async synthesizeOpenAI(text, args) {
+        const voice = args.voice && OPENAI_VOICES.includes(args.voice) ? args.voice : 'nova';
+        const model = args.model || 'tts-1';
+        const format = args.format || 'mp3';
+        const response = await fetch(`${this.config.openaiBaseUrl}/audio/speech`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.config.openaiApiKey}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                model,
+                voice,
+                input: text,
+                response_format: format,
+                speed: args.speed,
+            }),
+        });
+        if (!response.ok) {
+            const err = await response.text();
+            return { success: false, error: `OpenAI TTS error (${response.status}): ${err.slice(0, 300)}` };
+        }
+        const buf = await response.arrayBuffer();
+        const audioBase64 = Buffer.from(buf).toString('base64');
+        const contentType = format === 'opus' ? 'audio/opus' : format === 'wav' ? 'audio/wav' : 'audio/mpeg';
+        const durationEstimateMs = Math.round((text.split(/\s+/).length / 150) * 60 * 1000);
+        return {
+            success: true,
+            output: { text, voice, model, provider: 'openai', audioBase64, contentType, durationEstimateMs },
+            contentType,
+        };
+    }
+    // ── ElevenLabs TTS ──
+    async synthesizeElevenLabs(text, args) {
+        const voiceId = ELEVENLABS_VOICES[(args.voice || 'rachel').toLowerCase()] || args.voice || ELEVENLABS_VOICES.rachel;
         const model = args.model || 'eleven_monolingual_v1';
+        const response = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`, {
+            method: 'POST',
+            headers: {
+                'xi-api-key': this.config.elevenLabsApiKey,
+                'Content-Type': 'application/json',
+                Accept: 'audio/mpeg',
+            },
+            body: JSON.stringify({
+                text,
+                model_id: model,
+                voice_settings: {
+                    stability: args.stability ?? 0.5,
+                    similarity_boost: args.similarity_boost ?? 0.75,
+                },
+            }),
+        });
+        if (!response.ok) {
+            const err = await response.text();
+            return { success: false, error: `ElevenLabs error (${response.status}): ${err.slice(0, 300)}` };
+        }
+        const buf = await response.arrayBuffer();
+        const audioBase64 = Buffer.from(buf).toString('base64');
+        const durationEstimateMs = Math.round((text.split(/\s+/).length / 150) * 60 * 1000);
+        return {
+            success: true,
+            output: {
+                text,
+                voice: args.voice || 'rachel',
+                model,
+                provider: 'elevenlabs',
+                audioBase64,
+                contentType: 'audio/mpeg',
+                durationEstimateMs,
+            },
+            contentType: 'audio/mpeg',
+        };
+    }
+    // ── Ollama TTS (local, experimental) ──
+    async synthesizeOllama(text, args) {
+        // Ollama doesn't natively support TTS yet, but some models (e.g., bark, piper)
+        // can be served via OpenAI-compatible endpoints. Try the OpenAI-compat path.
+        const voice = args.voice || 'default';
+        const model = args.model || 'tts'; // User must have a TTS model loaded
         try {
-            const response = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`, {
+            const response = await fetch(`${this.config.ollamaBaseUrl}/v1/audio/speech`, {
                 method: 'POST',
-                headers: { 'xi-api-key': this.apiKey, 'Content-Type': 'application/json', Accept: 'audio/mpeg' },
-                body: JSON.stringify({
-                    text,
-                    model_id: model,
-                    voice_settings: { stability: args.stability ?? 0.5, similarity_boost: args.similarity_boost ?? 0.75 },
-                }),
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ model, voice, input: text }),
             });
             if (!response.ok) {
-                const err = await response.text();
-                return { success: false, error: `ElevenLabs error (${response.status}): ${err}` };
+                return {
+                    success: false,
+                    error: `Ollama TTS not available (${response.status}). Ollama doesn't natively support TTS yet. ` +
+                        'Set OPENAI_API_KEY or ELEVENLABS_API_KEY for cloud TTS, or use a dedicated local TTS server.',
+                };
             }
             const buf = await response.arrayBuffer();
             const audioBase64 = Buffer.from(buf).toString('base64');
             const durationEstimateMs = Math.round((text.split(/\s+/).length / 150) * 60 * 1000);
             return {
                 success: true,
-                output: { text, voice: args.voice || 'rachel', model, audioBase64, contentType: 'audio/mpeg', durationEstimateMs },
+                output: { text, voice, model, provider: 'ollama', audioBase64, contentType: 'audio/mpeg', durationEstimateMs },
                 contentType: 'audio/mpeg',
             };
         }
-        catch (err) {
-            return { success: false, error: `TTS failed: ${err.message}` };
+        catch {
+            return {
+                success: false,
+                error: 'Ollama TTS endpoint not reachable. Ollama doesn\'t natively support TTS yet. ' +
+                    'Set OPENAI_API_KEY or ELEVENLABS_API_KEY for cloud TTS.',
+            };
         }
     }
 }

package/dist/tools/textToSpeech.js.map

@@ -1 +1 @@
-{"version":3,"file":"textToSpeech.js","sourceRoot":"","sources":["../../src/tools/textToSpeech.ts"],"names":[],"mappings":"AAAA;;GAEG;AAqBH,MAAM,MAAM,GAA2B;IACrC,MAAM,EAAE,sBAAsB;IAC9B,IAAI,EAAE,sBAAsB;IAC5B,KAAK,EAAE,sBAAsB;IAC7B,MAAM,EAAE,sBAAsB;IAC9B,IAAI,EAAE,sBAAsB;IAC5B,MAAM,EAAE,sBAAsB;IAC9B,IAAI,EAAE,sBAAsB;IAC5B,GAAG,EAAE,sBAAsB;CAC5B,CAAC;AAEF,MAAM,OAAO,gBAAgB;IAClB,EAAE,GAAG,mBAAmB,CAAC;IACzB,IAAI,GAAG,gBAAgB,CAAC;IACxB,WAAW,GAAG,gBAAgB,CAAC;IAC/B,WAAW,GAClB,6EAA6E;QAC7E,+DAA+D,CAAC;IACzD,QAAQ,GAAG,OAAO,CAAC;IACnB,OAAO,GAAG,OAAO,CAAC;IAClB,cAAc,GAAG,KAAK,CAAC;IAEvB,WAAW,GAAqB;QACvC,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,kCAAkC,EAAE;YACzE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,mBAAmB,EAAE;YAC9E,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,uBAAuB,EAAE;YAC3D,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE;YACnE,gBAAgB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE;SAC5E;QACD,QAAQ,EAAE,CAAC,MAAM,CAAC;KACnB,CAAC;IAEO,oBAAoB,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAE3C,MAAM,CAAS;IAEvB,YAAY,MAAe;QACzB,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,EAAE,CAAC;IAC/D,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAc,EAAE,QAA8B;QAC1D,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,oCAAoC,EAAE,CAAC;QAEzF,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC;QAC9F,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,uBAAuB,CAAC;QAEpD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,+CAA+C,OAAO,EAAE,EAAE;gBACrF,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,EAAE,YAAY,EAAE;gBAChG,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,IAAI;oBACJ,QAAQ,EAAE,KAAK;oBACf,cAAc,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,GAAG,EAAE,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,IAAI,IAAI,EAAE;iBACtG,CAAC;aACH,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAClC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,qBAAqB,QAAQ,CAAC,MAAM,MAAM,GAAG,EAAE,EAAE,CAAC;YACpF,CAAC;YAED,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;YACzC,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACxD,MAAM,kBAAkB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;YAEpF,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,kBAAkB,EAAE;gBAClH,WAAW,EAAE,YAAY;aAC1B,CAAC;QACJ,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,eAAe,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC;QACjE,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"textToSpeech.js","sourceRoot":"","sources":["../../src/tools/textToSpeech.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AA+BH,2CAA2C;AAC3C,MAAM,iBAAiB,GAA2B;IAChD,MAAM,EAAE,sBAAsB;IAC9B,IAAI,EAAE,sBAAsB;IAC5B,KAAK,EAAE,sBAAsB;IAC7B,MAAM,EAAE,sBAAsB;IAC9B,IAAI,EAAE,sBAAsB;IAC5B,MAAM,EAAE,sBAAsB;IAC9B,IAAI,EAAE,sBAAsB;IAC5B,GAAG,EAAE,sBAAsB;CAC5B,CAAC;AAEF,6BAA6B;AAC7B,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;AAU5E,MAAM,OAAO,gBAAgB;IAClB,EAAE,GAAG,uBAAuB,CAAC;IAC7B,IAAI,GAAG,gBAAgB,CAAC;IACxB,WAAW,GAAG,gBAAgB,CAAC;IAC/B,WAAW,GAClB,8GAA8G;QAC9G,mFAAmF;QACnF,8EAA8E,CAAC;IACxE,QAAQ,GAAG,OAAO,CAAC;IACnB,OAAO,GAAG,OAAO,CAAC;IAClB,cAAc,GAAG,KAAK,CAAC;IAEvB,WAAW,GAAqB;QACvC,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,4CAA4C,EAAE;YACnF,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,WAAW,EACT,yEAAyE;oBACzE,8EAA8E;oBAC9E,uBAAuB;aAC1B;YACD,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,oHAAoH;aAClI;YACD,QAAQ,EAAE;gBACR,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,CAAC,QAAQ,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,CAAC;gBAChD,WAAW,EAAE,gEAAgE;aAC9E;YACD,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,qCAAqC,EAAE;YAC1G,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,WAAW,EAAE,mCAAmC,EAAE;YACvG,gBAAgB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,WAAW,EAAE,oCAAoC,EAAE;YAC/G,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,WAAW,EAAE,sBAAsB,EAAE;SAC7G;QACD,QAAQ,EAAE,CAAC,MAAM,CAAC;KACnB,CAAC;IAEO,oBAAoB,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAE3C,MAAM,CAAY;IAE1B,YAAY,MAAkB;QAC5B,IAAI,CAAC,MAAM,GAAG;YACZ,YAAY,EAAE,MAAM,EAAE,YAAY,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE;YACtE,aAAa,EAAE,MAAM,EAAE,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,2BAA2B;YAClG,gBAAgB,EAAE,MAAM,EAAE,gBAAgB,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,EAAE;YAClF,aAAa,EAAE,MAAM,EAAE,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,wBAAwB;YAC/F,eAAe,EAAE,MAAM,EAAE,eAAe,IAAK,OAAO,CAAC,GAAG,CAAC,YAA4B,IAAI,MAAM;SAChG,CAAC;IACJ,CAAC;IAEO,eAAe,CAAC,SAAuB;QAC7C,MAAM,IAAI,GAAG,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,MAAM,CAAC;QAChE,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YACpB,gDAAgD;YAChD,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY;gBAAE,OAAO,QAAQ,CAAC;YACnE,IAAI,IAAI,KAAK,YAAY,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB;gBAAE,OAAO,YAAY,CAAC;YAC/E,IAAI,IAAI,KAAK,QAAQ;gBAAE,OAAO,QAAQ,CAAC;YACvC,8DAA8D;QAChE,CAAC;QAED,6EAA6E;QAC7E,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY;YAAE,OAAO,QAAQ,CAAC;QAC9C,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB;YAAE,OAAO,YAAY,CAAC;QACtD,OAAO,QAAQ,CAAC,CAAC,iDAAiD;IACpE,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAc,EAAE,QAA8B;QAC1D,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAErD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EACH,mHAAmH;oBACnH,kGAAkG;aACrG,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,QAAQ,QAAQ,EAAE,CAAC;gBACjB,KAAK,QAAQ;oBACX,OAAO,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBACjD,KAAK,YAAY;oBACf,OAAO,MAAM,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBACrD,KAAK,QAAQ;oBACX,OAAO,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBACjD;oBACE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,yBAAyB,QAAQ,EAAE,EAAE,CAAC;YAC1E,CAAC;QACH,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,eAAe,QAAQ,MAAM,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC;QAC/E,CAAC;IACH,CAAC;IAED,mBAAmB;IAEX,KAAK,CAAC,gBAAgB,CAAC,IAAY,EAAE,IAAc;QACzD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;QACrF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QAEpC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,eAAe,EAAE;YACxE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE;gBACnD,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK;gBACL,KAAK;gBACL,KAAK,EAAE,IAAI;gBACX,eAAe,EAAE,MAAM;gBACvB,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAClC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,qBAAqB,QAAQ,CAAC,MAAM,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC;QAClG,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;QACzC,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACxD,MAAM,WAAW,GAAG,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC;QACrG,MAAM,kBAAkB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;QAEpF,OAAO;YACL,OAAO,EAAE,IAAI;YACb,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,kBAAkB,EAAE;YAChG,WAAW;SACZ,CAAC;IACJ,CAAC;IAED,uBAAuB;IAEf,KAAK,CAAC,oBAAoB,CAAC,IAAY,EAAE,IAAc;QAC7D,MAAM,OAAO,GAAG,iBAAiB,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,IAAI,CAAC,KAAK,IAAI,iBAAiB,CAAC,MAAM,CAAC;QACpH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,uBAAuB,CAAC;QAEpD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,+CAA+C,OAAO,EAAE,EAAE;YACrF,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAiB;gBAC3C,cAAc,EAAE,kBAAkB;gBAClC,MAAM,EAAE,YAAY;aACrB;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,IAAI;gBACJ,QAAQ,EAAE,KAAK;gBACf,cAAc,EAAE;oBACd,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,GAAG;oBAChC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,IAAI,IAAI;iBAChD;aACF,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAClC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,qBAAqB,QAAQ,CAAC,MAAM,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC;QAClG,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;QACzC,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACxD,MAAM,kBAAkB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;QAEpF,OAAO;YACL,OAAO,EAAE,IAAI;YACb,MAAM,EAAE;gBACN,IAAI;gBACJ,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,QAAQ;gBAC7B,KAAK;gBACL,QAAQ,EAAE,YAAY;gBACtB,WAAW;gBACX,WAAW,EAAE,YAAY;gBACzB,kBAAkB;aACnB;YACD,WAAW,EAAE,YAAY;SAC1B,CAAC;IACJ,CAAC;IAED,yCAAyC;IAEjC,KAAK,CAAC,gBAAgB,CAAC,IAAY,EAAE,IAAc;QACzD,+EAA+E;QAC/E,6EAA6E;QAC7E,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,SAAS,CAAC;QACtC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,CAAC,oCAAoC;QAEvE,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,kBAAkB,EAAE;gBAC3E,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;aACpD,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EACH,6BAA6B,QAAQ,CAAC,MAAM,8CAA8C;wBAC1F,8FAA8F;iBACjG,CAAC;YACJ,CAAC;YAED,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;YACzC,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACxD,MAAM,kBAAkB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;YAEpF,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,kBAAkB,EAAE;gBAC9G,WAAW,EAAE,YAAY;aAC1B,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EACH,+EAA+E;oBAC/E,yDAAyD;aAC5D,CAAC;QACJ,CAAC;IACH,CAAC;CACF"}

package/manifest.json

@@ -2,19 +2,34 @@
   "$schema": "https://agentos.sh/schemas/extension-manifest-v1.json",
   "id": "com.framers.media.voice-synthesis",
   "name": "Voice Synthesis Extension",
-  "version": "1.0.0",
-  "description": "Text-to-speech synthesis via ElevenLabs for generating agent voice audio",
+  "version": "2.0.0",
+  "description": "Voice synthesis and transcription tools via OpenAI, ElevenLabs, Deepgram, and local Ollama/Whisper-compatible runtimes",
   "author": { "name": "Frame.dev" },
   "license": "MIT",
-  "keywords": ["tts", "voice", "speech", "elevenlabs", "audio"],
+  "keywords": ["tts", "stt", "voice", "speech", "transcription", "openai", "elevenlabs", "deepgram", "ollama", "whisper", "audio"],
   "agentosVersion": "^2.0.0",
   "categories": ["media", "communication"],
   "extensions": [
-    { "kind": "tool", "id": "textToSpeech", "displayName": "Text to Speech", "entry": "./src/tools/textToSpeech.ts" }
+    { "kind": "tool", "id": "text_to_speech", "displayName": "Text to Speech", "entry": "./dist/tools/textToSpeech.js" },
+    { "kind": "tool", "id": "speech_to_text", "displayName": "Speech to Text", "entry": "./dist/tools/speechToText.js" }
   ],
   "configuration": {
     "properties": {
-      "elevenlabs.apiKey": { "type": "string", "secret": true }
+      "openai.apiKey": { "type": "string", "secret": true },
+      "openai.baseUrl": { "type": "string" },
+      "deepgram.apiKey": { "type": "string", "secret": true },
+      "deepgram.baseUrl": { "type": "string" },
+      "elevenlabs.apiKey": { "type": "string", "secret": true },
+      "ollama.baseUrl": { "type": "string" },
+      "whisperLocal.baseUrl": { "type": "string" },
+      "tts.defaultProvider": {
+        "type": "string",
+        "enum": ["auto", "openai", "elevenlabs", "ollama"]
+      },
+      "stt.defaultProvider": {
+        "type": "string",
+        "enum": ["auto", "openai", "deepgram", "whisper-local"]
+      }
     }
   }
 }

package/package.json

@@ -1,32 +1,58 @@
 {
   "name": "@framers/agentos-ext-voice-synthesis",
-  "version": "1.0.0",
-  "description": "ElevenLabs text-to-speech synthesis tool for AgentOS",
+  "version": "2.0.1",
+  "description": "Voice synthesis and transcription tools for AgentOS via OpenAI, ElevenLabs, Deepgram, and local Ollama/Whisper-compatible runtimes",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "type": "module",
   "sideEffects": false,
   "exports": {
-    ".": { "import": "./dist/index.js", "types": "./dist/index.d.ts" },
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
     "./manifest.json": "./manifest.json",
     "./package.json": "./package.json"
   },
-  "scripts": {
-    "build": "tsc",
-    "test": "vitest run",
-    "test:watch": "vitest",
-    "test:coverage": "vitest run --coverage",
-    "lint": "eslint src --ext .ts",
-    "typecheck": "tsc --noEmit",
-    "clean": "rimraf dist"
+  "keywords": [
+    "agentos",
+    "extension",
+    "tts",
+    "stt",
+    "voice",
+    "transcription",
+    "openai",
+    "elevenlabs",
+    "deepgram",
+    "ollama",
+    "whisper",
+    "speech",
+    "media"
+  ],
+  "author": {
+    "name": "Framers AI",
+    "email": "team@frame.dev",
+    "url": "https://frame.dev"
   },
-  "keywords": ["agentos", "extension", "tts", "voice", "elevenlabs", "speech", "media"],
-  "author": { "name": "Framers AI", "email": "team@frame.dev", "url": "https://frame.dev" },
-  "contributors": [{ "name": "Johnny Dunn", "email": "johnnyfived@protonmail.com", "url": "https://github.com/jddunn" }],
+  "contributors": [
+    {
+      "name": "Johnny Dunn",
+      "email": "johnnyfived@protonmail.com",
+      "url": "https://github.com/jddunn"
+    }
+  ],
   "license": "MIT",
-  "repository": { "type": "git", "url": "https://github.com/framersai/agentos-extensions.git", "directory": "registry/curated/media/voice-synthesis" },
-  "publishConfig": { "access": "public" },
-  "peerDependencies": { "@framers/agentos": "^0.1.0" },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/framersai/agentos-extensions.git",
+    "directory": "registry/curated/media/voice-synthesis"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "peerDependencies": {
+    "@framers/agentos": "^0.1.0"
+  },
   "devDependencies": {
     "@framers/agentos": "^0.1.0",
     "@types/node": "^20.12.12",
@@ -34,5 +60,14 @@
     "rimraf": "^5.0.7",
     "typescript": "^5.4.5",
     "vitest": "^1.6.0"
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
+    "lint": "eslint src --ext .ts",
+    "typecheck": "tsc --noEmit",
+    "clean": "rimraf dist"
   }
-}
+}

package/src/index.ts

@@ -1,30 +1,92 @@
 /**
- * Voice Synthesis Extension Pack — ElevenLabs TTS for agents.
+ * Voice Synthesis Extension Pack — TTS + STT voice tools for agents.
+ *
+ * Supports: OpenAI TTS/STT, ElevenLabs TTS, Ollama (local TTS).
+ * Auto-detects available provider from API keys.
  */
 
-import { TextToSpeechTool } from './tools/textToSpeech.js';
+import { TextToSpeechTool, type TTSConfig, type TTSProvider } from './tools/textToSpeech.js';
+import { SpeechToTextTool, type STTConfig, type STTProvider } from './tools/speechToText.js';
 
 export interface VoiceSynthesisExtensionOptions {
   elevenLabsApiKey?: string;
+  openaiApiKey?: string;
+  openaiBaseUrl?: string;
+  deepgramApiKey?: string;
+  deepgramBaseUrl?: string;
+  ollamaBaseUrl?: string;
+  whisperLocalBaseUrl?: string;
+  defaultProvider?: TTSProvider;
+  defaultSttProvider?: STTProvider;
   priority?: number;
 }
 
 export function createExtensionPack(context: any) {
   const options = (context.options || {}) as VoiceSynthesisExtensionOptions;
-  const apiKey = options.elevenLabsApiKey || context.getSecret?.('elevenlabs.apiKey') || process.env.ELEVENLABS_API_KEY;
-  const tool = new TextToSpeechTool(apiKey);
+
+  const config: TTSConfig = {
+    openaiApiKey: options.openaiApiKey || context.getSecret?.('openai.apiKey') || process.env.OPENAI_API_KEY,
+    openaiBaseUrl: options.openaiBaseUrl || process.env.OPENAI_BASE_URL,
+    elevenLabsApiKey: options.elevenLabsApiKey || context.getSecret?.('elevenlabs.apiKey') || process.env.ELEVENLABS_API_KEY,
+    ollamaBaseUrl: options.ollamaBaseUrl || process.env.OLLAMA_BASE_URL,
+    defaultProvider: options.defaultProvider || (process.env.TTS_PROVIDER as TTSProvider) || 'auto',
+  };
+  const sttConfig: STTConfig = {
+    openaiApiKey: options.openaiApiKey || context.getSecret?.('openai.apiKey') || process.env.OPENAI_API_KEY,
+    openaiBaseUrl: options.openaiBaseUrl || process.env.OPENAI_BASE_URL,
+    deepgramApiKey: options.deepgramApiKey || context.getSecret?.('deepgram.apiKey') || process.env.DEEPGRAM_API_KEY,
+    deepgramBaseUrl: options.deepgramBaseUrl || process.env.DEEPGRAM_BASE_URL,
+    whisperLocalBaseUrl: options.whisperLocalBaseUrl || process.env.WHISPER_LOCAL_BASE_URL,
+    defaultProvider: options.defaultSttProvider || (process.env.STT_PROVIDER as STTProvider) || 'auto',
+  };
+
+  const ttsTool = new TextToSpeechTool(config);
+  const sttTool = new SpeechToTextTool(sttConfig);
+
+  const providers: string[] = [];
+  if (config.openaiApiKey) providers.push('OpenAI');
+  if (config.elevenLabsApiKey) providers.push('ElevenLabs');
+  providers.push('Ollama (local fallback)');
+  const sttProviders: string[] = [];
+  if (sttConfig.openaiApiKey) sttProviders.push('OpenAI Whisper');
+  if (sttConfig.deepgramApiKey) sttProviders.push('Deepgram');
+  if (sttConfig.whisperLocalBaseUrl) sttProviders.push('Whisper-local');
+
+  const features: string[] = [
+    `TTS: ${providers.join(', ')}`,
+    `STT: ${sttProviders.length > 0 ? sttProviders.join(', ') : 'not configured'}`,
+  ];
 
   return {
     name: '@framers/agentos-ext-voice-synthesis',
-    version: '1.0.0',
+    version: '2.0.0',
     descriptors: [
-      { id: 'textToSpeech', kind: 'tool' as const, priority: options.priority || 50, payload: tool, requiredSecrets: [{ id: 'elevenlabs.apiKey' }] },
+      {
+        id: ttsTool.name,
+        kind: 'tool' as const,
+        priority: options.priority || 50,
+        payload: ttsTool,
+        requiredSecrets: [],
+      },
+      {
+        id: sttTool.name,
+        kind: 'tool' as const,
+        priority: options.priority || 50,
+        payload: sttTool,
+        requiredSecrets: [],
+      },
     ],
-    onActivate: async () => context.logger?.info('Voice Synthesis Extension activated'),
-    onDeactivate: async () => context.logger?.info('Voice Synthesis Extension deactivated'),
+    onActivate: async () => {
+      context.logger?.info?.(`Voice Synthesis activated — ${features.join(' | ')}`);
+    },
+    onDeactivate: async () => {
+      context.logger?.info?.('Voice Synthesis deactivated');
+    },
   };
 }
 
 export { TextToSpeechTool };
-export type { TTSInput, TTSOutput } from './tools/textToSpeech.js';
+export { SpeechToTextTool };
+export type { TTSInput, TTSOutput, TTSConfig, TTSProvider } from './tools/textToSpeech.js';
+export type { STTInput, STTOutput, STTConfig, STTProvider } from './tools/speechToText.js';
 export default createExtensionPack;