@jambonz/speech-utils 0.0.24 → 0.0.25

package/lib/synth-audio.js

@@ -38,6 +38,7 @@ const {AudioEncoding} = require('../stubs/riva/proto/riva_audio_pb');
 const debug = require('debug')('jambonz:realtimedb-helpers');
 const EXPIRES = (process.env.JAMBONES_TTS_CACHE_DURATION_MINS || 4 * 60) * 60; // cache tts for 4 hours
 const TMP_FOLDER = '/tmp';
+const OpenAI = require('openai');
 
 
 const trimTrailingSilence = (buffer) => {
@@ -83,7 +84,7 @@ async function synthAudio(client, logger, stats, { account_sid,
   logger = logger || noopLogger;
 
   assert.ok(['google', 'aws', 'polly', 'microsoft',
-    'wellsaid', 'nuance', 'nvidia', 'ibm', 'elevenlabs'].includes(vendor) ||
+    'wellsaid', 'nuance', 'nvidia', 'ibm', 'elevenlabs', 'whisper'].includes(vendor) ||
   vendor.startsWith('custom'),
   `synthAudio supported vendors are google, aws, microsoft, nuance, nvidia and wellsaid, not ${vendor}`);
   if ('google' === vendor) {
@@ -117,6 +118,14 @@ async function synthAudio(client, logger, stats, { account_sid,
     language = 'en-US'; // WellSaid only supports English atm
     assert.ok(voice, 'synthAudio requires voice when wellsaid is used');
     assert.ok(!text.startsWith('<speak'), 'wellsaid does not support SSML tags');
+  } else if ('elevenlabs' === vendor) {
+    assert.ok(voice, 'synthAudio requires voice when elevenlabs is used');
+    assert.ok(credentials.api_key, 'synthAudio requires api_key when elevenlabs is used');
+    assert.ok(credentials.model_id, 'synthAudio requires model_id when elevenlabs is used');
+  } else if ('whisper' === vendor) {
+    assert.ok(voice, 'synthAudio requires voice when whisper is used');
+    assert.ok(credentials.model_id, 'synthAudio requires model when whisper is used');
+    assert.ok(credentials.api_key, 'synthAudio requires api_key when whisper is used');
   } else  if (vendor.startsWith('custom')) {
     assert.ok(credentials.custom_tts_url, `synthAudio requires custom_tts_url in credentials when ${vendor} is used`);
   }
@@ -187,6 +196,9 @@ async function synthAudio(client, logger, stats, { account_sid,
       case 'elevenlabs':
         audioBuffer = await synthElevenlabs(logger, {credentials, stats, language, voice, text, filePath});
         break;
+      case 'whisper':
+        audioBuffer = await synthWhisper(logger, {credentials, stats, voice, text});
+        break;
       case vendor.startsWith('custom') ? vendor : 'cant_match_value':
         ({ audioBuffer, filePath } = await synthCustomVendor(logger,
           {credentials, stats, language, voice, text, filePath}));
@@ -591,11 +603,35 @@ const synthElevenlabs = async(logger, {credentials, stats, language, voice, text
     });
     return mp3;
   } catch (err) {
-    logger.info({err}, 'synthEvenlabs returned error');
+    logger.info({err}, 'synth Elevenlabs returned error');
+    stats.increment('tts.count', ['vendor:elevenlabs', 'accepted:no']);
     throw err;
   }
 };
 
+const synthWhisper = async(logger, {credentials, stats, voice, text}) => {
+  const {api_key, model_id, baseURL, timeout} = credentials;
+  try {
+    const openai = new OpenAI.OpenAI({
+      apiKey: api_key,
+      timeout: timeout || 5000,
+      ...(baseURL && {baseURL})
+    });
+
+    const mp3 = await openai.audio.speech.create({
+      model: model_id,
+      voice,
+      input: text,
+      response_format: 'mp3'
+    });
+    return Buffer.from(await mp3.arrayBuffer());
+  } catch (err) {
+    logger.info({err}, 'synth whisper returned error');
+    stats.increment('tts.count', ['vendor:openai', 'accepted:no']);
+    throw err;
+  }
+}
+;
 const getFileExtFromMime = (mime) => {
   switch (mime) {
     case 'audio/wav':

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jambonz/speech-utils",
-  "version": "0.0.24",
+  "version": "0.0.25",
   "description": "TTS-related speech utilities for jambonz",
   "main": "index.js",
   "author": "Dave Horton",
@@ -32,8 +32,9 @@
     "form-urlencoded": "^6.1.0",
     "google-protobuf": "^3.21.2",
     "ibm-watson": "^8.0.0",
-    "microsoft-cognitiveservices-speech-sdk": "^1.31.0",
     "ioredis": "^5.3.2",
+    "microsoft-cognitiveservices-speech-sdk": "^1.31.0",
+    "openai": "^4.16.2",
     "undici": "^5.21.0"
   },
   "devDependencies": {

package/test/synth.js

@@ -38,7 +38,7 @@ test('Google speech synth tests', async(t) => {
         },
       },
       language: 'en-GB',
-      gender: 'MALE',
+      gender: 'FEMALE',
       text: 'This is a test.  This is only a test',
       salt: 'foo.bar',
     });
@@ -53,7 +53,7 @@ test('Google speech synth tests', async(t) => {
         },
       },
       language: 'en-GB',
-      gender: 'MALE',
+      gender: 'FEMALE',
       text: 'This is a test.  This is only a test',
     });
     t.ok(opts.servedFromCache, `successfully retrieved cached google audio from ${opts.filePath}`);
@@ -68,7 +68,7 @@ test('Google speech synth tests', async(t) => {
       },
       disableTtsCache: true,
       language: 'en-GB',
-      gender: 'MALE',
+      gender: 'FEMALE',
       text: 'This is a test.  This is only a test',
     });
     t.ok(!opts.servedFromCache, `successfully synthesized google audio regardless of current cache to ${opts.filePath}`);
@@ -450,7 +450,7 @@ test('Elevenlabs speech synth tests', async(t) => {
   const {synthAudio, client} = fn(opts, logger);
 
   if (!process.env.ELEVENLABS_API_KEY || !process.env.ELEVENLABS_VOICE_ID || !process.env.ELEVENLABS_MODEL_ID) {
-    t.pass('skipping IBM Watson speech synth tests since IBM_TTS_API_KEY or IBM_TTS_API_KEY not provided');
+    t.pass('skipping ElevenLabs speech synth tests since ELEVENLABS_API_KEY or ELEVENLABS_VOICE_ID or ELEVENLABS_MODEL_ID not provided');
     return t.end();
   }
   const text = 'Hi there and welcome to jambones!';
@@ -474,6 +474,35 @@ test('Elevenlabs speech synth tests', async(t) => {
   client.quit();
 })
 
+test('whisper speech synth tests', async(t) => {
+  const fn = require('..');
+  const {synthAudio, client} = fn(opts, logger);
+
+  if (!process.env.OPENAI_API_KEY) {
+    t.pass('skipping OPENAI speech synth tests since OPENAI_API_KEY not provided');
+    return t.end();
+  }
+  const text = 'Hi there and welcome to jambones!';
+  try {
+    let opts = await synthAudio(stats, {
+      vendor: 'whisper',
+      credentials: {
+        api_key: process.env.OPENAI_API_KEY,
+        model_id: 'tts-1'
+      },
+      language: 'en-US',
+      voice: 'alloy',
+      text,
+    });
+    t.ok(!opts.servedFromCache, `successfully synthesized whisper audio to ${opts.filePath}`);
+
+  } catch (err) {
+    console.error(JSON.stringify(err));
+    t.end(err);
+  }
+  client.quit();
+})
+
 test('TTS Cache tests', async(t) => {
   const fn = require('..');
   const {purgeTtsCache, getTtsSize, client} = fn(opts, logger);