@jambonz/speech-utils 0.1.21 → 0.1.24

package/lib/get-aws-sts-token.js

@@ -7,22 +7,23 @@ const CACHE_EXPIRY = process.env.AWS_STS_SESSION_RESET_EXPIRY || (EXPIRY - 600);
 
 async function getAwsAuthToken(
   logger, createHash, retrieveHash,
-  {accessKeyId, secretAccessKey, region, roleArn}) {
+  {speech_credential_sid, accessKeyId, secretAccessKey, region, roleArn}) {
   logger = logger || noopLogger;
   try {
-    const key = makeAwsKey(roleArn || accessKeyId);
+    // if incase instance profile is used, speech_credential_sid will be used as key to lookup cache
+    const key = makeAwsKey(roleArn || accessKeyId || speech_credential_sid);
     const obj = await retrieveHash(key);
     if (obj) return {...obj, servedFromCache: true};
-
+    /* access token not found in cache, so generate it using STS */
     let data;
+    let expiry = CACHE_EXPIRY;
     if (roleArn) {
       const stsClient = new STSClient({ region });
       const roleToAssume = { RoleArn: roleArn, RoleSessionName: 'Jambonz_Speech', DurationSeconds: EXPIRY};
       const command = new AssumeRoleCommand(roleToAssume);
 
       data = await stsClient.send(command);
-    } else {
-      /* access token not found in cache, so generate it using STS */
+    } else if (accessKeyId) {
       const stsClient = new STSClient({
         region,
         credentials: {
@@ -32,6 +33,26 @@ async function getAwsAuthToken(
       });
       const command = new GetSessionTokenCommand({DurationSeconds: EXPIRY});
       data = await stsClient.send(command);
+    } else {
+      // instance profile is used.
+      const stsClient = new STSClient({ region });
+      const cred = await stsClient.config.credentials();
+      // method in the AWS SDK automatically fetches credentials using the default credential
+      // provider chain. If the credentials come from an instance profile or an environment
+      // variable, their expiration is controlled by AWS and not explicitly by our code.
+      if (cred && cred.expiration) {
+        const currentTime = new Date();
+        const expiryTime = new Date(cred.expiration);
+        const remainingTimeInSeconds = Math.round((expiryTime - currentTime) / 1000);
+        expiry = remainingTimeInSeconds;
+      }
+      data = {
+        Credentials: {
+          AccessKeyId: cred.accessKeyId,
+          SecretAccessKey: cred.secretAccessKey,
+          SessionToken: cred.sessionToken
+        }
+      };
     }
 
     const credentials = {
@@ -40,9 +61,11 @@ async function getAwsAuthToken(
       sessionToken: data.Credentials.SessionToken,
       securityToken: data.Credentials.SessionToken
     };
-
-    createHash(key, credentials, CACHE_EXPIRY)
-      .catch((err) => logger.error(err, `Error saving hash for key ${key}`));
+    // Only cache if expiry is good
+    if (expiry > 0) {
+      createHash(key, credentials, expiry)
+        .catch((err) => logger.error(err, `Error saving hash for key ${key}`));
+    }
 
     return {...credentials, servedFromCache: false};
   } catch (err) {

package/lib/synth-audio.js

@@ -3,6 +3,7 @@ const fs = require('fs');
 const bent = require('bent');
 const ttsGoogle = require('@google-cloud/text-to-speech');
 const { PollyClient, SynthesizeSpeechCommand } = require('@aws-sdk/client-polly');
+const { CartesiaClient } = require('@cartesia/cartesia-js');
 
 const sdk = require('microsoft-cognitiveservices-speech-sdk');
 const TextToSpeechV1 = require('ibm-watson/text-to-speech/v1');
@@ -96,7 +97,7 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
   logger = logger || noopLogger;
 
   assert.ok(['google', 'aws', 'polly', 'microsoft', 'wellsaid', 'nuance', 'nvidia', 'ibm', 'elevenlabs',
-    'whisper', 'deepgram', 'playht', 'rimelabs', 'verbio'].includes(vendor) ||
+    'whisper', 'deepgram', 'playht', 'rimelabs', 'verbio', 'cartesia'].includes(vendor) ||
   vendor.startsWith('custom'),
   `synthAudio supported vendors are google, aws, microsoft, nuance, nvidia and wellsaid ..etc, not ${vendor}`);
   if ('google' === vendor) {
@@ -157,7 +158,9 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
     if (!credentials.deepgram_tts_uri) {
       assert.ok(credentials.api_key, 'synthAudio requires api_key when deepgram is used');
     }
-
+  } else if ('cartesia' === vendor) {
+    assert.ok(credentials.api_key, 'synthAudio requires api_key when cartesia is used');
+    assert.ok(credentials.model_id, 'synthAudio requires model_id when cartesia is used');
   }
 
   const key = makeSynthKey({
@@ -170,6 +173,8 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
     renderForCaching
   });
   let filePath;
+  // used only for custom vendor
+  let fileExtension;
   filePath = makeFilePath({vendor, voice, key, salt, renderForCaching});
   debug(`synth key is ${key}`);
   let cached;
@@ -201,7 +206,17 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
     debug('result WAS found in cache');
     servedFromCache = true;
     stats.increment('tts.cache.requests', ['found:yes']);
-    audioBuffer = Buffer.from(cached, 'base64');
+    if (vendor.startsWith('custom')) {
+      // custom vendors support multiple mime types such as: mp3, wav, r8, r16 ...etc,
+      // mime type/file extension is available when http response has header Content-type.
+      // In cache, file extension is store together with audiBuffer in a json.
+      // Normal cache audio will be base64 string
+      const payload = JSON.parse(cached);
+      filePath = filePath.replace(/\.[^\.]*$/g, payload.fileExtension);
+      audioBuffer = Buffer.from(payload.audioBuffer, 'base64');
+    } else {
+      audioBuffer = Buffer.from(cached, 'base64');
+    }
     client.expire(key, EXPIRES).catch((err) => logger.info(err, 'Error setting expires'));
   }
   if (!cached) {
@@ -249,6 +264,11 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
           credentials, options, stats, language, voice, text, renderForCaching, disableTtsStreaming, filePath
         });
         break;
+      case 'cartesia':
+        audioBuffer = await synthCartesia(logger, {
+          credentials, options, stats, language, voice, text, renderForCaching, disableTtsStreaming, filePath
+        });
+        break;
       case 'rimelabs':
         audioBuffer = await synthRimelabs(logger, {
           credentials, options, stats, language, voice, text, renderForCaching, disableTtsStreaming, filePath
@@ -268,7 +288,7 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
           renderForCaching, disableTtsStreaming});
         break;
       case vendor.startsWith('custom') ? vendor : 'cant_match_value':
-        ({ audioBuffer, filePath } = await synthCustomVendor(logger,
+        ({ audioBuffer, filePath, fileExtension } = await synthCustomVendor(logger,
           {credentials, stats, language, voice, text, filePath}));
         break;
       default:
@@ -282,7 +302,14 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
     debug(`tts rtt time for ${text.length} chars on ${vendorLabel}: ${rtt}`);
     logger.info(`tts rtt time for ${text.length} chars on ${vendorLabel}: ${rtt}`);
 
-    client.setex(key, EXPIRES, audioBuffer.toString('base64'))
+    const base64Audio = audioBuffer.toString('base64');
+    const cacheContent = vendor.startsWith('custom') ?
+      JSON.stringify({
+        audioBuffer: base64Audio,
+        fileExtension
+      }) : base64Audio;
+
+    client.setex(key, EXPIRES, cacheContent)
       .catch((err) => logger.error(err, `error calling setex on key ${key}`));
   }
 
@@ -729,9 +756,11 @@ const synthCustomVendor = async(logger, {credentials, stats, language, voice, te
     const regex = /\.[^\.]*$/g;
     const mime = response.headers['content-type'];
     const buffer = await response.arrayBuffer();
+    const fileExtension = getFileExtFromMime(mime);
     return {
       audioBuffer: buffer,
-      filePath: filePath.replace(regex, getFileExtFromMime(mime))
+      filePath: filePath.replace(regex, fileExtension),
+      fileExtension
     };
   } catch (err) {
     logger.info({err}, `Vendor ${vendor} returned error`);
@@ -1046,6 +1075,71 @@ const synthDeepgram = async(logger, {credentials, stats, model, text, renderForC
   }
 };
 
+const synthCartesia = async(logger, {
+  credentials, options, stats, voice, language, text, renderForCaching, disableTtsStreaming
+}) => {
+  const {api_key, model_id, embedding, options: credOpts} = credentials;
+  const opts = !!options && Object.keys(options).length !== 0 ? options : JSON.parse(credOpts || '{}');
+
+  if (!JAMBONES_DISABLE_TTS_STREAMING && !renderForCaching && !disableTtsStreaming) {
+    let params = '';
+    params += `{api_key=${api_key}`;
+    params += `,model_id=${model_id}`;
+    params += ',vendor=cartesia';
+    params += `,voice=${voice}`;
+    params += ',write_cache_file=1';
+    params += `,language=${language}`;
+    params += `,voice_mode=${embedding ? 'embedding' : 'id'}`;
+    if (embedding) params += `,embedding=${embedding}`;
+    if (opts.speed) params += `,speed=${opts.speed}`;
+    if (opts.emotion) params += `,emotion=${opts.emotion}`;
+    params += '}';
+
+    return {
+      filePath: `say:${params}${text.replace(/\n/g, ' ').replace(/\r/g, ' ')}`,
+      servedFromCache: false,
+      rtt: 0
+    };
+  }
+
+  try {
+    const client = new CartesiaClient({ apiKey: api_key });
+    const mp3 = await client.tts.bytes({
+      modelId: model_id,
+      transcript: text,
+      voice: {
+        mode: embedding ? 'embedding' : 'id',
+        ...(embedding ?
+          {
+            embedding: embedding.split(',').map(Number)
+          } :
+          {
+            id: voice
+          }
+        ),
+        ...(opts.speed || opts.emotion && {
+          experimentalControls: {
+            ...(opts.speed && {speed: opts.speed}),
+            ...(opts.emotion && {emotion: opts.emotion}),
+          }
+        })
+      },
+      language: language,
+      outputFormat: {
+        container: 'mp3',
+        bitRate: 128000,
+        sampleRate: 8000
+      },
+    });
+    return Buffer.from(mp3);
+  } catch (err) {
+    logger.info({err}, 'synth Cartesia returned error');
+    stats.increment('tts.count', ['vendor:cartesia', 'accepted:no']);
+    throw err;
+  }
+
+};
+
 const getFileExtFromMime = (mime) => {
   switch (mime) {
     case 'audio/wav':

package/lib/utils.js

@@ -50,6 +50,7 @@ function getFileExtension({vendor, voice, renderForCaching = false}) {
     case 'elevenlabs':
     case 'rimelabs':
     case 'playht':
+    case 'cartesia':
       if (renderForCaching || JAMBONES_DISABLE_TTS_STREAMING) {
         return mp3Extension;
       } else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jambonz/speech-utils",
-  "version": "0.1.21",
+  "version": "0.1.24",
   "description": "TTS-related speech utilities for jambonz",
   "main": "index.js",
   "author": "Dave Horton",
@@ -28,6 +28,7 @@
   "dependencies": {
     "@aws-sdk/client-polly": "^3.496.0",
     "@aws-sdk/client-sts": "^3.496.0",
+    "@cartesia/cartesia-js": "^2.1.0",
     "@google-cloud/text-to-speech": "^5.5.0",
     "@grpc/grpc-js": "^1.9.14",
     "@jambonz/realtimedb-helpers": "^0.8.7",

package/test/synth.js

@@ -554,6 +554,7 @@ test('Custom Vendor speech synth tests', async(t) => {
       text: 'This is a test.  This is only a test',
     });
     t.ok(!opts.servedFromCache, `successfully synthesized custom vendor audio to ${opts.filePath}`);
+    t.ok(opts.filePath.endsWith('wav'), 'audio is cached as wav file');
     let obj = await getJSON(`http://127.0.0.1:3100/lastRequest/somethingnew`);
     t.ok(obj.headers.Authorization == 'Bearer some_jwt_token', 'Custom Vendor Authentication Header is correct');
     t.ok(obj.body.language == 'en-US', 'Custom Vendor Language is correct');
@@ -561,6 +562,21 @@ test('Custom Vendor speech synth tests', async(t) => {
     t.ok(obj.body.type == 'text', 'Custom Vendor type is correct');
     t.ok(obj.body.text == 'This is a test.  This is only a test', 'Custom Vendor text is correct');
 
+    // Checking if cache is stored with wav format
+    opts = await synthAudio(stats, {
+      vendor: 'custom:somethingnew',
+      credentials: {
+        use_for_tts: 1,
+        custom_tts_url: "http://127.0.0.1:3100/somethingnew",
+        auth_token: 'some_jwt_token'
+      },
+      language: 'en-US',
+      voice: 'English-US.Female-1',
+      text: 'This is a test.  This is only a test',
+    });
+    t.ok(opts.servedFromCache, `successfully get custom vendor cached audio to ${opts.filePath}`);
+    t.ok(opts.filePath.endsWith('wav'), 'audio is cached as wav file');
+
     opts = await synthAudio(stats, {
       vendor: 'custom:somethingnew2',
       credentials: {
@@ -670,6 +686,40 @@ test('PlayHT3.0 speech synth tests', async(t) => {
   await testPlayHT(t, 'Play3.0');
 });
 
+test('Cartesia speech synth tests', async(t) => {
+  const fn = require('..');
+  const {synthAudio, client} = fn(opts, logger);
+
+  if (!process.env.CARTESIA_API_KEY) {
+    t.pass('skipping Cartesia speech synth tests since CARTESIA_API_KEY is not provided');
+    return t.end();
+  }
+  const text = 'Hi there and welcome to jambones! ' + Date.now();
+  try {
+    const opts = await synthAudio(stats, {
+      vendor: 'cartesia',
+      credentials: {
+        api_key: process.env.CARTESIA_API_KEY,
+        model_id: 'sonic-english',
+        options: JSON.stringify({
+          speed: 1,
+          emotion: 'female_happy',
+        })
+      },
+      language: 'en',
+      voice: '694f9389-aac1-45b6-b726-9d9369183238',
+      text,
+      renderForCaching: true
+    });
+    t.ok(!opts.servedFromCache, `successfully playht eleven audio to ${opts.filePath}`);
+
+  } catch (err) {
+    console.error(JSON.stringify(err));
+    t.end(err);
+  }
+  client.quit();
+});
+
 test('rimelabs speech synth tests', async(t) => {
   const fn = require('..');
   const {synthAudio, client} = fn(opts, logger);