@jambonz/speech-utils 1.0.23 → 1.0.24

package/lib/synth-audio.js

@@ -1,5 +1,6 @@
 const assert = require('assert');
 const fs = require('fs');
+const crypto = require('crypto');
 const bent = require('bent');
 const ttsGoogle = require('@google-cloud/text-to-speech');
 const { PollyClient, SynthesizeSpeechCommand } = require('@aws-sdk/client-polly');
@@ -79,9 +80,10 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
   let rtt;
   logger = logger || noopLogger;
 
+  let mistralRefAudio;
   assert.ok(['google', 'aws', 'polly', 'microsoft', 'wellsaid', 'nvidia', 'elevenlabs',
     'whisper', 'deepgram', 'deepgramflux', 'rimelabs', 'cartesia', 'gradium', 'kugelaudio', 'nineninesix', 'inworld',
-    'resemble', 'murf', 'xai', 'fishaudio', 'speechify', 'giggy', 'svara']
+    'resemble', 'murf', 'xai', 'fishaudio', 'speechify', 'giggy', 'mistral', 'svara']
     .includes(vendor) ||
   vendor.startsWith('custom'),
   `synthAudio supported vendors are google, aws, microsoft, nvidia and wellsaid ..etc, not ${vendor}`);
@@ -145,6 +147,11 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
   } else if ('giggy' === vendor) {
     assert.ok(voice, 'synthAudio requires voice when giggy is used');
     assert.ok(credentials.api_key, 'synthAudio requires api_key when giggy is used');
+  } else if ('mistral' === vendor) {
+    mistralRefAudio = options?.ref_audio || parseCredOptions(credentials.options).ref_audio;
+    /* a one-off cloned voice (ref_audio) replaces the voice */
+    assert.ok(voice || mistralRefAudio, 'synthAudio requires voice or ref_audio when mistral is used');
+    assert.ok(credentials.api_key, 'synthAudio requires api_key when mistral is used');
   } else if ('svara' === vendor) {
     assert.ok(voice, 'synthAudio requires voice when svara is used');
     assert.ok(credentials.api_key, 'synthAudio requires api_key when svara is used');
@@ -171,7 +178,10 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
     account_sid,
     vendor,
     language: language || '',
-    voice: voice || deploymentId,
+    /* a cloned voice is identified by its reference clip */
+    voice: mistralRefAudio ?
+      `ref_audio:${crypto.createHash('sha1').update(mistralRefAudio).digest('hex')}` :
+      voice || deploymentId,
     engine,
     // model or model_id is used to identify the tts cache.
     model: model || credentials.model_id,
@@ -261,6 +271,11 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
           credentials, options, stats, voice, key, text, renderForCaching, disableTtsStreaming,
           disableTtsCache});
         break;
+      case 'mistral':
+        audioData = await synthMistral(logger, {
+          credentials, options, stats, voice, key, text, renderForCaching, disableTtsStreaming,
+          disableTtsCache});
+        break;
       case 'svara':
         audioData = await synthSvara(logger, {
           credentials, options, stats, language, voice, key, text, renderForCaching, disableTtsStreaming,
@@ -1863,6 +1878,102 @@ const synthSvara = async(logger, {
   }
 };
 
+/* Mistral Voxtral: no language field, the voice carries it; output is mono 24 kHz */
+/* credential Extra Options are stored as a JSON string or an object */
+const parseCredOptions = (opts) => {
+  if (typeof opts !== 'string') return opts || {};
+  try {
+    return JSON.parse(opts);
+  } catch {
+    return {};
+  }
+};
+
+const synthMistral = async(logger, {
+  credentials, options, stats, voice, key, text, renderForCaching, disableTtsStreaming,
+  disableTtsCache
+}) => {
+  const {api_key, model_id} = credentials;
+  const {
+    api_uri, expressiveness, prompt_cache_key, ref_audio, phoneme_hints, conversation_history
+  } = {...parseCredOptions(credentials.options), ...options};
+  const isSet = (v) => v !== null && v !== undefined;
+  const asJson = (v) => (typeof v === 'string' ? v : JSON.stringify(v));
+  const model = options?.model_id || model_id || 'voxtral-mini-tts-3-0';
+
+  /* default to using the streaming interface, unless disabled by env var OR we want just a cache file */
+  if (!JAMBONES_DISABLE_TTS_STREAMING && !renderForCaching && !disableTtsStreaming) {
+    let params = '{';
+    params += `api_key=${api_key}`;
+    params += `,playback_id=${key}`;
+    params += ',vendor=mistral';
+    if (voice) params += `,voice=${voice}`;
+    params += `,write_cache_file=${disableTtsCache ? 0 : 1}`;
+    params += `,model_id=${model}`;
+    if (api_uri) params += `,api_uri=${api_uri}`;
+    if (isSet(expressiveness)) params += `,expressiveness=${expressiveness}`;
+    if (prompt_cache_key) params += `,prompt_cache_key=${prompt_cache_key}`;
+    if (ref_audio) params += `,ref_audio=${ref_audio}`;
+    if (isSet(phoneme_hints)) params += `,phoneme_hints=${asJson(phoneme_hints)}`;
+    if (isSet(conversation_history)) params += `,conversation_history=${asJson(conversation_history)}`;
+    params += '}';
+
+    return {
+      filePath: `say:${params}${text.replace(/\n/g, ' ').replace(/\r/g, ' ')}`,
+      servedFromCache: false,
+      rtt: 0
+    };
+  }
+
+  try {
+    const sampleRate = 24000;
+    const host = (api_uri || 'api.mistral.ai').replace(/^[a-z]+:\/\//, '').replace(/\/$/, '');
+    /* http:// or ws:// in api_uri means a plain-text proxy, as mediajam reads it */
+    const scheme = /^(http|ws):\/\//.test(api_uri || '') ? 'http' : 'https';
+    const post = bent(`${scheme}://${host}`, 'POST', 'json', {
+      'Authorization': `Bearer ${api_key}`,
+      'Content-Type': 'application/json'
+    });
+    /* invalid JSON is dropped, as on the streaming path */
+    const parseJson = (v) => {
+      if (typeof v !== 'string') return v;
+      try {
+        return JSON.parse(v);
+      } catch {
+        return undefined;
+      }
+    };
+    const hints = isSet(phoneme_hints) ? parseJson(phoneme_hints) : undefined;
+    const history = isSet(conversation_history) && !ref_audio ? parseJson(conversation_history) : undefined;
+    const {audio_data} = await post('/v1/audio/speech', {
+      model,
+      input: text,
+      response_format: 'pcm',
+      ...(ref_audio ? {ref_audio} : {voice_id: voice}),
+      ...(isSet(expressiveness) && {expressiveness: Number(expressiveness)}),
+      ...(prompt_cache_key && {prompt_cache_key}),
+      ...(isSet(hints) && {phoneme_hints: hints}),
+      ...(isSet(history) && {conversation_history: history})
+    });
+    /* Voxtral pcm is float32; the r24 cache is s16 */
+    const f32 = Buffer.from(audio_data, 'base64');
+    const audioContent = Buffer.alloc(Math.floor(f32.length / 4) * 2);
+    for (let i = 0; i < audioContent.length / 2; i++) {
+      const v = Math.max(-1, Math.min(1, f32.readFloatLE(i * 4)));
+      audioContent.writeInt16LE(Math.round(v * 32767), i * 2);
+    }
+    return {
+      audioContent,
+      extension: 'r24',
+      sampleRate
+    };
+  } catch (err) {
+    logger.info({err}, 'synth mistral returned error');
+    stats.increment('tts.count', ['vendor:mistral', 'accepted:no']);
+    throw err;
+  }
+};
+
 /* nineninesix.ai — a Cartesia-compatible API, but only raw/wav come back
    (mp3 is rejected), so the cache render asks for wav rather than mp3. */
 /* fish.audio — msgpack websocket for streaming, and a POST endpoint for the cache

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jambonz/speech-utils",
-  "version": "1.0.23",
+  "version": "1.0.24",
   "engines": {
     "node": ">=22"
   },

package/test/synth.js

@@ -1206,6 +1206,35 @@ test('svara speech synth tests', async(t) => {
   client.quit();
 });
 
+test('mistral speech synth tests', async(t) => {
+  const fn = require('..');
+  const {synthAudio, client} = fn(opts, logger);
+
+  if (!process.env.MISTRAL_API_KEY) {
+    t.pass('skipping mistral speech synth tests since MISTRAL_API_KEY is not provided');
+    return t.end();
+  }
+  const text = 'Hi there and welcome to jambonz! ' + Date.now();
+  try {
+    const opts = await synthAudio(stats, {
+      vendor: 'mistral',
+      credentials: {
+        api_key: process.env.MISTRAL_API_KEY
+      },
+      language: 'en-US',
+      voice: process.env.MISTRAL_VOICE_ID || 'voice_478',
+      text,
+      renderForCaching: true
+    });
+    t.ok(!opts.servedFromCache, `successfully synthed mistral audio to ${opts.filePath}`);
+
+  } catch (err) {
+    console.error(JSON.stringify(err));
+    t.end(err);
+  }
+  client.quit();
+});
+
 test('fishaudio speech synth tests', async(t) => {
   const fn = require('..');
   const {synthAudio, client} = fn(opts, logger);