@jambonz/speech-utils 1.0.3 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/publish.yml +2 -2
- package/index.js +0 -2
- package/lib/get-tts-voices.js +2 -92
- package/lib/synth-audio.js +48 -262
- package/lib/utils.js +1 -101
- package/package.json +1 -3
- package/test/index.js +0 -1
- package/test/list-voices.js +0 -93
- package/test/synth.js +0 -158
- package/build_stubs.sh +0 -11
- package/lib/get-nuance-access-token.js +0 -49
- package/lib/get-verbio-token.js +0 -51
- package/protos/nuance/synthesizer.proto +0 -332
- package/stubs/nuance/synthesizer_grpc_pb.js +0 -104
- package/stubs/nuance/synthesizer_pb.js +0 -6815
- package/test/nuance.js +0 -81
package/index.js
CHANGED
|
@@ -14,8 +14,6 @@ module.exports = (opts, logger) => {
|
|
|
14
14
|
purgeTtsCache: require('./lib/purge-tts-cache').bind(null, client, logger),
|
|
15
15
|
addFileToCache: require('./lib/add-file-to-cache').bind(null, client, logger),
|
|
16
16
|
synthAudio: require('./lib/synth-audio').bind(null, client, createHash, retrieveHash, logger),
|
|
17
|
-
getVerbioAccessToken: require('./lib/get-verbio-token').bind(null, client, logger),
|
|
18
|
-
getNuanceAccessToken: require('./lib/get-nuance-access-token').bind(null, client, logger),
|
|
19
17
|
|
|
20
18
|
getAwsAuthToken: require('./lib/get-aws-sts-token').bind(null, logger, createHash, retrieveHash),
|
|
21
19
|
getTtsVoices: require('./lib/get-tts-voices').bind(null, client, createHash, retrieveHash, logger),
|
package/lib/get-tts-voices.js
CHANGED
|
@@ -1,74 +1,8 @@
|
|
|
1
1
|
const assert = require('assert');
|
|
2
|
-
const {noopLogger
|
|
3
|
-
const getNuanceAccessToken = require('./get-nuance-access-token');
|
|
4
|
-
const getVerbioAccessToken = require('./get-verbio-token');
|
|
5
|
-
const {GetVoicesRequest, Voice} = require('../stubs/nuance/synthesizer_pb');
|
|
2
|
+
const {noopLogger} = require('./utils');
|
|
6
3
|
const ttsGoogle = require('@google-cloud/text-to-speech');
|
|
7
4
|
const { PollyClient, DescribeVoicesCommand } = require('@aws-sdk/client-polly');
|
|
8
5
|
const getAwsAuthToken = require('./get-aws-sts-token');
|
|
9
|
-
const {Pool} = require('undici');
|
|
10
|
-
const { HTTP_TIMEOUT } = require('./config');
|
|
11
|
-
const verbioVoicePool = new Pool('https://us.rest.speechcenter.verbio.com');
|
|
12
|
-
|
|
13
|
-
const getNuanceVoices = async(client, logger, credentials) => {
|
|
14
|
-
const {client_id: clientId, secret: secret, nuance_tts_uri} = credentials;
|
|
15
|
-
|
|
16
|
-
return new Promise(async(resolve, reject) => {
|
|
17
|
-
/* get a nuance access token */
|
|
18
|
-
let token, nuanceClient;
|
|
19
|
-
try {
|
|
20
|
-
if (nuance_tts_uri) {
|
|
21
|
-
nuanceClient = await createKryptonClient(nuance_tts_uri);
|
|
22
|
-
}
|
|
23
|
-
else {
|
|
24
|
-
const access_token = await getNuanceAccessToken(client, logger, clientId, secret, 'tts');
|
|
25
|
-
token = access_token.access_token;
|
|
26
|
-
nuanceClient = await createNuanceClient(token);
|
|
27
|
-
}
|
|
28
|
-
} catch (err) {
|
|
29
|
-
logger.error({err}, 'getTtsVoices: error retrieving access token');
|
|
30
|
-
return reject(err);
|
|
31
|
-
}
|
|
32
|
-
/* retrieve all voices */
|
|
33
|
-
const v = new Voice();
|
|
34
|
-
const request = new GetVoicesRequest();
|
|
35
|
-
request.setVoice(v);
|
|
36
|
-
|
|
37
|
-
nuanceClient.getVoices(request, (err, response) => {
|
|
38
|
-
if (err) {
|
|
39
|
-
logger.error({err, clientId, secret, token}, 'getTtsVoices: error retrieving voices');
|
|
40
|
-
return reject(err);
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
/* return all the voices that are not restricted and eliminate duplicates */
|
|
44
|
-
const voices = response.getVoicesList()
|
|
45
|
-
.map((v) => {
|
|
46
|
-
return {
|
|
47
|
-
language: v.getLanguage(),
|
|
48
|
-
name: v.getName(),
|
|
49
|
-
model: v.getModel(),
|
|
50
|
-
gender: v.getGender() === 1 ? 'male' : 'female',
|
|
51
|
-
restricted: v.getRestricted()
|
|
52
|
-
};
|
|
53
|
-
});
|
|
54
|
-
const v = voices
|
|
55
|
-
.filter((v) => v.restricted === false)
|
|
56
|
-
.map((v) => {
|
|
57
|
-
delete v.restricted;
|
|
58
|
-
return v;
|
|
59
|
-
})
|
|
60
|
-
.sort((a, b) => {
|
|
61
|
-
if (a.language < b.language) return -1;
|
|
62
|
-
if (a.language > b.language) return 1;
|
|
63
|
-
if (a.name < b.name) return -1;
|
|
64
|
-
return 1;
|
|
65
|
-
});
|
|
66
|
-
const arr = [...new Set(v.map((v) => JSON.stringify(v)))]
|
|
67
|
-
.map((v) => JSON.parse(v));
|
|
68
|
-
resolve(arr);
|
|
69
|
-
});
|
|
70
|
-
});
|
|
71
|
-
};
|
|
72
6
|
|
|
73
7
|
const getGoogleVoices = async(_client, logger, credentials) => {
|
|
74
8
|
const client = new ttsGoogle.TextToSpeechClient({credentials});
|
|
@@ -109,26 +43,6 @@ const getAwsVoices = async(_client, createHash, retrieveHash, logger, credential
|
|
|
109
43
|
}
|
|
110
44
|
};
|
|
111
45
|
|
|
112
|
-
const getVerbioVoices = async(client, logger, credentials) => {
|
|
113
|
-
try {
|
|
114
|
-
const access_token = await getVerbioAccessToken(client, logger, credentials);
|
|
115
|
-
const { body} = await verbioVoicePool.request({
|
|
116
|
-
path: '/api/v1/voices',
|
|
117
|
-
method: 'GET',
|
|
118
|
-
headers: {
|
|
119
|
-
'Authorization': `Bearer ${access_token.access_token}`,
|
|
120
|
-
'User-Agent': 'jambonz'
|
|
121
|
-
},
|
|
122
|
-
timeout: HTTP_TIMEOUT,
|
|
123
|
-
followRedirects: false
|
|
124
|
-
});
|
|
125
|
-
return await body.json();
|
|
126
|
-
} catch (err) {
|
|
127
|
-
logger.info({err}, 'getVerbioVoices - failed to list voices for Verbio');
|
|
128
|
-
throw err;
|
|
129
|
-
}
|
|
130
|
-
};
|
|
131
|
-
|
|
132
46
|
/**
|
|
133
47
|
* Synthesize speech to an mp3 file, and also cache the generated speech
|
|
134
48
|
* in redis (base64 format) for 24 hours so as to avoid unnecessarily paying
|
|
@@ -148,19 +62,15 @@ const getVerbioVoices = async(client, logger, credentials) => {
|
|
|
148
62
|
async function getTtsVoices(client, createHash, retrieveHash, logger, {vendor, credentials}) {
|
|
149
63
|
logger = logger || noopLogger;
|
|
150
64
|
|
|
151
|
-
assert.ok(['
|
|
65
|
+
assert.ok(['google', 'aws', 'polly'].includes(vendor),
|
|
152
66
|
`getTtsVoices not supported for vendor ${vendor}`);
|
|
153
67
|
|
|
154
68
|
switch (vendor) {
|
|
155
|
-
case 'nuance':
|
|
156
|
-
return getNuanceVoices(client, logger, credentials);
|
|
157
69
|
case 'google':
|
|
158
70
|
return getGoogleVoices(client, logger, credentials);
|
|
159
71
|
case 'aws':
|
|
160
72
|
case 'polly':
|
|
161
73
|
return getAwsVoices(client, createHash, retrieveHash, logger, credentials);
|
|
162
|
-
case 'verbio':
|
|
163
|
-
return getVerbioVoices(client, logger, credentials);
|
|
164
74
|
default:
|
|
165
75
|
break;
|
|
166
76
|
}
|
package/lib/synth-audio.js
CHANGED
|
@@ -16,26 +16,10 @@ const {
|
|
|
16
16
|
} = sdk;
|
|
17
17
|
const {
|
|
18
18
|
makeSynthKey,
|
|
19
|
-
createNuanceClient,
|
|
20
|
-
createKryptonClient,
|
|
21
19
|
createRivaClient,
|
|
22
20
|
noopLogger,
|
|
23
|
-
makeFilePath
|
|
24
|
-
makePlayhtKey
|
|
21
|
+
makeFilePath
|
|
25
22
|
} = require('./utils');
|
|
26
|
-
const getNuanceAccessToken = require('./get-nuance-access-token');
|
|
27
|
-
const getVerbioAccessToken = require('./get-verbio-token');
|
|
28
|
-
const {
|
|
29
|
-
SynthesisRequest,
|
|
30
|
-
Voice,
|
|
31
|
-
AudioFormat,
|
|
32
|
-
AudioParameters,
|
|
33
|
-
PCM,
|
|
34
|
-
Input,
|
|
35
|
-
Text,
|
|
36
|
-
SSML,
|
|
37
|
-
EventParameters
|
|
38
|
-
} = require('../stubs/nuance/synthesizer_pb');
|
|
39
23
|
const {SynthesizeSpeechRequest} = require('../stubs/riva/proto/riva_tts_pb');
|
|
40
24
|
const {AudioEncoding} = require('../stubs/riva/proto/riva_audio_pb');
|
|
41
25
|
const debug = require('debug')('jambonz:realtimedb-helpers');
|
|
@@ -95,10 +79,10 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
|
|
|
95
79
|
let rtt;
|
|
96
80
|
logger = logger || noopLogger;
|
|
97
81
|
|
|
98
|
-
assert.ok(['google', 'aws', 'polly', 'microsoft', 'wellsaid', '
|
|
99
|
-
'whisper', 'deepgram', '
|
|
82
|
+
assert.ok(['google', 'aws', 'polly', 'microsoft', 'wellsaid', 'nvidia', 'elevenlabs',
|
|
83
|
+
'whisper', 'deepgram', 'rimelabs', 'cartesia', 'inworld', 'resemble'].includes(vendor) ||
|
|
100
84
|
vendor.startsWith('custom'),
|
|
101
|
-
`synthAudio supported vendors are google, aws, microsoft,
|
|
85
|
+
`synthAudio supported vendors are google, aws, microsoft, nvidia and wellsaid ..etc, not ${vendor}`);
|
|
102
86
|
if ('google' === vendor) {
|
|
103
87
|
assert.ok(language, 'synthAudio requires language when google is used');
|
|
104
88
|
}
|
|
@@ -109,13 +93,6 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
|
|
|
109
93
|
assert.ok(language || deploymentId, 'synthAudio requires language when microsoft is used');
|
|
110
94
|
assert.ok(voice || deploymentId, 'synthAudio requires voice when microsoft is used');
|
|
111
95
|
}
|
|
112
|
-
else if ('nuance' === vendor) {
|
|
113
|
-
assert.ok(voice, 'synthAudio requires voice when nuance is used');
|
|
114
|
-
if (!credentials.nuance_tts_uri) {
|
|
115
|
-
assert.ok(credentials.client_id, 'synthAudio requires client_id in credentials when nuance is used');
|
|
116
|
-
assert.ok(credentials.secret, 'synthAudio requires client_id in credentials when nuance is used');
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
96
|
else if ('nvidia' === vendor) {
|
|
120
97
|
assert.ok(voice, 'synthAudio requires voice when nvidia is used');
|
|
121
98
|
assert.ok(language, 'synthAudio requires language when nvidia is used');
|
|
@@ -129,11 +106,6 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
|
|
|
129
106
|
assert.ok(voice, 'synthAudio requires voice when elevenlabs is used');
|
|
130
107
|
assert.ok(credentials.api_key, 'synthAudio requires api_key when elevenlabs is used');
|
|
131
108
|
assert.ok(credentials.model_id, 'synthAudio requires model_id when elevenlabs is used');
|
|
132
|
-
} else if ('playht' === vendor) {
|
|
133
|
-
assert.ok(voice, 'synthAudio requires voice when playht is used');
|
|
134
|
-
assert.ok(credentials.api_key, 'synthAudio requires api_key when playht is used');
|
|
135
|
-
assert.ok(credentials.user_id, 'synthAudio requires user_id when playht is used');
|
|
136
|
-
assert.ok(credentials.voice_engine, 'synthAudio requires voice_engine when playht is used');
|
|
137
109
|
} else if ('inworld' === vendor) {
|
|
138
110
|
assert.ok(voice, 'synthAudio requires voice when inworld is used');
|
|
139
111
|
assert.ok(credentials.api_key, 'synthAudio requires api_key when inworld is used');
|
|
@@ -148,10 +120,6 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
|
|
|
148
120
|
assert.ok(credentials.api_key, 'synthAudio requires api_key when whisper is used');
|
|
149
121
|
} else if (vendor.startsWith('custom')) {
|
|
150
122
|
assert.ok(credentials.custom_tts_url, `synthAudio requires custom_tts_url in credentials when ${vendor} is used`);
|
|
151
|
-
} else if ('verbio' === vendor) {
|
|
152
|
-
assert.ok(voice, 'synthAudio requires voice when verbio is used');
|
|
153
|
-
assert.ok(credentials.client_id, 'synthAudio requires client_id when verbio is used');
|
|
154
|
-
assert.ok(credentials.client_secret, 'synthAudio requires client_secret when verbio is used');
|
|
155
123
|
} else if ('deepgram' === vendor) {
|
|
156
124
|
if (!credentials.deepgram_tts_uri) {
|
|
157
125
|
assert.ok(credentials.api_key, 'synthAudio requires api_key when deepgram is used');
|
|
@@ -216,10 +184,6 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
|
|
|
216
184
|
audioData = await synthMicrosoft(logger, {credentials, stats, language, voice, key, text, deploymentId,
|
|
217
185
|
renderForCaching, disableTtsStreaming, disableTtsCache});
|
|
218
186
|
break;
|
|
219
|
-
case 'nuance':
|
|
220
|
-
model = model || 'enhanced';
|
|
221
|
-
audioData = await synthNuance(client, logger, {credentials, stats, voice, model, key, text});
|
|
222
|
-
break;
|
|
223
187
|
case 'nvidia':
|
|
224
188
|
audioData = await synthNvidia(client, logger, {credentials, stats, language, voice, model, key, text,
|
|
225
189
|
renderForCaching, disableTtsStreaming, disableTtsCache});
|
|
@@ -232,11 +196,6 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
|
|
|
232
196
|
credentials, options, stats, language, voice, key, text, renderForCaching, disableTtsStreaming,
|
|
233
197
|
disableTtsCache});
|
|
234
198
|
break;
|
|
235
|
-
case 'playht':
|
|
236
|
-
audioData = await synthPlayHT(client, logger, {
|
|
237
|
-
credentials, options, stats, language, voice, key, text, renderForCaching, disableTtsStreaming,
|
|
238
|
-
disableTtsCache});
|
|
239
|
-
break;
|
|
240
199
|
case 'cartesia':
|
|
241
200
|
audioData = await synthCartesia(logger, {
|
|
242
201
|
credentials, options, stats, language, voice, key, text, renderForCaching, disableTtsStreaming,
|
|
@@ -257,11 +216,6 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
|
|
|
257
216
|
credentials, stats, voice, key, text, instructions, renderForCaching, disableTtsStreaming,
|
|
258
217
|
disableTtsCache});
|
|
259
218
|
break;
|
|
260
|
-
case 'verbio':
|
|
261
|
-
audioData = await synthVerbio(client, logger, {
|
|
262
|
-
credentials, stats, voice, key, text, renderForCaching, disableTtsStreaming, disableTtsCache});
|
|
263
|
-
if (audioData?.filePath) return audioData;
|
|
264
|
-
break;
|
|
265
219
|
case 'deepgram':
|
|
266
220
|
audioData = await synthDeepgram(logger, {credentials, stats, model, key, text,
|
|
267
221
|
renderForCaching, disableTtsStreaming, disableTtsCache});
|
|
@@ -316,6 +270,7 @@ const synthPolly = async(createHash, retrieveHash, logger,
|
|
|
316
270
|
params += `,playback_id=${key}`;
|
|
317
271
|
params += `,write_cache_file=${disableTtsCache ? 0 : 1}`;
|
|
318
272
|
params += ',vendor=aws';
|
|
273
|
+
params += `,voice=${voice}`;
|
|
319
274
|
if (accessKeyId && secretAccessKey) {
|
|
320
275
|
if (accessKeyId) params += `,accessKeyId=${accessKeyId}`;
|
|
321
276
|
if (secretAccessKey) params += `,secretAccessKey=${secretAccessKey}`;
|
|
@@ -724,70 +679,6 @@ const synthWellSaid = async(logger, {credentials, stats, language, voice, gender
|
|
|
724
679
|
}
|
|
725
680
|
};
|
|
726
681
|
|
|
727
|
-
const synthNuance = async(client, logger, {credentials, stats, voice, model, text}) => {
|
|
728
|
-
let nuanceClient;
|
|
729
|
-
const {client_id, secret, nuance_tts_uri} = credentials;
|
|
730
|
-
if (nuance_tts_uri) {
|
|
731
|
-
nuanceClient = await createKryptonClient(nuance_tts_uri);
|
|
732
|
-
}
|
|
733
|
-
else {
|
|
734
|
-
/* get a nuance access token */
|
|
735
|
-
const {access_token} = await getNuanceAccessToken(client, logger, client_id, secret, 'tts');
|
|
736
|
-
nuanceClient = await createNuanceClient(access_token);
|
|
737
|
-
}
|
|
738
|
-
|
|
739
|
-
const v = new Voice();
|
|
740
|
-
const p = new AudioParameters();
|
|
741
|
-
const f = new AudioFormat();
|
|
742
|
-
const pcm = new PCM();
|
|
743
|
-
const params = new EventParameters();
|
|
744
|
-
const request = new SynthesisRequest();
|
|
745
|
-
const input = new Input();
|
|
746
|
-
|
|
747
|
-
if (text.startsWith('<speak')) {
|
|
748
|
-
const ssml = new SSML();
|
|
749
|
-
ssml.setText(text);
|
|
750
|
-
input.setSsml(ssml);
|
|
751
|
-
}
|
|
752
|
-
else {
|
|
753
|
-
const t = new Text();
|
|
754
|
-
t.setText(text);
|
|
755
|
-
input.setText(t);
|
|
756
|
-
}
|
|
757
|
-
const sampleRate = 8000;
|
|
758
|
-
pcm.setSampleRateHz(sampleRate);
|
|
759
|
-
f.setPcm(pcm);
|
|
760
|
-
p.setAudioFormat(f);
|
|
761
|
-
v.setName(voice);
|
|
762
|
-
v.setModel(model);
|
|
763
|
-
request.setVoice(v);
|
|
764
|
-
request.setAudioParams(p);
|
|
765
|
-
request.setInput(input);
|
|
766
|
-
request.setEventParams(params);
|
|
767
|
-
request.setUserId('jambonz');
|
|
768
|
-
|
|
769
|
-
return new Promise((resolve, reject) => {
|
|
770
|
-
nuanceClient.unarySynthesize(request, (err, response) => {
|
|
771
|
-
if (err) {
|
|
772
|
-
console.error(err);
|
|
773
|
-
return reject(err);
|
|
774
|
-
}
|
|
775
|
-
const status = response.getStatus();
|
|
776
|
-
const code = status.getCode();
|
|
777
|
-
if (code !== 200) {
|
|
778
|
-
const message = status.getMessage();
|
|
779
|
-
const details = status.getDetails();
|
|
780
|
-
return reject({code, message, details});
|
|
781
|
-
}
|
|
782
|
-
resolve({
|
|
783
|
-
audioContent: Buffer.from(response.getAudio()),
|
|
784
|
-
extension: 'r8',
|
|
785
|
-
sampleRate
|
|
786
|
-
});
|
|
787
|
-
});
|
|
788
|
-
});
|
|
789
|
-
};
|
|
790
|
-
|
|
791
682
|
const synthNvidia = async(client, logger, {
|
|
792
683
|
credentials, stats, language, voice, model, key, text, renderForCaching, disableTtsStreaming, disableTtsCache
|
|
793
684
|
}) => {
|
|
@@ -953,101 +844,6 @@ const synthElevenlabs = async(logger, {
|
|
|
953
844
|
}
|
|
954
845
|
};
|
|
955
846
|
|
|
956
|
-
const synthPlayHT = async(client, logger, {
|
|
957
|
-
credentials, options, stats, voice, language, key, text, renderForCaching, disableTtsStreaming, disableTtsCache
|
|
958
|
-
}) => {
|
|
959
|
-
const {api_key, user_id, voice_engine, playht_tts_uri, options: credOpts} = credentials;
|
|
960
|
-
const opts = !!options && Object.keys(options).length !== 0 ? options : JSON.parse(credOpts || '{}');
|
|
961
|
-
|
|
962
|
-
let synthesizeUrl = playht_tts_uri ? `${playht_tts_uri}/api/v2/tts/stream` : 'https://api.play.ht/api/v2/tts/stream';
|
|
963
|
-
// If model is play3.0, the synthesizeUrl is got from authentication endpoint
|
|
964
|
-
if (voice_engine === 'Play3.0') {
|
|
965
|
-
try {
|
|
966
|
-
const post = bent('https://api.play.ht', 'POST', 'json', 201, {
|
|
967
|
-
'AUTHORIZATION': api_key,
|
|
968
|
-
'X-USER-ID': user_id,
|
|
969
|
-
'Accept': 'application/json'
|
|
970
|
-
});
|
|
971
|
-
const key = makePlayhtKey(api_key);
|
|
972
|
-
const url = await client.get(key);
|
|
973
|
-
if (!url) {
|
|
974
|
-
const {inference_address, expires_at_ms} = await post('/api/v3/auth');
|
|
975
|
-
synthesizeUrl = inference_address;
|
|
976
|
-
const expiry = Math.floor((expires_at_ms - Date.now()) / 1000 - 30);
|
|
977
|
-
await client.set(key, inference_address, 'EX', expiry);
|
|
978
|
-
} else {
|
|
979
|
-
// Use cached URL
|
|
980
|
-
synthesizeUrl = url;
|
|
981
|
-
}
|
|
982
|
-
} catch (err) {
|
|
983
|
-
logger.info({err}, 'synth PlayHT returned error for authentication version 3.0');
|
|
984
|
-
stats.increment('tts.count', ['vendor:playht', 'accepted:no']);
|
|
985
|
-
throw err;
|
|
986
|
-
}
|
|
987
|
-
}
|
|
988
|
-
|
|
989
|
-
/* default to using the streaming interface, unless disabled by env var OR we want just a cache file */
|
|
990
|
-
if (!JAMBONES_DISABLE_TTS_STREAMING && !renderForCaching && !disableTtsStreaming) {
|
|
991
|
-
let params = '{';
|
|
992
|
-
params += `api_key=${api_key}`;
|
|
993
|
-
params += `,playback_id=${key}`;
|
|
994
|
-
params += `,user_id=${user_id}`;
|
|
995
|
-
params += ',vendor=playht';
|
|
996
|
-
params += `,voice=${voice}`;
|
|
997
|
-
params += `,voice_engine=${voice_engine}`;
|
|
998
|
-
params += `,synthesize_url=${synthesizeUrl}`;
|
|
999
|
-
params += `,write_cache_file=${disableTtsCache ? 0 : 1}`;
|
|
1000
|
-
params += `,language=${language}`;
|
|
1001
|
-
if (opts.quality) params += `,quality=${opts.quality}`;
|
|
1002
|
-
if (opts.speed) params += `,speed=${opts.speed}`;
|
|
1003
|
-
if (opts.seed) params += `,style=${opts.seed}`;
|
|
1004
|
-
if (opts.temperature) params += `,temperature=${opts.temperature}`;
|
|
1005
|
-
if (opts.emotion) params += `,emotion=${opts.emotion}`;
|
|
1006
|
-
if (opts.voice_guidance) params += `,voice_guidance=${opts.voice_guidance}`;
|
|
1007
|
-
if (opts.style_guidance) params += `,style_guidance=${opts.style_guidance}`;
|
|
1008
|
-
if (opts.text_guidance) params += `,text_guidance=${opts.text_guidance}`;
|
|
1009
|
-
if (opts.top_p) params += `,top_p=${opts.top_p}`;
|
|
1010
|
-
if (opts.repetition_penalty) params += `,repetition_penalty=${opts.repetition_penalty}`;
|
|
1011
|
-
params += '}';
|
|
1012
|
-
|
|
1013
|
-
return {
|
|
1014
|
-
filePath: `say:${params}${text.replace(/\n/g, ' ').replace(/\r/g, ' ')}`,
|
|
1015
|
-
servedFromCache: false,
|
|
1016
|
-
rtt: 0
|
|
1017
|
-
};
|
|
1018
|
-
}
|
|
1019
|
-
|
|
1020
|
-
try {
|
|
1021
|
-
const post = bent('POST', 'buffer', {
|
|
1022
|
-
...(voice_engine !== 'Play3.0' && {
|
|
1023
|
-
'AUTHORIZATION': api_key,
|
|
1024
|
-
'X-USER-ID': user_id,
|
|
1025
|
-
}),
|
|
1026
|
-
'Accept': 'audio/mpeg',
|
|
1027
|
-
'Content-Type': 'application/json'
|
|
1028
|
-
});
|
|
1029
|
-
|
|
1030
|
-
const audioContent = await post(synthesizeUrl, {
|
|
1031
|
-
text,
|
|
1032
|
-
...(voice_engine === 'Play3.0' && { language }),
|
|
1033
|
-
voice,
|
|
1034
|
-
voice_engine,
|
|
1035
|
-
output_format: 'mp3',
|
|
1036
|
-
sample_rate: 8000,
|
|
1037
|
-
...opts
|
|
1038
|
-
});
|
|
1039
|
-
return {
|
|
1040
|
-
audioContent,
|
|
1041
|
-
extension: 'mp3',
|
|
1042
|
-
sampleRate: 8000
|
|
1043
|
-
};
|
|
1044
|
-
} catch (err) {
|
|
1045
|
-
logger.info({err}, 'synth PlayHT returned error');
|
|
1046
|
-
stats.increment('tts.count', ['vendor:playht', 'accepted:no']);
|
|
1047
|
-
throw err;
|
|
1048
|
-
}
|
|
1049
|
-
};
|
|
1050
|
-
|
|
1051
847
|
const synthInworld = async(logger, {
|
|
1052
848
|
credentials, options, stats, voice, key, text, renderForCaching, disableTtsStreaming, disableTtsCache
|
|
1053
849
|
}) => {
|
|
@@ -1173,54 +969,6 @@ const synthRimelabs = async(logger, {
|
|
|
1173
969
|
throw err;
|
|
1174
970
|
}
|
|
1175
971
|
};
|
|
1176
|
-
const synthVerbio = async(client, logger, {
|
|
1177
|
-
credentials, stats, voice, key, text, renderForCaching, disableTtsStreaming, disableTtsCache
|
|
1178
|
-
}) => {
|
|
1179
|
-
//https://doc.speechcenter.verbio.com/#tag/Text-To-Speech-REST-API
|
|
1180
|
-
if (text.length > 2000) {
|
|
1181
|
-
throw new Error('Verbio cannot synthesize for the text length larger than 2000 characters');
|
|
1182
|
-
}
|
|
1183
|
-
const token = await getVerbioAccessToken(client, logger, credentials);
|
|
1184
|
-
if (!process.env.JAMBONES_DISABLE_TTS_STREAMING && !renderForCaching && !disableTtsStreaming) {
|
|
1185
|
-
let params = '{';
|
|
1186
|
-
params += `access_token=${token.access_token}`;
|
|
1187
|
-
params += `,playback_id=${key}`;
|
|
1188
|
-
params += ',vendor=verbio';
|
|
1189
|
-
params += `,voice=${voice}`;
|
|
1190
|
-
params += `,write_cache_file=${disableTtsCache ? 0 : 1}`;
|
|
1191
|
-
params += '}';
|
|
1192
|
-
|
|
1193
|
-
return {
|
|
1194
|
-
filePath: `say:${params}${text.replace(/\n/g, ' ')}`,
|
|
1195
|
-
servedFromCache: false,
|
|
1196
|
-
rtt: 0
|
|
1197
|
-
};
|
|
1198
|
-
}
|
|
1199
|
-
|
|
1200
|
-
try {
|
|
1201
|
-
const post = bent('https://us.rest.speechcenter.verbio.com', 'POST', 'buffer', {
|
|
1202
|
-
'Authorization': `Bearer ${token.access_token}`,
|
|
1203
|
-
'User-Agent': 'jambonz',
|
|
1204
|
-
'Content-Type': 'application/json'
|
|
1205
|
-
});
|
|
1206
|
-
const audioContent = await post('/api/v1/synthesize', {
|
|
1207
|
-
voice_id: voice,
|
|
1208
|
-
output_sample_rate: '8k',
|
|
1209
|
-
output_encoding: 'pcm16',
|
|
1210
|
-
text
|
|
1211
|
-
});
|
|
1212
|
-
return {
|
|
1213
|
-
audioContent,
|
|
1214
|
-
extension: 'r8',
|
|
1215
|
-
sampleRate: 8000
|
|
1216
|
-
};
|
|
1217
|
-
} catch (err) {
|
|
1218
|
-
logger.info({err}, 'synth Verbio returned error');
|
|
1219
|
-
stats.increment('tts.count', ['vendor:verbio', 'accepted:no']);
|
|
1220
|
-
throw err;
|
|
1221
|
-
}
|
|
1222
|
-
};
|
|
1223
|
-
|
|
1224
972
|
const synthWhisper = async(logger, {credentials, stats, voice, key, text, instructions,
|
|
1225
973
|
renderForCaching, disableTtsStreaming, disableTtsCache}) => {
|
|
1226
974
|
const {api_key, model_id, baseURL, timeout, speed} = credentials;
|
|
@@ -1343,6 +1091,18 @@ const synthCartesia = async(logger, {
|
|
|
1343
1091
|
try {
|
|
1344
1092
|
const client = new CartesiaClient({ apiKey: api_key });
|
|
1345
1093
|
const sampleRate = 48000;
|
|
1094
|
+
|
|
1095
|
+
// omit a control unless explicitly provided (0 is a valid value, so test for nullish only)
|
|
1096
|
+
const has = (v) => v !== null && v !== undefined;
|
|
1097
|
+
/* Voice controls are model-family specific:
|
|
1098
|
+
- sonic-2 takes `experimentalControls` (emotion is an array, no volume).
|
|
1099
|
+
- sonic-3 family (sonic-3, sonic-3.5, pinned sonic-3.x snapshots) takes
|
|
1100
|
+
`generationConfig` (emotion is a string, volume supported). Match the same
|
|
1101
|
+
"starts with sonic-3" predicate the freeswitch streaming module uses
|
|
1102
|
+
(mod_cartesia_tts_streaming: strncmp(model_id, "sonic-3", ...)), so cached
|
|
1103
|
+
and streamed synthesis behave identically.
|
|
1104
|
+
Older models (sonic, sonic-english, sonic-multilingual, sonic-2024-*) take neither. */
|
|
1105
|
+
const isSonic3 = model_id?.startsWith('sonic-3');
|
|
1346
1106
|
const mp3Stream = await client.tts.bytes({
|
|
1347
1107
|
modelId: model_id,
|
|
1348
1108
|
transcript: text,
|
|
@@ -1358,16 +1118,16 @@ const synthCartesia = async(logger, {
|
|
|
1358
1118
|
),
|
|
1359
1119
|
...(model_id === 'sonic-2' && (opts.speed || opts.emotion) && {
|
|
1360
1120
|
experimentalControls: {
|
|
1361
|
-
...(opts.speed
|
|
1121
|
+
...(has(opts.speed) && {speed: opts.speed}),
|
|
1362
1122
|
...(opts.emotion && {emotion: [opts.emotion]}),
|
|
1363
1123
|
}
|
|
1364
1124
|
}),
|
|
1365
1125
|
},
|
|
1366
|
-
...(
|
|
1126
|
+
...(isSonic3 && (has(opts.speed) || has(opts.emotion) || has(opts.volume)) && {
|
|
1367
1127
|
generationConfig: {
|
|
1368
|
-
...(opts.volume
|
|
1369
|
-
...(opts.speed
|
|
1370
|
-
...(opts.emotion
|
|
1128
|
+
...(has(opts.volume) && {volume: opts.volume}),
|
|
1129
|
+
...(has(opts.speed) && {speed: opts.speed}),
|
|
1130
|
+
...(has(opts.emotion) && {emotion: opts.emotion}),
|
|
1371
1131
|
}
|
|
1372
1132
|
}),
|
|
1373
1133
|
language: language,
|
|
@@ -1391,6 +1151,32 @@ const synthCartesia = async(logger, {
|
|
|
1391
1151
|
sampleRate
|
|
1392
1152
|
};
|
|
1393
1153
|
} catch (err) {
|
|
1154
|
+
/* Cartesia's tts.bytes() uses a streaming response, so on an HTTP error the SDK
|
|
1155
|
+
throws a CartesiaError whose `body` is an unconsumed stream-wrapper object
|
|
1156
|
+
(async-iterable, yielding Uint8Array chunks) rather than the parsed error
|
|
1157
|
+
JSON. Read it so callers get a meaningful message instead of a serialized
|
|
1158
|
+
stream object. */
|
|
1159
|
+
if (err && err.body && typeof err.body !== 'string' && typeof err.body[Symbol.asyncIterator] === 'function') {
|
|
1160
|
+
try {
|
|
1161
|
+
const chunks = [];
|
|
1162
|
+
for await (const chunk of err.body) {
|
|
1163
|
+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
1164
|
+
}
|
|
1165
|
+
const text = Buffer.concat(chunks).toString('utf8');
|
|
1166
|
+
if (text) {
|
|
1167
|
+
let parsed;
|
|
1168
|
+
try {
|
|
1169
|
+
parsed = JSON.parse(text);
|
|
1170
|
+
} catch {
|
|
1171
|
+
parsed = null;
|
|
1172
|
+
}
|
|
1173
|
+
err.message = (parsed && (parsed.error || parsed.message)) || text;
|
|
1174
|
+
err.body = parsed || text;
|
|
1175
|
+
}
|
|
1176
|
+
} catch (readErr) {
|
|
1177
|
+
logger.info({readErr}, 'synth Cartesia: failed to read error response body');
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
1394
1180
|
logger.info({err}, 'synth Cartesia returned error');
|
|
1395
1181
|
stats.increment('tts.count', ['vendor:cartesia', 'accepted:no']);
|
|
1396
1182
|
throw err;
|