@jambonz/speech-utils 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +0 -2
- package/lib/get-tts-voices.js +2 -92
- package/lib/synth-audio.js +5 -257
- package/lib/utils.js +1 -101
- package/package.json +1 -3
- package/test/index.js +0 -1
- package/test/list-voices.js +0 -93
- package/test/synth.js +0 -158
- package/build_stubs.sh +0 -11
- package/lib/get-nuance-access-token.js +0 -49
- package/lib/get-verbio-token.js +0 -51
- package/protos/nuance/synthesizer.proto +0 -332
- package/stubs/nuance/synthesizer_grpc_pb.js +0 -104
- package/stubs/nuance/synthesizer_pb.js +0 -6815
- package/test/nuance.js +0 -81
package/index.js
CHANGED
|
@@ -14,8 +14,6 @@ module.exports = (opts, logger) => {
|
|
|
14
14
|
purgeTtsCache: require('./lib/purge-tts-cache').bind(null, client, logger),
|
|
15
15
|
addFileToCache: require('./lib/add-file-to-cache').bind(null, client, logger),
|
|
16
16
|
synthAudio: require('./lib/synth-audio').bind(null, client, createHash, retrieveHash, logger),
|
|
17
|
-
getVerbioAccessToken: require('./lib/get-verbio-token').bind(null, client, logger),
|
|
18
|
-
getNuanceAccessToken: require('./lib/get-nuance-access-token').bind(null, client, logger),
|
|
19
17
|
|
|
20
18
|
getAwsAuthToken: require('./lib/get-aws-sts-token').bind(null, logger, createHash, retrieveHash),
|
|
21
19
|
getTtsVoices: require('./lib/get-tts-voices').bind(null, client, createHash, retrieveHash, logger),
|
package/lib/get-tts-voices.js
CHANGED
|
@@ -1,74 +1,8 @@
|
|
|
1
1
|
const assert = require('assert');
|
|
2
|
-
const {noopLogger
|
|
3
|
-
const getNuanceAccessToken = require('./get-nuance-access-token');
|
|
4
|
-
const getVerbioAccessToken = require('./get-verbio-token');
|
|
5
|
-
const {GetVoicesRequest, Voice} = require('../stubs/nuance/synthesizer_pb');
|
|
2
|
+
const {noopLogger} = require('./utils');
|
|
6
3
|
const ttsGoogle = require('@google-cloud/text-to-speech');
|
|
7
4
|
const { PollyClient, DescribeVoicesCommand } = require('@aws-sdk/client-polly');
|
|
8
5
|
const getAwsAuthToken = require('./get-aws-sts-token');
|
|
9
|
-
const {Pool} = require('undici');
|
|
10
|
-
const { HTTP_TIMEOUT } = require('./config');
|
|
11
|
-
const verbioVoicePool = new Pool('https://us.rest.speechcenter.verbio.com');
|
|
12
|
-
|
|
13
|
-
const getNuanceVoices = async(client, logger, credentials) => {
|
|
14
|
-
const {client_id: clientId, secret: secret, nuance_tts_uri} = credentials;
|
|
15
|
-
|
|
16
|
-
return new Promise(async(resolve, reject) => {
|
|
17
|
-
/* get a nuance access token */
|
|
18
|
-
let token, nuanceClient;
|
|
19
|
-
try {
|
|
20
|
-
if (nuance_tts_uri) {
|
|
21
|
-
nuanceClient = await createKryptonClient(nuance_tts_uri);
|
|
22
|
-
}
|
|
23
|
-
else {
|
|
24
|
-
const access_token = await getNuanceAccessToken(client, logger, clientId, secret, 'tts');
|
|
25
|
-
token = access_token.access_token;
|
|
26
|
-
nuanceClient = await createNuanceClient(token);
|
|
27
|
-
}
|
|
28
|
-
} catch (err) {
|
|
29
|
-
logger.error({err}, 'getTtsVoices: error retrieving access token');
|
|
30
|
-
return reject(err);
|
|
31
|
-
}
|
|
32
|
-
/* retrieve all voices */
|
|
33
|
-
const v = new Voice();
|
|
34
|
-
const request = new GetVoicesRequest();
|
|
35
|
-
request.setVoice(v);
|
|
36
|
-
|
|
37
|
-
nuanceClient.getVoices(request, (err, response) => {
|
|
38
|
-
if (err) {
|
|
39
|
-
logger.error({err, clientId, secret, token}, 'getTtsVoices: error retrieving voices');
|
|
40
|
-
return reject(err);
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
/* return all the voices that are not restricted and eliminate duplicates */
|
|
44
|
-
const voices = response.getVoicesList()
|
|
45
|
-
.map((v) => {
|
|
46
|
-
return {
|
|
47
|
-
language: v.getLanguage(),
|
|
48
|
-
name: v.getName(),
|
|
49
|
-
model: v.getModel(),
|
|
50
|
-
gender: v.getGender() === 1 ? 'male' : 'female',
|
|
51
|
-
restricted: v.getRestricted()
|
|
52
|
-
};
|
|
53
|
-
});
|
|
54
|
-
const v = voices
|
|
55
|
-
.filter((v) => v.restricted === false)
|
|
56
|
-
.map((v) => {
|
|
57
|
-
delete v.restricted;
|
|
58
|
-
return v;
|
|
59
|
-
})
|
|
60
|
-
.sort((a, b) => {
|
|
61
|
-
if (a.language < b.language) return -1;
|
|
62
|
-
if (a.language > b.language) return 1;
|
|
63
|
-
if (a.name < b.name) return -1;
|
|
64
|
-
return 1;
|
|
65
|
-
});
|
|
66
|
-
const arr = [...new Set(v.map((v) => JSON.stringify(v)))]
|
|
67
|
-
.map((v) => JSON.parse(v));
|
|
68
|
-
resolve(arr);
|
|
69
|
-
});
|
|
70
|
-
});
|
|
71
|
-
};
|
|
72
6
|
|
|
73
7
|
const getGoogleVoices = async(_client, logger, credentials) => {
|
|
74
8
|
const client = new ttsGoogle.TextToSpeechClient({credentials});
|
|
@@ -109,26 +43,6 @@ const getAwsVoices = async(_client, createHash, retrieveHash, logger, credential
|
|
|
109
43
|
}
|
|
110
44
|
};
|
|
111
45
|
|
|
112
|
-
const getVerbioVoices = async(client, logger, credentials) => {
|
|
113
|
-
try {
|
|
114
|
-
const access_token = await getVerbioAccessToken(client, logger, credentials);
|
|
115
|
-
const { body} = await verbioVoicePool.request({
|
|
116
|
-
path: '/api/v1/voices',
|
|
117
|
-
method: 'GET',
|
|
118
|
-
headers: {
|
|
119
|
-
'Authorization': `Bearer ${access_token.access_token}`,
|
|
120
|
-
'User-Agent': 'jambonz'
|
|
121
|
-
},
|
|
122
|
-
timeout: HTTP_TIMEOUT,
|
|
123
|
-
followRedirects: false
|
|
124
|
-
});
|
|
125
|
-
return await body.json();
|
|
126
|
-
} catch (err) {
|
|
127
|
-
logger.info({err}, 'getVerbioVoices - failed to list voices for Verbio');
|
|
128
|
-
throw err;
|
|
129
|
-
}
|
|
130
|
-
};
|
|
131
|
-
|
|
132
46
|
/**
|
|
133
47
|
* Synthesize speech to an mp3 file, and also cache the generated speech
|
|
134
48
|
* in redis (base64 format) for 24 hours so as to avoid unnecessarily paying
|
|
@@ -148,19 +62,15 @@ const getVerbioVoices = async(client, logger, credentials) => {
|
|
|
148
62
|
async function getTtsVoices(client, createHash, retrieveHash, logger, {vendor, credentials}) {
|
|
149
63
|
logger = logger || noopLogger;
|
|
150
64
|
|
|
151
|
-
assert.ok(['
|
|
65
|
+
assert.ok(['google', 'aws', 'polly'].includes(vendor),
|
|
152
66
|
`getTtsVoices not supported for vendor ${vendor}`);
|
|
153
67
|
|
|
154
68
|
switch (vendor) {
|
|
155
|
-
case 'nuance':
|
|
156
|
-
return getNuanceVoices(client, logger, credentials);
|
|
157
69
|
case 'google':
|
|
158
70
|
return getGoogleVoices(client, logger, credentials);
|
|
159
71
|
case 'aws':
|
|
160
72
|
case 'polly':
|
|
161
73
|
return getAwsVoices(client, createHash, retrieveHash, logger, credentials);
|
|
162
|
-
case 'verbio':
|
|
163
|
-
return getVerbioVoices(client, logger, credentials);
|
|
164
74
|
default:
|
|
165
75
|
break;
|
|
166
76
|
}
|
package/lib/synth-audio.js
CHANGED
|
@@ -16,26 +16,10 @@ const {
|
|
|
16
16
|
} = sdk;
|
|
17
17
|
const {
|
|
18
18
|
makeSynthKey,
|
|
19
|
-
createNuanceClient,
|
|
20
|
-
createKryptonClient,
|
|
21
19
|
createRivaClient,
|
|
22
20
|
noopLogger,
|
|
23
|
-
makeFilePath
|
|
24
|
-
makePlayhtKey
|
|
21
|
+
makeFilePath
|
|
25
22
|
} = require('./utils');
|
|
26
|
-
const getNuanceAccessToken = require('./get-nuance-access-token');
|
|
27
|
-
const getVerbioAccessToken = require('./get-verbio-token');
|
|
28
|
-
const {
|
|
29
|
-
SynthesisRequest,
|
|
30
|
-
Voice,
|
|
31
|
-
AudioFormat,
|
|
32
|
-
AudioParameters,
|
|
33
|
-
PCM,
|
|
34
|
-
Input,
|
|
35
|
-
Text,
|
|
36
|
-
SSML,
|
|
37
|
-
EventParameters
|
|
38
|
-
} = require('../stubs/nuance/synthesizer_pb');
|
|
39
23
|
const {SynthesizeSpeechRequest} = require('../stubs/riva/proto/riva_tts_pb');
|
|
40
24
|
const {AudioEncoding} = require('../stubs/riva/proto/riva_audio_pb');
|
|
41
25
|
const debug = require('debug')('jambonz:realtimedb-helpers');
|
|
@@ -95,10 +79,10 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
|
|
|
95
79
|
let rtt;
|
|
96
80
|
logger = logger || noopLogger;
|
|
97
81
|
|
|
98
|
-
assert.ok(['google', 'aws', 'polly', 'microsoft', 'wellsaid', '
|
|
99
|
-
'whisper', 'deepgram', '
|
|
82
|
+
assert.ok(['google', 'aws', 'polly', 'microsoft', 'wellsaid', 'nvidia', 'elevenlabs',
|
|
83
|
+
'whisper', 'deepgram', 'rimelabs', 'cartesia', 'inworld', 'resemble'].includes(vendor) ||
|
|
100
84
|
vendor.startsWith('custom'),
|
|
101
|
-
`synthAudio supported vendors are google, aws, microsoft,
|
|
85
|
+
`synthAudio supported vendors are google, aws, microsoft, nvidia and wellsaid ..etc, not ${vendor}`);
|
|
102
86
|
if ('google' === vendor) {
|
|
103
87
|
assert.ok(language, 'synthAudio requires language when google is used');
|
|
104
88
|
}
|
|
@@ -109,13 +93,6 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
|
|
|
109
93
|
assert.ok(language || deploymentId, 'synthAudio requires language when microsoft is used');
|
|
110
94
|
assert.ok(voice || deploymentId, 'synthAudio requires voice when microsoft is used');
|
|
111
95
|
}
|
|
112
|
-
else if ('nuance' === vendor) {
|
|
113
|
-
assert.ok(voice, 'synthAudio requires voice when nuance is used');
|
|
114
|
-
if (!credentials.nuance_tts_uri) {
|
|
115
|
-
assert.ok(credentials.client_id, 'synthAudio requires client_id in credentials when nuance is used');
|
|
116
|
-
assert.ok(credentials.secret, 'synthAudio requires client_id in credentials when nuance is used');
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
96
|
else if ('nvidia' === vendor) {
|
|
120
97
|
assert.ok(voice, 'synthAudio requires voice when nvidia is used');
|
|
121
98
|
assert.ok(language, 'synthAudio requires language when nvidia is used');
|
|
@@ -129,11 +106,6 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
|
|
|
129
106
|
assert.ok(voice, 'synthAudio requires voice when elevenlabs is used');
|
|
130
107
|
assert.ok(credentials.api_key, 'synthAudio requires api_key when elevenlabs is used');
|
|
131
108
|
assert.ok(credentials.model_id, 'synthAudio requires model_id when elevenlabs is used');
|
|
132
|
-
} else if ('playht' === vendor) {
|
|
133
|
-
assert.ok(voice, 'synthAudio requires voice when playht is used');
|
|
134
|
-
assert.ok(credentials.api_key, 'synthAudio requires api_key when playht is used');
|
|
135
|
-
assert.ok(credentials.user_id, 'synthAudio requires user_id when playht is used');
|
|
136
|
-
assert.ok(credentials.voice_engine, 'synthAudio requires voice_engine when playht is used');
|
|
137
109
|
} else if ('inworld' === vendor) {
|
|
138
110
|
assert.ok(voice, 'synthAudio requires voice when inworld is used');
|
|
139
111
|
assert.ok(credentials.api_key, 'synthAudio requires api_key when inworld is used');
|
|
@@ -148,10 +120,6 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
|
|
|
148
120
|
assert.ok(credentials.api_key, 'synthAudio requires api_key when whisper is used');
|
|
149
121
|
} else if (vendor.startsWith('custom')) {
|
|
150
122
|
assert.ok(credentials.custom_tts_url, `synthAudio requires custom_tts_url in credentials when ${vendor} is used`);
|
|
151
|
-
} else if ('verbio' === vendor) {
|
|
152
|
-
assert.ok(voice, 'synthAudio requires voice when verbio is used');
|
|
153
|
-
assert.ok(credentials.client_id, 'synthAudio requires client_id when verbio is used');
|
|
154
|
-
assert.ok(credentials.client_secret, 'synthAudio requires client_secret when verbio is used');
|
|
155
123
|
} else if ('deepgram' === vendor) {
|
|
156
124
|
if (!credentials.deepgram_tts_uri) {
|
|
157
125
|
assert.ok(credentials.api_key, 'synthAudio requires api_key when deepgram is used');
|
|
@@ -216,10 +184,6 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
|
|
|
216
184
|
audioData = await synthMicrosoft(logger, {credentials, stats, language, voice, key, text, deploymentId,
|
|
217
185
|
renderForCaching, disableTtsStreaming, disableTtsCache});
|
|
218
186
|
break;
|
|
219
|
-
case 'nuance':
|
|
220
|
-
model = model || 'enhanced';
|
|
221
|
-
audioData = await synthNuance(client, logger, {credentials, stats, voice, model, key, text});
|
|
222
|
-
break;
|
|
223
187
|
case 'nvidia':
|
|
224
188
|
audioData = await synthNvidia(client, logger, {credentials, stats, language, voice, model, key, text,
|
|
225
189
|
renderForCaching, disableTtsStreaming, disableTtsCache});
|
|
@@ -232,11 +196,6 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
|
|
|
232
196
|
credentials, options, stats, language, voice, key, text, renderForCaching, disableTtsStreaming,
|
|
233
197
|
disableTtsCache});
|
|
234
198
|
break;
|
|
235
|
-
case 'playht':
|
|
236
|
-
audioData = await synthPlayHT(client, logger, {
|
|
237
|
-
credentials, options, stats, language, voice, key, text, renderForCaching, disableTtsStreaming,
|
|
238
|
-
disableTtsCache});
|
|
239
|
-
break;
|
|
240
199
|
case 'cartesia':
|
|
241
200
|
audioData = await synthCartesia(logger, {
|
|
242
201
|
credentials, options, stats, language, voice, key, text, renderForCaching, disableTtsStreaming,
|
|
@@ -257,11 +216,6 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
|
|
|
257
216
|
credentials, stats, voice, key, text, instructions, renderForCaching, disableTtsStreaming,
|
|
258
217
|
disableTtsCache});
|
|
259
218
|
break;
|
|
260
|
-
case 'verbio':
|
|
261
|
-
audioData = await synthVerbio(client, logger, {
|
|
262
|
-
credentials, stats, voice, key, text, renderForCaching, disableTtsStreaming, disableTtsCache});
|
|
263
|
-
if (audioData?.filePath) return audioData;
|
|
264
|
-
break;
|
|
265
219
|
case 'deepgram':
|
|
266
220
|
audioData = await synthDeepgram(logger, {credentials, stats, model, key, text,
|
|
267
221
|
renderForCaching, disableTtsStreaming, disableTtsCache});
|
|
@@ -316,6 +270,7 @@ const synthPolly = async(createHash, retrieveHash, logger,
|
|
|
316
270
|
params += `,playback_id=${key}`;
|
|
317
271
|
params += `,write_cache_file=${disableTtsCache ? 0 : 1}`;
|
|
318
272
|
params += ',vendor=aws';
|
|
273
|
+
params += `,voice=${voice}`;
|
|
319
274
|
if (accessKeyId && secretAccessKey) {
|
|
320
275
|
if (accessKeyId) params += `,accessKeyId=${accessKeyId}`;
|
|
321
276
|
if (secretAccessKey) params += `,secretAccessKey=${secretAccessKey}`;
|
|
@@ -724,70 +679,6 @@ const synthWellSaid = async(logger, {credentials, stats, language, voice, gender
|
|
|
724
679
|
}
|
|
725
680
|
};
|
|
726
681
|
|
|
727
|
-
const synthNuance = async(client, logger, {credentials, stats, voice, model, text}) => {
|
|
728
|
-
let nuanceClient;
|
|
729
|
-
const {client_id, secret, nuance_tts_uri} = credentials;
|
|
730
|
-
if (nuance_tts_uri) {
|
|
731
|
-
nuanceClient = await createKryptonClient(nuance_tts_uri);
|
|
732
|
-
}
|
|
733
|
-
else {
|
|
734
|
-
/* get a nuance access token */
|
|
735
|
-
const {access_token} = await getNuanceAccessToken(client, logger, client_id, secret, 'tts');
|
|
736
|
-
nuanceClient = await createNuanceClient(access_token);
|
|
737
|
-
}
|
|
738
|
-
|
|
739
|
-
const v = new Voice();
|
|
740
|
-
const p = new AudioParameters();
|
|
741
|
-
const f = new AudioFormat();
|
|
742
|
-
const pcm = new PCM();
|
|
743
|
-
const params = new EventParameters();
|
|
744
|
-
const request = new SynthesisRequest();
|
|
745
|
-
const input = new Input();
|
|
746
|
-
|
|
747
|
-
if (text.startsWith('<speak')) {
|
|
748
|
-
const ssml = new SSML();
|
|
749
|
-
ssml.setText(text);
|
|
750
|
-
input.setSsml(ssml);
|
|
751
|
-
}
|
|
752
|
-
else {
|
|
753
|
-
const t = new Text();
|
|
754
|
-
t.setText(text);
|
|
755
|
-
input.setText(t);
|
|
756
|
-
}
|
|
757
|
-
const sampleRate = 8000;
|
|
758
|
-
pcm.setSampleRateHz(sampleRate);
|
|
759
|
-
f.setPcm(pcm);
|
|
760
|
-
p.setAudioFormat(f);
|
|
761
|
-
v.setName(voice);
|
|
762
|
-
v.setModel(model);
|
|
763
|
-
request.setVoice(v);
|
|
764
|
-
request.setAudioParams(p);
|
|
765
|
-
request.setInput(input);
|
|
766
|
-
request.setEventParams(params);
|
|
767
|
-
request.setUserId('jambonz');
|
|
768
|
-
|
|
769
|
-
return new Promise((resolve, reject) => {
|
|
770
|
-
nuanceClient.unarySynthesize(request, (err, response) => {
|
|
771
|
-
if (err) {
|
|
772
|
-
console.error(err);
|
|
773
|
-
return reject(err);
|
|
774
|
-
}
|
|
775
|
-
const status = response.getStatus();
|
|
776
|
-
const code = status.getCode();
|
|
777
|
-
if (code !== 200) {
|
|
778
|
-
const message = status.getMessage();
|
|
779
|
-
const details = status.getDetails();
|
|
780
|
-
return reject({code, message, details});
|
|
781
|
-
}
|
|
782
|
-
resolve({
|
|
783
|
-
audioContent: Buffer.from(response.getAudio()),
|
|
784
|
-
extension: 'r8',
|
|
785
|
-
sampleRate
|
|
786
|
-
});
|
|
787
|
-
});
|
|
788
|
-
});
|
|
789
|
-
};
|
|
790
|
-
|
|
791
682
|
const synthNvidia = async(client, logger, {
|
|
792
683
|
credentials, stats, language, voice, model, key, text, renderForCaching, disableTtsStreaming, disableTtsCache
|
|
793
684
|
}) => {
|
|
@@ -953,101 +844,6 @@ const synthElevenlabs = async(logger, {
|
|
|
953
844
|
}
|
|
954
845
|
};
|
|
955
846
|
|
|
956
|
-
const synthPlayHT = async(client, logger, {
|
|
957
|
-
credentials, options, stats, voice, language, key, text, renderForCaching, disableTtsStreaming, disableTtsCache
|
|
958
|
-
}) => {
|
|
959
|
-
const {api_key, user_id, voice_engine, playht_tts_uri, options: credOpts} = credentials;
|
|
960
|
-
const opts = !!options && Object.keys(options).length !== 0 ? options : JSON.parse(credOpts || '{}');
|
|
961
|
-
|
|
962
|
-
let synthesizeUrl = playht_tts_uri ? `${playht_tts_uri}/api/v2/tts/stream` : 'https://api.play.ht/api/v2/tts/stream';
|
|
963
|
-
// If model is play3.0, the synthesizeUrl is got from authentication endpoint
|
|
964
|
-
if (voice_engine === 'Play3.0') {
|
|
965
|
-
try {
|
|
966
|
-
const post = bent('https://api.play.ht', 'POST', 'json', 201, {
|
|
967
|
-
'AUTHORIZATION': api_key,
|
|
968
|
-
'X-USER-ID': user_id,
|
|
969
|
-
'Accept': 'application/json'
|
|
970
|
-
});
|
|
971
|
-
const key = makePlayhtKey(api_key);
|
|
972
|
-
const url = await client.get(key);
|
|
973
|
-
if (!url) {
|
|
974
|
-
const {inference_address, expires_at_ms} = await post('/api/v3/auth');
|
|
975
|
-
synthesizeUrl = inference_address;
|
|
976
|
-
const expiry = Math.floor((expires_at_ms - Date.now()) / 1000 - 30);
|
|
977
|
-
await client.set(key, inference_address, 'EX', expiry);
|
|
978
|
-
} else {
|
|
979
|
-
// Use cached URL
|
|
980
|
-
synthesizeUrl = url;
|
|
981
|
-
}
|
|
982
|
-
} catch (err) {
|
|
983
|
-
logger.info({err}, 'synth PlayHT returned error for authentication version 3.0');
|
|
984
|
-
stats.increment('tts.count', ['vendor:playht', 'accepted:no']);
|
|
985
|
-
throw err;
|
|
986
|
-
}
|
|
987
|
-
}
|
|
988
|
-
|
|
989
|
-
/* default to using the streaming interface, unless disabled by env var OR we want just a cache file */
|
|
990
|
-
if (!JAMBONES_DISABLE_TTS_STREAMING && !renderForCaching && !disableTtsStreaming) {
|
|
991
|
-
let params = '{';
|
|
992
|
-
params += `api_key=${api_key}`;
|
|
993
|
-
params += `,playback_id=${key}`;
|
|
994
|
-
params += `,user_id=${user_id}`;
|
|
995
|
-
params += ',vendor=playht';
|
|
996
|
-
params += `,voice=${voice}`;
|
|
997
|
-
params += `,voice_engine=${voice_engine}`;
|
|
998
|
-
params += `,synthesize_url=${synthesizeUrl}`;
|
|
999
|
-
params += `,write_cache_file=${disableTtsCache ? 0 : 1}`;
|
|
1000
|
-
params += `,language=${language}`;
|
|
1001
|
-
if (opts.quality) params += `,quality=${opts.quality}`;
|
|
1002
|
-
if (opts.speed) params += `,speed=${opts.speed}`;
|
|
1003
|
-
if (opts.seed) params += `,style=${opts.seed}`;
|
|
1004
|
-
if (opts.temperature) params += `,temperature=${opts.temperature}`;
|
|
1005
|
-
if (opts.emotion) params += `,emotion=${opts.emotion}`;
|
|
1006
|
-
if (opts.voice_guidance) params += `,voice_guidance=${opts.voice_guidance}`;
|
|
1007
|
-
if (opts.style_guidance) params += `,style_guidance=${opts.style_guidance}`;
|
|
1008
|
-
if (opts.text_guidance) params += `,text_guidance=${opts.text_guidance}`;
|
|
1009
|
-
if (opts.top_p) params += `,top_p=${opts.top_p}`;
|
|
1010
|
-
if (opts.repetition_penalty) params += `,repetition_penalty=${opts.repetition_penalty}`;
|
|
1011
|
-
params += '}';
|
|
1012
|
-
|
|
1013
|
-
return {
|
|
1014
|
-
filePath: `say:${params}${text.replace(/\n/g, ' ').replace(/\r/g, ' ')}`,
|
|
1015
|
-
servedFromCache: false,
|
|
1016
|
-
rtt: 0
|
|
1017
|
-
};
|
|
1018
|
-
}
|
|
1019
|
-
|
|
1020
|
-
try {
|
|
1021
|
-
const post = bent('POST', 'buffer', {
|
|
1022
|
-
...(voice_engine !== 'Play3.0' && {
|
|
1023
|
-
'AUTHORIZATION': api_key,
|
|
1024
|
-
'X-USER-ID': user_id,
|
|
1025
|
-
}),
|
|
1026
|
-
'Accept': 'audio/mpeg',
|
|
1027
|
-
'Content-Type': 'application/json'
|
|
1028
|
-
});
|
|
1029
|
-
|
|
1030
|
-
const audioContent = await post(synthesizeUrl, {
|
|
1031
|
-
text,
|
|
1032
|
-
...(voice_engine === 'Play3.0' && { language }),
|
|
1033
|
-
voice,
|
|
1034
|
-
voice_engine,
|
|
1035
|
-
output_format: 'mp3',
|
|
1036
|
-
sample_rate: 8000,
|
|
1037
|
-
...opts
|
|
1038
|
-
});
|
|
1039
|
-
return {
|
|
1040
|
-
audioContent,
|
|
1041
|
-
extension: 'mp3',
|
|
1042
|
-
sampleRate: 8000
|
|
1043
|
-
};
|
|
1044
|
-
} catch (err) {
|
|
1045
|
-
logger.info({err}, 'synth PlayHT returned error');
|
|
1046
|
-
stats.increment('tts.count', ['vendor:playht', 'accepted:no']);
|
|
1047
|
-
throw err;
|
|
1048
|
-
}
|
|
1049
|
-
};
|
|
1050
|
-
|
|
1051
847
|
const synthInworld = async(logger, {
|
|
1052
848
|
credentials, options, stats, voice, key, text, renderForCaching, disableTtsStreaming, disableTtsCache
|
|
1053
849
|
}) => {
|
|
@@ -1173,54 +969,6 @@ const synthRimelabs = async(logger, {
|
|
|
1173
969
|
throw err;
|
|
1174
970
|
}
|
|
1175
971
|
};
|
|
1176
|
-
const synthVerbio = async(client, logger, {
|
|
1177
|
-
credentials, stats, voice, key, text, renderForCaching, disableTtsStreaming, disableTtsCache
|
|
1178
|
-
}) => {
|
|
1179
|
-
//https://doc.speechcenter.verbio.com/#tag/Text-To-Speech-REST-API
|
|
1180
|
-
if (text.length > 2000) {
|
|
1181
|
-
throw new Error('Verbio cannot synthesize for the text length larger than 2000 characters');
|
|
1182
|
-
}
|
|
1183
|
-
const token = await getVerbioAccessToken(client, logger, credentials);
|
|
1184
|
-
if (!process.env.JAMBONES_DISABLE_TTS_STREAMING && !renderForCaching && !disableTtsStreaming) {
|
|
1185
|
-
let params = '{';
|
|
1186
|
-
params += `access_token=${token.access_token}`;
|
|
1187
|
-
params += `,playback_id=${key}`;
|
|
1188
|
-
params += ',vendor=verbio';
|
|
1189
|
-
params += `,voice=${voice}`;
|
|
1190
|
-
params += `,write_cache_file=${disableTtsCache ? 0 : 1}`;
|
|
1191
|
-
params += '}';
|
|
1192
|
-
|
|
1193
|
-
return {
|
|
1194
|
-
filePath: `say:${params}${text.replace(/\n/g, ' ')}`,
|
|
1195
|
-
servedFromCache: false,
|
|
1196
|
-
rtt: 0
|
|
1197
|
-
};
|
|
1198
|
-
}
|
|
1199
|
-
|
|
1200
|
-
try {
|
|
1201
|
-
const post = bent('https://us.rest.speechcenter.verbio.com', 'POST', 'buffer', {
|
|
1202
|
-
'Authorization': `Bearer ${token.access_token}`,
|
|
1203
|
-
'User-Agent': 'jambonz',
|
|
1204
|
-
'Content-Type': 'application/json'
|
|
1205
|
-
});
|
|
1206
|
-
const audioContent = await post('/api/v1/synthesize', {
|
|
1207
|
-
voice_id: voice,
|
|
1208
|
-
output_sample_rate: '8k',
|
|
1209
|
-
output_encoding: 'pcm16',
|
|
1210
|
-
text
|
|
1211
|
-
});
|
|
1212
|
-
return {
|
|
1213
|
-
audioContent,
|
|
1214
|
-
extension: 'r8',
|
|
1215
|
-
sampleRate: 8000
|
|
1216
|
-
};
|
|
1217
|
-
} catch (err) {
|
|
1218
|
-
logger.info({err}, 'synth Verbio returned error');
|
|
1219
|
-
stats.increment('tts.count', ['vendor:verbio', 'accepted:no']);
|
|
1220
|
-
throw err;
|
|
1221
|
-
}
|
|
1222
|
-
};
|
|
1223
|
-
|
|
1224
972
|
const synthWhisper = async(logger, {credentials, stats, voice, key, text, instructions,
|
|
1225
973
|
renderForCaching, disableTtsStreaming, disableTtsCache}) => {
|
|
1226
974
|
const {api_key, model_id, baseURL, timeout, speed} = credentials;
|
package/lib/utils.js
CHANGED
|
@@ -1,20 +1,7 @@
|
|
|
1
1
|
const crypto = require('crypto');
|
|
2
|
-
const {SynthesizerClient} = require('../stubs/nuance/synthesizer_grpc_pb');
|
|
3
2
|
const {RivaSpeechSynthesisClient} = require('../stubs/riva/proto/riva_tts_grpc_pb');
|
|
4
|
-
const {Pool} = require('undici');
|
|
5
|
-
const pool = new Pool('https://auth.crt.nuance.com');
|
|
6
|
-
const NUANCE_AUTH_ENDPOINT = 'tts.api.nuance.com:443';
|
|
7
3
|
const grpc = require('@grpc/grpc-js');
|
|
8
|
-
const
|
|
9
|
-
const { TMP_FOLDER, HTTP_TIMEOUT } = require('./config');
|
|
10
|
-
|
|
11
|
-
const debug = require('debug')('jambonz:realtimedb-helpers');
|
|
12
|
-
/**
|
|
13
|
-
* Future TODO: cache recently used connections to providers
|
|
14
|
-
* to avoid connection overhead during a call.
|
|
15
|
-
* Will need to periodically age them out to avoid memory leaks.
|
|
16
|
-
*/
|
|
17
|
-
//const nuanceClientMap = new Map();
|
|
4
|
+
const { TMP_FOLDER } = require('./config');
|
|
18
5
|
|
|
19
6
|
function makeSynthKey({
|
|
20
7
|
account_sid = '',
|
|
@@ -45,90 +32,12 @@ const noopLogger = {
|
|
|
45
32
|
error: () => {}
|
|
46
33
|
};
|
|
47
34
|
|
|
48
|
-
const toBase64 = (str) => Buffer.from(str || '', 'utf8').toString('base64');
|
|
49
|
-
|
|
50
|
-
function makeBasicAuthHeader(username, password) {
|
|
51
|
-
if (!username || !password) return {};
|
|
52
|
-
const creds = `${encodeURIComponent(username)}:${password || ''}`;
|
|
53
|
-
const header = `Basic ${toBase64(creds)}`;
|
|
54
|
-
return {Authorization: header};
|
|
55
|
-
}
|
|
56
|
-
|
|
57
35
|
function makeAwsKey(awsAccessKeyId) {
|
|
58
36
|
const hash = crypto.createHash('sha1');
|
|
59
37
|
hash.update(awsAccessKeyId);
|
|
60
38
|
return `aws:${hash.digest('hex')}`;
|
|
61
39
|
}
|
|
62
40
|
|
|
63
|
-
function makePlayhtKey(apiKey) {
|
|
64
|
-
const hash = crypto.createHash('sha1');
|
|
65
|
-
hash.update(apiKey);
|
|
66
|
-
return `playht:${hash.digest('hex')}`;
|
|
67
|
-
}
|
|
68
|
-
function makeVerbioKey(client_id) {
|
|
69
|
-
const hash = crypto.createHash('sha1');
|
|
70
|
-
hash.update(client_id);
|
|
71
|
-
return `verbio:${hash.digest('hex')}`;
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
function makeNuanceKey(clientId, secret, scope) {
|
|
75
|
-
const hash = crypto.createHash('sha1');
|
|
76
|
-
hash.update(`${clientId}:${secret}:${scope}`);
|
|
77
|
-
return `nuance:${hash.digest('hex')}`;
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
const getNuanceAccessToken = async(clientId, secret, scope = 'asr tts') => {
|
|
81
|
-
const payload = {
|
|
82
|
-
grant_type: 'client_credentials',
|
|
83
|
-
scope
|
|
84
|
-
};
|
|
85
|
-
const auth = makeBasicAuthHeader(clientId, secret);
|
|
86
|
-
const {statusCode, headers, body} = await pool.request({
|
|
87
|
-
path: '/oauth2/token',
|
|
88
|
-
method: 'POST',
|
|
89
|
-
headers: {
|
|
90
|
-
...auth,
|
|
91
|
-
'Content-Type': 'application/x-www-form-urlencoded'
|
|
92
|
-
},
|
|
93
|
-
body: formurlencoded(payload),
|
|
94
|
-
timeout: HTTP_TIMEOUT,
|
|
95
|
-
followRedirects: false
|
|
96
|
-
});
|
|
97
|
-
|
|
98
|
-
if (200 !== statusCode) {
|
|
99
|
-
debug({statusCode, headers, body: body.text()}, 'error fetching access token from Nuance');
|
|
100
|
-
const err = new Error();
|
|
101
|
-
err.statusCode = statusCode;
|
|
102
|
-
throw err;
|
|
103
|
-
}
|
|
104
|
-
const json = await body.json();
|
|
105
|
-
return json.access_token;
|
|
106
|
-
};
|
|
107
|
-
|
|
108
|
-
const createKryptonClient = async(uri) => {
|
|
109
|
-
const client = new SynthesizerClient(uri, grpc.credentials.createInsecure());
|
|
110
|
-
return client;
|
|
111
|
-
};
|
|
112
|
-
|
|
113
|
-
const createNuanceClient = async(access_token) => {
|
|
114
|
-
|
|
115
|
-
//if (nuanceClientMap.has(access_token)) return nuanceClientMap.get(access_token);
|
|
116
|
-
|
|
117
|
-
const generateMetadata = (params, callback) => {
|
|
118
|
-
var metadata = new grpc.Metadata();
|
|
119
|
-
metadata.add('authorization', `Bearer ${access_token}`);
|
|
120
|
-
callback(null, metadata);
|
|
121
|
-
};
|
|
122
|
-
|
|
123
|
-
const sslCreds = grpc.credentials.createSsl();
|
|
124
|
-
const authCreds = grpc.credentials.createFromMetadataGenerator(generateMetadata);
|
|
125
|
-
const combined_creds = grpc.credentials.combineChannelCredentials(sslCreds, authCreds);
|
|
126
|
-
const client = new SynthesizerClient(NUANCE_AUTH_ENDPOINT, combined_creds);
|
|
127
|
-
|
|
128
|
-
//if (process.env.NUANCE_CACHE_TTS_CONNECTIONS) nuanceClientMap.set(access_token, client);
|
|
129
|
-
return client;
|
|
130
|
-
};
|
|
131
|
-
|
|
132
41
|
const createRivaClient = async(rivaUri) => {
|
|
133
42
|
const client = new RivaSpeechSynthesisClient(rivaUri, grpc.credentials.createInsecure());
|
|
134
43
|
return client;
|
|
@@ -136,17 +45,8 @@ const createRivaClient = async(rivaUri) => {
|
|
|
136
45
|
|
|
137
46
|
module.exports = {
|
|
138
47
|
makeSynthKey,
|
|
139
|
-
makeNuanceKey,
|
|
140
|
-
|
|
141
|
-
makePlayhtKey,
|
|
142
48
|
makeAwsKey,
|
|
143
|
-
makeVerbioKey,
|
|
144
|
-
getNuanceAccessToken,
|
|
145
|
-
createNuanceClient,
|
|
146
|
-
createKryptonClient,
|
|
147
49
|
createRivaClient,
|
|
148
|
-
makeBasicAuthHeader,
|
|
149
|
-
NUANCE_AUTH_ENDPOINT,
|
|
150
50
|
noopLogger,
|
|
151
51
|
makeFilePath
|
|
152
52
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jambonz/speech-utils",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.6",
|
|
4
4
|
"description": "TTS-related speech utilities for jambonz",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"author": "Dave Horton",
|
|
@@ -13,7 +13,6 @@
|
|
|
13
13
|
"coverage": "nyc --reporter html --report-dir ./coverage npm run test",
|
|
14
14
|
"jslint": "eslint index.js lib",
|
|
15
15
|
"jslint:fix": "npm run jslint --fix",
|
|
16
|
-
"build": "./build_stubs.sh",
|
|
17
16
|
"prepare": "husky"
|
|
18
17
|
},
|
|
19
18
|
"repository": {
|
|
@@ -35,7 +34,6 @@
|
|
|
35
34
|
"@jambonz/realtimedb-helpers": "^0.8.7",
|
|
36
35
|
"bent": "^7.3.12",
|
|
37
36
|
"debug": "^4.3.4",
|
|
38
|
-
"form-urlencoded": "^6.1.4",
|
|
39
37
|
"google-protobuf": "^3.21.2",
|
|
40
38
|
"microsoft-cognitiveservices-speech-sdk": "1.38.0",
|
|
41
39
|
"openai": "^4.98.0",
|