@rimelabs/sdk 0.1.0-alpha.13 → 0.1.0-alpha.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +183 -86
- package/dist/client.d.ts +2 -2
- package/dist/client.js +2 -1
- package/dist/realtime/protocol.js +13 -13
- package/dist/realtime/session.js +7 -7
- package/dist/stt/client.d.ts +1 -2
- package/dist/stt/client.js +3 -7
- package/dist/tts/audio.d.ts +9 -0
- package/dist/tts/audio.js +42 -3
- package/dist/tts/buffer.d.ts +3 -0
- package/dist/tts/buffer.js +53 -0
- package/dist/tts/client.d.ts +9 -0
- package/dist/tts/client.js +29 -6
- package/dist/tts/stream.d.ts +5 -1
- package/dist/tts/stream.js +11 -3
- package/dist/tts/transport.d.ts +2 -1
- package/dist/tts/transport.js +8 -6
- package/docs/realtime.md +3 -0
- package/docs/stt.md +2 -2
- package/docs/tts.md +49 -10
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.1.0-alpha.15](https://github.com/rimelabs/rime-sdk/compare/typescript-v0.1.0-alpha.14...typescript-v0.1.0-alpha.15) (2026-10-13)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
* **realtime:** align Prism protocol names and contract fixtures ([#68](https://github.com/rimelabs/rime-sdk/issues/68)) ([806d8a4](https://github.com/rimelabs/rime-sdk/commit/806d8a4811f0baf6b220487dfc6ccd21cfe8ea18))
|
|
9
|
+
|
|
10
|
+
## [0.1.0-alpha.14](https://github.com/rimelabs/rime-sdk/compare/typescript-v0.1.0-alpha.13...typescript-v0.1.0-alpha.14) (2026-10-12)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
### Features
|
|
14
|
+
|
|
15
|
+
* **tts:** add sample rates, time-scale control, and buffered audio output ([1856d0d](https://github.com/rimelabs/rime-sdk/commit/1856d0d6c00a46d7917b61d73bc7f2a8526c8bb0))
|
|
16
|
+
|
|
3
17
|
## [0.1.0-alpha.13](https://github.com/rimelabs/rime-sdk/compare/typescript-v0.1.0-alpha.12...typescript-v0.1.0-alpha.13) (2026-10-11)
|
|
4
18
|
|
|
5
19
|
|
package/README.md
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
# Rime SDK for Node.js
|
|
1
|
+
# Rime SDK for TypeScript and Node.js
|
|
2
2
|
|
|
3
|
-
Generate speech with
|
|
4
|
-
|
|
3
|
+
Generate speech with Coda or Mist v3, transcribe audio with STT, and build spoken
|
|
4
|
+
conversations with Prism through the `Rime` client. Your application handles
|
|
5
5
|
audio capture and playback.
|
|
6
6
|
|
|
7
|
-
##
|
|
7
|
+
## Installation
|
|
8
8
|
|
|
9
|
-
Requires Node.js 22 or later
|
|
10
|
-
|
|
9
|
+
Requires **Node.js 22 or later**. Install the alpha package from npm. It uses
|
|
10
|
+
ESM, includes TypeScript types, and runs on a server; browsers are not supported.
|
|
11
11
|
|
|
12
12
|
```sh
|
|
13
13
|
mkdir rime-quickstart
|
|
@@ -17,83 +17,14 @@ npm install @rimelabs/sdk wavefile
|
|
|
17
17
|
export RIME_API_KEY="your-api-key"
|
|
18
18
|
```
|
|
19
19
|
|
|
20
|
-
Get
|
|
21
|
-
|
|
22
|
-
|
|
20
|
+
For an existing project, skip the first three commands. Get an API key from the
|
|
21
|
+
[Rime dashboard](https://app.rime.ai/). A key is required: `new Rime()` reads
|
|
22
|
+
`RIME_API_KEY`, or you can pass `new Rime({ apiKey: "..." })` to override it.
|
|
23
|
+
The SDK does not load `.env` files. Shell commands below assume macOS or Linux.
|
|
23
24
|
|
|
24
|
-
|
|
25
|
-
The additional
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
## TTS: save speech
|
|
29
|
-
|
|
30
|
-
Save this as `tts.mjs`. It writes a playable `speech.wav` file with no audio-device dependencies.
|
|
31
|
-
|
|
32
|
-
```javascript
|
|
33
|
-
import { writeFileSync } from "node:fs";
|
|
34
|
-
import wavefile from "wavefile";
|
|
35
|
-
import { Rime } from "@rimelabs/sdk";
|
|
36
|
-
|
|
37
|
-
const client = new Rime({ timeout: 60 });
|
|
38
|
-
try {
|
|
39
|
-
const chunks = [];
|
|
40
|
-
for await (const chunk of client.tts.synthesize(
|
|
41
|
-
"Your appointment is confirmed for tomorrow.",
|
|
42
|
-
)) {
|
|
43
|
-
chunks.push(Buffer.from(chunk));
|
|
44
|
-
}
|
|
45
|
-
const pcm = Buffer.concat(chunks);
|
|
46
|
-
const samples = Array.from({ length: pcm.length / 2 }, (_, i) =>
|
|
47
|
-
pcm.readInt16LE(i * 2),
|
|
48
|
-
);
|
|
49
|
-
const wav = new wavefile.WaveFile();
|
|
50
|
-
wav.fromScratch(1, 24000, "16", samples);
|
|
51
|
-
writeFileSync("speech.wav", wav.toBuffer());
|
|
52
|
-
console.log("Saved speech.wav");
|
|
53
|
-
} finally {
|
|
54
|
-
await client.close();
|
|
55
|
-
}
|
|
56
|
-
```
|
|
57
|
-
|
|
58
|
-
Run `node tts.mjs`, then open `speech.wav` in your audio player.
|
|
59
|
-
This short example collects the audio in memory before writing the file.
|
|
60
|
-
For long responses, write audio chunks as they arrive.
|
|
61
|
-
|
|
62
|
-
Coda is the default model. Use `new Rime({ model: "mistv3" })` for Mist v3, or pass
|
|
63
|
-
`{ voice: "your-voice" }` as the second argument to `client.tts.synthesize(...)` to select a voice.
|
|
64
|
-
The default output is mono 24 kHz PCM16. The example adds its WAV header.
|
|
65
|
-
Use `client.tts.synthesize(text)` for a complete string. To supply text from an LLM,
|
|
66
|
-
pass an async iterable of strings to `client.tts.stream(chunks)`. Both methods
|
|
67
|
-
return streaming audio. Rime automatically combines text fragments into
|
|
68
|
-
sentences for speech.
|
|
69
|
-
|
|
70
|
-
## STT quick start
|
|
71
|
-
|
|
72
|
-
Stream one utterance of headerless signed PCM16 little-endian audio. Specify the
|
|
73
|
-
spoken language; exhaust the audio source when the utterance ends. Each update
|
|
74
|
-
replaces the complete current transcript.
|
|
75
|
-
|
|
76
|
-
```javascript
|
|
77
|
-
import { createReadStream } from "node:fs";
|
|
78
|
-
import { Rime } from "@rimelabs/sdk";
|
|
79
|
-
|
|
80
|
-
async function* audioChunks() {
|
|
81
|
-
yield* createReadStream("utterance.pcm");
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
const client = new Rime();
|
|
85
|
-
try {
|
|
86
|
-
const transcript = client.stt.stream(audioChunks(), {
|
|
87
|
-
language: "en",
|
|
88
|
-
});
|
|
89
|
-
for await (const update of transcript) console.log(update.kind, update.text);
|
|
90
|
-
} finally {
|
|
91
|
-
await client.close();
|
|
92
|
-
}
|
|
93
|
-
```
|
|
94
|
-
|
|
95
|
-
Input defaults to 16 kHz mono. The [STT guide](https://github.com/rimelabs/rime-sdk/blob/main/typescript/docs/stt.md)
|
|
96
|
-
covers other PCM formats, written/verbatim output, recognition hints and cancellation.
|
|
25
|
+
Examples use JavaScript `.mjs` files so you can run them without a TypeScript build.
|
|
26
|
+
The additional `wavefile` dependency is only needed by the Prism example; TTS and
|
|
27
|
+
STT use the SDK and Node.js built-ins.
|
|
97
28
|
|
|
98
29
|
## Prism: send recorded speech
|
|
99
30
|
|
|
@@ -196,10 +127,176 @@ stop playback on interruption, and report the actual played position to Prism.
|
|
|
196
127
|
Keep the API key on your server when you add a browser interface.
|
|
197
128
|
TTS settings do not configure Prism sessions.
|
|
198
129
|
|
|
130
|
+
## TTS
|
|
131
|
+
|
|
132
|
+
Use `client.tts.synthesize(text, options)` for a complete string, or
|
|
133
|
+
`client.tts.stream(textChunks, options)` for an async iterable of strings.
|
|
134
|
+
Both return an `AudioStream` immediately; consume it with `for await`, without
|
|
135
|
+
awaiting the method call. For complete audio in memory, await
|
|
136
|
+
`client.tts.synthesizeToBuffer(text, options)` to receive a Node.js `Buffer`.
|
|
137
|
+
Text is required and must contain non-whitespace characters.
|
|
138
|
+
|
|
139
|
+
### Client options
|
|
140
|
+
|
|
141
|
+
These optional `new Rime({...})` fields configure TTS. They do not configure STT or Prism.
|
|
142
|
+
|
|
143
|
+
| Option | Purpose and values | Required | Default |
|
|
144
|
+
| --- | --- | --- | --- |
|
|
145
|
+
| `model` | Select `"coda"` or `"mistv3"`. | No | `"coda"` |
|
|
146
|
+
| `endpoint` | Override the TTS TLS hostname and optional port, such as `"coda.api.rime.ai:443"`; no scheme or path. | No | `coda.api.rime.ai:443` for Coda; `mist.api.rime.ai:443` for Mist v3. |
|
|
147
|
+
| `timeout` | Overall TTS timeout in positive seconds, such as `60`; `null` disables it. | No | No overall timeout. |
|
|
148
|
+
|
|
149
|
+
### Synthesis options
|
|
150
|
+
|
|
151
|
+
All three TTS methods accept the same `SynthesisOptions` fields. Import
|
|
152
|
+
`AudioFormat` from `@rimelabs/sdk` when selecting an encoding.
|
|
153
|
+
|
|
154
|
+
| Option | Purpose and values | Required | Default |
|
|
155
|
+
| --- | --- | --- | --- |
|
|
156
|
+
| `voice` | Voice name for the selected model, such as `"clementine"` or `"astra"`. | No | `"clementine"` for Coda; `"astra"` for Mist v3. |
|
|
157
|
+
| `language` | Spoken language code, such as `"en"` or `"es"`; the service validates support. | No | `"en"` |
|
|
158
|
+
| `audioFormat` | Output encoding: `AudioFormat.PCM`, `.WAV`, `.MP3`, `.PCM_24000`, or `.MULAW_8000`. | No | Raw PCM16 at 24 kHz (`PCM_24000`). |
|
|
159
|
+
| `sampleRate` | Requested output rate in Hz: `8000`, `16000`, or `24000`. | No | `24000`; `8000` for `MULAW_8000`. |
|
|
160
|
+
| `timeScaleFactor` | Duration multiplier from `0.4` to `2.5`; below `1` speeds speech up, above `1` slows it down. | No | Omitted; normal duration (`1`). |
|
|
161
|
+
| `timestamps` | `true` requests word timings; supported only with Mist v3. | No | `false` |
|
|
162
|
+
| `customLexicon` | Array of `{ spelling, pronunciation }` entries for Coda; pronunciations use space-separated X-SAMPA strings. | No | Empty array. |
|
|
163
|
+
| `timeout` | Override the overall timeout with positive seconds, or `null` to disable it. | No | Inherits the client timeout. |
|
|
164
|
+
|
|
165
|
+
Discover valid values with `await client.voices.list({ language: "en" })` and
|
|
166
|
+
`await client.languages.list()`. Omit the voice language filter to list all voices
|
|
167
|
+
for the selected model. Both calls accept an options object with `timeout` in
|
|
168
|
+
seconds; discovery has an additional ten-second maximum.
|
|
169
|
+
|
|
170
|
+
### Example: save an IVR greeting
|
|
171
|
+
|
|
172
|
+
Save as `tts.mjs`. This writes a complete 16 kHz WAV file, ready for playback or upload.
|
|
173
|
+
|
|
174
|
+
```javascript
|
|
175
|
+
import { writeFile } from "node:fs/promises";
|
|
176
|
+
import { AudioFormat, Rime } from "@rimelabs/sdk";
|
|
177
|
+
|
|
178
|
+
const client = new Rime({ timeout: 60 });
|
|
179
|
+
try {
|
|
180
|
+
const audio = await client.tts.synthesizeToBuffer(
|
|
181
|
+
"Thank you for calling. Your appointment is confirmed for tomorrow.",
|
|
182
|
+
{
|
|
183
|
+
audioFormat: AudioFormat.WAV,
|
|
184
|
+
sampleRate: 16000,
|
|
185
|
+
timeScaleFactor: 0.9,
|
|
186
|
+
},
|
|
187
|
+
);
|
|
188
|
+
await writeFile("greeting.wav", audio);
|
|
189
|
+
console.log("Saved greeting.wav");
|
|
190
|
+
} finally {
|
|
191
|
+
await client.close();
|
|
192
|
+
}
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
Run `node tts.mjs`, then open `greeting.wav` in an audio player.
|
|
196
|
+
|
|
197
|
+
### Audio and stream behavior
|
|
198
|
+
|
|
199
|
+
All output is mono. PCM is signed 16-bit little-endian audio without a header;
|
|
200
|
+
WAV contains PCM16; MP3 is encoded by the service. PCM/WAV rates are requested
|
|
201
|
+
from the server. MP3 headers determine the actual encoded rate. The fixed
|
|
202
|
+
`PCM_24000` and `MULAW_8000` presets reject conflicting sample rates; mu-law is
|
|
203
|
+
converted locally from 24 kHz PCM. Read `audio.sampleRate` on an `AudioStream`
|
|
204
|
+
for the requested rate.
|
|
205
|
+
|
|
206
|
+
The buffer helper waits for successful completion and finalizes WAV lengths.
|
|
207
|
+
Its fixed limit is **64 MiB per call**, including headers; exceeding it throws
|
|
208
|
+
`RimeResourceLimitError` without returning partial audio. WAV/MP3 stream chunks
|
|
209
|
+
are fragments of one file. Use the buffer helper when you need a complete WAV.
|
|
210
|
+
|
|
211
|
+
For incremental text, send fragments as they arrive; the service combines them
|
|
212
|
+
into sentences. Consume streaming audio to the end to confirm success, since an
|
|
213
|
+
error can follow earlier audio. Use `await audio.cancel()` when stopping early
|
|
214
|
+
and `await client.close()` in a `finally` block to release the client. Synthesis
|
|
215
|
+
is not retried automatically, and overall timeouts include pauses by the caller;
|
|
216
|
+
connection and progress limits also apply.
|
|
217
|
+
|
|
218
|
+
For timestamps, enable `timestamps: true`, consume the audio stream fully, then
|
|
219
|
+
call `await audio.timestamps()`. The result has `status` and `spans` with word
|
|
220
|
+
text and start/end seconds. Nonzero alignment status does not fail the audio.
|
|
221
|
+
The buffer helper returns only bytes, so use an audio stream to access timings.
|
|
222
|
+
Custom lexicons apply to the whole request; the service validates pronunciations
|
|
223
|
+
and model/language support.
|
|
224
|
+
|
|
225
|
+
## STT
|
|
226
|
+
|
|
227
|
+
Use `client.stt.stream(audioChunks, options)` to transcribe one utterance.
|
|
228
|
+
The required source is an async iterable of headerless signed PCM16 little-endian
|
|
229
|
+
`Uint8Array` chunks (`Buffer` also works). Ending the source ends the utterance;
|
|
230
|
+
your application decides when to stop.
|
|
231
|
+
|
|
232
|
+
STT uses streaming gRPC over TLS to `stt.api.rime.ai:443`, with the client's API
|
|
233
|
+
key. There is no STT model, endpoint, or protocol selector. TTS client settings,
|
|
234
|
+
including its timeout, do not apply to STT.
|
|
235
|
+
|
|
236
|
+
### Transcription options
|
|
237
|
+
|
|
238
|
+
Pass these fields in the required `TranscriptionOptions` object.
|
|
239
|
+
|
|
240
|
+
| Option | Purpose and values | Required | Default |
|
|
241
|
+
| --- | --- | --- | --- |
|
|
242
|
+
| `language` | Spoken BCP-47 language tag, such as `"en"` or `"es"`; passed to the service. | Yes | None; no automatic detection. |
|
|
243
|
+
| `mode` | `"written"` normalizes written text; `"verbatim"` prefers spoken wording. | No | `"written"` |
|
|
244
|
+
| `contextTerms` | Array of recognition hints, such as `["Rime"]`; hints do not force a match. | No | Empty array. |
|
|
245
|
+
| `inputFormat` | Object with `sampleRate` (`8000`, `16000`, `24000`, `48000`), `channels` (`1` or `2`), and `encoding` (only `"pcm_s16le"`). | No | 16 kHz mono PCM16; each field is optional. |
|
|
246
|
+
| `timeout` | Overall timeout in positive seconds, including source/consumer pauses; `null` disables it. | No | No overall timeout; independent of the client timeout. |
|
|
247
|
+
| `signal` | An `AbortSignal`, such as `controller.signal`, to cancel this utterance. | No | No signal. |
|
|
248
|
+
|
|
249
|
+
### Example: transcribe a recording
|
|
250
|
+
|
|
251
|
+
Save a short recording as `utterance.pcm`: **16 kHz, mono, signed PCM16
|
|
252
|
+
little-endian, with no WAV header**. Save this as `stt.mjs`.
|
|
253
|
+
|
|
254
|
+
```javascript
|
|
255
|
+
import { createReadStream } from "node:fs";
|
|
256
|
+
import { Rime } from "@rimelabs/sdk";
|
|
257
|
+
|
|
258
|
+
async function* audioChunks() {
|
|
259
|
+
yield* createReadStream("utterance.pcm");
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
const client = new Rime();
|
|
263
|
+
try {
|
|
264
|
+
const transcript = client.stt.stream(audioChunks(), {
|
|
265
|
+
language: "en",
|
|
266
|
+
timeout: 120,
|
|
267
|
+
});
|
|
268
|
+
for await (const update of transcript) {
|
|
269
|
+
console.log(update.kind, update.text);
|
|
270
|
+
}
|
|
271
|
+
} finally {
|
|
272
|
+
await client.close();
|
|
273
|
+
}
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
Run `node stt.mjs`. Each partial update replaces the **complete current
|
|
277
|
+
transcript**; do not append snapshots. A final update confirms successful
|
|
278
|
+
completion and may be empty for silence. Errors do not turn partials into finals.
|
|
279
|
+
|
|
280
|
+
The SDK converts supported input formats to 16 kHz mono. Stereo must be
|
|
281
|
+
interleaved. WAV headers, MP3, and mu-law input are not accepted. Chunks may split
|
|
282
|
+
sample frames, but an incomplete final frame fails the request. The SDK does not
|
|
283
|
+
provide microphone capture, automatic end-of-speech detection, word timings, or
|
|
284
|
+
diarization. Recognition is not retried automatically.
|
|
285
|
+
|
|
286
|
+
Use `await transcript.cancel()` or the supplied `AbortSignal` to stop one
|
|
287
|
+
utterance. Closing the client cancels all its operations. Audio sources must
|
|
288
|
+
yield without blocking the event loop. Connection, acceptance, and completion
|
|
289
|
+
limits still apply when the overall timeout is disabled.
|
|
290
|
+
|
|
199
291
|
## Help and licenses
|
|
200
292
|
|
|
201
|
-
See [
|
|
202
|
-
|
|
293
|
+
See the [TTS guide](https://github.com/rimelabs/rime-sdk/blob/main/typescript/docs/tts.md),
|
|
294
|
+
[STT guide](https://github.com/rimelabs/rime-sdk/blob/main/typescript/docs/stt.md), and
|
|
295
|
+
[Prism guide](https://github.com/rimelabs/rime-sdk/blob/main/typescript/docs/realtime.md)
|
|
296
|
+
for more examples. Visit [Rime documentation](https://docs.rime.ai/) for service
|
|
297
|
+
information, or report SDK issues on [GitHub](https://github.com/rimelabs/rime-sdk/issues).
|
|
298
|
+
Include the SDK version and the error's `requestId` when available; omit API keys.
|
|
203
299
|
|
|
204
|
-
SDK code uses the MIT license. The Prism protocol module uses
|
|
205
|
-
|
|
300
|
+
SDK code uses the [MIT license](LICENSE). The Prism protocol module uses
|
|
301
|
+
[Apache 2.0](LICENSE-PRISM); see its [notices](NOTICE-PRISM).
|
|
302
|
+
License texts and notices are included in the distribution.
|
package/dist/client.d.ts
CHANGED
|
@@ -11,8 +11,6 @@ export interface RimeOptions {
|
|
|
11
11
|
model?: string;
|
|
12
12
|
endpoint?: string | null;
|
|
13
13
|
timeout?: number | null;
|
|
14
|
-
/** Recognition hostname and optional port; defaults to stt.api.rime.ai:443. */
|
|
15
|
-
sttEndpoint?: string | null;
|
|
16
14
|
}
|
|
17
15
|
export declare class Rime {
|
|
18
16
|
private readonly credentials;
|
|
@@ -30,6 +28,8 @@ export declare class Rime {
|
|
|
30
28
|
readonly tts: {
|
|
31
29
|
/** Synthesize complete text and return streaming audio. Do not await this call. */
|
|
32
30
|
synthesize: (text: string, options?: SynthesisOptions) => AudioStream;
|
|
31
|
+
/** Await complete audio bytes, finalizing WAV lengths. Fixed output limit: 64 MiB. */
|
|
32
|
+
synthesizeToBuffer: (text: string, options?: SynthesisOptions) => Promise<Buffer>;
|
|
33
33
|
/** Supply incremental text and return streaming audio. Do not await this call. */
|
|
34
34
|
stream: (text: TextSource, options?: SynthesisOptions) => AudioStream;
|
|
35
35
|
};
|
package/dist/client.js
CHANGED
|
@@ -25,7 +25,7 @@ export class Rime {
|
|
|
25
25
|
const defaultTimeout = timeout(options.timeout);
|
|
26
26
|
const deployment = resolve(options.model ?? "coda", options.endpoint);
|
|
27
27
|
this.credentials = new Credentials(key, deployment);
|
|
28
|
-
this.sttClient = new STT(this.credentials,
|
|
28
|
+
this.sttClient = new STT(this.credentials, () => this.checkOpen());
|
|
29
29
|
this.stt = {
|
|
30
30
|
stream: (audio, options) => this.sttClient.stream(audio, options),
|
|
31
31
|
};
|
|
@@ -36,6 +36,7 @@ export class Rime {
|
|
|
36
36
|
this.ttsClient = new TTS(this.credentials, deployment, defaultTimeout, () => this.checkOpen());
|
|
37
37
|
this.tts = {
|
|
38
38
|
synthesize: (text, options) => this.ttsClient.synthesize(text, options),
|
|
39
|
+
synthesizeToBuffer: (text, options) => this.ttsClient.synthesizeToBuffer(text, options),
|
|
39
40
|
stream: (text, options) => this.ttsClient.stream(text, options),
|
|
40
41
|
};
|
|
41
42
|
this.voices = this.ttsClient.voices;
|
|
@@ -66,15 +66,15 @@ function duration(value, field) {
|
|
|
66
66
|
}
|
|
67
67
|
function session(value) {
|
|
68
68
|
const view = object(value, "session");
|
|
69
|
-
const waits = view.
|
|
69
|
+
const waits = view.prism_tool_waits === undefined
|
|
70
70
|
? {}
|
|
71
|
-
: object(view.
|
|
72
|
-
if (view.
|
|
71
|
+
: object(view.prism_tool_waits, "session.prism_tool_waits");
|
|
72
|
+
if (view.prism_tool_waits !== undefined &&
|
|
73
73
|
(!("result_timeout_s" in waits) || !("continuation_timeout_s" in waits)))
|
|
74
|
-
invalid("session.
|
|
75
|
-
const interrupt = view.
|
|
74
|
+
invalid("session.prism_tool_waits");
|
|
75
|
+
const interrupt = view.prism_effective_interrupt_response;
|
|
76
76
|
if (interrupt !== undefined && typeof interrupt !== "boolean")
|
|
77
|
-
invalid("session.
|
|
77
|
+
invalid("session.prism_effective_interrupt_response");
|
|
78
78
|
return {
|
|
79
79
|
id: id(view.id, "session.id"),
|
|
80
80
|
voice: view.voice === undefined ? "" : string(view.voice, "session.voice"),
|
|
@@ -189,7 +189,7 @@ function decodeValue(value, sessionId) {
|
|
|
189
189
|
contentIndex: integer(event.content_index, "content_index"),
|
|
190
190
|
};
|
|
191
191
|
break;
|
|
192
|
-
case "
|
|
192
|
+
case "prism.typed_input.ready":
|
|
193
193
|
payload = { kind: "input.ready" };
|
|
194
194
|
break;
|
|
195
195
|
case "input_audio_buffer.speech_started":
|
|
@@ -244,10 +244,10 @@ function decodeValue(value, sessionId) {
|
|
|
244
244
|
const response = object(event.response, "response"), metadata = response.metadata == null
|
|
245
245
|
? {}
|
|
246
246
|
: object(response.metadata, "metadata");
|
|
247
|
-
const cause = metadata.
|
|
247
|
+
const cause = metadata.prism_cause === undefined
|
|
248
248
|
? undefined
|
|
249
|
-
: string(metadata.
|
|
250
|
-
const parent = optionalString(metadata.
|
|
249
|
+
: string(metadata.prism_cause, "prism_cause");
|
|
250
|
+
const parent = optionalString(metadata.prism_parent_response_id, "prism_parent_response_id");
|
|
251
251
|
payload = {
|
|
252
252
|
kind: "response.started",
|
|
253
253
|
response: ref(response.id),
|
|
@@ -261,7 +261,7 @@ function decodeValue(value, sessionId) {
|
|
|
261
261
|
? "proactive"
|
|
262
262
|
: "unknown",
|
|
263
263
|
parent: parent === null ? null : ref(parent),
|
|
264
|
-
inputItemId: optionalString(metadata.
|
|
264
|
+
inputItemId: optionalString(metadata.prism_input_item_id, "prism_input_item_id"),
|
|
265
265
|
};
|
|
266
266
|
break;
|
|
267
267
|
}
|
|
@@ -282,7 +282,7 @@ function decodeValue(value, sessionId) {
|
|
|
282
282
|
? status
|
|
283
283
|
: "unknown",
|
|
284
284
|
reason: optionalString(details.reason, "reason"),
|
|
285
|
-
drainToken: optionalString(metadata.
|
|
285
|
+
drainToken: optionalString(metadata.prism_drain_token, "prism_drain_token"),
|
|
286
286
|
};
|
|
287
287
|
break;
|
|
288
288
|
}
|
|
@@ -348,7 +348,7 @@ function decodeValue(value, sessionId) {
|
|
|
348
348
|
}
|
|
349
349
|
return {
|
|
350
350
|
eventId: event.event_id === undefined ? "" : string(event.event_id, "event_id"),
|
|
351
|
-
requestId: optionalString(event.
|
|
351
|
+
requestId: optionalString(event.prism_request_event_id, "prism_request_event_id"),
|
|
352
352
|
payload,
|
|
353
353
|
};
|
|
354
354
|
}
|
package/dist/realtime/session.js
CHANGED
|
@@ -348,21 +348,21 @@ export class RealtimeSession {
|
|
|
348
348
|
}
|
|
349
349
|
async sendText(value, options = {}) {
|
|
350
350
|
text(value);
|
|
351
|
-
return this.operation(options, (signal) => this.create({
|
|
351
|
+
return this.operation(options, (signal) => this.create({ prism_input_text: value, metadata: { prism_cause: "user_text" } }, signal, p.requests.text));
|
|
352
352
|
}
|
|
353
353
|
async requestReply(options = {}) {
|
|
354
354
|
const body = {
|
|
355
|
-
metadata: {
|
|
355
|
+
metadata: { prism_cause: "proactive" },
|
|
356
356
|
};
|
|
357
357
|
if (options.instruction != null) {
|
|
358
358
|
text(options.instruction);
|
|
359
|
-
body.
|
|
359
|
+
body.prism_instruction = options.instruction;
|
|
360
360
|
}
|
|
361
361
|
if (options.toolCall != null) {
|
|
362
362
|
const state = this.response(options.toolCall);
|
|
363
363
|
if (state.calls.get(options.toolCall.callId) === null)
|
|
364
364
|
throw new RimeInputError("Record the tool result before requesting its report");
|
|
365
|
-
body.
|
|
365
|
+
body.prism_call_id = options.toolCall.callId;
|
|
366
366
|
}
|
|
367
367
|
return this.operation(options, (signal) => this.create(body, signal));
|
|
368
368
|
}
|
|
@@ -434,8 +434,8 @@ export class RealtimeSession {
|
|
|
434
434
|
return (await this.request(p.requests.create, {
|
|
435
435
|
response: {
|
|
436
436
|
metadata: {
|
|
437
|
-
|
|
438
|
-
|
|
437
|
+
prism_cause: "tool_continuation",
|
|
438
|
+
prism_parent_response_id: ref.responseId,
|
|
439
439
|
},
|
|
440
440
|
},
|
|
441
441
|
}, signal, undefined, state)).response;
|
|
@@ -524,7 +524,7 @@ export class RealtimeSession {
|
|
|
524
524
|
this.check();
|
|
525
525
|
if (state.token)
|
|
526
526
|
await this.send({
|
|
527
|
-
type: "
|
|
527
|
+
type: "prism.playback.drained",
|
|
528
528
|
response_id: state.ref.responseId,
|
|
529
529
|
drain_token: state.token,
|
|
530
530
|
played_ms: played,
|
package/dist/stt/client.d.ts
CHANGED
|
@@ -3,11 +3,10 @@ import { TranscriptStream } from "./stream.js";
|
|
|
3
3
|
import type { AudioSource, TranscriptionOptions } from "./types.js";
|
|
4
4
|
export declare class STT {
|
|
5
5
|
private readonly credentials;
|
|
6
|
-
private readonly endpoint;
|
|
7
6
|
private readonly checkOpen;
|
|
8
7
|
private connection;
|
|
9
8
|
private streams;
|
|
10
|
-
constructor(credentials: Credentials,
|
|
9
|
+
constructor(credentials: Credentials, checkOpen: () => void);
|
|
11
10
|
stream(source: AudioSource, options: TranscriptionOptions): TranscriptStream;
|
|
12
11
|
close(): Promise<void>;
|
|
13
12
|
}
|
package/dist/stt/client.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { connect, metadata } from "../grpc.js";
|
|
2
|
-
import {
|
|
2
|
+
import { timeout } from "../validation.js";
|
|
3
3
|
import { RimeInputError } from "../errors.js";
|
|
4
4
|
import { resolvePCMFormat } from "../pcm.js";
|
|
5
5
|
import { transport, requireSchema } from "./transport.js";
|
|
@@ -7,13 +7,11 @@ import { TranscriptStream } from "./stream.js";
|
|
|
7
7
|
import { policy } from "./policy.js";
|
|
8
8
|
export class STT {
|
|
9
9
|
credentials;
|
|
10
|
-
endpoint;
|
|
11
10
|
checkOpen;
|
|
12
11
|
connection = null;
|
|
13
12
|
streams = new Set();
|
|
14
|
-
constructor(credentials,
|
|
13
|
+
constructor(credentials, checkOpen) {
|
|
15
14
|
this.credentials = credentials;
|
|
16
|
-
this.endpoint = endpoint;
|
|
17
15
|
this.checkOpen = checkOpen;
|
|
18
16
|
}
|
|
19
17
|
stream(source, options) {
|
|
@@ -29,9 +27,7 @@ export class STT {
|
|
|
29
27
|
if (!Array.isArray(terms) || terms.some((term) => typeof term !== "string"))
|
|
30
28
|
throw new RimeInputError("contextTerms must be an array of strings");
|
|
31
29
|
const format = resolvePCMFormat(options.inputFormat);
|
|
32
|
-
const target =
|
|
33
|
-
? policy.target
|
|
34
|
-
: endpointAddress(this.endpoint).target;
|
|
30
|
+
const target = policy.target;
|
|
35
31
|
const stream = new TranscriptStream({
|
|
36
32
|
checkOpen: this.checkOpen,
|
|
37
33
|
forget: (stream) => this.streams.delete(stream),
|
package/dist/tts/audio.d.ts
CHANGED
|
@@ -1,11 +1,20 @@
|
|
|
1
1
|
export declare class AudioFormat {
|
|
2
2
|
readonly encoding: string;
|
|
3
|
+
/** Fixed preset rate, or zero when selected per request. See AudioStream.sampleRate. */
|
|
3
4
|
readonly sampleRate: number;
|
|
4
5
|
readonly channels: number;
|
|
6
|
+
static readonly PCM: Readonly<AudioFormat>;
|
|
7
|
+
static readonly WAV: Readonly<AudioFormat>;
|
|
8
|
+
static readonly MP3: Readonly<AudioFormat>;
|
|
5
9
|
static readonly PCM_24000: Readonly<AudioFormat>;
|
|
6
10
|
static readonly MULAW_8000: Readonly<AudioFormat>;
|
|
7
11
|
private constructor();
|
|
8
12
|
}
|
|
13
|
+
export declare function wireFormat(format: AudioFormat): string;
|
|
14
|
+
export declare function resolveAudio(format: AudioFormat | null | undefined, sampleRate: number | null | undefined, timeScaleFactor: number | null | undefined): {
|
|
15
|
+
format: AudioFormat;
|
|
16
|
+
sampleRate: number;
|
|
17
|
+
};
|
|
9
18
|
export declare class Converter {
|
|
10
19
|
private profile;
|
|
11
20
|
private tail;
|
package/dist/tts/audio.js
CHANGED
|
@@ -1,16 +1,52 @@
|
|
|
1
|
-
import { RimeAudioFormatError } from "../errors.js";
|
|
1
|
+
import { RimeAudioFormatError, RimeInputError } from "../errors.js";
|
|
2
2
|
export class AudioFormat {
|
|
3
3
|
encoding;
|
|
4
4
|
sampleRate;
|
|
5
5
|
channels;
|
|
6
|
+
static PCM = Object.freeze(new AudioFormat("pcm_s16le", 0, 1));
|
|
7
|
+
static WAV = Object.freeze(new AudioFormat("wav", 0, 1));
|
|
8
|
+
static MP3 = Object.freeze(new AudioFormat("mp3", 0, 1));
|
|
6
9
|
static PCM_24000 = Object.freeze(new AudioFormat("pcm_s16le", 24000, 1));
|
|
7
10
|
static MULAW_8000 = Object.freeze(new AudioFormat("mulaw", 8000, 1));
|
|
8
|
-
constructor(encoding,
|
|
11
|
+
constructor(encoding,
|
|
12
|
+
/** Fixed preset rate, or zero when selected per request. See AudioStream.sampleRate. */
|
|
13
|
+
sampleRate, channels) {
|
|
9
14
|
this.encoding = encoding;
|
|
10
15
|
this.sampleRate = sampleRate;
|
|
11
16
|
this.channels = channels;
|
|
12
17
|
}
|
|
13
18
|
}
|
|
19
|
+
export function wireFormat(format) {
|
|
20
|
+
if (format === AudioFormat.WAV)
|
|
21
|
+
return "audio/wav";
|
|
22
|
+
if (format === AudioFormat.MP3)
|
|
23
|
+
return "audio/mpeg";
|
|
24
|
+
return "audio/pcm";
|
|
25
|
+
}
|
|
26
|
+
export function resolveAudio(format, sampleRate, timeScaleFactor) {
|
|
27
|
+
format ??= AudioFormat.PCM;
|
|
28
|
+
if (![
|
|
29
|
+
AudioFormat.PCM,
|
|
30
|
+
AudioFormat.PCM_24000,
|
|
31
|
+
AudioFormat.MULAW_8000,
|
|
32
|
+
AudioFormat.WAV,
|
|
33
|
+
AudioFormat.MP3,
|
|
34
|
+
].includes(format))
|
|
35
|
+
throw new RimeAudioFormatError("Select a named AudioFormat");
|
|
36
|
+
sampleRate ??= format.sampleRate || 24000;
|
|
37
|
+
if (![8000, 16000, 24000].includes(sampleRate))
|
|
38
|
+
throw new RimeAudioFormatError("sampleRate must be 8000, 16000, or 24000");
|
|
39
|
+
if (format.sampleRate && sampleRate !== format.sampleRate)
|
|
40
|
+
throw new RimeAudioFormatError("sampleRate conflicts with the fixed AudioFormat preset");
|
|
41
|
+
if (timeScaleFactor != null &&
|
|
42
|
+
(!Number.isFinite(timeScaleFactor) ||
|
|
43
|
+
timeScaleFactor < 0.4 ||
|
|
44
|
+
timeScaleFactor > 2.5))
|
|
45
|
+
throw new RimeInputError("timeScaleFactor must be a finite number from 0.4 to 2.5");
|
|
46
|
+
if (format === AudioFormat.PCM && sampleRate === 24000)
|
|
47
|
+
format = AudioFormat.PCM_24000;
|
|
48
|
+
return { format, sampleRate };
|
|
49
|
+
}
|
|
14
50
|
const raw = Array.from({ length: 63 }, (_, i) => (i === 31
|
|
15
51
|
? (2 * 3400) / 24000
|
|
16
52
|
: Math.sin(((2 * Math.PI * 3400) / 24000) * (i - 31)) /
|
|
@@ -47,12 +83,15 @@ export class Converter {
|
|
|
47
83
|
return null;
|
|
48
84
|
}
|
|
49
85
|
process(input, final = false) {
|
|
86
|
+
if (this.profile === AudioFormat.WAV || this.profile === AudioFormat.MP3)
|
|
87
|
+
return Buffer.from(input);
|
|
50
88
|
const data = Buffer.concat([this.tail, input]);
|
|
51
89
|
const size = Math.floor(data.length / 2) * 2;
|
|
52
90
|
this.tail = Buffer.from(data.subarray(size));
|
|
53
91
|
if (final && this.tail.length)
|
|
54
92
|
throw new RimeAudioFormatError("Incomplete final PCM sample frame");
|
|
55
|
-
if (this.profile === AudioFormat.
|
|
93
|
+
if (this.profile === AudioFormat.PCM ||
|
|
94
|
+
this.profile === AudioFormat.PCM_24000)
|
|
56
95
|
return data.subarray(0, size);
|
|
57
96
|
const result = [];
|
|
58
97
|
for (let i = 0; i < size; i += 2) {
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { RimeAudioFormatError } from "../errors.js";
|
|
2
|
+
export const maxBufferBytes = 64 * 1024 * 1024;
|
|
3
|
+
/** Validate mono PCM16 WAV and replace unknown streaming chunk lengths. */
|
|
4
|
+
export function finalizeWav(data, sampleRate, requestId) {
|
|
5
|
+
const invalid = () => new RimeAudioFormatError("Expected a complete mono PCM16 WAV at the requested sample rate", requestId);
|
|
6
|
+
if (data.length < 12 ||
|
|
7
|
+
data.toString("latin1", 0, 4) !== "RIFF" ||
|
|
8
|
+
data.toString("latin1", 8, 12) !== "WAVE")
|
|
9
|
+
throw invalid();
|
|
10
|
+
if (data.length - 8 > 0xffffffff)
|
|
11
|
+
throw new RimeAudioFormatError("Audio exceeds the WAV size limit", requestId);
|
|
12
|
+
if (![0xffffffff, data.length - 8].includes(data.readUInt32LE(4)))
|
|
13
|
+
throw invalid();
|
|
14
|
+
let offset = 12;
|
|
15
|
+
let foundFormat = false;
|
|
16
|
+
let dataSizeOffset;
|
|
17
|
+
let dataSize = 0;
|
|
18
|
+
while (offset < data.length) {
|
|
19
|
+
if (data.length - offset < 8)
|
|
20
|
+
throw invalid();
|
|
21
|
+
const name = data.toString("latin1", offset, offset + 4);
|
|
22
|
+
let size = data.readUInt32LE(offset + 4);
|
|
23
|
+
const start = offset + 8;
|
|
24
|
+
if (name === "data" && size === 0xffffffff)
|
|
25
|
+
size = data.length - start;
|
|
26
|
+
const end = start + size;
|
|
27
|
+
if (end > data.length)
|
|
28
|
+
throw invalid();
|
|
29
|
+
if (name === "fmt ") {
|
|
30
|
+
if (foundFormat ||
|
|
31
|
+
size < 16 ||
|
|
32
|
+
data.readUInt16LE(start) !== 1 ||
|
|
33
|
+
data.readUInt16LE(start + 2) !== 1 ||
|
|
34
|
+
data.readUInt32LE(start + 4) !== sampleRate ||
|
|
35
|
+
data.readUInt32LE(start + 8) !== sampleRate * 2 ||
|
|
36
|
+
data.readUInt16LE(start + 12) !== 2 ||
|
|
37
|
+
data.readUInt16LE(start + 14) !== 16)
|
|
38
|
+
throw invalid();
|
|
39
|
+
foundFormat = true;
|
|
40
|
+
}
|
|
41
|
+
else if (name === "data") {
|
|
42
|
+
if (!foundFormat || dataSizeOffset !== undefined || size % 2)
|
|
43
|
+
throw invalid();
|
|
44
|
+
dataSizeOffset = offset + 4;
|
|
45
|
+
dataSize = size;
|
|
46
|
+
}
|
|
47
|
+
offset = end + (size % 2);
|
|
48
|
+
}
|
|
49
|
+
if (offset !== data.length || dataSizeOffset === undefined)
|
|
50
|
+
throw invalid();
|
|
51
|
+
data.writeUInt32LE(data.length - 8, 4);
|
|
52
|
+
data.writeUInt32LE(dataSize, dataSizeOffset);
|
|
53
|
+
}
|
package/dist/tts/client.d.ts
CHANGED
|
@@ -7,6 +7,10 @@ export interface SynthesisOptions {
|
|
|
7
7
|
voice?: string | null;
|
|
8
8
|
language?: string;
|
|
9
9
|
audioFormat?: AudioFormat | null;
|
|
10
|
+
/** Requested output rate: 8000, 16000, or 24000 Hz. Defaults to 24000 for PCM/WAV/MP3. */
|
|
11
|
+
sampleRate?: number | null;
|
|
12
|
+
/** Duration multiplier from 0.4 to 2.5. Below 1 speeds speech up; above 1 slows it down. */
|
|
13
|
+
timeScaleFactor?: number | null;
|
|
10
14
|
/** Request final word timestamps. Supported only with model="mistv3". */
|
|
11
15
|
timestamps?: boolean;
|
|
12
16
|
/** Request-wide pronunciation overrides; requires a phoneme-capable Coda deployment. */
|
|
@@ -37,6 +41,11 @@ export declare class TTS {
|
|
|
37
41
|
private prepare;
|
|
38
42
|
/** Synthesize complete text and return an audio stream. Do not await this call. */
|
|
39
43
|
synthesize(text: string, options?: SynthesisOptions): AudioStream;
|
|
44
|
+
/** Await complete audio in memory, finalizing WAV lengths. Never returns partial audio.
|
|
45
|
+
* The fixed output limit is 64 MiB (67,108,864 bytes); exceeding it raises
|
|
46
|
+
* RimeResourceLimitError. Use streaming synthesis for larger outputs or timestamps.
|
|
47
|
+
*/
|
|
48
|
+
synthesizeToBuffer(text: string, options?: SynthesisOptions): Promise<Buffer>;
|
|
40
49
|
/** Stream text chunks into synthesis and return an audio stream. Do not await this call. */
|
|
41
50
|
stream(text: TextSource, options?: SynthesisOptions): AudioStream;
|
|
42
51
|
private discover;
|
package/dist/tts/client.js
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { AudioFormat } from "./audio.js";
|
|
1
|
+
import { AudioFormat, resolveAudio } from "./audio.js";
|
|
2
|
+
import { maxBufferBytes, finalizeWav } from "./buffer.js";
|
|
2
3
|
import { snapshot } from "./lexicon.js";
|
|
3
4
|
import { AudioStream, constructionKey } from "./stream.js";
|
|
4
5
|
import { policy, timeout, nonempty } from "./policy.js";
|
|
5
6
|
import { abortable } from "../cancellation.js";
|
|
6
7
|
import { transport, connect, metadata, discover, } from "./transport.js";
|
|
7
|
-
import { RimeInputError,
|
|
8
|
+
import { RimeInputError, RimeResourceLimitError, RimeCancelledError, RimeTimeoutError, RimeUnavailableError, } from "../errors.js";
|
|
8
9
|
export class TTS {
|
|
9
10
|
credentials;
|
|
10
11
|
deployment;
|
|
@@ -50,6 +51,30 @@ export class TTS {
|
|
|
50
51
|
yield text;
|
|
51
52
|
})(), options);
|
|
52
53
|
}
|
|
54
|
+
/** Await complete audio in memory, finalizing WAV lengths. Never returns partial audio.
|
|
55
|
+
* The fixed output limit is 64 MiB (67,108,864 bytes); exceeding it raises
|
|
56
|
+
* RimeResourceLimitError. Use streaming synthesis for larger outputs or timestamps.
|
|
57
|
+
*/
|
|
58
|
+
async synthesizeToBuffer(text, options = {}) {
|
|
59
|
+
const audio = this.synthesize(text, options);
|
|
60
|
+
const chunks = [];
|
|
61
|
+
let size = 0;
|
|
62
|
+
try {
|
|
63
|
+
for await (const chunk of audio) {
|
|
64
|
+
if (chunk.byteLength > maxBufferBytes - size)
|
|
65
|
+
throw new RimeResourceLimitError("Audio exceeds the 64 MiB buffer limit", audio.requestId);
|
|
66
|
+
chunks.push(chunk);
|
|
67
|
+
size += chunk.byteLength;
|
|
68
|
+
}
|
|
69
|
+
const data = Buffer.concat(chunks, size);
|
|
70
|
+
if (audio.format === AudioFormat.WAV)
|
|
71
|
+
finalizeWav(data, audio.sampleRate, audio.requestId);
|
|
72
|
+
return data;
|
|
73
|
+
}
|
|
74
|
+
finally {
|
|
75
|
+
await audio.cancel();
|
|
76
|
+
}
|
|
77
|
+
}
|
|
53
78
|
/** Stream text chunks into synthesis and return an audio stream. Do not await this call. */
|
|
54
79
|
stream(text, options = {}) {
|
|
55
80
|
this.checkOpen();
|
|
@@ -67,14 +92,12 @@ export class TTS {
|
|
|
67
92
|
? this.deployment.defaultVoice
|
|
68
93
|
: nonempty(options.voice, "voice");
|
|
69
94
|
const language = nonempty(options.language ?? "en", "language");
|
|
70
|
-
const format = options.audioFormat
|
|
71
|
-
if (format !== AudioFormat.PCM_24000 && format !== AudioFormat.MULAW_8000)
|
|
72
|
-
throw new RimeAudioFormatError("Select a named AudioFormat profile");
|
|
95
|
+
const { format, sampleRate } = resolveAudio(options.audioFormat, options.sampleRate, options.timeScaleFactor);
|
|
73
96
|
const audio = new AudioStream(constructionKey, {
|
|
74
97
|
prepare: (s) => this.prepare(s),
|
|
75
98
|
checkOpen: () => this.checkOpen(),
|
|
76
99
|
forget: (s) => this.streams.delete(s),
|
|
77
|
-
}, text, voice, language, format, timeout(options.timeout, this.defaultTimeout), timestamps, lexicon);
|
|
100
|
+
}, text, voice, language, format, timeout(options.timeout, this.defaultTimeout), timestamps, lexicon, sampleRate, options.timeScaleFactor ?? undefined);
|
|
78
101
|
this.streams.add(audio);
|
|
79
102
|
return audio;
|
|
80
103
|
}
|
package/dist/tts/stream.d.ts
CHANGED
|
@@ -18,6 +18,8 @@ export declare class AudioStream implements AsyncIterableIterator<Uint8Array> {
|
|
|
18
18
|
private timeout;
|
|
19
19
|
private timestampsRequested;
|
|
20
20
|
private customLexicon;
|
|
21
|
+
private sampleRateValue;
|
|
22
|
+
private timeScaleFactor?;
|
|
21
23
|
private readonly controller;
|
|
22
24
|
private readonly queue;
|
|
23
25
|
private worker;
|
|
@@ -35,8 +37,10 @@ export declare class AudioStream implements AsyncIterableIterator<Uint8Array> {
|
|
|
35
37
|
private readonly operationId;
|
|
36
38
|
private bytesReceived;
|
|
37
39
|
private bytesDelivered;
|
|
38
|
-
constructor(key: symbol, owner: StreamOwner, source: TextSource | null, voice: string, language: string, formatValue: AudioFormat, timeout: number | null, timestampsRequested?: boolean, customLexicon?: readonly PronunciationEntry[]);
|
|
40
|
+
constructor(key: symbol, owner: StreamOwner, source: TextSource | null, voice: string, language: string, formatValue: AudioFormat, timeout: number | null, timestampsRequested?: boolean, customLexicon?: readonly PronunciationEntry[], sampleRateValue?: number, timeScaleFactor?: number | undefined);
|
|
39
41
|
get format(): AudioFormat;
|
|
42
|
+
/** Requested output rate. MP3 headers describe the encoded rate. */
|
|
43
|
+
get sampleRate(): number;
|
|
40
44
|
get requestId(): string | null;
|
|
41
45
|
/** Read requested timestamps after consuming all audio; never drains audio.
|
|
42
46
|
* Calling before iteration completes rejects with RimeInputError. Synthesis
|
package/dist/tts/stream.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { debuglog } from "node:util";
|
|
3
3
|
const log = debuglog("rime-sdk");
|
|
4
|
-
import { Converter } from "./audio.js";
|
|
4
|
+
import { AudioFormat, Converter, wireFormat } from "./audio.js";
|
|
5
5
|
import { policy } from "./policy.js";
|
|
6
6
|
import { abortable } from "../cancellation.js";
|
|
7
7
|
import { ByteQueue } from "./queue.js";
|
|
@@ -17,6 +17,8 @@ export class AudioStream {
|
|
|
17
17
|
timeout;
|
|
18
18
|
timestampsRequested;
|
|
19
19
|
customLexicon;
|
|
20
|
+
sampleRateValue;
|
|
21
|
+
timeScaleFactor;
|
|
20
22
|
controller = new AbortController();
|
|
21
23
|
queue = new ByteQueue(policy.outputBytes, policy.outputChunkBytes);
|
|
22
24
|
worker = null;
|
|
@@ -34,7 +36,7 @@ export class AudioStream {
|
|
|
34
36
|
operationId = randomUUID();
|
|
35
37
|
bytesReceived = 0;
|
|
36
38
|
bytesDelivered = 0;
|
|
37
|
-
constructor(key, owner, source, voice, language, formatValue, timeout, timestampsRequested = false, customLexicon = []) {
|
|
39
|
+
constructor(key, owner, source, voice, language, formatValue, timeout, timestampsRequested = false, customLexicon = [], sampleRateValue = 24000, timeScaleFactor) {
|
|
38
40
|
this.owner = owner;
|
|
39
41
|
this.source = source;
|
|
40
42
|
this.voice = voice;
|
|
@@ -43,12 +45,18 @@ export class AudioStream {
|
|
|
43
45
|
this.timeout = timeout;
|
|
44
46
|
this.timestampsRequested = timestampsRequested;
|
|
45
47
|
this.customLexicon = customLexicon;
|
|
48
|
+
this.sampleRateValue = sampleRateValue;
|
|
49
|
+
this.timeScaleFactor = timeScaleFactor;
|
|
46
50
|
if (key !== constructionKey)
|
|
47
51
|
throw new TypeError("AudioStream is returned by client.tts.synthesize() or client.tts.stream()");
|
|
48
52
|
}
|
|
49
53
|
get format() {
|
|
50
54
|
return this.formatValue;
|
|
51
55
|
}
|
|
56
|
+
/** Requested output rate. MP3 headers describe the encoded rate. */
|
|
57
|
+
get sampleRate() {
|
|
58
|
+
return this.sampleRateValue;
|
|
59
|
+
}
|
|
52
60
|
get requestId() {
|
|
53
61
|
return this.call?.requestId ?? null;
|
|
54
62
|
}
|
|
@@ -218,7 +226,7 @@ export class AudioStream {
|
|
|
218
226
|
try {
|
|
219
227
|
const prepared = await this.owner.prepare(this.controller.signal);
|
|
220
228
|
this.call = new SynthesisCall(prepared, this.controller.signal);
|
|
221
|
-
await this.call.start(this.voice, this.language, this.timestampsRequested, this.customLexicon);
|
|
229
|
+
await this.call.start(this.voice, this.language, this.timestampsRequested, this.customLexicon, wireFormat(this.format), this.format === AudioFormat.MULAW_8000 ? 24000 : this.sampleRate, this.timeScaleFactor);
|
|
222
230
|
tasks = [this.produce(), this.read()];
|
|
223
231
|
await Promise.all(tasks);
|
|
224
232
|
this.queue.finish();
|
package/dist/tts/transport.d.ts
CHANGED
|
@@ -14,6 +14,7 @@ export declare const transport: {
|
|
|
14
14
|
export declare function connect(client: grpc.Client, signal: AbortSignal): Promise<void>;
|
|
15
15
|
export declare function discover(prepared: PreparedConnection, kind: DiscoveryKind, language: string | null | undefined, signal: AbortSignal): Promise<string[]>;
|
|
16
16
|
export declare class SynthesisCall {
|
|
17
|
+
private audioFormat;
|
|
17
18
|
private timestampsRequested;
|
|
18
19
|
private readonly timestampTrailer;
|
|
19
20
|
private readonly rpc;
|
|
@@ -21,7 +22,7 @@ export declare class SynthesisCall {
|
|
|
21
22
|
get requestId(): string | null;
|
|
22
23
|
get done(): boolean;
|
|
23
24
|
cancel(): void;
|
|
24
|
-
start(voice: string, language: string, timestamps?: boolean, customLexicon?: readonly PronunciationEntry[]): Promise<void>;
|
|
25
|
+
start(voice: string, language: string, timestamps?: boolean, customLexicon?: readonly PronunciationEntry[], audioFormat?: string, sampleRate?: number, timeScaleFactor?: number): Promise<void>;
|
|
25
26
|
timestampResult(): import("./timestamps.js").TimestampResult;
|
|
26
27
|
write(text: string): Promise<void>;
|
|
27
28
|
finishInput(): void;
|
package/dist/tts/transport.js
CHANGED
|
@@ -22,7 +22,7 @@ export function connect(client, signal) {
|
|
|
22
22
|
function openStream(client, auth) {
|
|
23
23
|
return client.makeBidiStreamRequest("/rime.TextToSpeech/SynthesizeStreaming", serializer(schema.StreamingSynthesisRequestSchema), deserializer(schema.SynthesisResponseStreamSchema), auth);
|
|
24
24
|
}
|
|
25
|
-
function header(voice, language, timestamps, customLexicon) {
|
|
25
|
+
function header(voice, language, timestamps, customLexicon, audioFormat, samplingRate, timeScaleFactor) {
|
|
26
26
|
return create(schema.StreamingSynthesisRequestSchema, {
|
|
27
27
|
payload: {
|
|
28
28
|
case: "header",
|
|
@@ -30,7 +30,7 @@ function header(voice, language, timestamps, customLexicon) {
|
|
|
30
30
|
speaker: voice,
|
|
31
31
|
language,
|
|
32
32
|
splitStrategy: schema.SplitStrategy.ACCUMULATE_SENTENCE,
|
|
33
|
-
audioParameters: { audioFormat
|
|
33
|
+
audioParameters: { audioFormat, samplingRate, timeScaleFactor },
|
|
34
34
|
timestamps: timestamps ? { enable: true } : undefined,
|
|
35
35
|
customLexicon: [...customLexicon],
|
|
36
36
|
}),
|
|
@@ -79,6 +79,7 @@ export async function discover(prepared, kind, language, signal) {
|
|
|
79
79
|
return response.languages;
|
|
80
80
|
}
|
|
81
81
|
export class SynthesisCall {
|
|
82
|
+
audioFormat = "audio/pcm";
|
|
82
83
|
timestampsRequested = false;
|
|
83
84
|
timestampTrailer = new TimestampTrailer();
|
|
84
85
|
rpc;
|
|
@@ -94,9 +95,10 @@ export class SynthesisCall {
|
|
|
94
95
|
cancel() {
|
|
95
96
|
this.rpc.cancel();
|
|
96
97
|
}
|
|
97
|
-
start(voice, language, timestamps = false, customLexicon = []) {
|
|
98
|
+
start(voice, language, timestamps = false, customLexicon = [], audioFormat = "audio/pcm", sampleRate = 24000, timeScaleFactor) {
|
|
98
99
|
this.timestampsRequested = timestamps;
|
|
99
|
-
|
|
100
|
+
this.audioFormat = audioFormat;
|
|
101
|
+
const message = header(voice, language, timestamps, customLexicon, audioFormat, sampleRate, timeScaleFactor);
|
|
100
102
|
return this.rpc.write(message);
|
|
101
103
|
}
|
|
102
104
|
timestampResult() {
|
|
@@ -109,8 +111,8 @@ export class SynthesisCall {
|
|
|
109
111
|
this.rpc.finishInput();
|
|
110
112
|
}
|
|
111
113
|
checkFormat(contentType) {
|
|
112
|
-
if (contentType !==
|
|
113
|
-
throw new errors.RimeAudioFormatError(
|
|
114
|
+
if (contentType !== this.audioFormat)
|
|
115
|
+
throw new errors.RimeAudioFormatError(`Expected ${this.audioFormat} from the service`, this.requestId);
|
|
114
116
|
}
|
|
115
117
|
async *audio() {
|
|
116
118
|
const headers = await this.rpc.headers();
|
package/docs/realtime.md
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
# Realtime / Prism for Node.js
|
|
2
2
|
|
|
3
|
+
The endpoint must use Rime's `prism.` events and `prism_` fields. Endpoints
|
|
4
|
+
that use the former `prsm` names are not supported.
|
|
5
|
+
|
|
3
6
|
Keep one event consumer running while your application sends input, runs tools,
|
|
4
7
|
and plays audio. The SDK manages the protocol state. Your application owns
|
|
5
8
|
microphone capture, tool execution, and playback.
|
package/docs/stt.md
CHANGED
|
@@ -52,8 +52,8 @@ split sample frames; the SDK carries incomplete bytes into the next chunk. A
|
|
|
52
52
|
truncated final frame raises `RimeAudioFormatError`, without committing the
|
|
53
53
|
utterance. Each gRPC audio payload is at most 64 KiB of 16 kHz mono PCM16.
|
|
54
54
|
|
|
55
|
-
|
|
56
|
-
|
|
55
|
+
Recognition connects to `stt.api.rime.ai:443` using TLS and the same API key as
|
|
56
|
+
the other features. No endpoint configuration is needed.
|
|
57
57
|
The existing `model`, `endpoint` and client `timeout` settings remain TTS settings.
|
|
58
58
|
There is no STT voice/model selector; the service selects its recognizer.
|
|
59
59
|
|
package/docs/tts.md
CHANGED
|
@@ -60,29 +60,68 @@ Model defaults still apply.
|
|
|
60
60
|
|
|
61
61
|
### Synthesis options
|
|
62
62
|
|
|
63
|
-
|
|
64
|
-
async iterable of strings
|
|
63
|
+
`synthesize` and `synthesizeToBuffer` require a nonblank string; `stream` requires
|
|
64
|
+
an async iterable of strings. All three accept the following settings:
|
|
65
65
|
|
|
66
66
|
| Option | Default | Meaning |
|
|
67
67
|
| --- | --- | --- |
|
|
68
68
|
| `voice` | Model's default voice | Voice name from `client.voices.list()` |
|
|
69
69
|
| `language` | `"en"` | Language code |
|
|
70
|
-
| `audioFormat` | `AudioFormat.
|
|
70
|
+
| `audioFormat` | `AudioFormat.PCM` (24 kHz) | Output encoding, imported from `@rimelabs/sdk` |
|
|
71
|
+
| `sampleRate` | `24000` (or fixed preset rate) | Requested output rate: `8000`, `16000`, or `24000` Hz |
|
|
72
|
+
| `timeScaleFactor` | Omitted (normal duration) | Duration multiplier from `0.4` to `2.5`; below 1 is faster, above 1 is slower |
|
|
71
73
|
| `timestamps` | `false` | Request final word timestamps; requires `model: "mistv3"` |
|
|
72
74
|
| `customLexicon` | `[]` | Array of pronunciation overrides for this request; requires Coda |
|
|
73
75
|
| `timeout` | Client setting | Seconds; pass `null` to disable the overall deadline |
|
|
74
76
|
|
|
75
77
|
### Audio formats
|
|
76
78
|
|
|
77
|
-
|
|
79
|
+
All output is mono. PCM and WAV use signed 16-bit little-endian samples.
|
|
78
80
|
|
|
79
|
-
| `AudioFormat`
|
|
80
|
-
| --- | --- | --- |
|
|
81
|
-
| `
|
|
82
|
-
| `
|
|
81
|
+
| `AudioFormat` | Encoding | Requested rate |
|
|
82
|
+
| --- | --- | --- |
|
|
83
|
+
| `PCM` | Headerless PCM16 | 8, 16, or 24 kHz |
|
|
84
|
+
| `WAV` | PCM16 WAV | 8, 16, or 24 kHz |
|
|
85
|
+
| `MP3` | MP3 (`audio/mpeg`) | 8, 16, or 24 kHz; encoded headers determine the actual rate |
|
|
86
|
+
| `PCM_24000` | Headerless PCM16 compatibility preset | Fixed 24 kHz |
|
|
87
|
+
| `MULAW_8000` | G.711 mu-law compatibility preset | Fixed 8 kHz |
|
|
88
|
+
|
|
89
|
+
PCM/WAV sample rates are requested from the server; the SDK does not resample
|
|
90
|
+
these formats. The existing mu-law preset still converts 24 kHz PCM locally.
|
|
91
|
+
An explicit rate conflicting with a fixed preset fails locally. For configurable
|
|
92
|
+
formats, `audio.format.sampleRate` is zero; use `audio.sampleRate` for the rate requested
|
|
93
|
+
by the operation. Existing default streams still report `PCM_24000` as their format.
|
|
94
|
+
|
|
95
|
+
PCM chunks contain complete sample frames. WAV and MP3 chunks are fragments of
|
|
96
|
+
one encoded stream, not independent files. Streaming WAV may contain unknown
|
|
97
|
+
lengths in its header; use `synthesizeToBuffer` for a completed IVR file.
|
|
98
|
+
|
|
99
|
+
### Complete audio in memory
|
|
100
|
+
|
|
101
|
+
Await `synthesizeToBuffer` to get all audio bytes. It uses the same streaming RPC,
|
|
102
|
+
waits for successful completion, and finalizes WAV file lengths. On failure it
|
|
103
|
+
raises an error and releases the stream; it never returns partial audio.
|
|
104
|
+
The output limit is fixed at **64 MiB (67,108,864 bytes)** per call, including
|
|
105
|
+
file headers. Exceeding it raises `RimeResourceLimitError` and returns no partial
|
|
106
|
+
audio. For mono PCM16/WAV, this is approximately 70 minutes at 8 kHz, 35 minutes
|
|
107
|
+
at 16 kHz, or 23 minutes at 24 kHz. Use streaming synthesis for larger outputs.
|
|
108
|
+
Collection can temporarily use more memory than the output limit.
|
|
109
|
+
Timeouts and client shutdown still apply. Use the streaming API to retrieve word timestamps.
|
|
83
110
|
|
|
84
|
-
|
|
85
|
-
|
|
111
|
+
```javascript
|
|
112
|
+
import { writeFile } from "node:fs/promises";
|
|
113
|
+
import { AudioFormat, Rime } from "@rimelabs/sdk";
|
|
114
|
+
|
|
115
|
+
const client = new Rime();
|
|
116
|
+
try {
|
|
117
|
+
const audio = await client.tts.synthesizeToBuffer("Thank you for calling.", {
|
|
118
|
+
audioFormat: AudioFormat.WAV, sampleRate: 16000, timeScaleFactor: 0.9,
|
|
119
|
+
});
|
|
120
|
+
await writeFile("greeting.wav", audio);
|
|
121
|
+
} finally {
|
|
122
|
+
await client.close();
|
|
123
|
+
}
|
|
124
|
+
```
|
|
86
125
|
|
|
87
126
|
### Timeouts
|
|
88
127
|
|