@rimelabs/sdk 0.1.0-alpha.10 → 0.1.0-alpha.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/README.md +38 -8
- package/dist/auth.d.ts +1 -0
- package/dist/auth.js +5 -2
- package/dist/cancellation.d.ts +5 -0
- package/dist/cancellation.js +36 -0
- package/dist/client.d.ts +11 -0
- package/dist/client.js +21 -5
- package/dist/grpc.d.ts +27 -0
- package/dist/grpc.js +175 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +1 -0
- package/dist/pcm.d.ts +21 -0
- package/dist/pcm.js +52 -0
- package/dist/realtime/async.d.ts +1 -5
- package/dist/realtime/async.js +1 -36
- package/dist/realtime/audio.d.ts +0 -11
- package/dist/realtime/audio.js +2 -45
- package/dist/realtime/session.js +2 -1
- package/dist/realtime/types.d.ts +2 -6
- package/dist/stt/audio.d.ts +15 -0
- package/dist/stt/audio.js +46 -0
- package/dist/stt/client.d.ts +13 -0
- package/dist/stt/client.js +73 -0
- package/dist/stt/policy.d.ts +11 -0
- package/dist/stt/policy.js +11 -0
- package/dist/stt/protocol.d.ts +13 -0
- package/dist/stt/protocol.js +61 -0
- package/dist/stt/queue.d.ts +16 -0
- package/dist/stt/queue.js +45 -0
- package/dist/stt/stream.d.ts +48 -0
- package/dist/stt/stream.js +224 -0
- package/dist/stt/transport.d.ts +13 -0
- package/dist/stt/transport.js +45 -0
- package/dist/stt/types.d.ts +34 -0
- package/dist/stt/types.js +1 -0
- package/dist/tts/client.d.ts +8 -0
- package/dist/tts/client.js +24 -7
- package/dist/tts/lexicon.d.ts +7 -0
- package/dist/tts/lexicon.js +23 -0
- package/dist/tts/policy.d.ts +2 -2
- package/dist/tts/policy.js +5 -28
- package/dist/tts/stream.d.ts +11 -2
- package/dist/tts/stream.js +42 -27
- package/dist/tts/timestamps.d.ts +24 -0
- package/dist/tts/timestamps.js +46 -0
- package/dist/tts/transport.d.ts +9 -13
- package/dist/tts/transport.js +43 -140
- package/dist/validation.d.ts +5 -0
- package/dist/validation.js +28 -0
- package/docs/stt.md +103 -0
- package/docs/tts.md +120 -7
- package/package.json +4 -5
- package/dist/tts/sentences.d.ts +0 -21
- package/dist/tts/sentences.js +0 -142
- package/vendor/LICENSE.blingfire +0 -21
- package/vendor/README.md +0 -11
- package/vendor/blingfire.cjs +0 -4
- package/vendor/blingfire.upstream.cjs +0 -1
- package/vendor/blingfire.wasm +0 -0
package/dist/tts/transport.js
CHANGED
|
@@ -1,56 +1,38 @@
|
|
|
1
|
+
import { connect as connectGrpc } from "../grpc.js";
|
|
2
|
+
export { metadata } from "../grpc.js";
|
|
3
|
+
import { BidiCall, serializer, deserializer, requestId, nativeError, } from "../grpc.js";
|
|
4
|
+
export { rpcError } from "../grpc.js";
|
|
1
5
|
import * as grpc from "@grpc/grpc-js";
|
|
2
|
-
import { create,
|
|
6
|
+
import { create, } from "@bufbuild/protobuf";
|
|
3
7
|
import * as schema from "@rimelabs/api";
|
|
4
8
|
import { policy } from "./policy.js";
|
|
5
9
|
import { abortable } from "../cancellation.js";
|
|
6
10
|
import * as errors from "../errors.js";
|
|
7
|
-
|
|
8
|
-
const classes = {
|
|
9
|
-
[grpc.status.UNAUTHENTICATED]: errors.RimeAuthenticationError,
|
|
10
|
-
[grpc.status.PERMISSION_DENIED]: errors.RimePermissionError,
|
|
11
|
-
[grpc.status.INVALID_ARGUMENT]: errors.RimeInputError,
|
|
12
|
-
[grpc.status.RESOURCE_EXHAUSTED]: errors.RimeResourceLimitError,
|
|
13
|
-
[grpc.status.UNAVAILABLE]: errors.RimeUnavailableError,
|
|
14
|
-
[grpc.status.DEADLINE_EXCEEDED]: errors.RimeTimeoutError,
|
|
15
|
-
[grpc.status.CANCELLED]: errors.RimeCancelledError,
|
|
16
|
-
};
|
|
17
|
-
const ErrorType = classes[code] ?? errors.RimeStreamError;
|
|
18
|
-
return new ErrorType(`Rime operation failed: ${grpc.status[code] ?? "UNKNOWN"}`, requestId);
|
|
19
|
-
}
|
|
11
|
+
import { TimestampTrailer } from "./timestamps.js";
|
|
20
12
|
export const transport = {
|
|
21
13
|
makeClient: (target) => new grpc.Client(target, grpc.credentials.createSsl(), {
|
|
22
14
|
"grpc.enable_retries": 0,
|
|
23
15
|
"grpc.max_receive_message_length": policy.receiveBytes,
|
|
24
|
-
"grpc.max_send_message_length":
|
|
16
|
+
"grpc.max_send_message_length": 131072,
|
|
25
17
|
}),
|
|
26
18
|
};
|
|
27
|
-
function
|
|
28
|
-
return (
|
|
29
|
-
}
|
|
30
|
-
function deserializer(desc) {
|
|
31
|
-
return (data) => fromBinary(desc, data);
|
|
32
|
-
}
|
|
33
|
-
export async function connect(client, signal) {
|
|
34
|
-
await abortable(new Promise((resolve, reject) => client.waitForReady(Date.now() + policy.connectionTimeout * 1000, (error) => error
|
|
35
|
-
? reject(new errors.RimeTimeoutError("Connection establishment timed out"))
|
|
36
|
-
: resolve())), signal);
|
|
37
|
-
}
|
|
38
|
-
export function metadata(value) {
|
|
39
|
-
const result = new grpc.Metadata();
|
|
40
|
-
result.set("authorization", value);
|
|
41
|
-
return result;
|
|
19
|
+
export function connect(client, signal) {
|
|
20
|
+
return connectGrpc(client, signal, policy.connectionTimeout);
|
|
42
21
|
}
|
|
43
22
|
function openStream(client, auth) {
|
|
44
23
|
return client.makeBidiStreamRequest("/rime.TextToSpeech/SynthesizeStreaming", serializer(schema.StreamingSynthesisRequestSchema), deserializer(schema.SynthesisResponseStreamSchema), auth);
|
|
45
24
|
}
|
|
46
|
-
function header(voice, language) {
|
|
25
|
+
function header(voice, language, timestamps, customLexicon) {
|
|
47
26
|
return create(schema.StreamingSynthesisRequestSchema, {
|
|
48
27
|
payload: {
|
|
49
28
|
case: "header",
|
|
50
29
|
value: create(schema.SynthesisRequestSchema, {
|
|
51
30
|
speaker: voice,
|
|
52
31
|
language,
|
|
32
|
+
splitStrategy: schema.SplitStrategy.ACCUMULATE_SENTENCE,
|
|
53
33
|
audioParameters: { audioFormat: "audio/pcm", samplingRate: 24000 },
|
|
34
|
+
timestamps: timestamps ? { enable: true } : undefined,
|
|
35
|
+
customLexicon: [...customLexicon],
|
|
54
36
|
}),
|
|
55
37
|
},
|
|
56
38
|
});
|
|
@@ -60,23 +42,6 @@ function textMessage(text) {
|
|
|
60
42
|
payload: { case: "textChunk", value: text },
|
|
61
43
|
});
|
|
62
44
|
}
|
|
63
|
-
function requestId(metadata) {
|
|
64
|
-
return metadata?.get("x-request-id")[0]?.toString() ?? null;
|
|
65
|
-
}
|
|
66
|
-
function nativeError(error, id) {
|
|
67
|
-
if (error instanceof errors.RimeError)
|
|
68
|
-
return error;
|
|
69
|
-
if (error &&
|
|
70
|
-
typeof error === "object" &&
|
|
71
|
-
"code" in error &&
|
|
72
|
-
typeof error.code === "number") {
|
|
73
|
-
const metadata = "metadata" in error ? error.metadata : null;
|
|
74
|
-
return rpcError(error.code, id ?? (metadata instanceof grpc.Metadata ? requestId(metadata) : null));
|
|
75
|
-
}
|
|
76
|
-
return new errors.RimeStreamError("Rime transport failed", id, {
|
|
77
|
-
cause: error,
|
|
78
|
-
});
|
|
79
|
-
}
|
|
80
45
|
async function unary(prepared, name, input, output, request, signal) {
|
|
81
46
|
let call;
|
|
82
47
|
let id = null;
|
|
@@ -114,115 +79,53 @@ export async function discover(prepared, kind, language, signal) {
|
|
|
114
79
|
return response.languages;
|
|
115
80
|
}
|
|
116
81
|
export class SynthesisCall {
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
statusPromise;
|
|
121
|
-
status = null;
|
|
122
|
-
id = null;
|
|
82
|
+
timestampsRequested = false;
|
|
83
|
+
timestampTrailer = new TimestampTrailer();
|
|
84
|
+
rpc;
|
|
123
85
|
constructor(prepared, signal) {
|
|
124
|
-
this.
|
|
125
|
-
this.call = openStream(prepared.client, prepared.metadata);
|
|
126
|
-
this.metadataPromise = new Promise((resolve, reject) => {
|
|
127
|
-
this.call.once("metadata", (metadata) => {
|
|
128
|
-
this.id = requestId(metadata);
|
|
129
|
-
resolve(metadata);
|
|
130
|
-
});
|
|
131
|
-
// A trailers-only response has no metadata event.
|
|
132
|
-
this.call.once("status", () => resolve(new grpc.Metadata()));
|
|
133
|
-
this.call.once("error", reject);
|
|
134
|
-
});
|
|
135
|
-
this.metadataPromise.catch(() => { });
|
|
136
|
-
this.statusPromise = new Promise((resolve) => {
|
|
137
|
-
this.call.once("status", (status) => {
|
|
138
|
-
this.status = status;
|
|
139
|
-
this.id ??= requestId(status.metadata);
|
|
140
|
-
resolve(status);
|
|
141
|
-
});
|
|
142
|
-
});
|
|
143
|
-
this.call.on("error", () => { });
|
|
86
|
+
this.rpc = new BidiCall(openStream(prepared.client, prepared.metadata), signal);
|
|
144
87
|
}
|
|
145
88
|
get requestId() {
|
|
146
|
-
return this.
|
|
89
|
+
return this.rpc.requestId;
|
|
147
90
|
}
|
|
148
91
|
get done() {
|
|
149
|
-
return this.
|
|
92
|
+
return this.rpc.done;
|
|
150
93
|
}
|
|
151
94
|
cancel() {
|
|
152
|
-
this.
|
|
95
|
+
this.rpc.cancel();
|
|
153
96
|
}
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
return
|
|
97
|
+
start(voice, language, timestamps = false, customLexicon = []) {
|
|
98
|
+
this.timestampsRequested = timestamps;
|
|
99
|
+
const message = header(voice, language, timestamps, customLexicon);
|
|
100
|
+
return this.rpc.write(message);
|
|
158
101
|
}
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
this.signal.throwIfAborted();
|
|
162
|
-
if (this.status) {
|
|
163
|
-
if (this.status.code !== grpc.status.OK)
|
|
164
|
-
throw rpcError(this.status.code, this.id);
|
|
165
|
-
throw new errors.RimeStreamError("The service completed before input finished", this.id);
|
|
166
|
-
}
|
|
167
|
-
await abortable(new Promise((resolve, reject) => {
|
|
168
|
-
this.call.write(message, (error) => error ? reject(error) : resolve());
|
|
169
|
-
}), this.signal);
|
|
170
|
-
}
|
|
171
|
-
catch (error) {
|
|
172
|
-
if (this.signal.aborted)
|
|
173
|
-
throw this.signal.reason;
|
|
174
|
-
if (error instanceof errors.RimeError)
|
|
175
|
-
throw error;
|
|
176
|
-
// A native write error can precede the final status event. Preserve the
|
|
177
|
-
// server rejection even if the reader has not consumed a response.
|
|
178
|
-
const status = await abortable(this.statusPromise, this.signal);
|
|
179
|
-
if (status.code !== grpc.status.OK)
|
|
180
|
-
throw rpcError(status.code, this.id);
|
|
181
|
-
throw this.error(error);
|
|
182
|
-
}
|
|
102
|
+
timestampResult() {
|
|
103
|
+
return this.timestampTrailer.result(this.requestId);
|
|
183
104
|
}
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
await this.writeMessage(header(voice, language));
|
|
187
|
-
}
|
|
188
|
-
async write(sentence) {
|
|
189
|
-
await this.writeMessage(textMessage(sentence));
|
|
105
|
+
write(text) {
|
|
106
|
+
return this.rpc.write(textMessage(text));
|
|
190
107
|
}
|
|
191
108
|
finishInput() {
|
|
192
|
-
|
|
193
|
-
this.call.end();
|
|
194
|
-
}
|
|
195
|
-
catch (error) {
|
|
196
|
-
throw this.error(error);
|
|
197
|
-
}
|
|
109
|
+
this.rpc.finishInput();
|
|
198
110
|
}
|
|
199
111
|
checkFormat(contentType) {
|
|
200
112
|
if (contentType !== "audio/pcm")
|
|
201
|
-
throw new errors.RimeAudioFormatError("Expected raw audio/pcm from the service", this.
|
|
113
|
+
throw new errors.RimeAudioFormatError("Expected raw audio/pcm from the service", this.requestId);
|
|
202
114
|
}
|
|
203
115
|
async *audio() {
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
const status = await abortable(this.statusPromise, this.signal);
|
|
217
|
-
if (status.code !== grpc.status.OK)
|
|
218
|
-
throw rpcError(status.code, this.id);
|
|
219
|
-
// Validate empty success only after giving a rejection its proper error.
|
|
220
|
-
this.checkFormat(contentType);
|
|
221
|
-
}
|
|
222
|
-
catch (error) {
|
|
223
|
-
if (this.signal.aborted)
|
|
224
|
-
throw this.signal.reason;
|
|
225
|
-
throw this.error(error);
|
|
116
|
+
const headers = await this.rpc.headers();
|
|
117
|
+
const contentType = headers.get("x-rime-audio-content-type")[0];
|
|
118
|
+
for await (const response of this.rpc.responses()) {
|
|
119
|
+
if (response.payload.case === "trailer" && this.timestampsRequested)
|
|
120
|
+
this.timestampTrailer.accept(response.payload.value);
|
|
121
|
+
if (response.payload.case !== "audio")
|
|
122
|
+
continue;
|
|
123
|
+
if (this.timestampTrailer.seen)
|
|
124
|
+
this.timestampTrailer.invalid = true;
|
|
125
|
+
if (response.payload.value.length)
|
|
126
|
+
this.checkFormat(contentType);
|
|
127
|
+
yield response.payload.value;
|
|
226
128
|
}
|
|
129
|
+
this.checkFormat(contentType);
|
|
227
130
|
}
|
|
228
131
|
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { RimeInputError } from "./errors.js";
|
|
2
|
+
export function endpointAddress(endpoint) {
|
|
3
|
+
const error = "endpoint must be a hostname with an optional port (1-65535), without a scheme or path";
|
|
4
|
+
if (typeof endpoint !== "string" || /\s/.test(endpoint))
|
|
5
|
+
throw new RimeInputError(error);
|
|
6
|
+
const [host, port, extra] = endpoint.split(":");
|
|
7
|
+
if (!host ||
|
|
8
|
+
host.length > 253 ||
|
|
9
|
+
extra !== undefined ||
|
|
10
|
+
host
|
|
11
|
+
.split(".")
|
|
12
|
+
.some((label) => !/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/i.test(label)) ||
|
|
13
|
+
(port !== undefined &&
|
|
14
|
+
(!/^[0-9]{1,5}$/.test(port) || Number(port) < 1 || Number(port) > 65535)))
|
|
15
|
+
throw new RimeInputError(error);
|
|
16
|
+
return {
|
|
17
|
+
target: `${host.toLowerCase()}:${port === undefined ? 443 : Number(port)}`,
|
|
18
|
+
audience: host.toLowerCase(),
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
export function timeout(value, inherited = null) {
|
|
22
|
+
if (value === undefined)
|
|
23
|
+
return inherited;
|
|
24
|
+
if (value !== null &&
|
|
25
|
+
(typeof value !== "number" || !Number.isFinite(value) || value <= 0))
|
|
26
|
+
throw new RimeInputError("timeout must be finite positive seconds or null");
|
|
27
|
+
return value;
|
|
28
|
+
}
|
package/docs/stt.md
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# Streaming speech recognition
|
|
2
|
+
|
|
3
|
+
`client.stt.stream` transcribes one utterance from an async source of `Uint8Array` chunks (including Node buffers).
|
|
4
|
+
The SDK handles the gRPC protocol, authentication, audio conversion, transcript
|
|
5
|
+
revisions, backpressure, and cancellation. Your application decides when speech
|
|
6
|
+
ends by exhausting the source.
|
|
7
|
+
|
|
8
|
+
```typescript
|
|
9
|
+
import { createReadStream } from "node:fs";
|
|
10
|
+
import { Rime } from "@rimelabs/sdk";
|
|
11
|
+
|
|
12
|
+
async function* audioChunks() {
|
|
13
|
+
yield* createReadStream("utterance.pcm");
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
const client = new Rime();
|
|
17
|
+
try {
|
|
18
|
+
const transcript = client.stt.stream(audioChunks(), {
|
|
19
|
+
language: "en",
|
|
20
|
+
inputFormat: { sampleRate: 16000, channels: 1 },
|
|
21
|
+
mode: "written",
|
|
22
|
+
contextTerms: ["Super-G", "USOC"],
|
|
23
|
+
timeout: 120,
|
|
24
|
+
});
|
|
25
|
+
for await (const update of transcript) {
|
|
26
|
+
if (update.kind === "partial") console.log("Current transcript:", update.text);
|
|
27
|
+
else console.log("Final:", update.text, "Language:", update.language);
|
|
28
|
+
}
|
|
29
|
+
console.log("Request:", transcript.requestId);
|
|
30
|
+
} finally {
|
|
31
|
+
await client.close();
|
|
32
|
+
}
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
The runnable [example](../examples/stt/stream.mjs) reads a PCM file
|
|
36
|
+
without loading it all into memory. Use Node.js 22 or later; browsers are not supported.
|
|
37
|
+
Open files inside the async generator so file errors are reported through the
|
|
38
|
+
SDK and resources are opened only after the service accepts the utterance.
|
|
39
|
+
|
|
40
|
+
## Options
|
|
41
|
+
|
|
42
|
+
| Option | Meaning |
|
|
43
|
+
| --- | --- |
|
|
44
|
+
| `language` | Required spoken BCP-47 tag. Passed unchanged to the service; no detection or SDK language allowlist. The deployed catalog currently supports English and Spanish. |
|
|
45
|
+
| `mode` | `"written"` (default) or `"verbatim"`. Formatting intent, not a guarantee of exact tokens. |
|
|
46
|
+
| `contextTerms` | Sequence of recognition hints, copied unchanged and in order. The service validates terms and prompt budgets. |
|
|
47
|
+
| `inputFormat` | `PCMFormat`: signed little-endian PCM16 at 8, 16, 24 or 48 kHz, mono or interleaved stereo. Defaults to 16 kHz mono. |
|
|
48
|
+
| `timeout` | Optional positive overall timeout in seconds. Includes connection time, source pauses and consumer pauses. `null` disables this SDK deadline, but not internal or server deadlines. |
|
|
49
|
+
|
|
50
|
+
Input contains raw samples, with no WAV header or compressed codec. Chunks may
|
|
51
|
+
split sample frames; the SDK carries incomplete bytes into the next chunk. A
|
|
52
|
+
truncated final frame raises `RimeAudioFormatError`, without committing the
|
|
53
|
+
utterance. Each gRPC audio payload is at most 64 KiB of 16 kHz mono PCM16.
|
|
54
|
+
|
|
55
|
+
The default recognition endpoint is `stt.api.rime.ai:443`, using TLS and the same
|
|
56
|
+
API key as the other features. `new Rime({ sttEndpoint: "host:443" })` overrides only STT.
|
|
57
|
+
The existing `model`, `endpoint` and client `timeout` settings remain TTS settings.
|
|
58
|
+
There is no STT voice/model selector; the service selects its recognizer.
|
|
59
|
+
|
|
60
|
+
## Results and completion
|
|
61
|
+
|
|
62
|
+
`TranscriptionPartial` and `TranscriptionFinal` are discriminated TypeScript types, exported from `@rimelabs/sdk`. Every update contains the complete current `text`.
|
|
63
|
+
**Replace the previous transcript. Do not concatenate partials.** Later partials
|
|
64
|
+
can change earlier words. `TranscriptionFinal.language` is the canonical language
|
|
65
|
+
selected by the service.
|
|
66
|
+
|
|
67
|
+
The source is not consumed until the server accepts the configuration. Audio
|
|
68
|
+
upload and transcript reading run concurrently. Decoding currently uses cumulative
|
|
69
|
+
two-second audio windows, so a partial is not promised for each audio packet.
|
|
70
|
+
Silence can finish successfully with empty final text and no partials.
|
|
71
|
+
|
|
72
|
+
Exhausting the source half-closes gRPC input. The SDK continues reading and emits
|
|
73
|
+
one final result only after a matching `done` and successful gRPC completion.
|
|
74
|
+
EOF, an error status, or cancellation cannot turn a partial into a final result.
|
|
75
|
+
No audio is automatically replayed after a connection failure.
|
|
76
|
+
|
|
77
|
+
## Lifetime and limits
|
|
78
|
+
|
|
79
|
+
Construction is lazy: starting iteration begins work. Consume each stream with
|
|
80
|
+
one reader. Breaking a `for await` loop calls the iterator's `return` and cancels
|
|
81
|
+
unfinished work. `await transcript.cancel()` and async disposal also cancel it.
|
|
82
|
+
Pass an `AbortSignal` through the stream's `signal` option to cancel just that
|
|
83
|
+
utterance. Client close cancels all STT, TTS and realtime operations; cancelling
|
|
84
|
+
one stream leaves its siblings running.
|
|
85
|
+
|
|
86
|
+
The SDK bounds the transcript queue to 16 updates, each with at most 64 KiB of
|
|
87
|
+
UTF-8 text, and bounds gRPC responses to 256 KiB. Slow consumers apply backpressure.
|
|
88
|
+
Internal connection and acceptance waits are 10 seconds; completion after source
|
|
89
|
+
exhaustion is bounded to 120 seconds. The completion timer stops after successful
|
|
90
|
+
gRPC completion. The optional overall timeout remains active until the final
|
|
91
|
+
result is consumed. An outstanding source `next()` cannot be forcibly stopped by
|
|
92
|
+
JavaScript; the SDK stops waiting for it and attempts `return()` for at most two
|
|
93
|
+
seconds. Sources that own devices should support cancellation themselves.
|
|
94
|
+
|
|
95
|
+
Server input-idle, utterance-size, request-duration and model-context limits still
|
|
96
|
+
apply. The current wire limit is 16 MiB per utterance; model context can impose a
|
|
97
|
+
smaller limit depending on audio and recognition terms. There is no SDK VAD,
|
|
98
|
+
server endpointing, word timing, diarization, or file-format detection.
|
|
99
|
+
|
|
100
|
+
Failures use the shared SDK error hierarchy: caller/input errors, cancellation,
|
|
101
|
+
authentication, permission, resource limits, timeouts, transient availability and
|
|
102
|
+
protocol failures stay distinct. `requestId` is retained on streams and failures
|
|
103
|
+
when the server supplies it. Keep it when reporting a problem.
|
package/docs/tts.md
CHANGED
|
@@ -1,12 +1,27 @@
|
|
|
1
1
|
# TTS for Node.js
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Send text as it becomes available; Rime automatically combines fragments into
|
|
4
|
+
sentences for speech. For nonblank input, ending the text stream flushes any
|
|
5
|
+
remaining text, even without final punctuation. Pausing the stream may delay
|
|
6
|
+
speech for an unfinished sentence.
|
|
7
|
+
|
|
8
|
+
If the source ends without any non-whitespace text, audio iteration rejects with
|
|
9
|
+
`RimeInputError`.
|
|
10
|
+
|
|
11
|
+
The SDK sends text in order with its whitespace preserved and streams audio back.
|
|
4
12
|
Your application owns playback. Start with the complete WAV example in the
|
|
5
13
|
[package quick start](../README.md#tts-save-speech).
|
|
6
14
|
|
|
15
|
+
## String input
|
|
16
|
+
|
|
17
|
+
Use `client.tts.synthesize(text)` for one nonblank string. It returns an audio
|
|
18
|
+
stream; it does not wait for all audio to be generated. Do not await this call.
|
|
19
|
+
Use the audio loop and cleanup shown in the package quick start.
|
|
20
|
+
|
|
7
21
|
## Stream incoming text
|
|
8
22
|
|
|
9
|
-
|
|
23
|
+
Use `client.tts.stream(chunks)` for an async iterable of strings. A plain string
|
|
24
|
+
belongs in `synthesize`, not `stream`. The source can supply partial sentences:
|
|
10
25
|
|
|
11
26
|
```javascript
|
|
12
27
|
async function* text() {
|
|
@@ -16,8 +31,8 @@ async function* text() {
|
|
|
16
31
|
```
|
|
17
32
|
|
|
18
33
|
Use `client.tts.stream(text())` with the same file write and cleanup.
|
|
19
|
-
|
|
20
|
-
|
|
34
|
+
Do not await `tts.stream()`; the first iterator read starts work. The quick start
|
|
35
|
+
collects chunks before writing a WAV file.
|
|
21
36
|
|
|
22
37
|
## Configuration
|
|
23
38
|
|
|
@@ -45,14 +60,16 @@ Model defaults still apply.
|
|
|
45
60
|
|
|
46
61
|
### Synthesis options
|
|
47
62
|
|
|
48
|
-
|
|
49
|
-
|
|
63
|
+
Both methods require text input: a nonblank string for `synthesize`, or an
|
|
64
|
+
async iterable of strings for `stream`. Both accept the following optional settings:
|
|
50
65
|
|
|
51
66
|
| Option | Default | Meaning |
|
|
52
67
|
| --- | --- | --- |
|
|
53
68
|
| `voice` | Model's default voice | Voice name from `client.voices.list()` |
|
|
54
69
|
| `language` | `"en"` | Language code |
|
|
55
70
|
| `audioFormat` | `AudioFormat.PCM_24000` | Output profile, imported from `@rimelabs/sdk` |
|
|
71
|
+
| `timestamps` | `false` | Request final word timestamps; requires `model: "mistv3"` |
|
|
72
|
+
| `customLexicon` | `[]` | Array of pronunciation overrides for this request; requires Coda |
|
|
56
73
|
| `timeout` | Client setting | Seconds; pass `null` to disable the overall deadline |
|
|
57
74
|
|
|
58
75
|
### Audio formats
|
|
@@ -74,6 +91,99 @@ connection and progress limits active. Discovery allows at most 10 seconds;
|
|
|
74
91
|
a shorter supplied timeout takes precedence. Omit an operation's `timeout` or
|
|
75
92
|
pass `undefined` to inherit the client setting.
|
|
76
93
|
|
|
94
|
+
## Custom pronunciations (Coda)
|
|
95
|
+
|
|
96
|
+
Pass words or phrases and their space-separated X-SAMPA pronunciations:
|
|
97
|
+
|
|
98
|
+
```javascript
|
|
99
|
+
import { writeFile } from "node:fs/promises";
|
|
100
|
+
import { Rime, RimeInputError } from "@rimelabs/sdk";
|
|
101
|
+
|
|
102
|
+
const lexicon = [{ spelling: "hello", pronunciation: 'h @ . " l oU' }];
|
|
103
|
+
const client = new Rime({ model: "coda", timeout: 60 });
|
|
104
|
+
try {
|
|
105
|
+
await writeFile("speech.pcm", client.tts.synthesize("Hello world.", {
|
|
106
|
+
language: "en",
|
|
107
|
+
customLexicon: lexicon,
|
|
108
|
+
}));
|
|
109
|
+
} catch (error) {
|
|
110
|
+
if (error instanceof RimeInputError) {
|
|
111
|
+
console.error(`Invalid synthesis request: ${error.message}; requestId=${error.requestId}`);
|
|
112
|
+
}
|
|
113
|
+
throw error;
|
|
114
|
+
} finally {
|
|
115
|
+
await client.close();
|
|
116
|
+
}
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
TypeScript callers can import the `PronunciationEntry` type from `@rimelabs/sdk`.
|
|
120
|
+
The SDK copies the array and entries when `tts.synthesize()` or `tts.stream()`
|
|
121
|
+
is called. The same lexicon applies to the entire request, including an async
|
|
122
|
+
text source; subsequent requests have their own options. Omit the option or
|
|
123
|
+
pass an empty array for default pronunciation.
|
|
124
|
+
|
|
125
|
+
The service matches whole words or phrases case-insensitively and applies the
|
|
126
|
+
pronunciation after text normalization. Longest matches take precedence; the
|
|
127
|
+
last entry for a repeated spelling wins. Inline `pronounce(...)` directives are
|
|
128
|
+
not supported.
|
|
129
|
+
|
|
130
|
+
Coda accepts custom pronunciations in German, English, Spanish, French,
|
|
131
|
+
Italian, and Portuguese, with up to 500 entries per request. The service validates
|
|
132
|
+
supported languages and pronunciation rules. Mist rejects nonempty lexicons.
|
|
133
|
+
|
|
134
|
+
Malformed option types and invalid Unicode in lexicon fields throw locally.
|
|
135
|
+
The service rejects unsupported models or languages, oversized lexicons, and
|
|
136
|
+
ill-formed entries with `RimeInputError`
|
|
137
|
+
before producing audio. Catch it around consuming the stream. Its `message`
|
|
138
|
+
preserves the offending spelling and checks such as `no-primary-stress` or
|
|
139
|
+
`unknown-phone`, and `requestId` identifies the request. The whole request fails;
|
|
140
|
+
entries are never silently skipped or repaired.
|
|
141
|
+
|
|
142
|
+
## Word timestamps (Mist v3)
|
|
143
|
+
|
|
144
|
+
Enable timestamps on a synthesis, consume all audio, then read its result:
|
|
145
|
+
|
|
146
|
+
```javascript
|
|
147
|
+
import { writeFile } from "node:fs/promises";
|
|
148
|
+
import { Rime } from "@rimelabs/sdk";
|
|
149
|
+
|
|
150
|
+
const client = new Rime({ model: "mistv3", timeout: 60 });
|
|
151
|
+
try {
|
|
152
|
+
const audio = client.tts.synthesize("Hello world.", { timestamps: true });
|
|
153
|
+
await writeFile("speech.pcm", audio);
|
|
154
|
+
const result = await audio.timestamps();
|
|
155
|
+
if (result.status.code === 0) {
|
|
156
|
+
for (const word of result.spans) {
|
|
157
|
+
console.log(word.text, word.start, word.end);
|
|
158
|
+
}
|
|
159
|
+
} else {
|
|
160
|
+
console.log("Timestamps unavailable:", result.status.code, result.status.message);
|
|
161
|
+
}
|
|
162
|
+
} finally {
|
|
163
|
+
await client.close();
|
|
164
|
+
}
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
`TimestampResult`, `TimestampStatus`, and `WordTimestamp` are exported types from
|
|
168
|
+
`@rimelabs/sdk`. The result is immutable. `spans` is an array of words with `text`,
|
|
169
|
+
`start`, and `end`; times are seconds from the beginning of the entire synthesis,
|
|
170
|
+
including when text arrives incrementally. Resampling to `MULAW_8000` does not
|
|
171
|
+
change these times. Words reflect normalized spoken text, so a number may become
|
|
172
|
+
several words. The model supplies the timings; the SDK does not estimate them.
|
|
173
|
+
|
|
174
|
+
Timestamps arrive once, after generation finishes. They are not incremental
|
|
175
|
+
events for live interruption handling. `timestamps()` never consumes audio or
|
|
176
|
+
waits for you to consume it: calling before successful iteration finishes, or
|
|
177
|
+
without enabling timestamps, rejects with `RimeInputError`. Results remain
|
|
178
|
+
accessible after a successfully consumed stream or its client is closed.
|
|
179
|
+
|
|
180
|
+
`status.code` is a numeric `google.rpc.Code`: `0` is success. Nonzero codes such as
|
|
181
|
+
`12` (unimplemented) or `4` (deadline exceeded) carry an explanation and no spans;
|
|
182
|
+
they do not fail successful audio. Missing or malformed timestamp responses reject
|
|
183
|
+
with `RimeStreamError` only when reading timestamps. A synthesis error or
|
|
184
|
+
cancellation also prevents reading a successful timestamp result. Enabling
|
|
185
|
+
timestamps with Coda throws `RimeInputError` before any request is sent.
|
|
186
|
+
|
|
77
187
|
## Discover voices and languages
|
|
78
188
|
|
|
79
189
|
List voices and languages while the client is open:
|
|
@@ -97,7 +207,7 @@ import { Rime, RimeError } from "@rimelabs/sdk";
|
|
|
97
207
|
let client;
|
|
98
208
|
try {
|
|
99
209
|
client = new Rime();
|
|
100
|
-
await writeFile("speech.pcm", client.tts.
|
|
210
|
+
await writeFile("speech.pcm", client.tts.synthesize("Hello. This is Rime."));
|
|
101
211
|
} catch (error) {
|
|
102
212
|
if (error instanceof RimeError) {
|
|
103
213
|
console.error(`Speech failed: ${error.message}; requestId=${error.requestId}`);
|
|
@@ -122,6 +232,9 @@ Each SDK error inherits from `RimeError`:
|
|
|
122
232
|
| `RimeCancelledError` | Operation cancelled |
|
|
123
233
|
| `RimeStreamError` | Other stream or transport failure |
|
|
124
234
|
|
|
235
|
+
Service errors retain their diagnostic message and request ID when provided.
|
|
236
|
+
Use the exception type for programmatic handling; message wording may change.
|
|
237
|
+
|
|
125
238
|
Partial audio can arrive before an error. Output is complete only after successful
|
|
126
239
|
iteration. Synthesis is not retried automatically.
|
|
127
240
|
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rimelabs/sdk",
|
|
3
|
-
"version": "0.1.0-alpha.
|
|
4
|
-
"description": "Rime streaming TTS and Prism realtime SDK for Node.js",
|
|
3
|
+
"version": "0.1.0-alpha.13",
|
|
4
|
+
"description": "Rime streaming TTS, STT and Prism realtime SDK for Node.js",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"engines": {
|
|
@@ -17,7 +17,6 @@
|
|
|
17
17
|
"files": [
|
|
18
18
|
"dist",
|
|
19
19
|
"docs",
|
|
20
|
-
"vendor",
|
|
21
20
|
"LICENSE",
|
|
22
21
|
"LICENSE-PRISM",
|
|
23
22
|
"NOTICE-PRISM",
|
|
@@ -29,12 +28,12 @@
|
|
|
29
28
|
"check": "tsc -p tsconfig.json --noEmit",
|
|
30
29
|
"test": "npm run build && node --test tests/*.test.mjs",
|
|
31
30
|
"prepack": "npm run build",
|
|
32
|
-
"lint": "prettier --check src tests
|
|
31
|
+
"lint": "prettier --check src tests examples/tts examples/realtime examples/common examples/browser examples/tests examples/stt"
|
|
33
32
|
},
|
|
34
33
|
"dependencies": {
|
|
35
34
|
"@bufbuild/protobuf": "^2.11.0",
|
|
36
35
|
"@grpc/grpc-js": "^1.14.0",
|
|
37
|
-
"@rimelabs/api": "0.
|
|
36
|
+
"@rimelabs/api": "0.4.0",
|
|
38
37
|
"ws": "^8.22.0"
|
|
39
38
|
},
|
|
40
39
|
"devDependencies": {
|
package/dist/tts/sentences.d.ts
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
interface BlingFire {
|
|
2
|
-
ready: Promise<BlingFire>;
|
|
3
|
-
_malloc(n: number): number;
|
|
4
|
-
_free(p: number): void;
|
|
5
|
-
_TextToSentences(input: number, length: number, output: number, capacity: number): number;
|
|
6
|
-
HEAPU8: Uint8Array;
|
|
7
|
-
}
|
|
8
|
-
export declare const ready: Promise<BlingFire>;
|
|
9
|
-
export declare class SentenceBuffer {
|
|
10
|
-
private limit;
|
|
11
|
-
private pending;
|
|
12
|
-
private committed;
|
|
13
|
-
private parts;
|
|
14
|
-
private untilScan;
|
|
15
|
-
private sincePunctuation;
|
|
16
|
-
private highSurrogate;
|
|
17
|
-
constructor(limit: number);
|
|
18
|
-
feed(fragment: string, final?: boolean): Generator<string>;
|
|
19
|
-
private scan;
|
|
20
|
-
}
|
|
21
|
-
export {};
|