@rimelabs/sdk 0.1.0-alpha.2 → 0.1.0-alpha.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +142 -0
- package/LICENSE-PRISM +201 -0
- package/NOTICE-PRISM +9 -0
- package/README.md +278 -114
- package/dist/auth.d.ts +9 -3
- package/dist/auth.js +12 -4
- package/dist/cancellation.d.ts +6 -0
- package/dist/cancellation.js +54 -0
- package/dist/client.d.ts +24 -23
- package/dist/client.js +41 -92
- package/dist/grpc.d.ts +27 -0
- package/dist/grpc.js +175 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +6 -2
- package/dist/pcm.d.ts +21 -0
- package/dist/pcm.js +52 -0
- package/dist/realtime/async.d.ts +23 -0
- package/dist/realtime/async.js +87 -0
- package/dist/realtime/audio.d.ts +2 -0
- package/dist/realtime/audio.js +11 -0
- package/dist/realtime/client.d.ts +12 -0
- package/dist/realtime/client.js +206 -0
- package/dist/realtime/conversation.d.ts +47 -0
- package/dist/realtime/conversation.js +309 -0
- package/dist/realtime/protocol.d.ts +84 -0
- package/dist/realtime/protocol.js +397 -0
- package/dist/realtime/session.d.ts +69 -0
- package/dist/realtime/session.js +744 -0
- package/dist/realtime/transport.d.ts +16 -0
- package/dist/realtime/transport.js +83 -0
- package/dist/realtime/types.d.ts +240 -0
- package/dist/realtime/types.js +10 -0
- package/dist/stt/audio.d.ts +15 -0
- package/dist/stt/audio.js +46 -0
- package/dist/stt/client.d.ts +17 -0
- package/dist/stt/client.js +121 -0
- package/dist/stt/policy.d.ts +11 -0
- package/dist/stt/policy.js +11 -0
- package/dist/stt/protocol.d.ts +14 -0
- package/dist/stt/protocol.js +76 -0
- package/dist/stt/queue.d.ts +16 -0
- package/dist/stt/queue.js +45 -0
- package/dist/stt/stream.d.ts +48 -0
- package/dist/stt/stream.js +224 -0
- package/dist/stt/transport.d.ts +15 -0
- package/dist/stt/transport.js +94 -0
- package/dist/stt/types.d.ts +51 -0
- package/dist/stt/types.js +1 -0
- package/dist/{audio.d.ts → tts/audio.d.ts} +9 -0
- package/dist/{audio.js → tts/audio.js} +44 -4
- package/dist/tts/buffer.d.ts +3 -0
- package/dist/tts/buffer.js +53 -0
- package/dist/tts/client.d.ts +53 -0
- package/dist/tts/client.js +151 -0
- package/dist/tts/lexicon.d.ts +7 -0
- package/dist/tts/lexicon.js +23 -0
- package/dist/{policy.d.ts → tts/policy.d.ts} +2 -3
- package/dist/tts/policy.js +41 -0
- package/dist/{stream.d.ts → tts/stream.d.ts} +15 -2
- package/dist/{stream.js → tts/stream.js} +54 -30
- package/dist/tts/timestamps.d.ts +24 -0
- package/dist/tts/timestamps.js +46 -0
- package/dist/{transport.d.ts → tts/transport.d.ts} +10 -13
- package/dist/tts/transport.js +133 -0
- package/dist/validation.d.ts +5 -0
- package/dist/validation.js +28 -0
- package/docs/realtime.md +426 -0
- package/docs/stt.md +186 -0
- package/docs/tts.md +284 -0
- package/package.json +14 -12
- package/dist/policy.js +0 -82
- package/dist/sentences.d.ts +0 -21
- package/dist/sentences.js +0 -142
- package/dist/transport.js +0 -223
- package/vendor/LICENSE.blingfire +0 -21
- package/vendor/README.md +0 -11
- package/vendor/blingfire.cjs +0 -4
- package/vendor/blingfire.upstream.cjs +0 -1
- package/vendor/blingfire.wasm +0 -0
- /package/dist/{queue.d.ts → tts/queue.d.ts} +0 -0
- /package/dist/{queue.js → tts/queue.js} +0 -0
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { AudioFormat } from "./audio.js";
|
|
2
|
+
import { type PronunciationEntry } from "./lexicon.js";
|
|
3
|
+
import { AudioStream, type TextSource } from "./stream.js";
|
|
4
|
+
import { Credentials } from "../auth.js";
|
|
5
|
+
import { policy } from "./policy.js";
|
|
6
|
+
export interface SynthesisOptions {
|
|
7
|
+
voice?: string | null;
|
|
8
|
+
language?: string;
|
|
9
|
+
audioFormat?: AudioFormat | null;
|
|
10
|
+
/** Requested output rate: 8000, 16000, or 24000 Hz. Defaults to 24000 for PCM/WAV/MP3. */
|
|
11
|
+
sampleRate?: number | null;
|
|
12
|
+
/** Duration multiplier from 0.4 to 2.5. Below 1 speeds speech up; above 1 slows it down. */
|
|
13
|
+
timeScaleFactor?: number | null;
|
|
14
|
+
/** Request final word timestamps. Supported only with model="mistv3". */
|
|
15
|
+
timestamps?: boolean;
|
|
16
|
+
/** Request-wide pronunciation overrides; requires a phoneme-capable Coda deployment. */
|
|
17
|
+
customLexicon?: readonly PronunciationEntry[];
|
|
18
|
+
timeout?: number | null;
|
|
19
|
+
}
|
|
20
|
+
export interface DiscoveryOptions {
|
|
21
|
+
timeout?: number | null;
|
|
22
|
+
}
|
|
23
|
+
export interface VoiceListOptions extends DiscoveryOptions {
|
|
24
|
+
language?: string | null;
|
|
25
|
+
}
|
|
26
|
+
export declare class TTS {
|
|
27
|
+
private readonly credentials;
|
|
28
|
+
private readonly deployment;
|
|
29
|
+
private readonly defaultTimeout;
|
|
30
|
+
private readonly checkOpen;
|
|
31
|
+
private client;
|
|
32
|
+
private streams;
|
|
33
|
+
private discoveryOperations;
|
|
34
|
+
readonly voices: {
|
|
35
|
+
list: (options?: VoiceListOptions) => Promise<string[]>;
|
|
36
|
+
};
|
|
37
|
+
readonly languages: {
|
|
38
|
+
list: (options?: DiscoveryOptions) => Promise<string[]>;
|
|
39
|
+
};
|
|
40
|
+
constructor(credentials: Credentials, deployment: Readonly<typeof policy>, defaultTimeout: number | null, checkOpen: () => void);
|
|
41
|
+
private prepare;
|
|
42
|
+
/** Synthesize complete text and return an audio stream. Do not await this call. */
|
|
43
|
+
synthesize(text: string, options?: SynthesisOptions): AudioStream;
|
|
44
|
+
/** Await complete audio in memory, finalizing WAV lengths. Never returns partial audio.
|
|
45
|
+
* The fixed output limit is 64 MiB (67,108,864 bytes); exceeding it raises
|
|
46
|
+
* RimeResourceLimitError. Use streaming synthesis for larger outputs or timestamps.
|
|
47
|
+
*/
|
|
48
|
+
synthesizeToBuffer(text: string, options?: SynthesisOptions): Promise<Buffer>;
|
|
49
|
+
/** Stream text chunks into synthesis and return an audio stream. Do not await this call. */
|
|
50
|
+
stream(text: TextSource, options?: SynthesisOptions): AudioStream;
|
|
51
|
+
private discover;
|
|
52
|
+
close(): Promise<void>;
|
|
53
|
+
}
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import { AudioFormat, resolveAudio } from "./audio.js";
|
|
2
|
+
import { maxBufferBytes, finalizeWav } from "./buffer.js";
|
|
3
|
+
import { snapshot } from "./lexicon.js";
|
|
4
|
+
import { AudioStream, constructionKey } from "./stream.js";
|
|
5
|
+
import { policy, timeout, nonempty } from "./policy.js";
|
|
6
|
+
import { abortable } from "../cancellation.js";
|
|
7
|
+
import { transport, connect, metadata, discover, } from "./transport.js";
|
|
8
|
+
import { RimeInputError, RimeResourceLimitError, RimeCancelledError, RimeTimeoutError, RimeUnavailableError, } from "../errors.js";
|
|
9
|
+
export class TTS {
|
|
10
|
+
credentials;
|
|
11
|
+
deployment;
|
|
12
|
+
defaultTimeout;
|
|
13
|
+
checkOpen;
|
|
14
|
+
client = null;
|
|
15
|
+
streams = new Set();
|
|
16
|
+
discoveryOperations = new Map();
|
|
17
|
+
voices;
|
|
18
|
+
languages;
|
|
19
|
+
constructor(credentials, deployment, defaultTimeout, checkOpen) {
|
|
20
|
+
this.credentials = credentials;
|
|
21
|
+
this.deployment = deployment;
|
|
22
|
+
this.defaultTimeout = defaultTimeout;
|
|
23
|
+
this.checkOpen = checkOpen;
|
|
24
|
+
this.voices = {
|
|
25
|
+
list: async (options = {}) => {
|
|
26
|
+
if (options.language !== undefined && options.language !== null)
|
|
27
|
+
nonempty(options.language, "language");
|
|
28
|
+
return this.discover("voices", options.language, options.timeout);
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
this.languages = {
|
|
32
|
+
list: (options = {}) => this.discover("languages", undefined, options.timeout),
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
async prepare(signal) {
|
|
36
|
+
this.checkOpen();
|
|
37
|
+
const auth = await this.credentials.metadata(signal);
|
|
38
|
+
this.checkOpen();
|
|
39
|
+
this.client ??= transport.makeClient(this.deployment.target);
|
|
40
|
+
await connect(this.client, signal);
|
|
41
|
+
return { client: this.client, metadata: metadata(auth) };
|
|
42
|
+
}
|
|
43
|
+
/** Synthesize complete text and return an audio stream. Do not await this call. */
|
|
44
|
+
synthesize(text, options = {}) {
|
|
45
|
+
this.checkOpen();
|
|
46
|
+
if (typeof text !== "string")
|
|
47
|
+
throw new RimeInputError("text must be a string; use stream() for an async text source");
|
|
48
|
+
if (!text.trim())
|
|
49
|
+
throw new RimeInputError("Text must contain non-whitespace characters");
|
|
50
|
+
return this.stream((async function* () {
|
|
51
|
+
yield text;
|
|
52
|
+
})(), options);
|
|
53
|
+
}
|
|
54
|
+
/** Await complete audio in memory, finalizing WAV lengths. Never returns partial audio.
|
|
55
|
+
* The fixed output limit is 64 MiB (67,108,864 bytes); exceeding it raises
|
|
56
|
+
* RimeResourceLimitError. Use streaming synthesis for larger outputs or timestamps.
|
|
57
|
+
*/
|
|
58
|
+
async synthesizeToBuffer(text, options = {}) {
|
|
59
|
+
const audio = this.synthesize(text, options);
|
|
60
|
+
const chunks = [];
|
|
61
|
+
let size = 0;
|
|
62
|
+
try {
|
|
63
|
+
for await (const chunk of audio) {
|
|
64
|
+
if (chunk.byteLength > maxBufferBytes - size)
|
|
65
|
+
throw new RimeResourceLimitError("Audio exceeds the 64 MiB buffer limit", audio.requestId);
|
|
66
|
+
chunks.push(chunk);
|
|
67
|
+
size += chunk.byteLength;
|
|
68
|
+
}
|
|
69
|
+
const data = Buffer.concat(chunks, size);
|
|
70
|
+
if (audio.format === AudioFormat.WAV)
|
|
71
|
+
finalizeWav(data, audio.sampleRate, audio.requestId);
|
|
72
|
+
return data;
|
|
73
|
+
}
|
|
74
|
+
finally {
|
|
75
|
+
await audio.cancel();
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
/** Stream text chunks into synthesis and return an audio stream. Do not await this call. */
|
|
79
|
+
stream(text, options = {}) {
|
|
80
|
+
this.checkOpen();
|
|
81
|
+
if (typeof text === "string" ||
|
|
82
|
+
!text ||
|
|
83
|
+
typeof text[Symbol.asyncIterator] !== "function")
|
|
84
|
+
throw new RimeInputError("text must be an async iterable of strings; use synthesize() for a string");
|
|
85
|
+
const lexicon = snapshot(options.customLexicon);
|
|
86
|
+
const timestamps = options.timestamps === undefined ? false : options.timestamps;
|
|
87
|
+
if (typeof timestamps !== "boolean")
|
|
88
|
+
throw new RimeInputError("timestamps must be a boolean");
|
|
89
|
+
if (timestamps && this.deployment.model !== "mistv3")
|
|
90
|
+
throw new RimeInputError("Word timestamps are supported only with model='mistv3'");
|
|
91
|
+
const voice = options.voice === undefined || options.voice === null
|
|
92
|
+
? this.deployment.defaultVoice
|
|
93
|
+
: nonempty(options.voice, "voice");
|
|
94
|
+
const language = nonempty(options.language ?? "en", "language");
|
|
95
|
+
const { format, sampleRate } = resolveAudio(options.audioFormat, options.sampleRate, options.timeScaleFactor);
|
|
96
|
+
const audio = new AudioStream(constructionKey, {
|
|
97
|
+
prepare: (s) => this.prepare(s),
|
|
98
|
+
checkOpen: () => this.checkOpen(),
|
|
99
|
+
forget: (s) => this.streams.delete(s),
|
|
100
|
+
}, text, voice, language, format, timeout(options.timeout, this.defaultTimeout), timestamps, lexicon, sampleRate, options.timeScaleFactor ?? undefined);
|
|
101
|
+
this.streams.add(audio);
|
|
102
|
+
return audio;
|
|
103
|
+
}
|
|
104
|
+
async discover(kind, language, override) {
|
|
105
|
+
this.checkOpen();
|
|
106
|
+
const budget = Math.min(timeout(override, this.defaultTimeout) ?? Infinity, policy.discoveryTimeout);
|
|
107
|
+
const controller = new AbortController();
|
|
108
|
+
let requestId = null;
|
|
109
|
+
const timer = setTimeout(() => controller.abort(new RimeTimeoutError("Discovery deadline expired")), budget * 1000);
|
|
110
|
+
const operation = (async () => {
|
|
111
|
+
try {
|
|
112
|
+
const prepared = await this.prepare(controller.signal);
|
|
113
|
+
for (let attempt = 0;; attempt++) {
|
|
114
|
+
requestId = null;
|
|
115
|
+
try {
|
|
116
|
+
return await discover(prepared, kind, language, controller.signal);
|
|
117
|
+
}
|
|
118
|
+
catch (error) {
|
|
119
|
+
if (!(error instanceof RimeUnavailableError) || attempt === 2)
|
|
120
|
+
throw error;
|
|
121
|
+
requestId = error.requestId;
|
|
122
|
+
await abortable(new Promise((resolve) => setTimeout(resolve, 50 * 2 ** attempt)), controller.signal);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
catch (error) {
|
|
127
|
+
if (error instanceof RimeTimeoutError && requestId !== null)
|
|
128
|
+
throw new RimeTimeoutError(error.message, requestId, {
|
|
129
|
+
cause: error.cause,
|
|
130
|
+
});
|
|
131
|
+
throw error;
|
|
132
|
+
}
|
|
133
|
+
finally {
|
|
134
|
+
clearTimeout(timer);
|
|
135
|
+
this.discoveryOperations.delete(controller);
|
|
136
|
+
}
|
|
137
|
+
})();
|
|
138
|
+
this.discoveryOperations.set(controller, operation);
|
|
139
|
+
return operation;
|
|
140
|
+
}
|
|
141
|
+
async close() {
|
|
142
|
+
const discoveries = [...this.discoveryOperations.values()];
|
|
143
|
+
for (const controller of this.discoveryOperations.keys())
|
|
144
|
+
controller.abort(new RimeCancelledError("Client closed during discovery"));
|
|
145
|
+
await Promise.all([
|
|
146
|
+
...[...this.streams].map((stream) => stream.cancel()),
|
|
147
|
+
Promise.allSettled(discoveries),
|
|
148
|
+
]);
|
|
149
|
+
this.client?.close();
|
|
150
|
+
}
|
|
151
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/** A word or phrase and its space-separated X-SAMPA pronunciation. */
|
|
2
|
+
export interface PronunciationEntry {
|
|
3
|
+
readonly spelling: string;
|
|
4
|
+
readonly pronunciation: string;
|
|
5
|
+
}
|
|
6
|
+
/** Copy request options now; model and linguistic validation belong to the service. */
|
|
7
|
+
export declare function snapshot(entries?: readonly PronunciationEntry[]): readonly PronunciationEntry[];
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { RimeInputError } from "../errors.js";
|
|
2
|
+
// Unicode mode treats valid surrogate pairs as one code point outside this range.
|
|
3
|
+
const unpairedSurrogate = /[\uD800-\uDFFF]/u;
|
|
4
|
+
/** Copy request options now; model and linguistic validation belong to the service. */
|
|
5
|
+
export function snapshot(entries = []) {
|
|
6
|
+
if (!Array.isArray(entries))
|
|
7
|
+
throw new RimeInputError("customLexicon must be an array of pronunciation entries");
|
|
8
|
+
// Array.from also visits holes, which are invalid entries.
|
|
9
|
+
return Object.freeze(Array.from(entries, (entry) => {
|
|
10
|
+
if (!entry ||
|
|
11
|
+
typeof entry !== "object" ||
|
|
12
|
+
typeof entry.spelling !== "string" ||
|
|
13
|
+
typeof entry.pronunciation !== "string")
|
|
14
|
+
throw new RimeInputError("Lexicon entries must have string spelling and pronunciation fields");
|
|
15
|
+
if (unpairedSurrogate.test(entry.spelling) ||
|
|
16
|
+
unpairedSurrogate.test(entry.pronunciation))
|
|
17
|
+
throw new RimeInputError("Lexicon spelling and pronunciation must contain valid Unicode");
|
|
18
|
+
return Object.freeze({
|
|
19
|
+
spelling: entry.spelling,
|
|
20
|
+
pronunciation: entry.pronunciation,
|
|
21
|
+
});
|
|
22
|
+
}));
|
|
23
|
+
}
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
+
export { timeout } from "../validation.js";
|
|
1
2
|
export declare const policy: {
|
|
3
|
+
model: string;
|
|
2
4
|
target: string;
|
|
3
5
|
audience: string;
|
|
4
6
|
defaultVoice: string;
|
|
5
7
|
exchangeUrl: string;
|
|
6
|
-
sentenceBytes: number;
|
|
7
8
|
sourceChars: number;
|
|
8
9
|
outputBytes: number;
|
|
9
10
|
outputChunkBytes: number;
|
|
@@ -16,6 +17,4 @@ export declare const policy: {
|
|
|
16
17
|
cleanupTimeout: number;
|
|
17
18
|
};
|
|
18
19
|
export declare function resolve(model: string, endpoint?: string | null): Readonly<typeof policy>;
|
|
19
|
-
export declare function timeout(value: unknown, inherited?: number | null): number | null;
|
|
20
20
|
export declare function nonempty(value: unknown, name: string): string;
|
|
21
|
-
export declare function abortable<T>(promise: PromiseLike<T>, signal: AbortSignal): Promise<T>;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { endpointAddress } from "../validation.js";
|
|
2
|
+
export { timeout } from "../validation.js";
|
|
3
|
+
import { RimeInputError } from "../errors.js";
|
|
4
|
+
const codaHostname = "coda.api.rime.ai";
|
|
5
|
+
export const policy = {
|
|
6
|
+
model: "coda",
|
|
7
|
+
target: `${codaHostname}:443`,
|
|
8
|
+
audience: codaHostname,
|
|
9
|
+
defaultVoice: "clementine",
|
|
10
|
+
exchangeUrl: "https://themis.api.rime.ai/v1/token",
|
|
11
|
+
sourceChars: 1024,
|
|
12
|
+
outputBytes: 96000,
|
|
13
|
+
outputChunkBytes: 9600,
|
|
14
|
+
receiveBytes: 4194304,
|
|
15
|
+
authTimeout: 10,
|
|
16
|
+
connectionTimeout: 10,
|
|
17
|
+
firstAudioTimeout: 30,
|
|
18
|
+
progressTimeout: 60,
|
|
19
|
+
discoveryTimeout: 10,
|
|
20
|
+
cleanupTimeout: 2,
|
|
21
|
+
};
|
|
22
|
+
export function resolve(model, endpoint) {
|
|
23
|
+
const deployment = { ...policy, model };
|
|
24
|
+
if (model === "mistv3") {
|
|
25
|
+
const hostname = "mist.api.rime.ai";
|
|
26
|
+
deployment.target = `${hostname}:443`;
|
|
27
|
+
deployment.audience = hostname;
|
|
28
|
+
deployment.defaultVoice = "astra";
|
|
29
|
+
}
|
|
30
|
+
else if (model !== "coda") {
|
|
31
|
+
throw new RimeInputError("model must be 'coda' or 'mistv3'");
|
|
32
|
+
}
|
|
33
|
+
if (endpoint === undefined || endpoint === null)
|
|
34
|
+
return deployment;
|
|
35
|
+
return { ...deployment, ...endpointAddress(endpoint) };
|
|
36
|
+
}
|
|
37
|
+
export function nonempty(value, name) {
|
|
38
|
+
if (typeof value !== "string" || !value.trim())
|
|
39
|
+
throw new RimeInputError(`${name} must be a non-empty string`);
|
|
40
|
+
return value;
|
|
41
|
+
}
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { AudioFormat } from "./audio.js";
|
|
2
|
+
import type { TimestampResult } from "./timestamps.js";
|
|
3
|
+
import type { PronunciationEntry } from "./lexicon.js";
|
|
2
4
|
import { type PreparedConnection } from "./transport.js";
|
|
3
5
|
export interface StreamOwner {
|
|
4
6
|
prepare(signal: AbortSignal): Promise<PreparedConnection>;
|
|
5
7
|
checkOpen(): void;
|
|
6
8
|
forget(stream: AudioStream): void;
|
|
7
9
|
}
|
|
8
|
-
export type TextSource =
|
|
10
|
+
export type TextSource = AsyncIterable<string>;
|
|
9
11
|
export declare const constructionKey: unique symbol;
|
|
10
12
|
export declare class AudioStream implements AsyncIterableIterator<Uint8Array> {
|
|
11
13
|
private owner;
|
|
@@ -14,6 +16,10 @@ export declare class AudioStream implements AsyncIterableIterator<Uint8Array> {
|
|
|
14
16
|
private language;
|
|
15
17
|
private formatValue;
|
|
16
18
|
private timeout;
|
|
19
|
+
private timestampsRequested;
|
|
20
|
+
private customLexicon;
|
|
21
|
+
private sampleRateValue;
|
|
22
|
+
private timeScaleFactor?;
|
|
17
23
|
private readonly controller;
|
|
18
24
|
private readonly queue;
|
|
19
25
|
private worker;
|
|
@@ -31,9 +37,16 @@ export declare class AudioStream implements AsyncIterableIterator<Uint8Array> {
|
|
|
31
37
|
private readonly operationId;
|
|
32
38
|
private bytesReceived;
|
|
33
39
|
private bytesDelivered;
|
|
34
|
-
constructor(key: symbol, owner: StreamOwner, source: TextSource | null, voice: string, language: string, formatValue: AudioFormat, timeout: number | null);
|
|
40
|
+
constructor(key: symbol, owner: StreamOwner, source: TextSource | null, voice: string, language: string, formatValue: AudioFormat, timeout: number | null, timestampsRequested?: boolean, customLexicon?: readonly PronunciationEntry[], sampleRateValue?: number, timeScaleFactor?: number | undefined);
|
|
35
41
|
get format(): AudioFormat;
|
|
42
|
+
/** Requested output rate. MP3 headers describe the encoded rate. */
|
|
43
|
+
get sampleRate(): number;
|
|
36
44
|
get requestId(): string | null;
|
|
45
|
+
/** Read requested timestamps after consuming all audio; never drains audio.
|
|
46
|
+
* Calling before iteration completes rejects with RimeInputError. Synthesis
|
|
47
|
+
* errors reject here too; alignment failures are returned in status.
|
|
48
|
+
*/
|
|
49
|
+
timestamps(): Promise<TimestampResult>;
|
|
37
50
|
private start;
|
|
38
51
|
private fail;
|
|
39
52
|
private produce;
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { debuglog } from "node:util";
|
|
3
3
|
const log = debuglog("rime-sdk");
|
|
4
|
-
import { Converter } from "./audio.js";
|
|
5
|
-
import { policy
|
|
4
|
+
import { AudioFormat, Converter, wireFormat } from "./audio.js";
|
|
5
|
+
import { policy } from "./policy.js";
|
|
6
|
+
import { abortable } from "../cancellation.js";
|
|
6
7
|
import { ByteQueue } from "./queue.js";
|
|
7
|
-
import { SentenceBuffer, ready } from "./sentences.js";
|
|
8
8
|
import { SynthesisCall } from "./transport.js";
|
|
9
|
-
import { RimeError, RimeInputError, RimeCancelledError, RimeTimeoutError, RimeStreamError, } from "
|
|
9
|
+
import { RimeError, RimeInputError, RimeCancelledError, RimeTimeoutError, RimeStreamError, } from "../errors.js";
|
|
10
10
|
export const constructionKey = Symbol("private stream constructor");
|
|
11
11
|
export class AudioStream {
|
|
12
12
|
owner;
|
|
@@ -15,6 +15,10 @@ export class AudioStream {
|
|
|
15
15
|
language;
|
|
16
16
|
formatValue;
|
|
17
17
|
timeout;
|
|
18
|
+
timestampsRequested;
|
|
19
|
+
customLexicon;
|
|
20
|
+
sampleRateValue;
|
|
21
|
+
timeScaleFactor;
|
|
18
22
|
controller = new AbortController();
|
|
19
23
|
queue = new ByteQueue(policy.outputBytes, policy.outputChunkBytes);
|
|
20
24
|
worker = null;
|
|
@@ -32,22 +36,43 @@ export class AudioStream {
|
|
|
32
36
|
operationId = randomUUID();
|
|
33
37
|
bytesReceived = 0;
|
|
34
38
|
bytesDelivered = 0;
|
|
35
|
-
constructor(key, owner, source, voice, language, formatValue, timeout) {
|
|
39
|
+
constructor(key, owner, source, voice, language, formatValue, timeout, timestampsRequested = false, customLexicon = [], sampleRateValue = 24000, timeScaleFactor) {
|
|
36
40
|
this.owner = owner;
|
|
37
41
|
this.source = source;
|
|
38
42
|
this.voice = voice;
|
|
39
43
|
this.language = language;
|
|
40
44
|
this.formatValue = formatValue;
|
|
41
45
|
this.timeout = timeout;
|
|
46
|
+
this.timestampsRequested = timestampsRequested;
|
|
47
|
+
this.customLexicon = customLexicon;
|
|
48
|
+
this.sampleRateValue = sampleRateValue;
|
|
49
|
+
this.timeScaleFactor = timeScaleFactor;
|
|
42
50
|
if (key !== constructionKey)
|
|
43
|
-
throw new TypeError("AudioStream is returned by client.tts.stream()");
|
|
51
|
+
throw new TypeError("AudioStream is returned by client.tts.synthesize() or client.tts.stream()");
|
|
44
52
|
}
|
|
45
53
|
get format() {
|
|
46
54
|
return this.formatValue;
|
|
47
55
|
}
|
|
56
|
+
/** Requested output rate. MP3 headers describe the encoded rate. */
|
|
57
|
+
get sampleRate() {
|
|
58
|
+
return this.sampleRateValue;
|
|
59
|
+
}
|
|
48
60
|
get requestId() {
|
|
49
61
|
return this.call?.requestId ?? null;
|
|
50
62
|
}
|
|
63
|
+
/** Read requested timestamps after consuming all audio; never drains audio.
|
|
64
|
+
* Calling before iteration completes rejects with RimeInputError. Synthesis
|
|
65
|
+
* errors reject here too; alignment failures are returned in status.
|
|
66
|
+
*/
|
|
67
|
+
async timestamps() {
|
|
68
|
+
if (!this.timestampsRequested)
|
|
69
|
+
throw new RimeInputError("Enable timestamps: true when creating the stream");
|
|
70
|
+
if (this.failure)
|
|
71
|
+
throw this.failure;
|
|
72
|
+
if (!this.finished)
|
|
73
|
+
throw new RimeInputError("Consume all audio before reading timestamps");
|
|
74
|
+
return this.call.timestampResult();
|
|
75
|
+
}
|
|
51
76
|
start() {
|
|
52
77
|
if (this.failure)
|
|
53
78
|
throw this.failure;
|
|
@@ -95,15 +120,11 @@ export class AudioStream {
|
|
|
95
120
|
async produce() {
|
|
96
121
|
const source = this.source;
|
|
97
122
|
let iterator;
|
|
98
|
-
|
|
123
|
+
let highSurrogate = "";
|
|
99
124
|
let meaningful = false;
|
|
100
125
|
try {
|
|
101
126
|
try {
|
|
102
|
-
iterator = (
|
|
103
|
-
? (async function* () {
|
|
104
|
-
yield source;
|
|
105
|
-
})()
|
|
106
|
-
: source)[Symbol.asyncIterator]();
|
|
127
|
+
iterator = source[Symbol.asyncIterator]();
|
|
107
128
|
}
|
|
108
129
|
catch (error) {
|
|
109
130
|
if (this.controller.signal.aborted)
|
|
@@ -130,35 +151,39 @@ export class AudioStream {
|
|
|
130
151
|
}
|
|
131
152
|
if (result.done)
|
|
132
153
|
break;
|
|
133
|
-
|
|
134
|
-
if (typeof chunk !== "string")
|
|
154
|
+
if (typeof result.value !== "string")
|
|
135
155
|
throw new RimeInputError("The text source must yield strings");
|
|
156
|
+
let chunk = highSurrogate + result.value;
|
|
157
|
+
highSurrogate = "";
|
|
158
|
+
// A JavaScript source may split one Unicode character across yields.
|
|
159
|
+
if (/[\uD800-\uDBFF]$/.test(chunk)) {
|
|
160
|
+
highSurrogate = chunk.slice(-1);
|
|
161
|
+
chunk = chunk.slice(0, -1);
|
|
162
|
+
}
|
|
136
163
|
if (chunk.trim())
|
|
137
164
|
meaningful = true;
|
|
138
|
-
//
|
|
165
|
+
// Bound wire messages without splitting a Unicode character.
|
|
139
166
|
for (let offset = 0; offset < chunk.length;) {
|
|
140
167
|
let end = Math.min(chunk.length, offset + policy.sourceChars);
|
|
141
168
|
if (end < chunk.length && /[\uD800-\uDBFF]/.test(chunk[end - 1]))
|
|
142
169
|
end++;
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
170
|
+
const part = chunk.slice(offset, end);
|
|
171
|
+
if (/[\uD800-\uDFFF]/u.test(part))
|
|
172
|
+
throw new RimeInputError("Text must contain valid Unicode");
|
|
173
|
+
if (!this.submitted)
|
|
174
|
+
this.progressAt = performance.now();
|
|
175
|
+
this.submitted = true;
|
|
176
|
+
await this.call.write(part);
|
|
177
|
+
this.progressAt = performance.now();
|
|
149
178
|
offset = end;
|
|
150
179
|
}
|
|
151
180
|
}
|
|
181
|
+
if (highSurrogate)
|
|
182
|
+
throw new RimeInputError("Text must contain valid Unicode");
|
|
152
183
|
if (!meaningful)
|
|
153
184
|
throw new RimeInputError("The text source contained no meaningful text");
|
|
154
|
-
for (const sentence of buffer.feed("", true)) {
|
|
155
|
-
if (!this.submitted)
|
|
156
|
-
this.progressAt = performance.now();
|
|
157
|
-
this.submitted = true;
|
|
158
|
-
await this.call.write(sentence);
|
|
159
|
-
}
|
|
160
|
-
this.call.finishInput();
|
|
161
185
|
this.inputDone = true;
|
|
186
|
+
this.call.finishInput();
|
|
162
187
|
}
|
|
163
188
|
catch (error) {
|
|
164
189
|
if (error instanceof RimeError || this.controller.signal.aborted)
|
|
@@ -199,10 +224,9 @@ export class AudioStream {
|
|
|
199
224
|
async run() {
|
|
200
225
|
let tasks = [];
|
|
201
226
|
try {
|
|
202
|
-
await abortable(ready, this.controller.signal);
|
|
203
227
|
const prepared = await this.owner.prepare(this.controller.signal);
|
|
204
228
|
this.call = new SynthesisCall(prepared, this.controller.signal);
|
|
205
|
-
await this.call.start(this.voice, this.language);
|
|
229
|
+
await this.call.start(this.voice, this.language, this.timestampsRequested, this.customLexicon, wireFormat(this.format), this.format === AudioFormat.MULAW_8000 ? 24000 : this.sampleRate, this.timeScaleFactor);
|
|
206
230
|
tasks = [this.produce(), this.read()];
|
|
207
231
|
await Promise.all(tasks);
|
|
208
232
|
this.queue.finish();
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { SynthesisResponseTrailer } from "@rimelabs/api";
|
|
2
|
+
/** A spoken, normalized word with start/end seconds from synthesis start. */
|
|
3
|
+
export interface WordTimestamp {
|
|
4
|
+
readonly text: string;
|
|
5
|
+
readonly start: number;
|
|
6
|
+
readonly end: number;
|
|
7
|
+
}
|
|
8
|
+
/** Alignment's google.rpc.Code (0 means OK) and service explanation. */
|
|
9
|
+
export interface TimestampStatus {
|
|
10
|
+
readonly code: number;
|
|
11
|
+
readonly message: string;
|
|
12
|
+
}
|
|
13
|
+
/** Final alignment status and words. Non-OK statuses carry no spans. */
|
|
14
|
+
export interface TimestampResult {
|
|
15
|
+
readonly status: TimestampStatus;
|
|
16
|
+
readonly spans: readonly WordTimestamp[];
|
|
17
|
+
}
|
|
18
|
+
export declare class TimestampTrailer {
|
|
19
|
+
seen: boolean;
|
|
20
|
+
invalid: boolean;
|
|
21
|
+
private value;
|
|
22
|
+
accept(trailer: SynthesisResponseTrailer): void;
|
|
23
|
+
result(requestId: string | null): TimestampResult;
|
|
24
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { RimeStreamError } from "../errors.js";
|
|
2
|
+
export class TimestampTrailer {
|
|
3
|
+
seen = false;
|
|
4
|
+
invalid = false;
|
|
5
|
+
value;
|
|
6
|
+
accept(trailer) {
|
|
7
|
+
if (this.seen)
|
|
8
|
+
this.invalid = true;
|
|
9
|
+
else {
|
|
10
|
+
this.seen = true;
|
|
11
|
+
this.value = trailer.timestamps;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
result(requestId) {
|
|
15
|
+
const malformed = () => new RimeStreamError("Invalid timestamp trailer", requestId);
|
|
16
|
+
if (this.invalid)
|
|
17
|
+
throw malformed();
|
|
18
|
+
const value = this.value;
|
|
19
|
+
if (!value)
|
|
20
|
+
throw new RimeStreamError("The service did not return requested timestamps", requestId);
|
|
21
|
+
if (!value.status || (value.status.code !== 0 && value.spans.length))
|
|
22
|
+
throw malformed();
|
|
23
|
+
const seconds = (duration) => {
|
|
24
|
+
if (!duration ||
|
|
25
|
+
duration.seconds < 0n ||
|
|
26
|
+
duration.seconds > 315576000000n ||
|
|
27
|
+
duration.nanos < 0 ||
|
|
28
|
+
duration.nanos >= 1000000000)
|
|
29
|
+
throw malformed();
|
|
30
|
+
return Number(duration.seconds) + duration.nanos / 1e9;
|
|
31
|
+
};
|
|
32
|
+
const spans = value.spans.map((span) => {
|
|
33
|
+
const start = seconds(span.start), end = seconds(span.end);
|
|
34
|
+
if (!span.text.trim() || end < start)
|
|
35
|
+
throw malformed();
|
|
36
|
+
return Object.freeze({ text: span.text, start, end });
|
|
37
|
+
});
|
|
38
|
+
return Object.freeze({
|
|
39
|
+
status: Object.freeze({
|
|
40
|
+
code: value.status.code,
|
|
41
|
+
message: value.status.message,
|
|
42
|
+
}),
|
|
43
|
+
spans: Object.freeze(spans),
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
}
|
|
@@ -1,33 +1,30 @@
|
|
|
1
|
+
export { metadata } from "../grpc.js";
|
|
2
|
+
export { rpcError } from "../grpc.js";
|
|
1
3
|
import * as grpc from "@grpc/grpc-js";
|
|
2
|
-
import
|
|
4
|
+
import type { PronunciationEntry } from "./lexicon.js";
|
|
3
5
|
export type Connection = grpc.Client;
|
|
4
6
|
export interface PreparedConnection {
|
|
5
7
|
client: Connection;
|
|
6
8
|
metadata: grpc.Metadata;
|
|
7
9
|
}
|
|
8
10
|
export type DiscoveryKind = "voices" | "languages";
|
|
9
|
-
export declare function rpcError(code: number, requestId?: string | null): errors.RimeError;
|
|
10
11
|
export declare const transport: {
|
|
11
12
|
makeClient: (target: string) => grpc.Client;
|
|
12
13
|
};
|
|
13
14
|
export declare function connect(client: grpc.Client, signal: AbortSignal): Promise<void>;
|
|
14
|
-
export declare function metadata(value: string): grpc.Metadata;
|
|
15
15
|
export declare function discover(prepared: PreparedConnection, kind: DiscoveryKind, language: string | null | undefined, signal: AbortSignal): Promise<string[]>;
|
|
16
16
|
export declare class SynthesisCall {
|
|
17
|
-
private
|
|
18
|
-
private
|
|
19
|
-
private readonly
|
|
20
|
-
private readonly
|
|
21
|
-
private status;
|
|
22
|
-
private id;
|
|
17
|
+
private audioFormat;
|
|
18
|
+
private timestampsRequested;
|
|
19
|
+
private readonly timestampTrailer;
|
|
20
|
+
private readonly rpc;
|
|
23
21
|
constructor(prepared: PreparedConnection, signal: AbortSignal);
|
|
24
22
|
get requestId(): string | null;
|
|
25
23
|
get done(): boolean;
|
|
26
24
|
cancel(): void;
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
write(sentence: string): Promise<void>;
|
|
25
|
+
start(voice: string, language: string, timestamps?: boolean, customLexicon?: readonly PronunciationEntry[], audioFormat?: string, sampleRate?: number, timeScaleFactor?: number): Promise<void>;
|
|
26
|
+
timestampResult(): import("./timestamps.js").TimestampResult;
|
|
27
|
+
write(text: string): Promise<void>;
|
|
31
28
|
finishInput(): void;
|
|
32
29
|
private checkFormat;
|
|
33
30
|
audio(): AsyncGenerator<Uint8Array>;
|