@rimelabs/sdk 0.1.0-alpha.2 → 0.1.0-alpha.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +155 -0
- package/LICENSE-PRISM +201 -0
- package/NOTICE-PRISM +9 -0
- package/README.md +279 -114
- package/dist/auth.d.ts +9 -3
- package/dist/auth.js +12 -4
- package/dist/cancellation.d.ts +6 -0
- package/dist/cancellation.js +54 -0
- package/dist/client.d.ts +24 -23
- package/dist/client.js +41 -92
- package/dist/grpc.d.ts +27 -0
- package/dist/grpc.js +175 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +6 -2
- package/dist/pcm.d.ts +21 -0
- package/dist/pcm.js +52 -0
- package/dist/realtime/async.d.ts +23 -0
- package/dist/realtime/async.js +87 -0
- package/dist/realtime/audio.d.ts +2 -0
- package/dist/realtime/audio.js +11 -0
- package/dist/realtime/client.d.ts +12 -0
- package/dist/realtime/client.js +139 -0
- package/dist/realtime/conversation.d.ts +47 -0
- package/dist/realtime/conversation.js +321 -0
- package/dist/realtime/faults.d.ts +2 -0
- package/dist/realtime/faults.js +25 -0
- package/dist/realtime/protocol.d.ts +99 -0
- package/dist/realtime/protocol.js +432 -0
- package/dist/realtime/session.d.ts +79 -0
- package/dist/realtime/session.js +808 -0
- package/dist/realtime/settings.d.ts +11 -0
- package/dist/realtime/settings.js +142 -0
- package/dist/realtime/transport.d.ts +16 -0
- package/dist/realtime/transport.js +86 -0
- package/dist/realtime/types.d.ts +292 -0
- package/dist/realtime/types.js +12 -0
- package/dist/stt/audio.d.ts +15 -0
- package/dist/stt/audio.js +46 -0
- package/dist/stt/client.d.ts +17 -0
- package/dist/stt/client.js +121 -0
- package/dist/stt/policy.d.ts +11 -0
- package/dist/stt/policy.js +11 -0
- package/dist/stt/protocol.d.ts +14 -0
- package/dist/stt/protocol.js +76 -0
- package/dist/stt/queue.d.ts +16 -0
- package/dist/stt/queue.js +45 -0
- package/dist/stt/stream.d.ts +48 -0
- package/dist/stt/stream.js +224 -0
- package/dist/stt/transport.d.ts +15 -0
- package/dist/stt/transport.js +94 -0
- package/dist/stt/types.d.ts +51 -0
- package/dist/stt/types.js +1 -0
- package/dist/{audio.d.ts → tts/audio.d.ts} +9 -0
- package/dist/{audio.js → tts/audio.js} +44 -4
- package/dist/tts/buffer.d.ts +3 -0
- package/dist/tts/buffer.js +53 -0
- package/dist/tts/client.d.ts +53 -0
- package/dist/tts/client.js +151 -0
- package/dist/tts/lexicon.d.ts +7 -0
- package/dist/tts/lexicon.js +23 -0
- package/dist/{policy.d.ts → tts/policy.d.ts} +2 -3
- package/dist/tts/policy.js +41 -0
- package/dist/{stream.d.ts → tts/stream.d.ts} +15 -2
- package/dist/{stream.js → tts/stream.js} +54 -30
- package/dist/tts/timestamps.d.ts +24 -0
- package/dist/tts/timestamps.js +46 -0
- package/dist/{transport.d.ts → tts/transport.d.ts} +10 -13
- package/dist/tts/transport.js +133 -0
- package/dist/validation.d.ts +5 -0
- package/dist/validation.js +28 -0
- package/docs/realtime.md +537 -0
- package/docs/stt.md +186 -0
- package/docs/tts.md +284 -0
- package/package.json +14 -12
- package/dist/policy.js +0 -82
- package/dist/sentences.d.ts +0 -21
- package/dist/sentences.js +0 -142
- package/dist/transport.js +0 -223
- package/vendor/LICENSE.blingfire +0 -21
- package/vendor/README.md +0 -11
- package/vendor/blingfire.cjs +0 -4
- package/vendor/blingfire.upstream.cjs +0 -1
- package/vendor/blingfire.wasm +0 -0
- /package/dist/{queue.d.ts → tts/queue.d.ts} +0 -0
- /package/dist/{queue.js → tts/queue.js} +0 -0
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { connect, metadata } from "../grpc.js";
|
|
2
|
+
import { timeout } from "../validation.js";
|
|
3
|
+
import { RimeInputError, RimeAudioFormatError, RimeResourceLimitError, RimeCancelledError, RimeTimeoutError, } from "../errors.js";
|
|
4
|
+
import { scope } from "../cancellation.js";
|
|
5
|
+
import { resolvePCMFormat } from "../pcm.js";
|
|
6
|
+
import { transport, requireSchema, transcribe, MAX_WAV_BYTES, } from "./transport.js";
|
|
7
|
+
import { TranscriptStream } from "./stream.js";
|
|
8
|
+
import { policy } from "./policy.js";
|
|
9
|
+
export class STT {
|
|
10
|
+
credentials;
|
|
11
|
+
checkOpen;
|
|
12
|
+
connection = null;
|
|
13
|
+
streams = new Set();
|
|
14
|
+
cancellation = new AbortController();
|
|
15
|
+
requests = new Set();
|
|
16
|
+
constructor(credentials, checkOpen) {
|
|
17
|
+
this.credentials = credentials;
|
|
18
|
+
this.checkOpen = checkOpen;
|
|
19
|
+
}
|
|
20
|
+
/** Transcribe complete WAV bytes (at most 16 MiB), without replay or retries. */
|
|
21
|
+
transcribe(audio, options) {
|
|
22
|
+
this.checkOpen();
|
|
23
|
+
if (!(audio instanceof Uint8Array) || audio.byteLength === 0)
|
|
24
|
+
throw new RimeAudioFormatError("audio must contain complete WAV bytes");
|
|
25
|
+
if (audio.byteLength > MAX_WAV_BYTES)
|
|
26
|
+
throw new RimeResourceLimitError("WAV input exceeds 16 MiB");
|
|
27
|
+
validateOptions(options);
|
|
28
|
+
const settings = {
|
|
29
|
+
...options,
|
|
30
|
+
contextTerms: [...(options.contextTerms ?? [])],
|
|
31
|
+
};
|
|
32
|
+
const data = Buffer.from(audio);
|
|
33
|
+
const deadline = scope([options.signal, this.cancellation.signal], timeout(options.timeout) ?? undefined, new RimeTimeoutError("Overall transcription deadline expired"));
|
|
34
|
+
const operation = (async () => {
|
|
35
|
+
try {
|
|
36
|
+
const prepared = await this.prepare(deadline.signal, policy.target);
|
|
37
|
+
return await transcribe(prepared.client, prepared.metadata, data, settings, deadline.signal);
|
|
38
|
+
}
|
|
39
|
+
catch (error) {
|
|
40
|
+
if (deadline.signal.aborted &&
|
|
41
|
+
!(error instanceof RimeTimeoutError) &&
|
|
42
|
+
!(error instanceof RimeCancelledError)) {
|
|
43
|
+
if (deadline.signal.reason instanceof RimeTimeoutError)
|
|
44
|
+
throw deadline.signal.reason;
|
|
45
|
+
throw new RimeCancelledError("Transcription cancelled");
|
|
46
|
+
}
|
|
47
|
+
throw error;
|
|
48
|
+
}
|
|
49
|
+
finally {
|
|
50
|
+
deadline.dispose();
|
|
51
|
+
}
|
|
52
|
+
})();
|
|
53
|
+
this.requests.add(operation);
|
|
54
|
+
void operation.then(() => this.requests.delete(operation), () => this.requests.delete(operation));
|
|
55
|
+
return operation;
|
|
56
|
+
}
|
|
57
|
+
async prepare(signal, target) {
|
|
58
|
+
this.checkOpen();
|
|
59
|
+
requireSchema();
|
|
60
|
+
signal.throwIfAborted();
|
|
61
|
+
const headers = metadata(this.credentials.authorization());
|
|
62
|
+
this.connection ??= transport.makeClient(target);
|
|
63
|
+
await connect(this.connection, signal, policy.connectionTimeout);
|
|
64
|
+
signal.throwIfAborted();
|
|
65
|
+
return { client: this.connection, metadata: headers };
|
|
66
|
+
}
|
|
67
|
+
stream(source, options) {
|
|
68
|
+
this.checkOpen();
|
|
69
|
+
if (!source || typeof source[Symbol.asyncIterator] !== "function")
|
|
70
|
+
throw new RimeInputError("audio must be an async iterable of Uint8Array chunks");
|
|
71
|
+
validateOptions(options);
|
|
72
|
+
const outputContract = options.outputContract ?? "revised_hypotheses";
|
|
73
|
+
if (outputContract !== "revised_hypotheses" &&
|
|
74
|
+
outputContract !== "append_fragments")
|
|
75
|
+
throw new RimeInputError("outputContract must be revised_hypotheses or append_fragments");
|
|
76
|
+
const mode = options.mode ?? "written";
|
|
77
|
+
const terms = options.contextTerms ?? [];
|
|
78
|
+
const format = resolvePCMFormat(options.inputFormat);
|
|
79
|
+
const target = policy.target;
|
|
80
|
+
const stream = new TranscriptStream({
|
|
81
|
+
checkOpen: this.checkOpen,
|
|
82
|
+
forget: (stream) => this.streams.delete(stream),
|
|
83
|
+
prepare: (signal) => this.prepare(signal, target),
|
|
84
|
+
}, source, {
|
|
85
|
+
language: options.language,
|
|
86
|
+
outputContract,
|
|
87
|
+
mode,
|
|
88
|
+
contextTerms: [...terms],
|
|
89
|
+
inputFormat: format,
|
|
90
|
+
timeout: timeout(options.timeout),
|
|
91
|
+
signal: options.signal,
|
|
92
|
+
});
|
|
93
|
+
this.streams.add(stream);
|
|
94
|
+
return stream;
|
|
95
|
+
}
|
|
96
|
+
async close() {
|
|
97
|
+
this.cancellation.abort(new RimeCancelledError("Client closed"));
|
|
98
|
+
try {
|
|
99
|
+
const results = await Promise.allSettled([
|
|
100
|
+
...[...this.streams].map((stream) => stream.cancel()),
|
|
101
|
+
...[...this.requests].map((request) => request.catch(() => { })),
|
|
102
|
+
]);
|
|
103
|
+
for (const result of results)
|
|
104
|
+
if (result.status === "rejected")
|
|
105
|
+
throw result.reason;
|
|
106
|
+
}
|
|
107
|
+
finally {
|
|
108
|
+
this.connection?.close();
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
function validateOptions(options) {
|
|
113
|
+
if (!options || typeof options.language !== "string")
|
|
114
|
+
throw new RimeInputError("language must be a string");
|
|
115
|
+
const mode = options.mode ?? "written";
|
|
116
|
+
if (mode !== "written" && mode !== "verbatim")
|
|
117
|
+
throw new RimeInputError("mode must be written or verbatim");
|
|
118
|
+
const terms = options.contextTerms ?? [];
|
|
119
|
+
if (!Array.isArray(terms) || terms.some((term) => typeof term !== "string"))
|
|
120
|
+
throw new RimeInputError("contextTerms must be an array of strings");
|
|
121
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** Independent resource and transport limits for recognition. Durations are seconds. */
|
|
2
|
+
export declare const policy: {
|
|
3
|
+
target: string;
|
|
4
|
+
connectionTimeout: number;
|
|
5
|
+
acceptanceTimeout: number;
|
|
6
|
+
completionTimeout: number;
|
|
7
|
+
cleanupTimeout: number;
|
|
8
|
+
receiveBytes: number;
|
|
9
|
+
transcriptBytes: number;
|
|
10
|
+
queuedUpdates: number;
|
|
11
|
+
};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** Independent resource and transport limits for recognition. Durations are seconds. */
|
|
2
|
+
export const policy = {
|
|
3
|
+
target: "stt.api.rime.ai:443",
|
|
4
|
+
connectionTimeout: 10,
|
|
5
|
+
acceptanceTimeout: 10,
|
|
6
|
+
completionTimeout: 120,
|
|
7
|
+
cleanupTimeout: 2,
|
|
8
|
+
receiveBytes: 1048576 + 65536,
|
|
9
|
+
transcriptBytes: 1048576,
|
|
10
|
+
queuedUpdates: 16,
|
|
11
|
+
};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import * as schema from "@rimelabs/api";
|
|
2
|
+
import type { TranscriptionPartial, TranscriptionDelta, TranscriptionFinal, TranscriptionOutputContract } from "./types.js";
|
|
3
|
+
/** Validate the wire sequence before exposing any final result. */
|
|
4
|
+
export declare class TranscriptState {
|
|
5
|
+
private readonly textLimit;
|
|
6
|
+
private readonly outputContract;
|
|
7
|
+
language: string | null;
|
|
8
|
+
private revision;
|
|
9
|
+
private text;
|
|
10
|
+
private final;
|
|
11
|
+
constructor(textLimit: number, outputContract?: TranscriptionOutputContract);
|
|
12
|
+
accept(message: schema.StreamingTranscriptionResponse, inputDone: boolean): TranscriptionPartial | TranscriptionDelta | null;
|
|
13
|
+
finish(): TranscriptionFinal;
|
|
14
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import * as schema from "@rimelabs/api";
|
|
2
|
+
import { RimeStreamError, RimeResourceLimitError } from "../errors.js";
|
|
3
|
+
/** Validate the wire sequence before exposing any final result. */
|
|
4
|
+
export class TranscriptState {
|
|
5
|
+
textLimit;
|
|
6
|
+
outputContract;
|
|
7
|
+
language = null;
|
|
8
|
+
revision = 0n;
|
|
9
|
+
text = "";
|
|
10
|
+
final = null;
|
|
11
|
+
constructor(textLimit, outputContract = "revised_hypotheses") {
|
|
12
|
+
this.textLimit = textLimit;
|
|
13
|
+
this.outputContract = outputContract;
|
|
14
|
+
}
|
|
15
|
+
accept(message, inputDone) {
|
|
16
|
+
const payload = message.payload;
|
|
17
|
+
if (payload.case === undefined)
|
|
18
|
+
return null;
|
|
19
|
+
if (this.final)
|
|
20
|
+
throw new RimeStreamError("Received a message after transcription completion");
|
|
21
|
+
if (payload.case === "accepted") {
|
|
22
|
+
const accepted = payload.value;
|
|
23
|
+
if (this.language !== null)
|
|
24
|
+
throw new RimeStreamError("Received duplicate transcription acceptance");
|
|
25
|
+
if (accepted.outputContract !==
|
|
26
|
+
(this.outputContract === "append_fragments"
|
|
27
|
+
? schema.StreamingOutputContract.APPEND_FRAGMENTS
|
|
28
|
+
: schema.StreamingOutputContract.REVISED_HYPOTHESES))
|
|
29
|
+
throw new RimeStreamError("The service accepted a different transcript contract");
|
|
30
|
+
if (!accepted.language?.tag || !accepted.language.source)
|
|
31
|
+
throw new RimeStreamError("The service did not confirm a resolved language");
|
|
32
|
+
this.language = accepted.language.tag;
|
|
33
|
+
return null;
|
|
34
|
+
}
|
|
35
|
+
if (this.language === null)
|
|
36
|
+
throw new RimeStreamError("Received a transcript before acceptance");
|
|
37
|
+
const append = this.outputContract === "append_fragments";
|
|
38
|
+
if (payload.case === "delta" && append) {
|
|
39
|
+
const text = payload.value.text;
|
|
40
|
+
if (Buffer.byteLength(this.text) + Buffer.byteLength(text) >
|
|
41
|
+
this.textLimit)
|
|
42
|
+
throw new RimeResourceLimitError("The transcript exceeds the SDK text limit");
|
|
43
|
+
this.text += text;
|
|
44
|
+
return { kind: "delta", text };
|
|
45
|
+
}
|
|
46
|
+
if (payload.case !== "hypothesis" && payload.case !== "done")
|
|
47
|
+
throw new RimeStreamError("Received an unexpected transcription response");
|
|
48
|
+
if (payload.case === "hypothesis" && append)
|
|
49
|
+
throw new RimeStreamError("Received an unexpected transcription response");
|
|
50
|
+
const value = payload.value;
|
|
51
|
+
if (Buffer.byteLength(value.text, "utf8") > this.textLimit)
|
|
52
|
+
throw new RimeResourceLimitError("The transcript exceeds the SDK text limit");
|
|
53
|
+
if (payload.case === "hypothesis") {
|
|
54
|
+
if (value.revision <= this.revision ||
|
|
55
|
+
(this.revision === 0n && value.revision !== 1n))
|
|
56
|
+
throw new RimeStreamError("Transcript revisions are not increasing from one");
|
|
57
|
+
this.revision = value.revision;
|
|
58
|
+
this.text = value.text;
|
|
59
|
+
return { kind: "partial", text: value.text };
|
|
60
|
+
}
|
|
61
|
+
if (!inputDone)
|
|
62
|
+
throw new RimeStreamError("The service completed before input finished");
|
|
63
|
+
if (value.revision !== this.revision || value.text !== this.text)
|
|
64
|
+
throw new RimeStreamError("Final transcript does not match prior updates");
|
|
65
|
+
const language = payload.value.language;
|
|
66
|
+
if (language?.tag !== this.language || !language.source)
|
|
67
|
+
throw new RimeStreamError("Final language does not match the accepted language");
|
|
68
|
+
this.final = { kind: "final", text: value.text, language: language.tag };
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
finish() {
|
|
72
|
+
if (!this.final)
|
|
73
|
+
throw new RimeStreamError("The service ended without a final transcript");
|
|
74
|
+
return this.final;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { TranscriptionUpdate } from "./types.js";
|
|
2
|
+
/** Bounded, atomic transcript events; failure wakes producers and consumers. */
|
|
3
|
+
export declare class TranscriptQueue {
|
|
4
|
+
private readonly limit;
|
|
5
|
+
private items;
|
|
6
|
+
private closed;
|
|
7
|
+
private error;
|
|
8
|
+
private waiters;
|
|
9
|
+
constructor(limit: number);
|
|
10
|
+
private wait;
|
|
11
|
+
private wake;
|
|
12
|
+
put(value: TranscriptionUpdate): Promise<void>;
|
|
13
|
+
get(): Promise<IteratorResult<TranscriptionUpdate>>;
|
|
14
|
+
finish(): void;
|
|
15
|
+
fail(error: Error): void;
|
|
16
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/** Bounded, atomic transcript events; failure wakes producers and consumers. */
|
|
2
|
+
export class TranscriptQueue {
|
|
3
|
+
limit;
|
|
4
|
+
items = [];
|
|
5
|
+
closed = false;
|
|
6
|
+
error = null;
|
|
7
|
+
waiters = new Set();
|
|
8
|
+
constructor(limit) {
|
|
9
|
+
this.limit = limit;
|
|
10
|
+
}
|
|
11
|
+
wait() {
|
|
12
|
+
return new Promise((resolve) => this.waiters.add(resolve));
|
|
13
|
+
}
|
|
14
|
+
wake() {
|
|
15
|
+
for (const resolve of this.waiters)
|
|
16
|
+
resolve();
|
|
17
|
+
this.waiters.clear();
|
|
18
|
+
}
|
|
19
|
+
async put(value) {
|
|
20
|
+
while (!this.closed && this.items.length >= this.limit)
|
|
21
|
+
await this.wait();
|
|
22
|
+
if (!this.closed) {
|
|
23
|
+
this.items.push(value);
|
|
24
|
+
this.wake();
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
async get() {
|
|
28
|
+
while (!this.closed && !this.items.length)
|
|
29
|
+
await this.wait();
|
|
30
|
+
if (this.error)
|
|
31
|
+
throw this.error;
|
|
32
|
+
const value = this.items.shift();
|
|
33
|
+
this.wake();
|
|
34
|
+
return value ? { done: false, value } : { done: true, value: undefined };
|
|
35
|
+
}
|
|
36
|
+
finish() {
|
|
37
|
+
this.closed = true;
|
|
38
|
+
this.wake();
|
|
39
|
+
}
|
|
40
|
+
fail(error) {
|
|
41
|
+
this.error ??= error;
|
|
42
|
+
this.items = [];
|
|
43
|
+
this.finish();
|
|
44
|
+
}
|
|
45
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import type * as grpc from "@grpc/grpc-js";
|
|
2
|
+
import type { AudioSource, TranscriptionOptions, TranscriptionUpdate } from "./types.js";
|
|
3
|
+
interface StreamOwner {
|
|
4
|
+
prepare(signal: AbortSignal): Promise<{
|
|
5
|
+
client: grpc.Client;
|
|
6
|
+
metadata: grpc.Metadata;
|
|
7
|
+
}>;
|
|
8
|
+
checkOpen(): void;
|
|
9
|
+
forget(stream: TranscriptStream): void;
|
|
10
|
+
}
|
|
11
|
+
/** Lazy, single-consumer stream. Source exhaustion commits input; return/cancel aborts it. */
|
|
12
|
+
export declare class TranscriptStream implements AsyncIterableIterator<TranscriptionUpdate> {
|
|
13
|
+
private readonly owner;
|
|
14
|
+
private source;
|
|
15
|
+
private readonly options;
|
|
16
|
+
private readonly controller;
|
|
17
|
+
private readonly queue;
|
|
18
|
+
private readonly timers;
|
|
19
|
+
private worker;
|
|
20
|
+
private workerFinished;
|
|
21
|
+
private call;
|
|
22
|
+
private failure;
|
|
23
|
+
private reading;
|
|
24
|
+
private finished;
|
|
25
|
+
private inputDone;
|
|
26
|
+
private accepted;
|
|
27
|
+
private readonly acceptance;
|
|
28
|
+
private admission;
|
|
29
|
+
private completion;
|
|
30
|
+
private readonly externalAbort;
|
|
31
|
+
constructor(owner: StreamOwner, source: AudioSource | null, options: TranscriptionOptions);
|
|
32
|
+
/** Server request identity, available once response headers arrive. */
|
|
33
|
+
get requestId(): string | null;
|
|
34
|
+
private deadline;
|
|
35
|
+
private start;
|
|
36
|
+
private fail;
|
|
37
|
+
private produce;
|
|
38
|
+
private read;
|
|
39
|
+
private run;
|
|
40
|
+
[Symbol.asyncIterator](): this;
|
|
41
|
+
next(): Promise<IteratorResult<TranscriptionUpdate>>;
|
|
42
|
+
private cleanup;
|
|
43
|
+
/** Cancel unfinished work and release owned resources; safe to repeat. */
|
|
44
|
+
cancel(): Promise<void>;
|
|
45
|
+
return(): Promise<IteratorResult<TranscriptionUpdate>>;
|
|
46
|
+
[Symbol.asyncDispose](): Promise<void>;
|
|
47
|
+
}
|
|
48
|
+
export {};
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
import { abortable, scope } from "../cancellation.js";
|
|
2
|
+
import { RimeError, RimeInputError, RimeCancelledError, RimeTimeoutError, RimeStreamError, } from "../errors.js";
|
|
3
|
+
import { InputAudio } from "./audio.js";
|
|
4
|
+
import { TranscriptQueue } from "./queue.js";
|
|
5
|
+
import { TranscriptState } from "./protocol.js";
|
|
6
|
+
import { TranscriptionCall } from "./transport.js";
|
|
7
|
+
import { policy } from "./policy.js";
|
|
8
|
+
/** Lazy, single-consumer stream. Source exhaustion commits input; return/cancel aborts it. */
|
|
9
|
+
export class TranscriptStream {
|
|
10
|
+
owner;
|
|
11
|
+
source;
|
|
12
|
+
options;
|
|
13
|
+
controller = new AbortController();
|
|
14
|
+
queue = new TranscriptQueue(policy.queuedUpdates);
|
|
15
|
+
timers = new Set();
|
|
16
|
+
worker = null;
|
|
17
|
+
workerFinished = false;
|
|
18
|
+
call = null;
|
|
19
|
+
failure = null;
|
|
20
|
+
reading = false;
|
|
21
|
+
finished = false;
|
|
22
|
+
inputDone = false;
|
|
23
|
+
accepted;
|
|
24
|
+
acceptance = new Promise((resolve) => {
|
|
25
|
+
this.accepted = resolve;
|
|
26
|
+
});
|
|
27
|
+
admission = null;
|
|
28
|
+
completion = null;
|
|
29
|
+
externalAbort = () => this.fail(new RimeCancelledError("Transcription cancelled"));
|
|
30
|
+
constructor(owner, source, options) {
|
|
31
|
+
this.owner = owner;
|
|
32
|
+
this.source = source;
|
|
33
|
+
this.options = options;
|
|
34
|
+
}
|
|
35
|
+
/** Server request identity, available once response headers arrive. */
|
|
36
|
+
get requestId() {
|
|
37
|
+
return this.call?.requestId ?? null;
|
|
38
|
+
}
|
|
39
|
+
deadline(seconds, message) {
|
|
40
|
+
const timer = scope([], seconds, new RimeTimeoutError(message));
|
|
41
|
+
timer.signal.addEventListener("abort", () => this.fail(timer.signal.reason), { once: true });
|
|
42
|
+
if (timer.signal.aborted)
|
|
43
|
+
this.fail(timer.signal.reason);
|
|
44
|
+
this.timers.add(timer);
|
|
45
|
+
return timer;
|
|
46
|
+
}
|
|
47
|
+
start() {
|
|
48
|
+
if (this.failure)
|
|
49
|
+
throw this.failure;
|
|
50
|
+
if (this.finished || this.worker)
|
|
51
|
+
return;
|
|
52
|
+
this.owner.checkOpen();
|
|
53
|
+
if (this.options.signal?.aborted) {
|
|
54
|
+
this.externalAbort();
|
|
55
|
+
throw this.failure;
|
|
56
|
+
}
|
|
57
|
+
this.options.signal?.addEventListener("abort", this.externalAbort, {
|
|
58
|
+
once: true,
|
|
59
|
+
});
|
|
60
|
+
if (this.options.timeout != null)
|
|
61
|
+
this.deadline(this.options.timeout, "Overall transcription deadline expired");
|
|
62
|
+
this.worker = this.run();
|
|
63
|
+
}
|
|
64
|
+
fail(error) {
|
|
65
|
+
if (this.failure || this.finished)
|
|
66
|
+
return;
|
|
67
|
+
if (error.requestId === null && this.requestId !== null)
|
|
68
|
+
error = new error.constructor(error.message, this.requestId, { cause: error.cause });
|
|
69
|
+
this.failure = error;
|
|
70
|
+
this.queue.fail(error);
|
|
71
|
+
for (const timer of this.timers)
|
|
72
|
+
timer.dispose();
|
|
73
|
+
this.controller.abort(error);
|
|
74
|
+
this.call?.cancel();
|
|
75
|
+
this.options.signal?.removeEventListener("abort", this.externalAbort);
|
|
76
|
+
if (!this.worker || this.workerFinished) {
|
|
77
|
+
this.source = null;
|
|
78
|
+
this.owner.forget(this);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
async produce() {
|
|
82
|
+
const signal = this.controller.signal;
|
|
83
|
+
await abortable(this.acceptance, signal);
|
|
84
|
+
const audio = new InputAudio(this.options.inputFormat);
|
|
85
|
+
let iterator;
|
|
86
|
+
try {
|
|
87
|
+
try {
|
|
88
|
+
iterator = this.source[Symbol.asyncIterator]();
|
|
89
|
+
}
|
|
90
|
+
catch (error) {
|
|
91
|
+
throw new RimeInputError("The audio source failed", this.requestId, {
|
|
92
|
+
cause: error,
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
for (;;) {
|
|
96
|
+
let result;
|
|
97
|
+
try {
|
|
98
|
+
result = await abortable(iterator.next(), signal);
|
|
99
|
+
}
|
|
100
|
+
catch (error) {
|
|
101
|
+
if (signal.aborted)
|
|
102
|
+
throw signal.reason;
|
|
103
|
+
throw new RimeInputError("The audio source failed", this.requestId, {
|
|
104
|
+
cause: error,
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
if (result.done)
|
|
108
|
+
break;
|
|
109
|
+
for (const part of audio.feed(result.value))
|
|
110
|
+
await this.call.writeAudio(part);
|
|
111
|
+
}
|
|
112
|
+
audio.finish();
|
|
113
|
+
this.inputDone = true;
|
|
114
|
+
this.completion = this.deadline(policy.completionTimeout, "Transcription completion timed out");
|
|
115
|
+
this.call.finishInput();
|
|
116
|
+
}
|
|
117
|
+
finally {
|
|
118
|
+
if (iterator?.return) {
|
|
119
|
+
const cleanup = scope([], policy.cleanupTimeout, new RimeTimeoutError("Audio source cleanup timed out"));
|
|
120
|
+
try {
|
|
121
|
+
await abortable(Promise.resolve().then(() => iterator.return()), cleanup.signal);
|
|
122
|
+
}
|
|
123
|
+
catch {
|
|
124
|
+
/* Caller-owned source cleanup cannot replace the operation outcome. */
|
|
125
|
+
}
|
|
126
|
+
finally {
|
|
127
|
+
cleanup.dispose();
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
async read() {
|
|
133
|
+
const state = new TranscriptState(policy.transcriptBytes, this.options.outputContract);
|
|
134
|
+
for await (const message of this.call.responses()) {
|
|
135
|
+
const update = state.accept(message, this.inputDone);
|
|
136
|
+
if (state.language !== null) {
|
|
137
|
+
this.admission?.dispose();
|
|
138
|
+
this.accepted();
|
|
139
|
+
}
|
|
140
|
+
if (update)
|
|
141
|
+
await this.queue.put(update);
|
|
142
|
+
}
|
|
143
|
+
return state.finish();
|
|
144
|
+
}
|
|
145
|
+
async run() {
|
|
146
|
+
let tasks = [];
|
|
147
|
+
try {
|
|
148
|
+
const prepared = await this.owner.prepare(this.controller.signal);
|
|
149
|
+
this.controller.signal.throwIfAborted();
|
|
150
|
+
this.call = new TranscriptionCall(prepared.client, prepared.metadata, this.controller.signal);
|
|
151
|
+
this.admission = this.deadline(policy.acceptanceTimeout, "Transcription acceptance timed out");
|
|
152
|
+
await this.call.start(this.options.language, this.options.mode ?? "written", this.options.contextTerms ?? [], this.options.outputContract);
|
|
153
|
+
const producer = this.produce(), reader = this.read();
|
|
154
|
+
tasks = [producer, reader];
|
|
155
|
+
const [, final] = await Promise.all([producer, reader]);
|
|
156
|
+
this.completion?.dispose();
|
|
157
|
+
await this.queue.put(final);
|
|
158
|
+
this.queue.finish();
|
|
159
|
+
}
|
|
160
|
+
catch (error) {
|
|
161
|
+
this.fail(error instanceof RimeError
|
|
162
|
+
? error
|
|
163
|
+
: new RimeStreamError("Transcription failed", this.requestId, {
|
|
164
|
+
cause: error,
|
|
165
|
+
}));
|
|
166
|
+
}
|
|
167
|
+
finally {
|
|
168
|
+
if (this.failure)
|
|
169
|
+
this.controller.abort(this.failure);
|
|
170
|
+
this.call?.cancel();
|
|
171
|
+
await Promise.allSettled(tasks);
|
|
172
|
+
this.source = null;
|
|
173
|
+
this.workerFinished = true;
|
|
174
|
+
if (this.failure)
|
|
175
|
+
this.owner.forget(this);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
[Symbol.asyncIterator]() {
|
|
179
|
+
return this;
|
|
180
|
+
}
|
|
181
|
+
async next() {
|
|
182
|
+
if (this.reading)
|
|
183
|
+
throw new RimeInputError("TranscriptStream permits only one concurrent reader");
|
|
184
|
+
this.reading = true;
|
|
185
|
+
try {
|
|
186
|
+
this.start();
|
|
187
|
+
if (this.finished)
|
|
188
|
+
return { done: true, value: undefined };
|
|
189
|
+
const result = await this.queue.get();
|
|
190
|
+
if (this.failure)
|
|
191
|
+
throw this.failure;
|
|
192
|
+
if (!result.done && result.value.kind === "final") {
|
|
193
|
+
await this.cleanup();
|
|
194
|
+
if (this.failure)
|
|
195
|
+
throw this.failure;
|
|
196
|
+
this.finished = true;
|
|
197
|
+
}
|
|
198
|
+
return result;
|
|
199
|
+
}
|
|
200
|
+
finally {
|
|
201
|
+
this.reading = false;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
async cleanup() {
|
|
205
|
+
await this.worker;
|
|
206
|
+
for (const timer of this.timers)
|
|
207
|
+
timer.dispose();
|
|
208
|
+
this.options.signal?.removeEventListener("abort", this.externalAbort);
|
|
209
|
+
this.owner.forget(this);
|
|
210
|
+
}
|
|
211
|
+
/** Cancel unfinished work and release owned resources; safe to repeat. */
|
|
212
|
+
async cancel() {
|
|
213
|
+
this.fail(new RimeCancelledError("Transcription cancelled"));
|
|
214
|
+
await this.cleanup();
|
|
215
|
+
this.source = null;
|
|
216
|
+
}
|
|
217
|
+
async return() {
|
|
218
|
+
await this.cancel();
|
|
219
|
+
return { done: true, value: undefined };
|
|
220
|
+
}
|
|
221
|
+
async [Symbol.asyncDispose]() {
|
|
222
|
+
await this.cancel();
|
|
223
|
+
}
|
|
224
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import * as grpc from "@grpc/grpc-js";
|
|
2
|
+
import * as schema from "@rimelabs/api";
|
|
3
|
+
import { BidiCall } from "../grpc.js";
|
|
4
|
+
import type { TranscriptionMode, TranscribeOptions, Transcription } from "./types.js";
|
|
5
|
+
export declare const MAX_WAV_BYTES: number;
|
|
6
|
+
export declare function requireSchema(): void;
|
|
7
|
+
export declare const transport: {
|
|
8
|
+
makeClient: (target: string) => grpc.Client;
|
|
9
|
+
};
|
|
10
|
+
export declare function transcribe(client: grpc.Client, auth: grpc.Metadata, audio: Uint8Array, options: TranscribeOptions, signal: AbortSignal): Promise<Transcription>;
|
|
11
|
+
export declare class TranscriptionCall extends BidiCall<schema.StreamingTranscriptionRequest, schema.StreamingTranscriptionResponse> {
|
|
12
|
+
constructor(client: grpc.Client, auth: grpc.Metadata, signal: AbortSignal);
|
|
13
|
+
start(language: string, mode: TranscriptionMode, contextTerms: readonly string[], outputContract?: import("./types.js").TranscriptionOutputContract): Promise<void>;
|
|
14
|
+
writeAudio(audio: Uint8Array): Promise<void>;
|
|
15
|
+
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import * as grpc from "@grpc/grpc-js";
|
|
2
|
+
import { create } from "@bufbuild/protobuf";
|
|
3
|
+
import * as schema from "@rimelabs/api";
|
|
4
|
+
import { BidiCall, serializer, deserializer, requestId, nativeError, } from "../grpc.js";
|
|
5
|
+
import { abortable, scope } from "../cancellation.js";
|
|
6
|
+
import { RimeInputError, RimeStreamError, RimeResourceLimitError, RimeTimeoutError, RimeCancelledError, } from "../errors.js";
|
|
7
|
+
import { policy } from "./policy.js";
|
|
8
|
+
export const MAX_WAV_BYTES = 16 * 1024 * 1024;
|
|
9
|
+
export function requireSchema() {
|
|
10
|
+
if (!schema.SpeechToText)
|
|
11
|
+
throw new RimeInputError("STT requires @rimelabs/api with SpeechToText definitions");
|
|
12
|
+
}
|
|
13
|
+
export const transport = {
|
|
14
|
+
makeClient: (target) => new grpc.Client(target, grpc.credentials.createSsl(), {
|
|
15
|
+
"grpc.enable_retries": 0,
|
|
16
|
+
"grpc.max_receive_message_length": policy.receiveBytes,
|
|
17
|
+
"grpc.max_send_message_length": MAX_WAV_BYTES + 65536,
|
|
18
|
+
}),
|
|
19
|
+
};
|
|
20
|
+
export async function transcribe(client, auth, audio, options, signal) {
|
|
21
|
+
const method = schema.SpeechToText.method.transcribe;
|
|
22
|
+
const deadline = scope([signal], policy.completionTimeout, new RimeTimeoutError("Transcription request timed out"));
|
|
23
|
+
let call;
|
|
24
|
+
let id = null;
|
|
25
|
+
try {
|
|
26
|
+
deadline.signal.throwIfAborted();
|
|
27
|
+
const result = await abortable(new Promise((resolve, reject) => {
|
|
28
|
+
call = client.makeUnaryRequest(`/${schema.SpeechToText.typeName}/${method.name}`, serializer(method.input), deserializer(method.output), create(method.input, {
|
|
29
|
+
audio,
|
|
30
|
+
options: {
|
|
31
|
+
language: options.language,
|
|
32
|
+
mode: options.mode === "verbatim"
|
|
33
|
+
? schema.TranscriptionMode.VERBATIM
|
|
34
|
+
: schema.TranscriptionMode.WRITTEN,
|
|
35
|
+
contextTerms: [...(options.contextTerms ?? [])],
|
|
36
|
+
},
|
|
37
|
+
}), auth, (error, response) => (error ? reject(error) : resolve(response)));
|
|
38
|
+
call.on("metadata", (headers) => {
|
|
39
|
+
id = requestId(headers);
|
|
40
|
+
});
|
|
41
|
+
call.on("status", (status) => {
|
|
42
|
+
id ??= requestId(status.metadata);
|
|
43
|
+
});
|
|
44
|
+
}), deadline.signal);
|
|
45
|
+
if (!result.language?.tag || !result.language.source)
|
|
46
|
+
throw new RimeStreamError("The service did not confirm a resolved language", id);
|
|
47
|
+
if (Buffer.byteLength(result.text) > policy.transcriptBytes)
|
|
48
|
+
throw new RimeResourceLimitError("The transcript exceeds the SDK text limit", id);
|
|
49
|
+
return { text: result.text, language: result.language.tag, requestId: id };
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
if (deadline.signal.aborted) {
|
|
53
|
+
const reason = deadline.signal.reason;
|
|
54
|
+
if (reason instanceof RimeTimeoutError)
|
|
55
|
+
throw new RimeTimeoutError(reason.message, id);
|
|
56
|
+
throw new RimeCancelledError("Transcription cancelled", id);
|
|
57
|
+
}
|
|
58
|
+
throw nativeError(error, id);
|
|
59
|
+
}
|
|
60
|
+
finally {
|
|
61
|
+
call?.cancel();
|
|
62
|
+
deadline.dispose();
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
export class TranscriptionCall extends BidiCall {
|
|
66
|
+
constructor(client, auth, signal) {
|
|
67
|
+
requireSchema();
|
|
68
|
+
const service = schema.SpeechToText;
|
|
69
|
+
const method = service.method.transcribeStreaming;
|
|
70
|
+
super(client.makeBidiStreamRequest(`/${service.typeName}/${method.name}`, serializer(method.input), deserializer(method.output), auth), signal);
|
|
71
|
+
}
|
|
72
|
+
start(language, mode, contextTerms, outputContract = "revised_hypotheses") {
|
|
73
|
+
return this.write(create(schema.StreamingTranscriptionRequestSchema, {
|
|
74
|
+
payload: {
|
|
75
|
+
case: "config",
|
|
76
|
+
value: {
|
|
77
|
+
language,
|
|
78
|
+
mode: mode === "verbatim"
|
|
79
|
+
? schema.TranscriptionMode.VERBATIM
|
|
80
|
+
: schema.TranscriptionMode.WRITTEN,
|
|
81
|
+
contextTerms: [...contextTerms],
|
|
82
|
+
outputContract: outputContract === "append_fragments"
|
|
83
|
+
? schema.StreamingOutputContract.APPEND_FRAGMENTS
|
|
84
|
+
: schema.StreamingOutputContract.REVISED_HYPOTHESES,
|
|
85
|
+
},
|
|
86
|
+
},
|
|
87
|
+
}));
|
|
88
|
+
}
|
|
89
|
+
writeAudio(audio) {
|
|
90
|
+
return this.write(create(schema.StreamingTranscriptionRequestSchema, {
|
|
91
|
+
payload: { case: "audio", value: audio },
|
|
92
|
+
}));
|
|
93
|
+
}
|
|
94
|
+
}
|