@rimelabs/sdk 0.1.0-alpha.17 → 0.1.0-alpha.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +3 -14
- package/dist/client.d.ts +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.js +1 -0
- package/dist/realtime/client.d.ts +1 -1
- package/dist/realtime/client.js +21 -2
- package/dist/realtime/conversation.d.ts +47 -0
- package/dist/realtime/conversation.js +309 -0
- package/dist/realtime/protocol.d.ts +3 -0
- package/dist/realtime/protocol.js +25 -0
- package/dist/realtime/session.d.ts +2 -0
- package/dist/realtime/session.js +16 -3
- package/dist/realtime/types.d.ts +17 -1
- package/dist/stt/client.js +5 -0
- package/dist/stt/policy.js +2 -2
- package/dist/stt/protocol.d.ts +4 -3
- package/dist/stt/protocol.js +18 -3
- package/dist/stt/stream.js +2 -2
- package/dist/stt/transport.d.ts +1 -1
- package/dist/stt/transport.js +4 -2
- package/dist/stt/types.d.ts +11 -2
- package/docs/realtime.md +177 -7
- package/docs/stt.md +48 -4
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,24 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.1.0-alpha.19](https://github.com/rimelabs/rime-sdk/compare/typescript-v0.1.0-alpha.18...typescript-v0.1.0-alpha.19) (2026-10-21)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
* add realtime conversation runners ([#89](https://github.com/rimelabs/rime-sdk/issues/89)) ([5447be7](https://github.com/rimelabs/rime-sdk/commit/5447be7713896ac8f68191cc957d82c58228296c))
|
|
9
|
+
|
|
10
|
+
## [0.1.0-alpha.18](https://github.com/rimelabs/rime-sdk/compare/typescript-v0.1.0-alpha.17...typescript-v0.1.0-alpha.18) (2026-10-21)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
### ⚠ BREAKING CHANGES
|
|
14
|
+
|
|
15
|
+
* Rust TranscriptionUpdate and TypeScript TranscriptionUpdate include Delta/delta; exhaustive handlers must handle the new variant.
|
|
16
|
+
|
|
17
|
+
### Features
|
|
18
|
+
|
|
19
|
+
* add append-fragment STT streaming across SDKs ([369f04d](https://github.com/rimelabs/rime-sdk/commit/369f04dadfea9d3d8593d14283d64e3a1f36e4d1))
|
|
20
|
+
* default realtime connections to public Prism and add optional turn timing controls ([#86](https://github.com/rimelabs/rime-sdk/issues/86)) ([0e07f68](https://github.com/rimelabs/rime-sdk/commit/0e07f6852bf0cefb60b8a42c48c2b76f95ac2362))
|
|
21
|
+
|
|
3
22
|
## [0.1.0-alpha.17](https://github.com/rimelabs/rime-sdk/compare/typescript-v0.1.0-alpha.16...typescript-v0.1.0-alpha.17) (2026-10-16)
|
|
4
23
|
|
|
5
24
|
|
package/README.md
CHANGED
|
@@ -28,16 +28,10 @@ STT use the SDK and Node.js built-ins.
|
|
|
28
28
|
|
|
29
29
|
## Prism: send recorded speech
|
|
30
30
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
31
|
+
With `RIME_API_KEY` set, the SDK connects to
|
|
32
|
+
`wss://prism.api.rime.ai/v1/realtime` and uses the server's default voice.
|
|
33
|
+
Pass `endpoint` or `voice` to select another deployment or voice.
|
|
34
34
|
|
|
35
|
-
```sh
|
|
36
|
-
export PRISM_URL="wss://your-prism-host/v1/realtime"
|
|
37
|
-
export PRISM_VOICE="your-deployment-voice"
|
|
38
|
-
```
|
|
39
|
-
|
|
40
|
-
You can omit `PRISM_VOICE` if the deployment has a default voice.
|
|
41
35
|
Record a short question and export it as `question.wav`, using **16 kHz, mono, signed 16-bit PCM WAV**.
|
|
42
36
|
Save the following as `prism.mjs` beside that file. It sends the recording at capture speed,
|
|
43
37
|
then sends silence so Prism can detect the end of speech. The spoken reply goes to `reply.wav`.
|
|
@@ -61,9 +55,6 @@ if (
|
|
|
61
55
|
const data = Buffer.from(input.data.samples);
|
|
62
56
|
if (!data.length || data.length % 2)
|
|
63
57
|
throw new Error("question.wav has no complete PCM16 audio");
|
|
64
|
-
const endpoint = process.env.PRISM_URL;
|
|
65
|
-
if (!endpoint)
|
|
66
|
-
throw new Error("Set PRISM_URL to your deployment's /v1/realtime URL");
|
|
67
58
|
|
|
68
59
|
const client = new Rime();
|
|
69
60
|
const stop = new AbortController();
|
|
@@ -75,8 +66,6 @@ let sending;
|
|
|
75
66
|
let reading;
|
|
76
67
|
try {
|
|
77
68
|
const session = await client.realtime.connect({
|
|
78
|
-
endpoint,
|
|
79
|
-
voice: process.env.PRISM_VOICE,
|
|
80
69
|
instructions: "Give short, clear answers.",
|
|
81
70
|
});
|
|
82
71
|
sending = (async () => {
|
package/dist/client.d.ts
CHANGED
|
@@ -23,7 +23,7 @@ export declare class Rime {
|
|
|
23
23
|
};
|
|
24
24
|
private readonly realtimeClient;
|
|
25
25
|
readonly realtime: {
|
|
26
|
-
connect: (options
|
|
26
|
+
connect: (options?: RealtimeConnectOptions) => Promise<RealtimeSession>;
|
|
27
27
|
};
|
|
28
28
|
private closed;
|
|
29
29
|
private closing;
|
package/dist/index.d.ts
CHANGED
|
@@ -7,4 +7,6 @@ export * from "./errors.js";
|
|
|
7
7
|
export { RealtimeSession } from "./realtime/session.js";
|
|
8
8
|
export * from "./realtime/types.js";
|
|
9
9
|
export { TranscriptStream } from "./stt/stream.js";
|
|
10
|
-
export type { AudioSource, TranscribeOptions, Transcription, TranscriptionOptions, TranscriptionMode, TranscriptionPartial, TranscriptionFinal, TranscriptionUpdate, } from "./stt/types.js";
|
|
10
|
+
export type { AudioSource, TranscribeOptions, Transcription, TranscriptionOptions, TranscriptionMode, TranscriptionPartial, TranscriptionDelta, TranscriptionOutputContract, TranscriptionFinal, TranscriptionUpdate, } from "./stt/types.js";
|
|
11
|
+
export { runConversation } from "./realtime/conversation.js";
|
|
12
|
+
export type { ConversationSource, ConversationOutput, ConversationPlayback, ConversationOptions, } from "./realtime/conversation.js";
|
package/dist/index.js
CHANGED
|
@@ -7,6 +7,6 @@ export declare class Realtime {
|
|
|
7
7
|
private readonly sessions;
|
|
8
8
|
private readonly opening;
|
|
9
9
|
constructor(credentials: Credentials, checkOpen: () => void);
|
|
10
|
-
connect(options
|
|
10
|
+
connect(options?: RealtimeConnectOptions): Promise<RealtimeSession>;
|
|
11
11
|
close(): Promise<void>;
|
|
12
12
|
}
|
package/dist/realtime/client.js
CHANGED
|
@@ -3,6 +3,7 @@ import { RimeCancelledError, RimeInputError, RimeUnavailableError, } from "../er
|
|
|
3
3
|
import { cancellation, scope } from "./async.js";
|
|
4
4
|
import { RealtimeSession } from "./session.js";
|
|
5
5
|
import { RealtimeConnection } from "./transport.js";
|
|
6
|
+
const DEFAULT_ENDPOINT = "wss://prism.api.rime.ai/v1/realtime";
|
|
6
7
|
function endpoint(value) {
|
|
7
8
|
let url;
|
|
8
9
|
try {
|
|
@@ -56,9 +57,9 @@ export class Realtime {
|
|
|
56
57
|
this.credentials = credentials;
|
|
57
58
|
this.checkOpen = checkOpen;
|
|
58
59
|
}
|
|
59
|
-
async connect(options) {
|
|
60
|
+
async connect(options = {}) {
|
|
60
61
|
this.checkOpen();
|
|
61
|
-
const address = endpoint(options.endpoint);
|
|
62
|
+
const address = endpoint(options.endpoint === undefined ? DEFAULT_ENDPOINT : options.endpoint);
|
|
62
63
|
const limits = timeouts(options.timeouts);
|
|
63
64
|
if ((options.model ?? "prism") !== "prism")
|
|
64
65
|
throw new RimeInputError("The realtime model must be prism");
|
|
@@ -103,6 +104,24 @@ export class Realtime {
|
|
|
103
104
|
};
|
|
104
105
|
}),
|
|
105
106
|
};
|
|
107
|
+
if (options.turnDetection !== undefined) {
|
|
108
|
+
const detection = options.turnDetection;
|
|
109
|
+
if (!detection ||
|
|
110
|
+
typeof detection !== "object" ||
|
|
111
|
+
Array.isArray(detection))
|
|
112
|
+
throw new RimeInputError("turnDetection must be an object");
|
|
113
|
+
if (detection.eagerness !== undefined) {
|
|
114
|
+
if (!["low", "medium", "high", "auto"].includes(detection.eagerness))
|
|
115
|
+
throw new RimeInputError("Turn eagerness must be low, medium, high, or auto");
|
|
116
|
+
settings.turn_detection.eagerness = detection.eagerness;
|
|
117
|
+
}
|
|
118
|
+
if (detection.endOfTurnTimeoutMs !== undefined) {
|
|
119
|
+
const value = detection.endOfTurnTimeoutMs;
|
|
120
|
+
if (!Number.isInteger(value) || value < 1000 || value > 10000)
|
|
121
|
+
throw new RimeInputError("End-of-turn timeout must be an integer from 1000 to 10000 ms");
|
|
122
|
+
settings.turn_detection.prism_end_of_turn_timeout_ms = value;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
106
125
|
for (const name of ["voice", "instructions"]) {
|
|
107
126
|
const value = options[name];
|
|
108
127
|
if (value !== undefined && value !== null) {
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { RealtimeSession } from "./session.js";
|
|
2
|
+
import type { AudioChunk, OutputRef, SessionEvent } from "./types.js";
|
|
3
|
+
export interface ConversationSource {
|
|
4
|
+
/** Return one chunk, or null at EOF. Abort must release a pending read. */
|
|
5
|
+
read(signal: AbortSignal): Promise<AudioChunk | null>;
|
|
6
|
+
}
|
|
7
|
+
export interface ConversationOutput {
|
|
8
|
+
/** Queue audio without blocking. Raise on overflow; never silently drop audio. */
|
|
9
|
+
write(chunk: AudioChunk): void;
|
|
10
|
+
/** Seal this output; completion still waits for the device to drain. */
|
|
11
|
+
end(): void;
|
|
12
|
+
/** Measured played milliseconds. Reject on measurement failure or abort.
|
|
13
|
+
* Abort stops the wait, without cancelling the device receipt. */
|
|
14
|
+
completion(signal: AbortSignal): Promise<number>;
|
|
15
|
+
}
|
|
16
|
+
export interface ConversationPlayback {
|
|
17
|
+
/** Queue an output in open order; audible playback must be FIFO.
|
|
18
|
+
* Keep completion and interruption state separate for each output.
|
|
19
|
+
* Do not mix outputs or play a later output ahead of an earlier one. */
|
|
20
|
+
open(output: OutputRef): ConversationOutput;
|
|
21
|
+
/** Stop selected outputs as one batch, including outputs after end().
|
|
22
|
+
* Discard their unplayed audio before starting any remaining output.
|
|
23
|
+
* Resume remaining outputs in FIFO order without waiting for new audio.
|
|
24
|
+
* Repeated interruptions are harmless; each receipt remains independent. */
|
|
25
|
+
interrupt(outputs: readonly OutputRef[]): void;
|
|
26
|
+
/** Confirm all playback has stopped and queued audio is discarded.
|
|
27
|
+
* Caller resources stay open. Reject and release the wait on abort.
|
|
28
|
+
* The runner supplies a separate signal with a five-second deadline and
|
|
29
|
+
* waits for cancellation cleanup before returning. */
|
|
30
|
+
stop(signal: AbortSignal): Promise<void>;
|
|
31
|
+
}
|
|
32
|
+
export interface ConversationOptions {
|
|
33
|
+
readonly source: ConversationSource;
|
|
34
|
+
readonly playback: ConversationPlayback;
|
|
35
|
+
/** Called inline for every event. Must be synchronous and nonblocking.
|
|
36
|
+
* Scheduled application work remains the caller's responsibility. */
|
|
37
|
+
readonly onEvent?: (event: SessionEvent) => void;
|
|
38
|
+
readonly signal?: AbortSignal;
|
|
39
|
+
}
|
|
40
|
+
/** Run until input EOF, session closure, failure, or abort.
|
|
41
|
+
* Owns the session's active lifetime and closes it on exit. The caller owns
|
|
42
|
+
* adapter resource contexts. Do not consume session.events or report playback
|
|
43
|
+
* elsewhere. Adapter operations must settle on abort. No input or callback
|
|
44
|
+
* queue is added. Limits: 64 responses, 128 outputs, 256 pending operations.
|
|
45
|
+
* Overflow fails the run. Callback and adapter failures reach the caller.
|
|
46
|
+
*/
|
|
47
|
+
export declare function runConversation(session: RealtimeSession, { source, playback, onEvent, signal }: ConversationOptions): Promise<void>;
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
import { abortable, scope } from "../cancellation.js";
|
|
2
|
+
import { RimeCancelledError, RimeInputError, RimeResourceLimitError, RimeTimeoutError, } from "../errors.js";
|
|
3
|
+
import { Deferred } from "./async.js";
|
|
4
|
+
import { isSessionCloseError } from "./session.js";
|
|
5
|
+
import { RimeRealtimeError } from "./types.js";
|
|
6
|
+
/** Run until input EOF, session closure, failure, or abort.
|
|
7
|
+
* Owns the session's active lifetime and closes it on exit. The caller owns
|
|
8
|
+
* adapter resource contexts. Do not consume session.events or report playback
|
|
9
|
+
* elsewhere. Adapter operations must settle on abort. No input or callback
|
|
10
|
+
* queue is added. Limits: 64 responses, 128 outputs, 256 pending operations.
|
|
11
|
+
* Overflow fails the run. Callback and adapter failures reach the caller.
|
|
12
|
+
*/
|
|
13
|
+
export async function runConversation(session, { source, playback, onEvent, signal }) {
|
|
14
|
+
const controller = new AbortController();
|
|
15
|
+
const running = controller.signal;
|
|
16
|
+
const receipts = new AbortController();
|
|
17
|
+
const abort = () => controller.abort(signal?.reason);
|
|
18
|
+
if (signal?.aborted)
|
|
19
|
+
abort();
|
|
20
|
+
else
|
|
21
|
+
signal?.addEventListener("abort", abort, { once: true });
|
|
22
|
+
const replies = new Map();
|
|
23
|
+
const parents = new Map();
|
|
24
|
+
const interrupted = new Set();
|
|
25
|
+
const tasks = new Set();
|
|
26
|
+
let fail;
|
|
27
|
+
let failed = false;
|
|
28
|
+
let failureReason;
|
|
29
|
+
const failure = new Promise((_, reject) => {
|
|
30
|
+
fail = (error) => {
|
|
31
|
+
if (!failed) {
|
|
32
|
+
failed = true;
|
|
33
|
+
failureReason = error;
|
|
34
|
+
reject(error);
|
|
35
|
+
}
|
|
36
|
+
};
|
|
37
|
+
});
|
|
38
|
+
void failure.catch(() => { });
|
|
39
|
+
function start(work, local) {
|
|
40
|
+
if (tasks.size >= 256)
|
|
41
|
+
throw new RimeResourceLimitError("Conversation pending operation limit exceeded");
|
|
42
|
+
const task = work();
|
|
43
|
+
tasks.add(task);
|
|
44
|
+
void task.then(() => tasks.delete(task), (error) => {
|
|
45
|
+
tasks.delete(task);
|
|
46
|
+
const expected = [local, running, receipts.signal].some((signal) => signal?.aborted &&
|
|
47
|
+
(error === signal.reason ||
|
|
48
|
+
error instanceof RimeCancelledError ||
|
|
49
|
+
(error instanceof DOMException && error.name === "AbortError")));
|
|
50
|
+
if (!expected && !isSessionCloseError(error))
|
|
51
|
+
fail(error);
|
|
52
|
+
});
|
|
53
|
+
return task;
|
|
54
|
+
}
|
|
55
|
+
async function measure(stream) {
|
|
56
|
+
const value = await stream.completion(receipts.signal);
|
|
57
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0)
|
|
58
|
+
throw new RimeInputError("Playback measurement must be finite nonnegative milliseconds");
|
|
59
|
+
return value;
|
|
60
|
+
}
|
|
61
|
+
function end(output) {
|
|
62
|
+
if (!output.sealed) {
|
|
63
|
+
output.sealed = true;
|
|
64
|
+
output.stream.end();
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
function cut(output) {
|
|
68
|
+
if (output.report)
|
|
69
|
+
return;
|
|
70
|
+
end(output);
|
|
71
|
+
output.report = start(async () => {
|
|
72
|
+
const played = await output.receipt;
|
|
73
|
+
running.throwIfAborted();
|
|
74
|
+
try {
|
|
75
|
+
await session.reportPlayback({
|
|
76
|
+
kind: "interrupted",
|
|
77
|
+
output: output.ref,
|
|
78
|
+
audioEndMs: Math.floor(played),
|
|
79
|
+
}, { signal: running });
|
|
80
|
+
}
|
|
81
|
+
catch (error) {
|
|
82
|
+
// The session also forwards the refusal to the application callback.
|
|
83
|
+
if (!(error instanceof RimeRealtimeError) ||
|
|
84
|
+
error.fault.scope !== "event" ||
|
|
85
|
+
error.fault.code !== "truncate_not_current")
|
|
86
|
+
throw error;
|
|
87
|
+
}
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
function finish(reply) {
|
|
91
|
+
const earlier = [];
|
|
92
|
+
for (const previous of replies.values()) {
|
|
93
|
+
if (previous === reply)
|
|
94
|
+
break;
|
|
95
|
+
earlier.push(previous.drained.promise);
|
|
96
|
+
}
|
|
97
|
+
const finishing = new AbortController();
|
|
98
|
+
reply.finish = finishing;
|
|
99
|
+
const cancel = () => finishing.abort(running.reason);
|
|
100
|
+
if (running.aborted)
|
|
101
|
+
cancel();
|
|
102
|
+
else
|
|
103
|
+
running.addEventListener("abort", cancel, { once: true });
|
|
104
|
+
start(async () => {
|
|
105
|
+
try {
|
|
106
|
+
let played = 0;
|
|
107
|
+
for (const output of reply.outputs.values())
|
|
108
|
+
played += await abortable(output.receipt, finishing.signal);
|
|
109
|
+
reply.drained.resolve();
|
|
110
|
+
if (reply.interrupted) {
|
|
111
|
+
for (const output of reply.outputs.values())
|
|
112
|
+
if (output.report)
|
|
113
|
+
await abortable(output.report, finishing.signal);
|
|
114
|
+
}
|
|
115
|
+
else {
|
|
116
|
+
// Prism's latest drain token clears all earlier playback too.
|
|
117
|
+
for (const drained of earlier)
|
|
118
|
+
await abortable(drained, finishing.signal);
|
|
119
|
+
await session.reportPlayback({ kind: "finished", response: reply.ref, playedMs: played }, { signal: finishing.signal });
|
|
120
|
+
}
|
|
121
|
+
finishing.signal.throwIfAborted();
|
|
122
|
+
replies.delete(reply.ref.responseId);
|
|
123
|
+
}
|
|
124
|
+
finally {
|
|
125
|
+
running.removeEventListener("abort", cancel);
|
|
126
|
+
}
|
|
127
|
+
}, finishing.signal);
|
|
128
|
+
}
|
|
129
|
+
function wasInterrupted(responseId) {
|
|
130
|
+
let current = responseId;
|
|
131
|
+
// The session retains at most 128 response references.
|
|
132
|
+
for (let count = 0; current !== undefined && count < 128; count++) {
|
|
133
|
+
if (interrupted.has(current))
|
|
134
|
+
return true;
|
|
135
|
+
current = parents.get(current);
|
|
136
|
+
}
|
|
137
|
+
return false;
|
|
138
|
+
}
|
|
139
|
+
function interrupt(selected) {
|
|
140
|
+
const affected = selected.filter((reply) => !reply.interrupted);
|
|
141
|
+
const outputs = affected.flatMap((reply) => [...reply.outputs.values()]);
|
|
142
|
+
for (const reply of affected) {
|
|
143
|
+
reply.interrupted = true;
|
|
144
|
+
if (parents.has(reply.ref.responseId))
|
|
145
|
+
interrupted.add(reply.ref.responseId);
|
|
146
|
+
reply.finish?.abort();
|
|
147
|
+
}
|
|
148
|
+
if (outputs.length)
|
|
149
|
+
playback.interrupt(outputs.map((output) => output.ref));
|
|
150
|
+
for (const output of outputs)
|
|
151
|
+
cut(output);
|
|
152
|
+
for (const reply of affected)
|
|
153
|
+
if (reply.ended)
|
|
154
|
+
finish(reply);
|
|
155
|
+
}
|
|
156
|
+
function outputFor(ref) {
|
|
157
|
+
const reply = replies.get(ref.response.responseId);
|
|
158
|
+
if (!reply || reply.ended)
|
|
159
|
+
return;
|
|
160
|
+
const key = JSON.stringify([ref.itemId, ref.outputIndex, ref.contentIndex]);
|
|
161
|
+
let output = reply.outputs.get(key);
|
|
162
|
+
if (!output) {
|
|
163
|
+
let count = 0;
|
|
164
|
+
for (const value of replies.values())
|
|
165
|
+
count += value.outputs.size;
|
|
166
|
+
if (count >= 128)
|
|
167
|
+
throw new RimeResourceLimitError("Conversation output limit exceeded");
|
|
168
|
+
const stream = playback.open(ref);
|
|
169
|
+
output = {
|
|
170
|
+
ref,
|
|
171
|
+
stream,
|
|
172
|
+
receipt: start(() => measure(stream)),
|
|
173
|
+
sealed: false,
|
|
174
|
+
};
|
|
175
|
+
reply.outputs.set(key, output);
|
|
176
|
+
if (reply.interrupted) {
|
|
177
|
+
playback.interrupt([ref]);
|
|
178
|
+
cut(output);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
return output;
|
|
182
|
+
}
|
|
183
|
+
async function send() {
|
|
184
|
+
while (!running.aborted) {
|
|
185
|
+
const chunk = await source.read(running);
|
|
186
|
+
if (chunk === null)
|
|
187
|
+
return;
|
|
188
|
+
await session.sendAudio(chunk, { signal: running });
|
|
189
|
+
}
|
|
190
|
+
running.throwIfAborted();
|
|
191
|
+
}
|
|
192
|
+
async function read() {
|
|
193
|
+
for await (const event of session.events) {
|
|
194
|
+
running.throwIfAborted();
|
|
195
|
+
const payload = event.payload;
|
|
196
|
+
switch (payload.kind) {
|
|
197
|
+
case "response.started": {
|
|
198
|
+
if (replies.size >= 64)
|
|
199
|
+
throw new RimeResourceLimitError("Conversation response limit exceeded");
|
|
200
|
+
const id = payload.response.responseId;
|
|
201
|
+
parents.set(id, payload.parent?.responseId);
|
|
202
|
+
if (parents.size > 128) {
|
|
203
|
+
const expired = parents.keys().next().value;
|
|
204
|
+
parents.delete(expired);
|
|
205
|
+
interrupted.delete(expired);
|
|
206
|
+
}
|
|
207
|
+
const blocked = wasInterrupted(id);
|
|
208
|
+
if (blocked)
|
|
209
|
+
interrupted.add(id);
|
|
210
|
+
replies.set(id, {
|
|
211
|
+
ref: payload.response,
|
|
212
|
+
outputs: new Map(),
|
|
213
|
+
interrupted: blocked,
|
|
214
|
+
ended: false,
|
|
215
|
+
drained: new Deferred(),
|
|
216
|
+
});
|
|
217
|
+
break;
|
|
218
|
+
}
|
|
219
|
+
case "message.started":
|
|
220
|
+
case "audio.delta":
|
|
221
|
+
case "audio.done": {
|
|
222
|
+
const output = outputFor(payload.output);
|
|
223
|
+
if (output) {
|
|
224
|
+
const reply = replies.get(payload.output.response.responseId);
|
|
225
|
+
if (payload.kind === "audio.delta" && !reply.interrupted)
|
|
226
|
+
output.stream.write(payload.audio);
|
|
227
|
+
else if (payload.kind === "audio.done")
|
|
228
|
+
end(output);
|
|
229
|
+
}
|
|
230
|
+
break;
|
|
231
|
+
}
|
|
232
|
+
case "speech.started":
|
|
233
|
+
if (session.info.interruptOnSpeech)
|
|
234
|
+
interrupt([...replies.values()]);
|
|
235
|
+
break;
|
|
236
|
+
case "response.abandoned": {
|
|
237
|
+
if (parents.has(payload.response.responseId))
|
|
238
|
+
interrupted.add(payload.response.responseId);
|
|
239
|
+
interrupt([...replies.values()].filter((reply) => wasInterrupted(reply.ref.responseId)));
|
|
240
|
+
break;
|
|
241
|
+
}
|
|
242
|
+
case "response.ended": {
|
|
243
|
+
const reply = replies.get(payload.response.responseId);
|
|
244
|
+
if (reply) {
|
|
245
|
+
reply.ended = true;
|
|
246
|
+
for (const output of reply.outputs.values())
|
|
247
|
+
end(output);
|
|
248
|
+
finish(reply);
|
|
249
|
+
}
|
|
250
|
+
break;
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
const result = onEvent?.(event);
|
|
254
|
+
if (result &&
|
|
255
|
+
typeof result.then === "function") {
|
|
256
|
+
void Promise.resolve(result).catch(() => { });
|
|
257
|
+
throw new RimeInputError("onEvent must be synchronous and nonblocking");
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
let original = false;
|
|
262
|
+
try {
|
|
263
|
+
running.throwIfAborted();
|
|
264
|
+
await abortable(Promise.race([start(send), start(read), failure]), running);
|
|
265
|
+
if (failed)
|
|
266
|
+
throw failureReason;
|
|
267
|
+
}
|
|
268
|
+
catch (error) {
|
|
269
|
+
if (!isSessionCloseError(error)) {
|
|
270
|
+
original = true;
|
|
271
|
+
throw error;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
finally {
|
|
275
|
+
let cleanupError;
|
|
276
|
+
// Stop input, event processing and reports, but keep receipt waits alive
|
|
277
|
+
// until the adapter confirms playback has stopped or the deadline expires.
|
|
278
|
+
controller.abort();
|
|
279
|
+
const shutdown = scope([], 5, new RimeTimeoutError("Playback shutdown confirmation timed out"));
|
|
280
|
+
try {
|
|
281
|
+
// Await the adapter itself, including cleanup after the deadline aborts.
|
|
282
|
+
// Racing this promise against abort would leave that cleanup running.
|
|
283
|
+
await playback.stop(shutdown.signal);
|
|
284
|
+
shutdown.signal.throwIfAborted();
|
|
285
|
+
}
|
|
286
|
+
catch (error) {
|
|
287
|
+
cleanupError = error;
|
|
288
|
+
}
|
|
289
|
+
finally {
|
|
290
|
+
shutdown.dispose();
|
|
291
|
+
}
|
|
292
|
+
receipts.abort();
|
|
293
|
+
try {
|
|
294
|
+
await session.close();
|
|
295
|
+
}
|
|
296
|
+
catch (error) {
|
|
297
|
+
cleanupError ??= error;
|
|
298
|
+
}
|
|
299
|
+
await Promise.allSettled(tasks);
|
|
300
|
+
signal?.removeEventListener("abort", abort);
|
|
301
|
+
if (!original) {
|
|
302
|
+
if (failed)
|
|
303
|
+
throw failureReason;
|
|
304
|
+
signal?.throwIfAborted();
|
|
305
|
+
if (cleanupError !== undefined)
|
|
306
|
+
throw cleanupError;
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
}
|
|
@@ -4,6 +4,8 @@ export interface SessionSettings {
|
|
|
4
4
|
input_audio_format: "pcm16";
|
|
5
5
|
turn_detection: {
|
|
6
6
|
interrupt_response: boolean;
|
|
7
|
+
eagerness?: t.TurnEagerness;
|
|
8
|
+
prism_end_of_turn_timeout_ms?: number;
|
|
7
9
|
};
|
|
8
10
|
tools: {
|
|
9
11
|
type: "function";
|
|
@@ -22,6 +24,7 @@ export interface SessionView {
|
|
|
22
24
|
interruptOnSpeech: boolean | null;
|
|
23
25
|
toolResultTimeoutS: number | null;
|
|
24
26
|
toolContinuationTimeoutS: number | null;
|
|
27
|
+
turnDetection: t.TurnDetectionInfo;
|
|
25
28
|
}
|
|
26
29
|
export interface SessionCreated {
|
|
27
30
|
kind: "session.created";
|
|
@@ -66,6 +66,30 @@ function duration(value, field) {
|
|
|
66
66
|
}
|
|
67
67
|
function session(value) {
|
|
68
68
|
const view = object(value, "session");
|
|
69
|
+
const detection = view.turn_detection === undefined
|
|
70
|
+
? {}
|
|
71
|
+
: object(view.turn_detection, "session.turn_detection");
|
|
72
|
+
let eagerness = null;
|
|
73
|
+
if (detection.eagerness !== undefined) {
|
|
74
|
+
const value = string(detection.eagerness, "session.turn_detection.eagerness");
|
|
75
|
+
eagerness =
|
|
76
|
+
value === "low" ||
|
|
77
|
+
value === "medium" ||
|
|
78
|
+
value === "high" ||
|
|
79
|
+
value === "auto"
|
|
80
|
+
? value
|
|
81
|
+
: "unknown";
|
|
82
|
+
}
|
|
83
|
+
let endOfTurnTimeoutMs = null;
|
|
84
|
+
if (detection.prism_end_of_turn_timeout_ms !== undefined) {
|
|
85
|
+
const value = detection.prism_end_of_turn_timeout_ms;
|
|
86
|
+
if (typeof value !== "number" ||
|
|
87
|
+
!Number.isInteger(value) ||
|
|
88
|
+
value < 1000 ||
|
|
89
|
+
value > 10000)
|
|
90
|
+
invalid("session.turn_detection.prism_end_of_turn_timeout_ms");
|
|
91
|
+
endOfTurnTimeoutMs = value;
|
|
92
|
+
}
|
|
69
93
|
const waits = view.prism_tool_waits === undefined
|
|
70
94
|
? {}
|
|
71
95
|
: object(view.prism_tool_waits, "session.prism_tool_waits");
|
|
@@ -79,6 +103,7 @@ function session(value) {
|
|
|
79
103
|
id: id(view.id, "session.id"),
|
|
80
104
|
voice: view.voice === undefined ? "" : string(view.voice, "session.voice"),
|
|
81
105
|
interruptOnSpeech: interrupt ?? null,
|
|
106
|
+
turnDetection: Object.freeze({ eagerness, endOfTurnTimeoutMs }),
|
|
82
107
|
toolResultTimeoutS: duration(waits.result_timeout_s, "result_timeout_s"),
|
|
83
108
|
toolContinuationTimeoutS: duration(waits.continuation_timeout_s, "continuation_timeout_s"),
|
|
84
109
|
};
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { RealtimeConnection } from "./transport.js";
|
|
2
2
|
import * as t from "./types.js";
|
|
3
3
|
import * as p from "./protocol.js";
|
|
4
|
+
/** @internal Recognize only failures created by intentional session closure. */
|
|
5
|
+
export declare function isSessionCloseError(error: unknown): boolean;
|
|
4
6
|
/** One conversation. Consume events concurrently with session operations. */
|
|
5
7
|
export declare class RealtimeSession {
|
|
6
8
|
private readonly socket;
|
package/dist/realtime/session.js
CHANGED
|
@@ -16,6 +16,11 @@ const NOT_READY = new Set([
|
|
|
16
16
|
"proactive_unavailable",
|
|
17
17
|
]);
|
|
18
18
|
const LIMIT = 128;
|
|
19
|
+
const closeErrors = new WeakSet();
|
|
20
|
+
/** @internal Recognize only failures created by intentional session closure. */
|
|
21
|
+
export function isSessionCloseError(error) {
|
|
22
|
+
return error instanceof Error && closeErrors.has(error);
|
|
23
|
+
}
|
|
19
24
|
class ResponseState {
|
|
20
25
|
ref;
|
|
21
26
|
done = new Flag();
|
|
@@ -109,6 +114,7 @@ export class RealtimeSession {
|
|
|
109
114
|
interruptOnSpeech: view.interruptOnSpeech ?? settings.turn_detection.interrupt_response,
|
|
110
115
|
toolResultTimeoutS: view.toolResultTimeoutS ?? null,
|
|
111
116
|
toolContinuationTimeoutS: view.toolContinuationTimeoutS ?? null,
|
|
117
|
+
turnDetection: view.turnDetection,
|
|
112
118
|
});
|
|
113
119
|
}
|
|
114
120
|
get info() {
|
|
@@ -151,8 +157,13 @@ export class RealtimeSession {
|
|
|
151
157
|
check() {
|
|
152
158
|
if (this.failure)
|
|
153
159
|
throw this.failure;
|
|
154
|
-
if (this.closed)
|
|
155
|
-
|
|
160
|
+
if (this.closed) {
|
|
161
|
+
// Reports can start after intentional close, before runner cleanup.
|
|
162
|
+
// Failed sessions retain failure and take the branch above.
|
|
163
|
+
const error = new RimeInputError("The realtime session is closed");
|
|
164
|
+
closeErrors.add(error);
|
|
165
|
+
throw error;
|
|
166
|
+
}
|
|
156
167
|
}
|
|
157
168
|
response(ref) {
|
|
158
169
|
this.check();
|
|
@@ -716,7 +727,9 @@ export class RealtimeSession {
|
|
|
716
727
|
if (!this.closing) {
|
|
717
728
|
this.closing = (async () => {
|
|
718
729
|
if (!this.closed) {
|
|
719
|
-
|
|
730
|
+
const error = new RimeStreamError("Realtime session closed");
|
|
731
|
+
closeErrors.add(error);
|
|
732
|
+
this.fail(error);
|
|
720
733
|
this.failure = null;
|
|
721
734
|
}
|
|
722
735
|
await this.socket.close();
|
package/dist/realtime/types.d.ts
CHANGED
|
@@ -19,13 +19,28 @@ export interface ToolDefinition {
|
|
|
19
19
|
readonly parameters: JsonObject;
|
|
20
20
|
readonly description?: string;
|
|
21
21
|
}
|
|
22
|
+
export type TurnEagerness = "low" | "medium" | "high" | "auto";
|
|
23
|
+
/** Omitted fields use the server's turn timing defaults. */
|
|
24
|
+
export interface TurnDetection {
|
|
25
|
+
/** How readily Prism decides that the user has finished speaking. */
|
|
26
|
+
readonly eagerness?: TurnEagerness;
|
|
27
|
+
/** Maximum silence before ending a turn: an integer from 1000 to 10000 ms. */
|
|
28
|
+
readonly endOfTurnTimeoutMs?: number;
|
|
29
|
+
}
|
|
30
|
+
/** Effective server settings. Null means the server did not publish the field. */
|
|
31
|
+
export interface TurnDetectionInfo {
|
|
32
|
+
readonly eagerness: TurnEagerness | "unknown" | null;
|
|
33
|
+
readonly endOfTurnTimeoutMs: number | null;
|
|
34
|
+
}
|
|
22
35
|
export interface RealtimeConnectOptions extends RealtimeOperationOptions {
|
|
23
|
-
|
|
36
|
+
/** Defaults to wss://prism.api.rime.ai/v1/realtime. */
|
|
37
|
+
readonly endpoint?: string;
|
|
24
38
|
readonly model?: "prism";
|
|
25
39
|
readonly voice?: string | null;
|
|
26
40
|
readonly instructions?: string | null;
|
|
27
41
|
readonly tools?: readonly ToolDefinition[];
|
|
28
42
|
readonly interruptOnSpeech?: boolean;
|
|
43
|
+
readonly turnDetection?: TurnDetection;
|
|
29
44
|
readonly timeouts?: RealtimeTimeouts;
|
|
30
45
|
}
|
|
31
46
|
export interface ReplyOptions extends RealtimeOperationOptions {
|
|
@@ -60,6 +75,7 @@ export interface SessionInfo {
|
|
|
60
75
|
readonly interruptOnSpeech: boolean;
|
|
61
76
|
readonly toolResultTimeoutS: number | null;
|
|
62
77
|
readonly toolContinuationTimeoutS: number | null;
|
|
78
|
+
readonly turnDetection: TurnDetectionInfo;
|
|
63
79
|
}
|
|
64
80
|
export interface RealtimeFault {
|
|
65
81
|
readonly code: string;
|
package/dist/stt/client.js
CHANGED
|
@@ -69,6 +69,10 @@ export class STT {
|
|
|
69
69
|
if (!source || typeof source[Symbol.asyncIterator] !== "function")
|
|
70
70
|
throw new RimeInputError("audio must be an async iterable of Uint8Array chunks");
|
|
71
71
|
validateOptions(options);
|
|
72
|
+
const outputContract = options.outputContract ?? "revised_hypotheses";
|
|
73
|
+
if (outputContract !== "revised_hypotheses" &&
|
|
74
|
+
outputContract !== "append_fragments")
|
|
75
|
+
throw new RimeInputError("outputContract must be revised_hypotheses or append_fragments");
|
|
72
76
|
const mode = options.mode ?? "written";
|
|
73
77
|
const terms = options.contextTerms ?? [];
|
|
74
78
|
const format = resolvePCMFormat(options.inputFormat);
|
|
@@ -79,6 +83,7 @@ export class STT {
|
|
|
79
83
|
prepare: (signal) => this.prepare(signal, target),
|
|
80
84
|
}, source, {
|
|
81
85
|
language: options.language,
|
|
86
|
+
outputContract,
|
|
82
87
|
mode,
|
|
83
88
|
contextTerms: [...terms],
|
|
84
89
|
inputFormat: format,
|
package/dist/stt/policy.js
CHANGED
package/dist/stt/protocol.d.ts
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import * as schema from "@rimelabs/api";
|
|
2
|
-
import type { TranscriptionPartial, TranscriptionFinal } from "./types.js";
|
|
2
|
+
import type { TranscriptionPartial, TranscriptionDelta, TranscriptionFinal, TranscriptionOutputContract } from "./types.js";
|
|
3
3
|
/** Validate the wire sequence before exposing any final result. */
|
|
4
4
|
export declare class TranscriptState {
|
|
5
5
|
private readonly textLimit;
|
|
6
|
+
private readonly outputContract;
|
|
6
7
|
language: string | null;
|
|
7
8
|
private revision;
|
|
8
9
|
private text;
|
|
9
10
|
private final;
|
|
10
|
-
constructor(textLimit: number);
|
|
11
|
-
accept(message: schema.StreamingTranscriptionResponse, inputDone: boolean): TranscriptionPartial | null;
|
|
11
|
+
constructor(textLimit: number, outputContract?: TranscriptionOutputContract);
|
|
12
|
+
accept(message: schema.StreamingTranscriptionResponse, inputDone: boolean): TranscriptionPartial | TranscriptionDelta | null;
|
|
12
13
|
finish(): TranscriptionFinal;
|
|
13
14
|
}
|
package/dist/stt/protocol.js
CHANGED
|
@@ -3,12 +3,14 @@ import { RimeStreamError, RimeResourceLimitError } from "../errors.js";
|
|
|
3
3
|
/** Validate the wire sequence before exposing any final result. */
|
|
4
4
|
export class TranscriptState {
|
|
5
5
|
textLimit;
|
|
6
|
+
outputContract;
|
|
6
7
|
language = null;
|
|
7
8
|
revision = 0n;
|
|
8
9
|
text = "";
|
|
9
10
|
final = null;
|
|
10
|
-
constructor(textLimit) {
|
|
11
|
+
constructor(textLimit, outputContract = "revised_hypotheses") {
|
|
11
12
|
this.textLimit = textLimit;
|
|
13
|
+
this.outputContract = outputContract;
|
|
12
14
|
}
|
|
13
15
|
accept(message, inputDone) {
|
|
14
16
|
const payload = message.payload;
|
|
@@ -21,7 +23,9 @@ export class TranscriptState {
|
|
|
21
23
|
if (this.language !== null)
|
|
22
24
|
throw new RimeStreamError("Received duplicate transcription acceptance");
|
|
23
25
|
if (accepted.outputContract !==
|
|
24
|
-
|
|
26
|
+
(this.outputContract === "append_fragments"
|
|
27
|
+
? schema.StreamingOutputContract.APPEND_FRAGMENTS
|
|
28
|
+
: schema.StreamingOutputContract.REVISED_HYPOTHESES))
|
|
25
29
|
throw new RimeStreamError("The service accepted a different transcript contract");
|
|
26
30
|
if (!accepted.language?.tag || !accepted.language.source)
|
|
27
31
|
throw new RimeStreamError("The service did not confirm a resolved language");
|
|
@@ -30,8 +34,19 @@ export class TranscriptState {
|
|
|
30
34
|
}
|
|
31
35
|
if (this.language === null)
|
|
32
36
|
throw new RimeStreamError("Received a transcript before acceptance");
|
|
37
|
+
const append = this.outputContract === "append_fragments";
|
|
38
|
+
if (payload.case === "delta" && append) {
|
|
39
|
+
const text = payload.value.text;
|
|
40
|
+
if (Buffer.byteLength(this.text) + Buffer.byteLength(text) >
|
|
41
|
+
this.textLimit)
|
|
42
|
+
throw new RimeResourceLimitError("The transcript exceeds the SDK text limit");
|
|
43
|
+
this.text += text;
|
|
44
|
+
return { kind: "delta", text };
|
|
45
|
+
}
|
|
33
46
|
if (payload.case !== "hypothesis" && payload.case !== "done")
|
|
34
47
|
throw new RimeStreamError("Received an unexpected transcription response");
|
|
48
|
+
if (payload.case === "hypothesis" && append)
|
|
49
|
+
throw new RimeStreamError("Received an unexpected transcription response");
|
|
35
50
|
const value = payload.value;
|
|
36
51
|
if (Buffer.byteLength(value.text, "utf8") > this.textLimit)
|
|
37
52
|
throw new RimeResourceLimitError("The transcript exceeds the SDK text limit");
|
|
@@ -46,7 +61,7 @@ export class TranscriptState {
|
|
|
46
61
|
if (!inputDone)
|
|
47
62
|
throw new RimeStreamError("The service completed before input finished");
|
|
48
63
|
if (value.revision !== this.revision || value.text !== this.text)
|
|
49
|
-
throw new RimeStreamError("Final transcript does not match
|
|
64
|
+
throw new RimeStreamError("Final transcript does not match prior updates");
|
|
50
65
|
const language = payload.value.language;
|
|
51
66
|
if (language?.tag !== this.language || !language.source)
|
|
52
67
|
throw new RimeStreamError("Final language does not match the accepted language");
|
package/dist/stt/stream.js
CHANGED
|
@@ -130,7 +130,7 @@ export class TranscriptStream {
|
|
|
130
130
|
}
|
|
131
131
|
}
|
|
132
132
|
async read() {
|
|
133
|
-
const state = new TranscriptState(policy.transcriptBytes);
|
|
133
|
+
const state = new TranscriptState(policy.transcriptBytes, this.options.outputContract);
|
|
134
134
|
for await (const message of this.call.responses()) {
|
|
135
135
|
const update = state.accept(message, this.inputDone);
|
|
136
136
|
if (state.language !== null) {
|
|
@@ -149,7 +149,7 @@ export class TranscriptStream {
|
|
|
149
149
|
this.controller.signal.throwIfAborted();
|
|
150
150
|
this.call = new TranscriptionCall(prepared.client, prepared.metadata, this.controller.signal);
|
|
151
151
|
this.admission = this.deadline(policy.acceptanceTimeout, "Transcription acceptance timed out");
|
|
152
|
-
await this.call.start(this.options.language, this.options.mode ?? "written", this.options.contextTerms ?? []);
|
|
152
|
+
await this.call.start(this.options.language, this.options.mode ?? "written", this.options.contextTerms ?? [], this.options.outputContract);
|
|
153
153
|
const producer = this.produce(), reader = this.read();
|
|
154
154
|
tasks = [producer, reader];
|
|
155
155
|
const [, final] = await Promise.all([producer, reader]);
|
package/dist/stt/transport.d.ts
CHANGED
|
@@ -10,6 +10,6 @@ export declare const transport: {
|
|
|
10
10
|
export declare function transcribe(client: grpc.Client, auth: grpc.Metadata, audio: Uint8Array, options: TranscribeOptions, signal: AbortSignal): Promise<Transcription>;
|
|
11
11
|
export declare class TranscriptionCall extends BidiCall<schema.StreamingTranscriptionRequest, schema.StreamingTranscriptionResponse> {
|
|
12
12
|
constructor(client: grpc.Client, auth: grpc.Metadata, signal: AbortSignal);
|
|
13
|
-
start(language: string, mode: TranscriptionMode, contextTerms: readonly string[]): Promise<void>;
|
|
13
|
+
start(language: string, mode: TranscriptionMode, contextTerms: readonly string[], outputContract?: import("./types.js").TranscriptionOutputContract): Promise<void>;
|
|
14
14
|
writeAudio(audio: Uint8Array): Promise<void>;
|
|
15
15
|
}
|
package/dist/stt/transport.js
CHANGED
|
@@ -69,7 +69,7 @@ export class TranscriptionCall extends BidiCall {
|
|
|
69
69
|
const method = service.method.transcribeStreaming;
|
|
70
70
|
super(client.makeBidiStreamRequest(`/${service.typeName}/${method.name}`, serializer(method.input), deserializer(method.output), auth), signal);
|
|
71
71
|
}
|
|
72
|
-
start(language, mode, contextTerms) {
|
|
72
|
+
start(language, mode, contextTerms, outputContract = "revised_hypotheses") {
|
|
73
73
|
return this.write(create(schema.StreamingTranscriptionRequestSchema, {
|
|
74
74
|
payload: {
|
|
75
75
|
case: "config",
|
|
@@ -79,7 +79,9 @@ export class TranscriptionCall extends BidiCall {
|
|
|
79
79
|
? schema.TranscriptionMode.VERBATIM
|
|
80
80
|
: schema.TranscriptionMode.WRITTEN,
|
|
81
81
|
contextTerms: [...contextTerms],
|
|
82
|
-
outputContract:
|
|
82
|
+
outputContract: outputContract === "append_fragments"
|
|
83
|
+
? schema.StreamingOutputContract.APPEND_FRAGMENTS
|
|
84
|
+
: schema.StreamingOutputContract.REVISED_HYPOTHESES,
|
|
83
85
|
},
|
|
84
86
|
},
|
|
85
87
|
}));
|
package/dist/stt/types.d.ts
CHANGED
|
@@ -3,7 +3,16 @@ import type { PCMFormat } from "../pcm.js";
|
|
|
3
3
|
export type TranscriptionMode = "written" | "verbatim";
|
|
4
4
|
/** One caller-ended utterance of headerless signed PCM16 little-endian audio. */
|
|
5
5
|
export type AudioSource = AsyncIterable<Uint8Array>;
|
|
6
|
+
/** Select replacement hypotheses (default) or permanent append fragments. */
|
|
7
|
+
export type TranscriptionOutputContract = "revised_hypotheses" | "append_fragments";
|
|
8
|
+
export interface TranscriptionDelta {
|
|
9
|
+
readonly kind: "delta";
|
|
10
|
+
/** Append exactly, preserving the server's newline separators. */
|
|
11
|
+
readonly text: string;
|
|
12
|
+
}
|
|
6
13
|
export interface TranscriptionOptions {
|
|
14
|
+
/** Defaults to revised_hypotheses. Each stream uses one fixed contract. */
|
|
15
|
+
outputContract?: TranscriptionOutputContract;
|
|
7
16
|
/** Required BCP-47 tag, passed unchanged to the service (for example "en"). */
|
|
8
17
|
language: string;
|
|
9
18
|
/** Formatting intent; defaults to written and does not guarantee exact tokens. */
|
|
@@ -31,9 +40,9 @@ export interface TranscriptionFinal {
|
|
|
31
40
|
/** Canonical BCP-47 tag selected by the service, for example "en". */
|
|
32
41
|
readonly language: string;
|
|
33
42
|
}
|
|
34
|
-
export type TranscriptionUpdate = TranscriptionPartial | TranscriptionFinal;
|
|
43
|
+
export type TranscriptionUpdate = TranscriptionPartial | TranscriptionDelta | TranscriptionFinal;
|
|
35
44
|
/** Options for a complete WAV recording. Audio is sent unchanged. */
|
|
36
|
-
export type TranscribeOptions = Omit<TranscriptionOptions, "inputFormat">;
|
|
45
|
+
export type TranscribeOptions = Omit<TranscriptionOptions, "inputFormat" | "outputContract">;
|
|
37
46
|
/** A successful whole-recording transcription. Silence may produce empty text. */
|
|
38
47
|
export interface Transcription {
|
|
39
48
|
readonly text: string;
|
package/docs/realtime.md
CHANGED
|
@@ -10,23 +10,159 @@ microphone capture, tool execution, and playback.
|
|
|
10
10
|
Start with the [recorded-speech quick start](../README.md#prism-send-recorded-speech)
|
|
11
11
|
to save a spoken reply without audio hardware.
|
|
12
12
|
|
|
13
|
-
For live conversations, keep the session open after
|
|
14
|
-
|
|
13
|
+
For live conversations, keep the session open after a response is cancelled.
|
|
14
|
+
Report playback from the output clock. Saving a file is not playback.
|
|
15
15
|
Typed turns also support mixed text and voice applications.
|
|
16
16
|
|
|
17
|
+
## Optional conversation runner
|
|
18
|
+
|
|
19
|
+
Use `runConversation` to connect your audio adapters once. It sends input,
|
|
20
|
+
queues output, handles interruption, reports measured playback, and stops its
|
|
21
|
+
operations when the conversation ends. The direct session methods remain available.
|
|
22
|
+
|
|
23
|
+
```ts
|
|
24
|
+
import { Rime, runConversation } from "@rimelabs/sdk";
|
|
25
|
+
|
|
26
|
+
const client = new Rime();
|
|
27
|
+
const controller = new AbortController();
|
|
28
|
+
try {
|
|
29
|
+
const session = await client.realtime.connect();
|
|
30
|
+
// microphone and speaker are adapters owned by your application.
|
|
31
|
+
await runConversation(session, {
|
|
32
|
+
source: microphone,
|
|
33
|
+
playback: speaker,
|
|
34
|
+
onEvent(event) {
|
|
35
|
+
if (event.payload.kind === "transcript.final")
|
|
36
|
+
console.log(event.payload.text);
|
|
37
|
+
},
|
|
38
|
+
signal: controller.signal,
|
|
39
|
+
});
|
|
40
|
+
} finally {
|
|
41
|
+
await client.close();
|
|
42
|
+
}
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
The input adapter implements `ConversationSource`.
|
|
46
|
+
`read(signal: AbortSignal): Promise<AudioChunk | null>` returns one chunk,
|
|
47
|
+
with an optional format, or `null` at EOF. An omitted format uses 16 kHz mono
|
|
48
|
+
PCM16. Abort must reject and release a pending
|
|
49
|
+
read. The runner awaits each `session.sendAudio` call before reading again. It
|
|
50
|
+
uses the session's existing PCM conversion and transport backpressure. Input
|
|
51
|
+
continues during output playback.
|
|
52
|
+
|
|
53
|
+
The playback adapter implements `ConversationPlayback`:
|
|
54
|
+
|
|
55
|
+
| Method | Contract |
|
|
56
|
+
| --- | --- |
|
|
57
|
+
| `open(output: OutputRef): ConversationOutput` | Queue this output after previously opened outputs. It can be opened before the first audio chunk. |
|
|
58
|
+
| `interrupt(outputs: readonly OutputRef[]): void` | Stop the selected outputs as one batch, discard their unplayed audio, then resume remaining output in queue order. |
|
|
59
|
+
| `stop(signal: AbortSignal): Promise<void>` | Confirm all playback has stopped and queued audio is discarded. Keep caller-owned resources open. |
|
|
60
|
+
|
|
61
|
+
Each `ConversationOutput` implements:
|
|
62
|
+
|
|
63
|
+
| Method | Contract |
|
|
64
|
+
| --- | --- |
|
|
65
|
+
| `write(chunk: AudioChunk): void` | Queue 24 kHz mono PCM16 without blocking. Bound the queue and throw on overflow. |
|
|
66
|
+
| `end(): void` | Accept no more audio. Let queued and device audio drain. |
|
|
67
|
+
| `completion(signal: AbortSignal): Promise<number>` | Return actual played milliseconds after drain or interruption. Reject if measurement fails or the signal aborts. |
|
|
68
|
+
|
|
69
|
+
All synchronous adapter methods must return promptly. Completion can be awaited
|
|
70
|
+
before `end()`. Keep receipts separate for each `OutputRef`, even when a new
|
|
71
|
+
response starts before an old receipt completes. Play outputs in the order they
|
|
72
|
+
were opened, using one FIFO audio queue. Do not mix outputs, interleave their
|
|
73
|
+
audio, or play later outputs before earlier outputs finish or are interrupted.
|
|
74
|
+
Prism assumes that interrupting a partially played reply leaves later replies
|
|
75
|
+
in the same turn unheard. Independent concurrent playback violates that assumption.
|
|
76
|
+
The runner passes all affected output references in one `interrupt` call. Stop
|
|
77
|
+
them and discard their queued audio before starting any remaining output. Then
|
|
78
|
+
resume valid output in queue order, without waiting for another chunk or event.
|
|
79
|
+
Interruption must work after `end()` and tolerate repeated calls. A remote
|
|
80
|
+
adapter must preserve the batch, for example with one message containing all
|
|
81
|
+
selected output IDs. Shutdown stops all output and must not resume playback.
|
|
82
|
+
Receipt delivery can be delayed or arrive out of order; audible playback cannot.
|
|
83
|
+
A receipt includes the device buffer where device timing is available. Bytes
|
|
84
|
+
received or queued do not measure playback. Zero is valid for an output that
|
|
85
|
+
played no audio.
|
|
86
|
+
|
|
87
|
+
Abort must release each pending adapter operation and remove its listeners.
|
|
88
|
+
Cancellation of a completion wait must not cancel the device receipt. The runner
|
|
89
|
+
stops input, event processing, and report submission on exit. It then awaits
|
|
90
|
+
`stop(signal)` before aborting remaining receipt waits. The stop signal is
|
|
91
|
+
independent of caller cancellation and has a five-second deadline. `stop` must
|
|
92
|
+
resolve only after local or remote playback has stopped, including device
|
|
93
|
+
buffers. Sending a stop command is not confirmation. It must reject and release
|
|
94
|
+
its wait when the stop signal aborts. The runner awaits this cancellation cleanup
|
|
95
|
+
before it returns, so cleanup can extend beyond the deadline. A missing confirmation raises
|
|
96
|
+
`RimeTimeoutError`; disconnects and other shutdown failures must reject too.
|
|
97
|
+
The caller opens and closes the adapter resources. The runner does not open
|
|
98
|
+
audio devices or host a browser.
|
|
99
|
+
|
|
100
|
+
`onEvent` is optional. It receives every event read from `session.events`, in
|
|
101
|
+
order, after the runner applies audio coordination. It must be synchronous and
|
|
102
|
+
nonblocking. Returning a promise is rejected. Callback exceptions reach the
|
|
103
|
+
caller. The callback can schedule application work. Continue to use
|
|
104
|
+
`submitToolResult` and `continueReply`; the runner neither registers nor executes
|
|
105
|
+
tools. The application must cancel and await its own tool tasks. Response
|
|
106
|
+
failures and nonterminal faults are forwarded for application handling. Session
|
|
107
|
+
failures reject the runner promise.
|
|
108
|
+
|
|
109
|
+
The runner is the only event reader. Do not consume `session.events` at the same
|
|
110
|
+
time, call `reportPlayback` independently, or send a second audio source. Other
|
|
111
|
+
session methods remain available for manual tools and application actions.
|
|
112
|
+
|
|
113
|
+
On confirmed `speech.started`, the runner uses the effective
|
|
114
|
+
`session.info.interruptOnSpeech` setting. It stops affected output, waits for the
|
|
115
|
+
interrupted receipt, and reports the measured position. It ignores late audio
|
|
116
|
+
for that response. A `speech.hint` alone does not stop playback. A server
|
|
117
|
+
`truncate_not_current` refusal is forwarded as an event; it does not stop a newer
|
|
118
|
+
response. Finished reports wait for generation completion and every output
|
|
119
|
+
receipt, including outstanding playback from earlier responses. Prism treats
|
|
120
|
+
the latest drain token as confirmation that all earlier playback has stopped.
|
|
121
|
+
An abandoned response also stops its dependent tool continuations, identified
|
|
122
|
+
by response parent references. Unrelated later turns remain eligible to play.
|
|
123
|
+
Late audio from an interrupted response or its continuations is discarded.
|
|
124
|
+
The runner retains at most 128 response parent references.
|
|
125
|
+
A report retains the measured duration of its own response. Waiting for earlier
|
|
126
|
+
playback does not block events or input, and does not wait for later responses.
|
|
127
|
+
|
|
128
|
+
Once called, the runner owns the session's active lifetime. Input EOF, explicit
|
|
129
|
+
session closure, abort, or failure stops all runner operations and closes the
|
|
130
|
+
session. EOF stops remaining playback and waits for shutdown confirmation.
|
|
131
|
+
Abort, including an abort received during cleanup, waits for playback shutdown
|
|
132
|
+
confirmation or its five-second deadline, then awaits adapter cancellation cleanup
|
|
133
|
+
before rejecting with `signal.reason`.
|
|
134
|
+
Explicit session closure ends normally even with playback reports pending;
|
|
135
|
+
remote connection failures reject with the session error.
|
|
136
|
+
Expected cancellation errors during cleanup are ignored. Other cleanup errors
|
|
137
|
+
reach the caller if there is no original failure. They do not replace an
|
|
138
|
+
existing failure or caller abort. The signal is optional. Use
|
|
139
|
+
an application deadline if an adapter can wait indefinitely.
|
|
140
|
+
|
|
141
|
+
The runner adds no input or callback queue. It retains at most 64 responses,
|
|
142
|
+
128 output streams, and 256 pending operations. Exceeding a limit throws
|
|
143
|
+
`RimeResourceLimitError`. The adapters must also bound their audio queues.
|
|
144
|
+
|
|
145
|
+
The [live voice example](https://github.com/rimelabs/rime-sdk/blob/main/typescript/examples/realtime/conversation.ts) supplies the
|
|
146
|
+
[browser audio adapter](https://github.com/rimelabs/rime-sdk/blob/main/typescript/examples/common/browser-audio.ts). Browser hosting
|
|
147
|
+
stays in the example. Its player uses `AudioContext.getOutputTimestamp()` for
|
|
148
|
+
receipts and shutdown confirmation. A disconnect cannot confirm a remote stop.
|
|
149
|
+
The [manual tool example](https://github.com/rimelabs/rime-sdk/blob/main/typescript/examples/realtime/manual-tools.ts) keeps
|
|
150
|
+
tool execution outside the runner.
|
|
151
|
+
|
|
17
152
|
## Connect
|
|
18
153
|
|
|
19
|
-
Use `await client.realtime.connect(
|
|
20
|
-
one conversation, and returns a `RealtimeSession`. Settings apply
|
|
21
|
-
new session to change them.
|
|
154
|
+
Use `await client.realtime.connect()` with `RIME_API_KEY` set. It opens a WebSocket,
|
|
155
|
+
initializes one conversation, and returns a `RealtimeSession`. Settings apply
|
|
156
|
+
once. Open a new session to change them.
|
|
22
157
|
|
|
23
158
|
| Option | Required? | Default and behavior |
|
|
24
159
|
| --- | --- | --- |
|
|
25
|
-
| `endpoint` |
|
|
160
|
+
| `endpoint` | No | Defaults to `wss://prism.api.rime.ai/v1/realtime`. Overrides must be full URLs; plain `ws://` is allowed only for localhost. |
|
|
26
161
|
| `voice` | If the server has no default voice | Omitted or `null` uses the server default. Supplied strings must be nonblank. |
|
|
27
162
|
| `instructions` | No | Omitted or `null` uses server defaults. Supplied strings must be nonblank. |
|
|
28
163
|
| `tools` | No | `[]`. Each tool requires `name` and a JSON Schema `parameters` object. `description` defaults to `""`. |
|
|
29
164
|
| `interruptOnSpeech` | No | `true`. Requests interruption when the server confirms user speech. |
|
|
165
|
+
| `turnDetection` | No | Omitted settings use the server's turn timing defaults. |
|
|
30
166
|
| `timeouts` | No | `{ connectS: 10, readyS: 30, requestS: 10 }`. Each field is optional. Units are seconds. |
|
|
31
167
|
| `model` | No | `"prism"`, the only accepted value. |
|
|
32
168
|
| `signal` | No | An `AbortSignal` that cancels connection setup. It does not cancel an established session. |
|
|
@@ -36,7 +172,8 @@ Credentials come from `new Rime({ apiKey: "..." })` or `RIME_API_KEY`. Prism use
|
|
|
36
172
|
voice, and timeout settings do not apply to Realtime.
|
|
37
173
|
|
|
38
174
|
After connection, `session.info` contains `sessionId`, `model`, `voice`,
|
|
39
|
-
`interruptOnSpeech`, `toolResultTimeoutS`, and
|
|
175
|
+
`interruptOnSpeech`, `turnDetection`, `toolResultTimeoutS`, and
|
|
176
|
+
`toolContinuationTimeoutS`.
|
|
40
177
|
Check the effective interruption setting there. Tool wait durations are total
|
|
41
178
|
server deadlines, not remaining time. A `null` duration can mean an omitted
|
|
42
179
|
field; it does not prove that no deadline exists.
|
|
@@ -45,6 +182,39 @@ Close the session in `finally` with `await session.close()`. Closing the parent
|
|
|
45
182
|
client closes all its TTS and Realtime operations. Both objects also support
|
|
46
183
|
`Symbol.asyncDispose`. Repeated close calls are safe.
|
|
47
184
|
|
|
185
|
+
## Optional turn timing
|
|
186
|
+
|
|
187
|
+
Pass a `TurnDetection` object to override how readily Prism ends a spoken turn.
|
|
188
|
+
Each field is optional; omitted fields use server defaults.
|
|
189
|
+
|
|
190
|
+
| Field | Values | Current server default |
|
|
191
|
+
| --- | --- | --- |
|
|
192
|
+
| `eagerness` | `"low"`, `"medium"`, `"high"`, `"auto"` | `"auto"`, equivalent to `"medium"` |
|
|
193
|
+
| `endOfTurnTimeoutMs` | Integer from 1,000 through 10,000 | 5,000 ms |
|
|
194
|
+
|
|
195
|
+
`"low"` waits for more certainty that the user has finished. `"high"` answers
|
|
196
|
+
sooner, with more risk of cutting in. The timeout is a maximum silence limit,
|
|
197
|
+
not a fixed delay before each reply. If the server falls back to its local
|
|
198
|
+
voice activity detector, it ends turns after 500 ms of silence instead.
|
|
199
|
+
|
|
200
|
+
```typescript
|
|
201
|
+
const session = await client.realtime.connect({
|
|
202
|
+
turnDetection: { eagerness: "low", endOfTurnTimeoutMs: 5000 },
|
|
203
|
+
});
|
|
204
|
+
try {
|
|
205
|
+
console.log(session.info.turnDetection);
|
|
206
|
+
// Send input and consume session.events here.
|
|
207
|
+
} finally {
|
|
208
|
+
await session.close();
|
|
209
|
+
}
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
`session.info.turnDetection` is a `TurnDetectionInfo` with the settings the
|
|
213
|
+
server reported in its initialization acknowledgment. A missing field is
|
|
214
|
+
`null`; the SDK does not substitute a requested value or assume a server
|
|
215
|
+
default. An unrecognized server eagerness becomes `"unknown"`.
|
|
216
|
+
`interruptOnSpeech` remains a separate connection option.
|
|
217
|
+
|
|
48
218
|
## Choose an operation
|
|
49
219
|
|
|
50
220
|
Every method returns a promise. Arguments without `?` are required.
|
package/docs/stt.md
CHANGED
|
@@ -63,7 +63,8 @@ try {
|
|
|
63
63
|
});
|
|
64
64
|
for await (const update of transcript) {
|
|
65
65
|
if (update.kind === "partial") console.log("Current transcript:", update.text);
|
|
66
|
-
else
|
|
66
|
+
else if (update.kind === "final")
|
|
67
|
+
console.log("Final:", update.text, "Language:", update.language);
|
|
67
68
|
}
|
|
68
69
|
console.log("Request:", transcript.requestId);
|
|
69
70
|
} finally {
|
|
@@ -76,6 +77,37 @@ without loading it all into memory. Use Node.js 22 or later; browsers are not su
|
|
|
76
77
|
Open files inside the async generator so file errors are reported through the
|
|
77
78
|
SDK and resources are opened only after the service accepts the utterance.
|
|
78
79
|
|
|
80
|
+
## Append-fragment streaming
|
|
81
|
+
|
|
82
|
+
Choose `outputContract: "append_fragments"` on
|
|
83
|
+
`client.stt.stream`. The default is `"revised_hypotheses"`; `transcribe()` does not
|
|
84
|
+
accept an output contract. Both streaming modes use gRPC only.
|
|
85
|
+
|
|
86
|
+
```javascript
|
|
87
|
+
let text = "";
|
|
88
|
+
const transcript = client.stt.stream(audioChunks(), {
|
|
89
|
+
language: "en", outputContract: "append_fragments",
|
|
90
|
+
});
|
|
91
|
+
for await (const update of transcript) {
|
|
92
|
+
if (update.kind === "delta") text += update.text;
|
|
93
|
+
else if (update.kind === "final") console.log(update.text);
|
|
94
|
+
}
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
`TranscriptionDelta` is a permanent fragment, not a replacement partial. The
|
|
98
|
+
server recognizes independent five-second audio windows; fragments after the
|
|
99
|
+
first include newline separators. Do not strip, add separators, or drop deltas.
|
|
100
|
+
The final text must match their concatenation. Empty speech can produce no
|
|
101
|
+
updates before an empty final. A final is delivered only after `done` and an OK
|
|
102
|
+
gRPC status, so a connection failure never promotes accumulated fragments to a
|
|
103
|
+
successful result.
|
|
104
|
+
|
|
105
|
+
Append mode suits longer recordings and consumers that cannot revise text. Send
|
|
106
|
+
recordings as fast as backpressure permits. It does not enable unlimited-length
|
|
107
|
+
streams: the service's 16 MiB audio and 120-second total-call limits still apply.
|
|
108
|
+
Split larger recordings into separate streams. Live voice agents usually prefer
|
|
109
|
+
revised hypotheses for earlier provisional text and corrections across windows.
|
|
110
|
+
|
|
79
111
|
## Options
|
|
80
112
|
|
|
81
113
|
| Option | Meaning |
|
|
@@ -83,6 +115,7 @@ SDK and resources are opened only after the service accepts the utterance.
|
|
|
83
115
|
| `language` | Required spoken BCP-47 tag. Passed unchanged to the service; no detection or SDK language allowlist. The deployed catalog currently supports English and Spanish. |
|
|
84
116
|
| `mode` | `"written"` (default) or `"verbatim"`. Formatting intent, not a guarantee of exact tokens. |
|
|
85
117
|
| `contextTerms` | Sequence of recognition hints, copied unchanged and in order. The service validates terms and prompt budgets. |
|
|
118
|
+
| `outputContract` | `"revised_hypotheses"` (default) or `"append_fragments"`. Streaming only. |
|
|
86
119
|
| `inputFormat` | `PCMFormat`: signed little-endian PCM16 at 8, 16, 24 or 48 kHz, mono or interleaved stereo. Defaults to 16 kHz mono. |
|
|
87
120
|
| `timeout` | Optional positive overall timeout in seconds. Includes connection time, source pauses and consumer pauses. `null` disables this SDK deadline, but not internal or server deadlines. |
|
|
88
121
|
|
|
@@ -98,11 +131,18 @@ There is no STT voice/model selector; the service selects its recognizer.
|
|
|
98
131
|
|
|
99
132
|
## Results and completion
|
|
100
133
|
|
|
101
|
-
`TranscriptionPartial` and `TranscriptionFinal` are discriminated
|
|
134
|
+
`TranscriptionPartial`, `TranscriptionDelta`, and `TranscriptionFinal` are discriminated types exported from `@rimelabs/sdk`. In the default revised-hypotheses mode, every update contains the complete current `text`.
|
|
102
135
|
**Replace the previous transcript. Do not concatenate partials.** Later partials
|
|
103
136
|
can change earlier words. `TranscriptionFinal.language` is the canonical language
|
|
104
137
|
selected by the service.
|
|
105
138
|
|
|
139
|
+
When upgrading, `TranscriptionUpdate` now includes `TranscriptionDelta`. Update
|
|
140
|
+
exhaustive handlers to handle `kind: "delta"`, even if you keep the default
|
|
141
|
+
revised-hypotheses contract. Only finals have `language`: check
|
|
142
|
+
`update.kind === "final"` before reading it, rather than assuming every
|
|
143
|
+
non-partial update is final. Append deltas exactly; replace text with partials
|
|
144
|
+
or the confirmed final.
|
|
145
|
+
|
|
106
146
|
The source is not consumed until the server accepts the configuration. Audio
|
|
107
147
|
upload and transcript reading run concurrently. Decoding currently uses cumulative
|
|
108
148
|
two-second audio windows, so a partial is not promised for each audio packet.
|
|
@@ -122,8 +162,12 @@ Pass an `AbortSignal` through the stream's `signal` option to cancel just that
|
|
|
122
162
|
utterance. Client close cancels all STT, TTS and realtime operations; cancelling
|
|
123
163
|
one stream leaves its siblings running.
|
|
124
164
|
|
|
125
|
-
The SDK bounds the transcript queue to 16 updates
|
|
126
|
-
|
|
165
|
+
The SDK bounds the transcript queue to 16 updates. In revised-hypotheses mode,
|
|
166
|
+
each replacement update is limited to 1 MiB of UTF-8 text; sizes are not summed
|
|
167
|
+
across revisions. In append-fragments mode, the concatenated text across all
|
|
168
|
+
deltas is limited to 1 MiB. The final transcript is also limited to 1 MiB in both
|
|
169
|
+
modes. gRPC responses are bounded to 1 MiB plus 64 KiB for the protobuf envelope.
|
|
170
|
+
Slow consumers apply backpressure.
|
|
127
171
|
Internal connection and acceptance waits are 10 seconds; completion after source
|
|
128
172
|
exhaustion is bounded to 120 seconds. The completion timer stops after successful
|
|
129
173
|
gRPC completion. The optional overall timeout remains active until the final
|