@rimelabs/sdk 0.1.0-alpha.17 → 0.1.0-alpha.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +3 -14
- package/dist/client.d.ts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/realtime/client.d.ts +1 -1
- package/dist/realtime/client.js +21 -2
- package/dist/realtime/protocol.d.ts +3 -0
- package/dist/realtime/protocol.js +25 -0
- package/dist/realtime/session.js +1 -0
- package/dist/realtime/types.d.ts +17 -1
- package/dist/stt/client.js +5 -0
- package/dist/stt/policy.js +2 -2
- package/dist/stt/protocol.d.ts +4 -3
- package/dist/stt/protocol.js +18 -3
- package/dist/stt/stream.js +2 -2
- package/dist/stt/transport.d.ts +1 -1
- package/dist/stt/transport.js +4 -2
- package/dist/stt/types.d.ts +11 -2
- package/docs/realtime.md +40 -5
- package/docs/stt.md +48 -4
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.1.0-alpha.18](https://github.com/rimelabs/rime-sdk/compare/typescript-v0.1.0-alpha.17...typescript-v0.1.0-alpha.18) (2026-10-21)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### ⚠ BREAKING CHANGES
|
|
7
|
+
|
|
8
|
+
* Rust TranscriptionUpdate and TypeScript TranscriptionUpdate include Delta/delta; exhaustive handlers must handle the new variant.
|
|
9
|
+
|
|
10
|
+
### Features
|
|
11
|
+
|
|
12
|
+
* add append-fragment STT streaming across SDKs ([369f04d](https://github.com/rimelabs/rime-sdk/commit/369f04dadfea9d3d8593d14283d64e3a1f36e4d1))
|
|
13
|
+
* default realtime connections to public Prism and add optional turn timing controls ([#86](https://github.com/rimelabs/rime-sdk/issues/86)) ([0e07f68](https://github.com/rimelabs/rime-sdk/commit/0e07f6852bf0cefb60b8a42c48c2b76f95ac2362))
|
|
14
|
+
|
|
3
15
|
## [0.1.0-alpha.17](https://github.com/rimelabs/rime-sdk/compare/typescript-v0.1.0-alpha.16...typescript-v0.1.0-alpha.17) (2026-10-16)
|
|
4
16
|
|
|
5
17
|
|
package/README.md
CHANGED
|
@@ -28,16 +28,10 @@ STT use the SDK and Node.js built-ins.
|
|
|
28
28
|
|
|
29
29
|
## Prism: send recorded speech
|
|
30
30
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
31
|
+
With `RIME_API_KEY` set, the SDK connects to
|
|
32
|
+
`wss://prism.api.rime.ai/v1/realtime` and uses the server's default voice.
|
|
33
|
+
Pass `endpoint` or `voice` to select another deployment or voice.
|
|
34
34
|
|
|
35
|
-
```sh
|
|
36
|
-
export PRISM_URL="wss://your-prism-host/v1/realtime"
|
|
37
|
-
export PRISM_VOICE="your-deployment-voice"
|
|
38
|
-
```
|
|
39
|
-
|
|
40
|
-
You can omit `PRISM_VOICE` if the deployment has a default voice.
|
|
41
35
|
Record a short question and export it as `question.wav`, using **16 kHz, mono, signed 16-bit PCM WAV**.
|
|
42
36
|
Save the following as `prism.mjs` beside that file. It sends the recording at capture speed,
|
|
43
37
|
then sends silence so Prism can detect the end of speech. The spoken reply goes to `reply.wav`.
|
|
@@ -61,9 +55,6 @@ if (
|
|
|
61
55
|
const data = Buffer.from(input.data.samples);
|
|
62
56
|
if (!data.length || data.length % 2)
|
|
63
57
|
throw new Error("question.wav has no complete PCM16 audio");
|
|
64
|
-
const endpoint = process.env.PRISM_URL;
|
|
65
|
-
if (!endpoint)
|
|
66
|
-
throw new Error("Set PRISM_URL to your deployment's /v1/realtime URL");
|
|
67
58
|
|
|
68
59
|
const client = new Rime();
|
|
69
60
|
const stop = new AbortController();
|
|
@@ -75,8 +66,6 @@ let sending;
|
|
|
75
66
|
let reading;
|
|
76
67
|
try {
|
|
77
68
|
const session = await client.realtime.connect({
|
|
78
|
-
endpoint,
|
|
79
|
-
voice: process.env.PRISM_VOICE,
|
|
80
69
|
instructions: "Give short, clear answers.",
|
|
81
70
|
});
|
|
82
71
|
sending = (async () => {
|
package/dist/client.d.ts
CHANGED
|
@@ -23,7 +23,7 @@ export declare class Rime {
|
|
|
23
23
|
};
|
|
24
24
|
private readonly realtimeClient;
|
|
25
25
|
readonly realtime: {
|
|
26
|
-
connect: (options
|
|
26
|
+
connect: (options?: RealtimeConnectOptions) => Promise<RealtimeSession>;
|
|
27
27
|
};
|
|
28
28
|
private closed;
|
|
29
29
|
private closing;
|
package/dist/index.d.ts
CHANGED
|
@@ -7,4 +7,4 @@ export * from "./errors.js";
|
|
|
7
7
|
export { RealtimeSession } from "./realtime/session.js";
|
|
8
8
|
export * from "./realtime/types.js";
|
|
9
9
|
export { TranscriptStream } from "./stt/stream.js";
|
|
10
|
-
export type { AudioSource, TranscribeOptions, Transcription, TranscriptionOptions, TranscriptionMode, TranscriptionPartial, TranscriptionFinal, TranscriptionUpdate, } from "./stt/types.js";
|
|
10
|
+
export type { AudioSource, TranscribeOptions, Transcription, TranscriptionOptions, TranscriptionMode, TranscriptionPartial, TranscriptionDelta, TranscriptionOutputContract, TranscriptionFinal, TranscriptionUpdate, } from "./stt/types.js";
|
|
@@ -7,6 +7,6 @@ export declare class Realtime {
|
|
|
7
7
|
private readonly sessions;
|
|
8
8
|
private readonly opening;
|
|
9
9
|
constructor(credentials: Credentials, checkOpen: () => void);
|
|
10
|
-
connect(options
|
|
10
|
+
connect(options?: RealtimeConnectOptions): Promise<RealtimeSession>;
|
|
11
11
|
close(): Promise<void>;
|
|
12
12
|
}
|
package/dist/realtime/client.js
CHANGED
|
@@ -3,6 +3,7 @@ import { RimeCancelledError, RimeInputError, RimeUnavailableError, } from "../er
|
|
|
3
3
|
import { cancellation, scope } from "./async.js";
|
|
4
4
|
import { RealtimeSession } from "./session.js";
|
|
5
5
|
import { RealtimeConnection } from "./transport.js";
|
|
6
|
+
const DEFAULT_ENDPOINT = "wss://prism.api.rime.ai/v1/realtime";
|
|
6
7
|
function endpoint(value) {
|
|
7
8
|
let url;
|
|
8
9
|
try {
|
|
@@ -56,9 +57,9 @@ export class Realtime {
|
|
|
56
57
|
this.credentials = credentials;
|
|
57
58
|
this.checkOpen = checkOpen;
|
|
58
59
|
}
|
|
59
|
-
async connect(options) {
|
|
60
|
+
async connect(options = {}) {
|
|
60
61
|
this.checkOpen();
|
|
61
|
-
const address = endpoint(options.endpoint);
|
|
62
|
+
const address = endpoint(options.endpoint === undefined ? DEFAULT_ENDPOINT : options.endpoint);
|
|
62
63
|
const limits = timeouts(options.timeouts);
|
|
63
64
|
if ((options.model ?? "prism") !== "prism")
|
|
64
65
|
throw new RimeInputError("The realtime model must be prism");
|
|
@@ -103,6 +104,24 @@ export class Realtime {
|
|
|
103
104
|
};
|
|
104
105
|
}),
|
|
105
106
|
};
|
|
107
|
+
if (options.turnDetection !== undefined) {
|
|
108
|
+
const detection = options.turnDetection;
|
|
109
|
+
if (!detection ||
|
|
110
|
+
typeof detection !== "object" ||
|
|
111
|
+
Array.isArray(detection))
|
|
112
|
+
throw new RimeInputError("turnDetection must be an object");
|
|
113
|
+
if (detection.eagerness !== undefined) {
|
|
114
|
+
if (!["low", "medium", "high", "auto"].includes(detection.eagerness))
|
|
115
|
+
throw new RimeInputError("Turn eagerness must be low, medium, high, or auto");
|
|
116
|
+
settings.turn_detection.eagerness = detection.eagerness;
|
|
117
|
+
}
|
|
118
|
+
if (detection.endOfTurnTimeoutMs !== undefined) {
|
|
119
|
+
const value = detection.endOfTurnTimeoutMs;
|
|
120
|
+
if (!Number.isInteger(value) || value < 1000 || value > 10000)
|
|
121
|
+
throw new RimeInputError("End-of-turn timeout must be an integer from 1000 to 10000 ms");
|
|
122
|
+
settings.turn_detection.prism_end_of_turn_timeout_ms = value;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
106
125
|
for (const name of ["voice", "instructions"]) {
|
|
107
126
|
const value = options[name];
|
|
108
127
|
if (value !== undefined && value !== null) {
|
|
@@ -4,6 +4,8 @@ export interface SessionSettings {
|
|
|
4
4
|
input_audio_format: "pcm16";
|
|
5
5
|
turn_detection: {
|
|
6
6
|
interrupt_response: boolean;
|
|
7
|
+
eagerness?: t.TurnEagerness;
|
|
8
|
+
prism_end_of_turn_timeout_ms?: number;
|
|
7
9
|
};
|
|
8
10
|
tools: {
|
|
9
11
|
type: "function";
|
|
@@ -22,6 +24,7 @@ export interface SessionView {
|
|
|
22
24
|
interruptOnSpeech: boolean | null;
|
|
23
25
|
toolResultTimeoutS: number | null;
|
|
24
26
|
toolContinuationTimeoutS: number | null;
|
|
27
|
+
turnDetection: t.TurnDetectionInfo;
|
|
25
28
|
}
|
|
26
29
|
export interface SessionCreated {
|
|
27
30
|
kind: "session.created";
|
|
@@ -66,6 +66,30 @@ function duration(value, field) {
|
|
|
66
66
|
}
|
|
67
67
|
function session(value) {
|
|
68
68
|
const view = object(value, "session");
|
|
69
|
+
const detection = view.turn_detection === undefined
|
|
70
|
+
? {}
|
|
71
|
+
: object(view.turn_detection, "session.turn_detection");
|
|
72
|
+
let eagerness = null;
|
|
73
|
+
if (detection.eagerness !== undefined) {
|
|
74
|
+
const value = string(detection.eagerness, "session.turn_detection.eagerness");
|
|
75
|
+
eagerness =
|
|
76
|
+
value === "low" ||
|
|
77
|
+
value === "medium" ||
|
|
78
|
+
value === "high" ||
|
|
79
|
+
value === "auto"
|
|
80
|
+
? value
|
|
81
|
+
: "unknown";
|
|
82
|
+
}
|
|
83
|
+
let endOfTurnTimeoutMs = null;
|
|
84
|
+
if (detection.prism_end_of_turn_timeout_ms !== undefined) {
|
|
85
|
+
const value = detection.prism_end_of_turn_timeout_ms;
|
|
86
|
+
if (typeof value !== "number" ||
|
|
87
|
+
!Number.isInteger(value) ||
|
|
88
|
+
value < 1000 ||
|
|
89
|
+
value > 10000)
|
|
90
|
+
invalid("session.turn_detection.prism_end_of_turn_timeout_ms");
|
|
91
|
+
endOfTurnTimeoutMs = value;
|
|
92
|
+
}
|
|
69
93
|
const waits = view.prism_tool_waits === undefined
|
|
70
94
|
? {}
|
|
71
95
|
: object(view.prism_tool_waits, "session.prism_tool_waits");
|
|
@@ -79,6 +103,7 @@ function session(value) {
|
|
|
79
103
|
id: id(view.id, "session.id"),
|
|
80
104
|
voice: view.voice === undefined ? "" : string(view.voice, "session.voice"),
|
|
81
105
|
interruptOnSpeech: interrupt ?? null,
|
|
106
|
+
turnDetection: Object.freeze({ eagerness, endOfTurnTimeoutMs }),
|
|
82
107
|
toolResultTimeoutS: duration(waits.result_timeout_s, "result_timeout_s"),
|
|
83
108
|
toolContinuationTimeoutS: duration(waits.continuation_timeout_s, "continuation_timeout_s"),
|
|
84
109
|
};
|
package/dist/realtime/session.js
CHANGED
|
@@ -109,6 +109,7 @@ export class RealtimeSession {
|
|
|
109
109
|
interruptOnSpeech: view.interruptOnSpeech ?? settings.turn_detection.interrupt_response,
|
|
110
110
|
toolResultTimeoutS: view.toolResultTimeoutS ?? null,
|
|
111
111
|
toolContinuationTimeoutS: view.toolContinuationTimeoutS ?? null,
|
|
112
|
+
turnDetection: view.turnDetection,
|
|
112
113
|
});
|
|
113
114
|
}
|
|
114
115
|
get info() {
|
package/dist/realtime/types.d.ts
CHANGED
|
@@ -19,13 +19,28 @@ export interface ToolDefinition {
|
|
|
19
19
|
readonly parameters: JsonObject;
|
|
20
20
|
readonly description?: string;
|
|
21
21
|
}
|
|
22
|
+
export type TurnEagerness = "low" | "medium" | "high" | "auto";
|
|
23
|
+
/** Omitted fields use the server's turn timing defaults. */
|
|
24
|
+
export interface TurnDetection {
|
|
25
|
+
/** How readily Prism decides that the user has finished speaking. */
|
|
26
|
+
readonly eagerness?: TurnEagerness;
|
|
27
|
+
/** Maximum silence before ending a turn: an integer from 1000 to 10000 ms. */
|
|
28
|
+
readonly endOfTurnTimeoutMs?: number;
|
|
29
|
+
}
|
|
30
|
+
/** Effective server settings. Null means the server did not publish the field. */
|
|
31
|
+
export interface TurnDetectionInfo {
|
|
32
|
+
readonly eagerness: TurnEagerness | "unknown" | null;
|
|
33
|
+
readonly endOfTurnTimeoutMs: number | null;
|
|
34
|
+
}
|
|
22
35
|
export interface RealtimeConnectOptions extends RealtimeOperationOptions {
|
|
23
|
-
|
|
36
|
+
/** Defaults to wss://prism.api.rime.ai/v1/realtime. */
|
|
37
|
+
readonly endpoint?: string;
|
|
24
38
|
readonly model?: "prism";
|
|
25
39
|
readonly voice?: string | null;
|
|
26
40
|
readonly instructions?: string | null;
|
|
27
41
|
readonly tools?: readonly ToolDefinition[];
|
|
28
42
|
readonly interruptOnSpeech?: boolean;
|
|
43
|
+
readonly turnDetection?: TurnDetection;
|
|
29
44
|
readonly timeouts?: RealtimeTimeouts;
|
|
30
45
|
}
|
|
31
46
|
export interface ReplyOptions extends RealtimeOperationOptions {
|
|
@@ -60,6 +75,7 @@ export interface SessionInfo {
|
|
|
60
75
|
readonly interruptOnSpeech: boolean;
|
|
61
76
|
readonly toolResultTimeoutS: number | null;
|
|
62
77
|
readonly toolContinuationTimeoutS: number | null;
|
|
78
|
+
readonly turnDetection: TurnDetectionInfo;
|
|
63
79
|
}
|
|
64
80
|
export interface RealtimeFault {
|
|
65
81
|
readonly code: string;
|
package/dist/stt/client.js
CHANGED
|
@@ -69,6 +69,10 @@ export class STT {
|
|
|
69
69
|
if (!source || typeof source[Symbol.asyncIterator] !== "function")
|
|
70
70
|
throw new RimeInputError("audio must be an async iterable of Uint8Array chunks");
|
|
71
71
|
validateOptions(options);
|
|
72
|
+
const outputContract = options.outputContract ?? "revised_hypotheses";
|
|
73
|
+
if (outputContract !== "revised_hypotheses" &&
|
|
74
|
+
outputContract !== "append_fragments")
|
|
75
|
+
throw new RimeInputError("outputContract must be revised_hypotheses or append_fragments");
|
|
72
76
|
const mode = options.mode ?? "written";
|
|
73
77
|
const terms = options.contextTerms ?? [];
|
|
74
78
|
const format = resolvePCMFormat(options.inputFormat);
|
|
@@ -79,6 +83,7 @@ export class STT {
|
|
|
79
83
|
prepare: (signal) => this.prepare(signal, target),
|
|
80
84
|
}, source, {
|
|
81
85
|
language: options.language,
|
|
86
|
+
outputContract,
|
|
82
87
|
mode,
|
|
83
88
|
contextTerms: [...terms],
|
|
84
89
|
inputFormat: format,
|
package/dist/stt/policy.js
CHANGED
package/dist/stt/protocol.d.ts
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import * as schema from "@rimelabs/api";
|
|
2
|
-
import type { TranscriptionPartial, TranscriptionFinal } from "./types.js";
|
|
2
|
+
import type { TranscriptionPartial, TranscriptionDelta, TranscriptionFinal, TranscriptionOutputContract } from "./types.js";
|
|
3
3
|
/** Validate the wire sequence before exposing any final result. */
|
|
4
4
|
export declare class TranscriptState {
|
|
5
5
|
private readonly textLimit;
|
|
6
|
+
private readonly outputContract;
|
|
6
7
|
language: string | null;
|
|
7
8
|
private revision;
|
|
8
9
|
private text;
|
|
9
10
|
private final;
|
|
10
|
-
constructor(textLimit: number);
|
|
11
|
-
accept(message: schema.StreamingTranscriptionResponse, inputDone: boolean): TranscriptionPartial | null;
|
|
11
|
+
constructor(textLimit: number, outputContract?: TranscriptionOutputContract);
|
|
12
|
+
accept(message: schema.StreamingTranscriptionResponse, inputDone: boolean): TranscriptionPartial | TranscriptionDelta | null;
|
|
12
13
|
finish(): TranscriptionFinal;
|
|
13
14
|
}
|
package/dist/stt/protocol.js
CHANGED
|
@@ -3,12 +3,14 @@ import { RimeStreamError, RimeResourceLimitError } from "../errors.js";
|
|
|
3
3
|
/** Validate the wire sequence before exposing any final result. */
|
|
4
4
|
export class TranscriptState {
|
|
5
5
|
textLimit;
|
|
6
|
+
outputContract;
|
|
6
7
|
language = null;
|
|
7
8
|
revision = 0n;
|
|
8
9
|
text = "";
|
|
9
10
|
final = null;
|
|
10
|
-
constructor(textLimit) {
|
|
11
|
+
constructor(textLimit, outputContract = "revised_hypotheses") {
|
|
11
12
|
this.textLimit = textLimit;
|
|
13
|
+
this.outputContract = outputContract;
|
|
12
14
|
}
|
|
13
15
|
accept(message, inputDone) {
|
|
14
16
|
const payload = message.payload;
|
|
@@ -21,7 +23,9 @@ export class TranscriptState {
|
|
|
21
23
|
if (this.language !== null)
|
|
22
24
|
throw new RimeStreamError("Received duplicate transcription acceptance");
|
|
23
25
|
if (accepted.outputContract !==
|
|
24
|
-
|
|
26
|
+
(this.outputContract === "append_fragments"
|
|
27
|
+
? schema.StreamingOutputContract.APPEND_FRAGMENTS
|
|
28
|
+
: schema.StreamingOutputContract.REVISED_HYPOTHESES))
|
|
25
29
|
throw new RimeStreamError("The service accepted a different transcript contract");
|
|
26
30
|
if (!accepted.language?.tag || !accepted.language.source)
|
|
27
31
|
throw new RimeStreamError("The service did not confirm a resolved language");
|
|
@@ -30,8 +34,19 @@ export class TranscriptState {
|
|
|
30
34
|
}
|
|
31
35
|
if (this.language === null)
|
|
32
36
|
throw new RimeStreamError("Received a transcript before acceptance");
|
|
37
|
+
const append = this.outputContract === "append_fragments";
|
|
38
|
+
if (payload.case === "delta" && append) {
|
|
39
|
+
const text = payload.value.text;
|
|
40
|
+
if (Buffer.byteLength(this.text) + Buffer.byteLength(text) >
|
|
41
|
+
this.textLimit)
|
|
42
|
+
throw new RimeResourceLimitError("The transcript exceeds the SDK text limit");
|
|
43
|
+
this.text += text;
|
|
44
|
+
return { kind: "delta", text };
|
|
45
|
+
}
|
|
33
46
|
if (payload.case !== "hypothesis" && payload.case !== "done")
|
|
34
47
|
throw new RimeStreamError("Received an unexpected transcription response");
|
|
48
|
+
if (payload.case === "hypothesis" && append)
|
|
49
|
+
throw new RimeStreamError("Received an unexpected transcription response");
|
|
35
50
|
const value = payload.value;
|
|
36
51
|
if (Buffer.byteLength(value.text, "utf8") > this.textLimit)
|
|
37
52
|
throw new RimeResourceLimitError("The transcript exceeds the SDK text limit");
|
|
@@ -46,7 +61,7 @@ export class TranscriptState {
|
|
|
46
61
|
if (!inputDone)
|
|
47
62
|
throw new RimeStreamError("The service completed before input finished");
|
|
48
63
|
if (value.revision !== this.revision || value.text !== this.text)
|
|
49
|
-
throw new RimeStreamError("Final transcript does not match
|
|
64
|
+
throw new RimeStreamError("Final transcript does not match prior updates");
|
|
50
65
|
const language = payload.value.language;
|
|
51
66
|
if (language?.tag !== this.language || !language.source)
|
|
52
67
|
throw new RimeStreamError("Final language does not match the accepted language");
|
package/dist/stt/stream.js
CHANGED
|
@@ -130,7 +130,7 @@ export class TranscriptStream {
|
|
|
130
130
|
}
|
|
131
131
|
}
|
|
132
132
|
async read() {
|
|
133
|
-
const state = new TranscriptState(policy.transcriptBytes);
|
|
133
|
+
const state = new TranscriptState(policy.transcriptBytes, this.options.outputContract);
|
|
134
134
|
for await (const message of this.call.responses()) {
|
|
135
135
|
const update = state.accept(message, this.inputDone);
|
|
136
136
|
if (state.language !== null) {
|
|
@@ -149,7 +149,7 @@ export class TranscriptStream {
|
|
|
149
149
|
this.controller.signal.throwIfAborted();
|
|
150
150
|
this.call = new TranscriptionCall(prepared.client, prepared.metadata, this.controller.signal);
|
|
151
151
|
this.admission = this.deadline(policy.acceptanceTimeout, "Transcription acceptance timed out");
|
|
152
|
-
await this.call.start(this.options.language, this.options.mode ?? "written", this.options.contextTerms ?? []);
|
|
152
|
+
await this.call.start(this.options.language, this.options.mode ?? "written", this.options.contextTerms ?? [], this.options.outputContract);
|
|
153
153
|
const producer = this.produce(), reader = this.read();
|
|
154
154
|
tasks = [producer, reader];
|
|
155
155
|
const [, final] = await Promise.all([producer, reader]);
|
package/dist/stt/transport.d.ts
CHANGED
|
@@ -10,6 +10,6 @@ export declare const transport: {
|
|
|
10
10
|
export declare function transcribe(client: grpc.Client, auth: grpc.Metadata, audio: Uint8Array, options: TranscribeOptions, signal: AbortSignal): Promise<Transcription>;
|
|
11
11
|
export declare class TranscriptionCall extends BidiCall<schema.StreamingTranscriptionRequest, schema.StreamingTranscriptionResponse> {
|
|
12
12
|
constructor(client: grpc.Client, auth: grpc.Metadata, signal: AbortSignal);
|
|
13
|
-
start(language: string, mode: TranscriptionMode, contextTerms: readonly string[]): Promise<void>;
|
|
13
|
+
start(language: string, mode: TranscriptionMode, contextTerms: readonly string[], outputContract?: import("./types.js").TranscriptionOutputContract): Promise<void>;
|
|
14
14
|
writeAudio(audio: Uint8Array): Promise<void>;
|
|
15
15
|
}
|
package/dist/stt/transport.js
CHANGED
|
@@ -69,7 +69,7 @@ export class TranscriptionCall extends BidiCall {
|
|
|
69
69
|
const method = service.method.transcribeStreaming;
|
|
70
70
|
super(client.makeBidiStreamRequest(`/${service.typeName}/${method.name}`, serializer(method.input), deserializer(method.output), auth), signal);
|
|
71
71
|
}
|
|
72
|
-
start(language, mode, contextTerms) {
|
|
72
|
+
start(language, mode, contextTerms, outputContract = "revised_hypotheses") {
|
|
73
73
|
return this.write(create(schema.StreamingTranscriptionRequestSchema, {
|
|
74
74
|
payload: {
|
|
75
75
|
case: "config",
|
|
@@ -79,7 +79,9 @@ export class TranscriptionCall extends BidiCall {
|
|
|
79
79
|
? schema.TranscriptionMode.VERBATIM
|
|
80
80
|
: schema.TranscriptionMode.WRITTEN,
|
|
81
81
|
contextTerms: [...contextTerms],
|
|
82
|
-
outputContract:
|
|
82
|
+
outputContract: outputContract === "append_fragments"
|
|
83
|
+
? schema.StreamingOutputContract.APPEND_FRAGMENTS
|
|
84
|
+
: schema.StreamingOutputContract.REVISED_HYPOTHESES,
|
|
83
85
|
},
|
|
84
86
|
},
|
|
85
87
|
}));
|
package/dist/stt/types.d.ts
CHANGED
|
@@ -3,7 +3,16 @@ import type { PCMFormat } from "../pcm.js";
|
|
|
3
3
|
export type TranscriptionMode = "written" | "verbatim";
|
|
4
4
|
/** One caller-ended utterance of headerless signed PCM16 little-endian audio. */
|
|
5
5
|
export type AudioSource = AsyncIterable<Uint8Array>;
|
|
6
|
+
/** Select replacement hypotheses (default) or permanent append fragments. */
|
|
7
|
+
export type TranscriptionOutputContract = "revised_hypotheses" | "append_fragments";
|
|
8
|
+
export interface TranscriptionDelta {
|
|
9
|
+
readonly kind: "delta";
|
|
10
|
+
/** Append exactly, preserving the server's newline separators. */
|
|
11
|
+
readonly text: string;
|
|
12
|
+
}
|
|
6
13
|
export interface TranscriptionOptions {
|
|
14
|
+
/** Defaults to revised_hypotheses. Each stream uses one fixed contract. */
|
|
15
|
+
outputContract?: TranscriptionOutputContract;
|
|
7
16
|
/** Required BCP-47 tag, passed unchanged to the service (for example "en"). */
|
|
8
17
|
language: string;
|
|
9
18
|
/** Formatting intent; defaults to written and does not guarantee exact tokens. */
|
|
@@ -31,9 +40,9 @@ export interface TranscriptionFinal {
|
|
|
31
40
|
/** Canonical BCP-47 tag selected by the service, for example "en". */
|
|
32
41
|
readonly language: string;
|
|
33
42
|
}
|
|
34
|
-
export type TranscriptionUpdate = TranscriptionPartial | TranscriptionFinal;
|
|
43
|
+
export type TranscriptionUpdate = TranscriptionPartial | TranscriptionDelta | TranscriptionFinal;
|
|
35
44
|
/** Options for a complete WAV recording. Audio is sent unchanged. */
|
|
36
|
-
export type TranscribeOptions = Omit<TranscriptionOptions, "inputFormat">;
|
|
45
|
+
export type TranscribeOptions = Omit<TranscriptionOptions, "inputFormat" | "outputContract">;
|
|
37
46
|
/** A successful whole-recording transcription. Silence may produce empty text. */
|
|
38
47
|
export interface Transcription {
|
|
39
48
|
readonly text: string;
|
package/docs/realtime.md
CHANGED
|
@@ -16,17 +16,18 @@ Typed turns also support mixed text and voice applications.
|
|
|
16
16
|
|
|
17
17
|
## Connect
|
|
18
18
|
|
|
19
|
-
Use `await client.realtime.connect(
|
|
20
|
-
one conversation, and returns a `RealtimeSession`. Settings apply
|
|
21
|
-
new session to change them.
|
|
19
|
+
Use `await client.realtime.connect()` with `RIME_API_KEY` set. It opens a WebSocket,
|
|
20
|
+
initializes one conversation, and returns a `RealtimeSession`. Settings apply
|
|
21
|
+
once. Open a new session to change them.
|
|
22
22
|
|
|
23
23
|
| Option | Required? | Default and behavior |
|
|
24
24
|
| --- | --- | --- |
|
|
25
|
-
| `endpoint` |
|
|
25
|
+
| `endpoint` | No | Defaults to `wss://prism.api.rime.ai/v1/realtime`. Overrides must be full URLs; plain `ws://` is allowed only for localhost. |
|
|
26
26
|
| `voice` | If the server has no default voice | Omitted or `null` uses the server default. Supplied strings must be nonblank. |
|
|
27
27
|
| `instructions` | No | Omitted or `null` uses server defaults. Supplied strings must be nonblank. |
|
|
28
28
|
| `tools` | No | `[]`. Each tool requires `name` and a JSON Schema `parameters` object. `description` defaults to `""`. |
|
|
29
29
|
| `interruptOnSpeech` | No | `true`. Requests interruption when the server confirms user speech. |
|
|
30
|
+
| `turnDetection` | No | Omitted settings use the server's turn timing defaults. |
|
|
30
31
|
| `timeouts` | No | `{ connectS: 10, readyS: 30, requestS: 10 }`. Each field is optional. Units are seconds. |
|
|
31
32
|
| `model` | No | `"prism"`, the only accepted value. |
|
|
32
33
|
| `signal` | No | An `AbortSignal` that cancels connection setup. It does not cancel an established session. |
|
|
@@ -36,7 +37,8 @@ Credentials come from `new Rime({ apiKey: "..." })` or `RIME_API_KEY`. Prism use
|
|
|
36
37
|
voice, and timeout settings do not apply to Realtime.
|
|
37
38
|
|
|
38
39
|
After connection, `session.info` contains `sessionId`, `model`, `voice`,
|
|
39
|
-
`interruptOnSpeech`, `toolResultTimeoutS`, and
|
|
40
|
+
`interruptOnSpeech`, `turnDetection`, `toolResultTimeoutS`, and
|
|
41
|
+
`toolContinuationTimeoutS`.
|
|
40
42
|
Check the effective interruption setting there. Tool wait durations are total
|
|
41
43
|
server deadlines, not remaining time. A `null` duration can mean an omitted
|
|
42
44
|
field; it does not prove that no deadline exists.
|
|
@@ -45,6 +47,39 @@ Close the session in `finally` with `await session.close()`. Closing the parent
|
|
|
45
47
|
client closes all its TTS and Realtime operations. Both objects also support
|
|
46
48
|
`Symbol.asyncDispose`. Repeated close calls are safe.
|
|
47
49
|
|
|
50
|
+
## Optional turn timing
|
|
51
|
+
|
|
52
|
+
Pass a `TurnDetection` object to override how readily Prism ends a spoken turn.
|
|
53
|
+
Each field is optional; omitted fields use server defaults.
|
|
54
|
+
|
|
55
|
+
| Field | Values | Current server default |
|
|
56
|
+
| --- | --- | --- |
|
|
57
|
+
| `eagerness` | `"low"`, `"medium"`, `"high"`, `"auto"` | `"auto"`, equivalent to `"medium"` |
|
|
58
|
+
| `endOfTurnTimeoutMs` | Integer from 1,000 through 10,000 | 5,000 ms |
|
|
59
|
+
|
|
60
|
+
`"low"` waits for more certainty that the user has finished. `"high"` answers
|
|
61
|
+
sooner, with more risk of cutting in. The timeout is a maximum silence limit,
|
|
62
|
+
not a fixed delay before each reply. If the server falls back to its local
|
|
63
|
+
voice activity detector, it ends turns after 500 ms of silence instead.
|
|
64
|
+
|
|
65
|
+
```typescript
|
|
66
|
+
const session = await client.realtime.connect({
|
|
67
|
+
turnDetection: { eagerness: "low", endOfTurnTimeoutMs: 5000 },
|
|
68
|
+
});
|
|
69
|
+
try {
|
|
70
|
+
console.log(session.info.turnDetection);
|
|
71
|
+
// Send input and consume session.events here.
|
|
72
|
+
} finally {
|
|
73
|
+
await session.close();
|
|
74
|
+
}
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
`session.info.turnDetection` is a `TurnDetectionInfo` with the settings the
|
|
78
|
+
server reported in its initialization acknowledgment. A missing field is
|
|
79
|
+
`null`; the SDK does not substitute a requested value or assume a server
|
|
80
|
+
default. An unrecognized server eagerness becomes `"unknown"`.
|
|
81
|
+
`interruptOnSpeech` remains a separate connection option.
|
|
82
|
+
|
|
48
83
|
## Choose an operation
|
|
49
84
|
|
|
50
85
|
Every method returns a promise. Arguments without `?` are required.
|
package/docs/stt.md
CHANGED
|
@@ -63,7 +63,8 @@ try {
|
|
|
63
63
|
});
|
|
64
64
|
for await (const update of transcript) {
|
|
65
65
|
if (update.kind === "partial") console.log("Current transcript:", update.text);
|
|
66
|
-
else
|
|
66
|
+
else if (update.kind === "final")
|
|
67
|
+
console.log("Final:", update.text, "Language:", update.language);
|
|
67
68
|
}
|
|
68
69
|
console.log("Request:", transcript.requestId);
|
|
69
70
|
} finally {
|
|
@@ -76,6 +77,37 @@ without loading it all into memory. Use Node.js 22 or later; browsers are not su
|
|
|
76
77
|
Open files inside the async generator so file errors are reported through the
|
|
77
78
|
SDK and resources are opened only after the service accepts the utterance.
|
|
78
79
|
|
|
80
|
+
## Append-fragment streaming
|
|
81
|
+
|
|
82
|
+
Choose `outputContract: "append_fragments"` on
|
|
83
|
+
`client.stt.stream`. The default is `"revised_hypotheses"`; `transcribe()` does not
|
|
84
|
+
accept an output contract. Both streaming modes use gRPC only.
|
|
85
|
+
|
|
86
|
+
```javascript
|
|
87
|
+
let text = "";
|
|
88
|
+
const transcript = client.stt.stream(audioChunks(), {
|
|
89
|
+
language: "en", outputContract: "append_fragments",
|
|
90
|
+
});
|
|
91
|
+
for await (const update of transcript) {
|
|
92
|
+
if (update.kind === "delta") text += update.text;
|
|
93
|
+
else if (update.kind === "final") console.log(update.text);
|
|
94
|
+
}
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
`TranscriptionDelta` is a permanent fragment, not a replacement partial. The
|
|
98
|
+
server recognizes independent five-second audio windows; fragments after the
|
|
99
|
+
first include newline separators. Do not strip, add separators, or drop deltas.
|
|
100
|
+
The final text must match their concatenation. Empty speech can produce no
|
|
101
|
+
updates before an empty final. A final is delivered only after `done` and an OK
|
|
102
|
+
gRPC status, so a connection failure never promotes accumulated fragments to a
|
|
103
|
+
successful result.
|
|
104
|
+
|
|
105
|
+
Append mode suits longer recordings and consumers that cannot revise text. Send
|
|
106
|
+
recordings as fast as backpressure permits. It does not enable unlimited-length
|
|
107
|
+
streams: the service's 16 MiB audio and 120-second total-call limits still apply.
|
|
108
|
+
Split larger recordings into separate streams. Live voice agents usually prefer
|
|
109
|
+
revised hypotheses for earlier provisional text and corrections across windows.
|
|
110
|
+
|
|
79
111
|
## Options
|
|
80
112
|
|
|
81
113
|
| Option | Meaning |
|
|
@@ -83,6 +115,7 @@ SDK and resources are opened only after the service accepts the utterance.
|
|
|
83
115
|
| `language` | Required spoken BCP-47 tag. Passed unchanged to the service; no detection or SDK language allowlist. The deployed catalog currently supports English and Spanish. |
|
|
84
116
|
| `mode` | `"written"` (default) or `"verbatim"`. Formatting intent, not a guarantee of exact tokens. |
|
|
85
117
|
| `contextTerms` | Sequence of recognition hints, copied unchanged and in order. The service validates terms and prompt budgets. |
|
|
118
|
+
| `outputContract` | `"revised_hypotheses"` (default) or `"append_fragments"`. Streaming only. |
|
|
86
119
|
| `inputFormat` | `PCMFormat`: signed little-endian PCM16 at 8, 16, 24 or 48 kHz, mono or interleaved stereo. Defaults to 16 kHz mono. |
|
|
87
120
|
| `timeout` | Optional positive overall timeout in seconds. Includes connection time, source pauses and consumer pauses. `null` disables this SDK deadline, but not internal or server deadlines. |
|
|
88
121
|
|
|
@@ -98,11 +131,18 @@ There is no STT voice/model selector; the service selects its recognizer.
|
|
|
98
131
|
|
|
99
132
|
## Results and completion
|
|
100
133
|
|
|
101
|
-
`TranscriptionPartial` and `TranscriptionFinal` are discriminated
|
|
134
|
+
`TranscriptionPartial`, `TranscriptionDelta`, and `TranscriptionFinal` are discriminated types exported from `@rimelabs/sdk`. In the default revised-hypotheses mode, every update contains the complete current `text`.
|
|
102
135
|
**Replace the previous transcript. Do not concatenate partials.** Later partials
|
|
103
136
|
can change earlier words. `TranscriptionFinal.language` is the canonical language
|
|
104
137
|
selected by the service.
|
|
105
138
|
|
|
139
|
+
When upgrading, `TranscriptionUpdate` now includes `TranscriptionDelta`. Update
|
|
140
|
+
exhaustive handlers to handle `kind: "delta"`, even if you keep the default
|
|
141
|
+
revised-hypotheses contract. Only finals have `language`: check
|
|
142
|
+
`update.kind === "final"` before reading it, rather than assuming every
|
|
143
|
+
non-partial update is final. Append deltas exactly; replace text with partials
|
|
144
|
+
or the confirmed final.
|
|
145
|
+
|
|
106
146
|
The source is not consumed until the server accepts the configuration. Audio
|
|
107
147
|
upload and transcript reading run concurrently. Decoding currently uses cumulative
|
|
108
148
|
two-second audio windows, so a partial is not promised for each audio packet.
|
|
@@ -122,8 +162,12 @@ Pass an `AbortSignal` through the stream's `signal` option to cancel just that
|
|
|
122
162
|
utterance. Client close cancels all STT, TTS and realtime operations; cancelling
|
|
123
163
|
one stream leaves its siblings running.
|
|
124
164
|
|
|
125
|
-
The SDK bounds the transcript queue to 16 updates
|
|
126
|
-
|
|
165
|
+
The SDK bounds the transcript queue to 16 updates. In revised-hypotheses mode,
|
|
166
|
+
each replacement update is limited to 1 MiB of UTF-8 text; sizes are not summed
|
|
167
|
+
across revisions. In append-fragments mode, the concatenated text across all
|
|
168
|
+
deltas is limited to 1 MiB. The final transcript is also limited to 1 MiB in both
|
|
169
|
+
modes. gRPC responses are bounded to 1 MiB plus 64 KiB for the protobuf envelope.
|
|
170
|
+
Slow consumers apply backpressure.
|
|
127
171
|
Internal connection and acceptance waits are 10 seconds; completion after source
|
|
128
172
|
exhaustion is bounded to 120 seconds. The completion timer stops after successful
|
|
129
173
|
gRPC completion. The optional overall timeout remains active until the final
|