@rimelabs/sdk 0.1.0-alpha.1 → 0.1.0-alpha.12

package/dist/realtime/transport.d.ts

@@ -0,0 +1,16 @@
+import { RimeError } from "../errors.js";
+/** Private transport. The public declarations do not require ws types. */
+export declare class RealtimeConnection {
+    private readonly socket;
+    private readonly opened;
+    private readonly ended;
+    private closing;
+    private connected;
+    readonly ready: Promise<void>;
+    onMessage: (raw: string) => void;
+    onFailure: (error: RimeError) => void;
+    constructor(endpoint: string, authorization: string);
+    private failure;
+    send(encoded: string): Promise<void>;
+    close(): Promise<void>;
+}

package/dist/realtime/transport.js

@@ -0,0 +1,83 @@
+import WebSocket from "ws";
+import { RimeAuthenticationError, RimeStreamError, RimeUnavailableError, } from "../errors.js";
+import { Deferred } from "./async.js";
+/** Private transport. The public declarations do not require ws types. */
+export class RealtimeConnection {
+    socket;
+    opened = new Deferred();
+    ended = new Deferred();
+    closing = null;
+    connected = false;
+    ready = this.opened.promise;
+    onMessage = () => { };
+    onFailure = () => { };
+    constructor(endpoint, authorization) {
+        this.socket = new WebSocket(endpoint, {
+            headers: { Authorization: authorization },
+            followRedirects: false,
+            perMessageDeflate: false,
+            maxPayload: 1024 * 1024,
+        });
+        this.socket.on("open", () => {
+            this.connected = true;
+            this.opened.resolve();
+        });
+        this.socket.on("unexpected-response", (request, response) => {
+            const error = [401, 403].includes(response.statusCode ?? 0)
+                ? new RimeAuthenticationError("Realtime endpoint rejected credentials")
+                : new RimeUnavailableError("Realtime endpoint rejected the connection");
+            this.failure(error);
+            response.destroy();
+            request.destroy();
+            this.socket.terminate();
+        });
+        this.socket.on("error", () => this.failure(!this.connected
+            ? new RimeUnavailableError("Realtime handshake failed")
+            : new RimeStreamError("Invalid event or lost realtime connection")));
+        this.socket.on("message", (data, binary) => {
+            if (binary) {
+                this.failure(new RimeStreamError("Expected a JSON text event"));
+                return;
+            }
+            this.onMessage(data.toString());
+        });
+        this.socket.on("close", () => {
+            this.ended.resolve();
+            this.failure(new RimeStreamError("Realtime connection closed; reconnect is not automatic"));
+        });
+    }
+    failure(error) {
+        this.opened.reject(error);
+        this.onFailure(error);
+    }
+    send(encoded) {
+        return new Promise((resolve, reject) => {
+            this.socket.send(encoded, (error) => {
+                if (error)
+                    reject(new RimeStreamError("Realtime write failed; outcome unknown"));
+                else
+                    resolve();
+            });
+        });
+    }
+    close() {
+        if (!this.closing) {
+            this.closing = (async () => {
+                if (this.socket.readyState === WebSocket.CLOSED)
+                    return;
+                const timer = setTimeout(() => this.socket.terminate(), 2000);
+                try {
+                    if (this.socket.readyState === WebSocket.CONNECTING)
+                        this.socket.terminate();
+                    else
+                        this.socket.close();
+                    await this.ended.promise;
+                }
+                finally {
+                    clearTimeout(timer);
+                }
+            })();
+        }
+        return this.closing;
+    }
+}

package/dist/realtime/types.d.ts

@@ -0,0 +1,186 @@
+import type { PCMFormat } from "../pcm.js";
+export type { PCMFormat } from "../pcm.js";
+import { RimeError, RimeTimeoutError } from "../errors.js";
+export type JsonValue = null | boolean | number | string | JsonValue[] | JsonObject;
+export interface JsonObject {
+    [key: string]: JsonValue;
+}
+/** All durations are positive, finite seconds. */
+export interface RealtimeTimeouts {
+    readonly connectS?: number;
+    readonly readyS?: number;
+    readonly requestS?: number;
+}
+export interface RealtimeOperationOptions {
+    readonly signal?: AbortSignal;
+}
+export interface ToolDefinition {
+    readonly name: string;
+    readonly parameters: JsonObject;
+    readonly description?: string;
+}
+export interface RealtimeConnectOptions extends RealtimeOperationOptions {
+    readonly endpoint: string;
+    readonly model?: "prism";
+    readonly voice?: string | null;
+    readonly instructions?: string | null;
+    readonly tools?: readonly ToolDefinition[];
+    readonly interruptOnSpeech?: boolean;
+    readonly timeouts?: RealtimeTimeouts;
+}
+export interface ReplyOptions extends RealtimeOperationOptions {
+    readonly instruction?: string | null;
+    readonly toolCall?: ToolCallRef | null;
+}
+export interface AudioChunk {
+    readonly data: Uint8Array;
+    readonly format?: PCMFormat;
+}
+export interface ResponseRef {
+    readonly sessionId: string;
+    readonly responseId: string;
+}
+export interface ItemRef {
+    readonly sessionId: string;
+    readonly itemId: string;
+}
+export interface ToolCallRef extends ResponseRef {
+    readonly callId: string;
+}
+export interface OutputRef {
+    readonly response: ResponseRef;
+    readonly itemId: string;
+    readonly outputIndex: number;
+    readonly contentIndex: number;
+}
+export interface SessionInfo {
+    readonly sessionId: string;
+    readonly model: "prism";
+    readonly voice: string;
+    readonly interruptOnSpeech: boolean;
+    readonly toolResultTimeoutS: number | null;
+    readonly toolContinuationTimeoutS: number | null;
+}
+export interface RealtimeFault {
+    readonly code: string;
+    readonly message: string;
+    readonly scope: "event" | "utterance" | "response" | "tool_roundtrip" | "session" | "unknown";
+    readonly requestId: string | null;
+    readonly responseId: string | null;
+    readonly itemId: string | null;
+    readonly callIds: readonly string[];
+    readonly correlationId: string | null;
+    readonly parameter: string | null;
+}
+export declare class RealtimeAdmissionTimeout extends RimeTimeoutError {
+}
+export declare class RimeRealtimeError extends RimeError {
+    readonly fault: RealtimeFault;
+    constructor(fault: RealtimeFault);
+}
+export interface InputReady {
+    readonly kind: "input.ready";
+}
+export interface SpeechStarted {
+    readonly kind: "speech.started";
+    readonly itemId: string;
+    readonly audioStartMs: number;
+}
+export interface SpeechStopped {
+    readonly kind: "speech.stopped";
+    readonly itemId: string;
+    readonly audioEndMs: number;
+}
+export interface InputCommitted {
+    readonly kind: "input.committed";
+    readonly itemId: string;
+}
+export interface TranscriptDelta {
+    readonly kind: "transcript.delta";
+    readonly itemId: string;
+    readonly delta: string;
+}
+export interface TranscriptFinal {
+    readonly kind: "transcript.final";
+    readonly itemId: string;
+    readonly text: string;
+}
+export interface TranscriptFailed {
+    readonly kind: "transcript.failed";
+    readonly itemId: string;
+    readonly error: RealtimeFault;
+}
+export interface ResponseStarted {
+    readonly kind: "response.started";
+    readonly response: ResponseRef;
+    readonly cause: "speech" | "text" | "proactive" | "tools" | "unknown";
+    readonly parent: ResponseRef | null;
+    readonly inputItemId: string | null;
+}
+export interface MessageStarted {
+    readonly kind: "message.started";
+    readonly output: OutputRef;
+}
+export interface TextDelta {
+    readonly kind: "text.delta";
+    readonly output: OutputRef;
+    readonly delta: string;
+}
+export interface TextDone {
+    readonly kind: "text.done";
+    readonly output: OutputRef;
+    readonly text: string;
+}
+export interface AudioDelta {
+    readonly kind: "audio.delta";
+    readonly output: OutputRef;
+    readonly audio: {
+        readonly data: Uint8Array;
+        readonly format: Required<PCMFormat>;
+    };
+}
+export interface AudioDone {
+    readonly kind: "audio.done";
+    readonly output: OutputRef;
+}
+export interface ToolCall {
+    readonly kind: "tool.call";
+    readonly call: ToolCallRef;
+    readonly itemId: string;
+    readonly name: string;
+    readonly arguments: JsonObject;
+}
+export interface ResponseAbandoned {
+    readonly kind: "response.abandoned";
+    readonly response: ResponseRef;
+}
+export interface ResponseEnded {
+    readonly kind: "response.ended";
+    readonly response: ResponseRef;
+    readonly status: "completed" | "cancelled" | "failed" | "unknown";
+    readonly reason: string | null;
+}
+export interface FaultEvent {
+    readonly kind: "error";
+    readonly error: RealtimeFault;
+}
+export type InputEvent = InputReady | SpeechStarted | SpeechStopped | InputCommitted | TranscriptDelta | TranscriptFinal | TranscriptFailed;
+export type ResponseEvent = ResponseStarted | ResponseAbandoned | MessageStarted | TextDelta | TextDone | AudioDelta | AudioDone | ToolCall | ResponseEnded;
+export interface SessionEvent {
+    readonly sessionId: string;
+    readonly eventId: string;
+    readonly requestId: string | null;
+    readonly payload: InputEvent | ResponseEvent | FaultEvent;
+}
+export interface PlaybackInterrupted {
+    readonly kind: "interrupted";
+    readonly output: OutputRef;
+    readonly audioEndMs: number;
+}
+export interface PlaybackFinished {
+    readonly kind: "finished";
+    readonly response: ResponseRef;
+    readonly playedMs?: number | null;
+    readonly audibleTailMs?: number | null;
+}
+export type PlaybackReport = PlaybackInterrupted | PlaybackFinished;

package/dist/realtime/types.js

@@ -0,0 +1,10 @@
+import { RimeError, RimeTimeoutError } from "../errors.js";
+export class RealtimeAdmissionTimeout extends RimeTimeoutError {
+}
+export class RimeRealtimeError extends RimeError {
+    fault;
+    constructor(fault) {
+        super(`${fault.code}: ${fault.message}`, fault.requestId);
+        this.fault = fault;
+    }
+}

package/dist/stt/audio.d.ts

@@ -0,0 +1,15 @@
+import { type PCMFormat } from "../pcm.js";
+/** One utterance of headerless signed PCM16 little-endian mono 16 kHz output.
+ * Fully consume each feed iterator before feeding the next chunk. Conversion
+ * advances one bounded source portion at a time so callers can apply backpressure.
+ */
+export declare class InputAudio {
+    private readonly converter;
+    private readonly frameBytes;
+    private pending;
+    private finished;
+    constructor(format?: PCMFormat);
+    feed(data: Uint8Array): Generator<Buffer>;
+    /** Check EOF before committing; never silently pad or truncate a final frame. */
+    finish(): void;
+}

package/dist/stt/audio.js

@@ -0,0 +1,46 @@
+import { RimeAudioFormatError, RimeInputError } from "../errors.js";
+import { InputConverter, resolvePCMFormat } from "../pcm.js";
+// Bound conversion work and temporary allocations for large source chunks.
+const sourceBytes = 16384;
+// Maximum decoded STT audio payload, excluding the protobuf envelope.
+const messageBytes = 65536;
+/** One utterance of headerless signed PCM16 little-endian mono 16 kHz output.
+ * Fully consume each feed iterator before feeding the next chunk. Conversion
+ * advances one bounded source portion at a time so callers can apply backpressure.
+ */
+export class InputAudio {
+    converter;
+    frameBytes;
+    pending = Buffer.alloc(0);
+    finished = false;
+    constructor(format) {
+        const resolved = resolvePCMFormat(format);
+        this.converter = new InputConverter(resolved);
+        this.frameBytes = 2 * resolved.channels;
+    }
+    *feed(data) {
+        if (this.finished)
+            throw new RimeInputError("The audio input is finished");
+        if (!(data instanceof Uint8Array))
+            throw new RimeAudioFormatError("The audio source must yield PCM16 Uint8Array chunks");
+        for (let offset = 0; offset < data.length; offset += sourceBytes) {
+            const part = Buffer.concat([
+                this.pending,
+                data.subarray(offset, offset + sourceBytes),
+            ]);
+            const complete = part.length - (part.length % this.frameBytes);
+            this.pending = Buffer.from(part.subarray(complete));
+            if (complete) {
+                const converted = this.converter.process(part.subarray(0, complete));
+                for (let start = 0; start < converted.length; start += messageBytes)
+                    yield converted.subarray(start, start + messageBytes);
+            }
+        }
+    }
+    /** Check EOF before committing; never silently pad or truncate a final frame. */
+    finish() {
+        this.finished = true;
+        if (this.pending.length)
+            throw new RimeAudioFormatError("The audio source ended with an incomplete PCM16 frame");
+    }
+}

package/dist/stt/client.d.ts

@@ -0,0 +1,13 @@
+import { Credentials } from "../auth.js";
+import { TranscriptStream } from "./stream.js";
+import type { AudioSource, TranscriptionOptions } from "./types.js";
+export declare class STT {
+    private readonly credentials;
+    private readonly endpoint;
+    private readonly checkOpen;
+    private connection;
+    private streams;
+    constructor(credentials: Credentials, endpoint: string | null | undefined, checkOpen: () => void);
+    stream(source: AudioSource, options: TranscriptionOptions): TranscriptStream;
+    close(): Promise<void>;
+}

package/dist/stt/client.js

@@ -0,0 +1,73 @@
+import { connect, metadata } from "../grpc.js";
+import { endpointAddress, timeout } from "../validation.js";
+import { RimeInputError } from "../errors.js";
+import { resolvePCMFormat } from "../pcm.js";
+import { transport, requireSchema } from "./transport.js";
+import { TranscriptStream } from "./stream.js";
+import { policy } from "./policy.js";
+export class STT {
+    credentials;
+    endpoint;
+    checkOpen;
+    connection = null;
+    streams = new Set();
+    constructor(credentials, endpoint, checkOpen) {
+        this.credentials = credentials;
+        this.endpoint = endpoint;
+        this.checkOpen = checkOpen;
+    }
+    stream(source, options) {
+        this.checkOpen();
+        if (!source || typeof source[Symbol.asyncIterator] !== "function")
+            throw new RimeInputError("audio must be an async iterable of Uint8Array chunks");
+        if (!options || typeof options.language !== "string")
+            throw new RimeInputError("language must be a string");
+        const mode = options.mode ?? "written";
+        if (mode !== "written" && mode !== "verbatim")
+            throw new RimeInputError("mode must be written or verbatim");
+        const terms = options.contextTerms ?? [];
+        if (!Array.isArray(terms) || terms.some((term) => typeof term !== "string"))
+            throw new RimeInputError("contextTerms must be an array of strings");
+        const format = resolvePCMFormat(options.inputFormat);
+        const target = this.endpoint == null
+            ? policy.target
+            : endpointAddress(this.endpoint).target;
+        const stream = new TranscriptStream({
+            checkOpen: this.checkOpen,
+            forget: (stream) => this.streams.delete(stream),
+            prepare: async (signal) => {
+                this.checkOpen();
+                requireSchema();
+                signal.throwIfAborted();
+                const headers = metadata(this.credentials.authorization());
+                this.connection ??= transport.makeClient(target);
+                await connect(this.connection, signal, policy.connectionTimeout);
+                signal.throwIfAborted();
+                return {
+                    client: this.connection,
+                    metadata: headers,
+                };
+            },
+        }, source, {
+            language: options.language,
+            mode,
+            contextTerms: [...terms],
+            inputFormat: format,
+            timeout: timeout(options.timeout),
+            signal: options.signal,
+        });
+        this.streams.add(stream);
+        return stream;
+    }
+    async close() {
+        try {
+            const results = await Promise.allSettled([...this.streams].map((stream) => stream.cancel()));
+            for (const result of results)
+                if (result.status === "rejected")
+                    throw result.reason;
+        }
+        finally {
+            this.connection?.close();
+        }
+    }
+}

package/dist/stt/policy.d.ts

@@ -0,0 +1,11 @@
+/** Independent resource and transport limits for recognition. Durations are seconds. */
+export declare const policy: {
+    target: string;
+    connectionTimeout: number;
+    acceptanceTimeout: number;
+    completionTimeout: number;
+    cleanupTimeout: number;
+    receiveBytes: number;
+    transcriptBytes: number;
+    queuedUpdates: number;
+};

package/dist/stt/policy.js

@@ -0,0 +1,11 @@
+/** Independent resource and transport limits for recognition. Durations are seconds. */
+export const policy = {
+    target: "stt.api.rime.ai:443",
+    connectionTimeout: 10,
+    acceptanceTimeout: 10,
+    completionTimeout: 120,
+    cleanupTimeout: 2,
+    receiveBytes: 262144,
+    transcriptBytes: 65536,
+    queuedUpdates: 16,
+};

package/dist/stt/protocol.d.ts

@@ -0,0 +1,13 @@
+import * as schema from "@rimelabs/api";
+import type { TranscriptionPartial, TranscriptionFinal } from "./types.js";
+/** Validate the wire sequence before exposing any final result. */
+export declare class TranscriptState {
+    private readonly textLimit;
+    language: string | null;
+    private revision;
+    private text;
+    private final;
+    constructor(textLimit: number);
+    accept(message: schema.StreamingTranscriptionResponse, inputDone: boolean): TranscriptionPartial | null;
+    finish(): TranscriptionFinal;
+}

package/dist/stt/protocol.js

@@ -0,0 +1,61 @@
+import * as schema from "@rimelabs/api";
+import { RimeStreamError, RimeResourceLimitError } from "../errors.js";
+/** Validate the wire sequence before exposing any final result. */
+export class TranscriptState {
+    textLimit;
+    language = null;
+    revision = 0n;
+    text = "";
+    final = null;
+    constructor(textLimit) {
+        this.textLimit = textLimit;
+    }
+    accept(message, inputDone) {
+        const payload = message.payload;
+        if (payload.case === undefined)
+            return null;
+        if (this.final)
+            throw new RimeStreamError("Received a message after transcription completion");
+        if (payload.case === "accepted") {
+            const accepted = payload.value;
+            if (this.language !== null)
+                throw new RimeStreamError("Received duplicate transcription acceptance");
+            if (accepted.outputContract !==
+                schema.StreamingOutputContract.REVISED_HYPOTHESES)
+                throw new RimeStreamError("The service accepted a different transcript contract");
+            if (!accepted.language?.tag || !accepted.language.source)
+                throw new RimeStreamError("The service did not confirm a resolved language");
+            this.language = accepted.language.tag;
+            return null;
+        }
+        if (this.language === null)
+            throw new RimeStreamError("Received a transcript before acceptance");
+        if (payload.case !== "hypothesis" && payload.case !== "done")
+            throw new RimeStreamError("Received an unexpected transcription response");
+        const value = payload.value;
+        if (Buffer.byteLength(value.text, "utf8") > this.textLimit)
+            throw new RimeResourceLimitError("The transcript exceeds the SDK text limit");
+        if (payload.case === "hypothesis") {
+            if (value.revision <= this.revision ||
+                (this.revision === 0n && value.revision !== 1n))
+                throw new RimeStreamError("Transcript revisions are not increasing from one");
+            this.revision = value.revision;
+            this.text = value.text;
+            return { kind: "partial", text: value.text };
+        }
+        if (!inputDone)
+            throw new RimeStreamError("The service completed before input finished");
+        if (value.revision !== this.revision || value.text !== this.text)
+            throw new RimeStreamError("Final transcript does not match the last hypothesis");
+        const language = payload.value.language;
+        if (language?.tag !== this.language || !language.source)
+            throw new RimeStreamError("Final language does not match the accepted language");
+        this.final = { kind: "final", text: value.text, language: language.tag };
+        return null;
+    }
+    finish() {
+        if (!this.final)
+            throw new RimeStreamError("The service ended without a final transcript");
+        return this.final;
+    }
+}

package/dist/stt/queue.d.ts

@@ -0,0 +1,16 @@
+import type { TranscriptionUpdate } from "./types.js";
+/** Bounded, atomic transcript events; failure wakes producers and consumers. */
+export declare class TranscriptQueue {
+    private readonly limit;
+    private items;
+    private closed;
+    private error;
+    private waiters;
+    constructor(limit: number);
+    private wait;
+    private wake;
+    put(value: TranscriptionUpdate): Promise<void>;
+    get(): Promise<IteratorResult<TranscriptionUpdate>>;
+    finish(): void;
+    fail(error: Error): void;
+}

package/dist/stt/queue.js

@@ -0,0 +1,45 @@
+/** Bounded, atomic transcript events; failure wakes producers and consumers. */
+export class TranscriptQueue {
+    limit;
+    items = [];
+    closed = false;
+    error = null;
+    waiters = new Set();
+    constructor(limit) {
+        this.limit = limit;
+    }
+    wait() {
+        return new Promise((resolve) => this.waiters.add(resolve));
+    }
+    wake() {
+        for (const resolve of this.waiters)
+            resolve();
+        this.waiters.clear();
+    }
+    async put(value) {
+        while (!this.closed && this.items.length >= this.limit)
+            await this.wait();
+        if (!this.closed) {
+            this.items.push(value);
+            this.wake();
+        }
+    }
+    async get() {
+        while (!this.closed && !this.items.length)
+            await this.wait();
+        if (this.error)
+            throw this.error;
+        const value = this.items.shift();
+        this.wake();
+        return value ? { done: false, value } : { done: true, value: undefined };
+    }
+    finish() {
+        this.closed = true;
+        this.wake();
+    }
+    fail(error) {
+        this.error ??= error;
+        this.items = [];
+        this.finish();
+    }
+}

package/dist/stt/stream.d.ts

@@ -0,0 +1,48 @@
+import type * as grpc from "@grpc/grpc-js";
+import type { AudioSource, TranscriptionOptions, TranscriptionUpdate } from "./types.js";
+interface StreamOwner {
+    prepare(signal: AbortSignal): Promise<{
+        client: grpc.Client;
+        metadata: grpc.Metadata;
+    }>;
+    checkOpen(): void;
+    forget(stream: TranscriptStream): void;
+}
+/** Lazy, single-consumer stream. Source exhaustion commits input; return/cancel aborts it. */
+export declare class TranscriptStream implements AsyncIterableIterator<TranscriptionUpdate> {
+    private readonly owner;
+    private source;
+    private readonly options;
+    private readonly controller;
+    private readonly queue;
+    private readonly timers;
+    private worker;
+    private workerFinished;
+    private call;
+    private failure;
+    private reading;
+    private finished;
+    private inputDone;
+    private accepted;
+    private readonly acceptance;
+    private admission;
+    private completion;
+    private readonly externalAbort;
+    constructor(owner: StreamOwner, source: AudioSource | null, options: TranscriptionOptions);
+    /** Server request identity, available once response headers arrive. */
+    get requestId(): string | null;
+    private deadline;
+    private start;
+    private fail;
+    private produce;
+    private read;
+    private run;
+    [Symbol.asyncIterator](): this;
+    next(): Promise<IteratorResult<TranscriptionUpdate>>;
+    private cleanup;
+    /** Cancel unfinished work and release owned resources; safe to repeat. */
+    cancel(): Promise<void>;
+    return(): Promise<IteratorResult<TranscriptionUpdate>>;
+    [Symbol.asyncDispose](): Promise<void>;
+}
+export {};