@rimelabs/sdk 0.1.0-alpha.10 → 0.1.0-alpha.13

package/dist/realtime/audio.js

@@ -1,14 +1,7 @@
 import { RimeAudioFormatError } from "../errors.js";
+import { resolvePCMFormat } from "../pcm.js";
 export function formatOf(chunk) {
-    const format = {
-        sampleRate: chunk.format?.sampleRate ?? 16000,
-        channels: chunk.format?.channels ?? 1,
-        encoding: chunk.format?.encoding ?? "pcm_s16le",
-    };
-    if (![8000, 16000, 24000, 48000].includes(format.sampleRate) ||
-        ![1, 2].includes(format.channels) ||
-        format.encoding !== "pcm_s16le")
-        throw new RimeAudioFormatError("Use PCM16 at 8, 16, 24 or 48 kHz, with one or two channels");
+    const format = resolvePCMFormat(chunk.format);
     if (!(chunk.data instanceof Uint8Array) ||
         chunk.data.length % (2 * format.channels))
         throw new RimeAudioFormatError("Audio must contain complete PCM16 frames as Uint8Array");
@@ -16,39 +9,3 @@ export function formatOf(chunk) {
         throw new RimeAudioFormatError("Send audio in chunks of at most 192000 bytes");
     return format;
 }
-/** Streaming linear interpolation, matching Python audioop.ratecv with no filter.
- * Integer phase remains bounded and preserves samples across chunk boundaries.
- */
-export class InputConverter {
-    format;
-    phase = -16000;
-    previous = 0;
-    constructor(format) {
-        this.format = format;
-    }
-    clone() {
-        const copy = new InputConverter(this.format);
-        copy.phase = this.phase;
-        copy.previous = this.previous;
-        return copy;
-    }
-    process(input) {
-        const data = Buffer.from(input.buffer, input.byteOffset, input.byteLength);
-        const samples = [];
-        for (let offset = 0; offset < data.length; offset += this.format.channels * 2) {
-            const current = this.format.channels === 1
-                ? data.readInt16LE(offset)
-                : Math.floor((data.readInt16LE(offset) + data.readInt16LE(offset + 2)) / 2);
-            this.phase += 16000;
-            while (this.phase >= 0) {
-                samples.push(Math.floor((this.previous * this.phase + current * (16000 - this.phase)) /
-                    16000));
-                this.phase -= this.format.sampleRate;
-            }
-            this.previous = current;
-        }
-        const output = Buffer.alloc(samples.length * 2);
-        samples.forEach((sample, index) => output.writeInt16LE(sample, index * 2));
-        return output;
-    }
-}

package/dist/realtime/session.js

@@ -6,7 +6,8 @@ import { setMaxListeners } from "node:events";
 import { abortable } from "../cancellation.js";
 import { RimeError, RimeInputError, RimeResourceLimitError, RimeStreamError, RimeTimeoutError, } from "../errors.js";
 import { Deferred, Flag, Mutex, cancellation, scope } from "./async.js";
-import { formatOf, InputConverter } from "./audio.js";
+import { formatOf } from "./audio.js";
+import { InputConverter } from "../pcm.js";
 import * as t from "./types.js";
 import * as p from "./protocol.js";
 const NOT_READY = new Set([

package/dist/realtime/types.d.ts

@@ -1,3 +1,5 @@
+import type { PCMFormat } from "../pcm.js";
+export type { PCMFormat } from "../pcm.js";
 import { RimeError, RimeTimeoutError } from "../errors.js";
 export type JsonValue = null | boolean | number | string | JsonValue[] | JsonObject;
 export interface JsonObject {
@@ -30,12 +32,6 @@ export interface ReplyOptions extends RealtimeOperationOptions {
     readonly instruction?: string | null;
     readonly toolCall?: ToolCallRef | null;
 }
-/** Omitted fields mean 16 kHz, mono, signed little-endian PCM16. */
-export interface PCMFormat {
-    readonly sampleRate?: 8000 | 16000 | 24000 | 48000;
-    readonly channels?: 1 | 2;
-    readonly encoding?: "pcm_s16le";
-}
 export interface AudioChunk {
     readonly data: Uint8Array;
     readonly format?: PCMFormat;

package/dist/stt/audio.d.ts

@@ -0,0 +1,15 @@
+import { type PCMFormat } from "../pcm.js";
+/** One utterance of headerless signed PCM16 little-endian mono 16 kHz output.
+ * Fully consume each feed iterator before feeding the next chunk. Conversion
+ * advances one bounded source portion at a time so callers can apply backpressure.
+ */
+export declare class InputAudio {
+    private readonly converter;
+    private readonly frameBytes;
+    private pending;
+    private finished;
+    constructor(format?: PCMFormat);
+    feed(data: Uint8Array): Generator<Buffer>;
+    /** Check EOF before committing; never silently pad or truncate a final frame. */
+    finish(): void;
+}

package/dist/stt/audio.js

@@ -0,0 +1,46 @@
+import { RimeAudioFormatError, RimeInputError } from "../errors.js";
+import { InputConverter, resolvePCMFormat } from "../pcm.js";
+// Bound conversion work and temporary allocations for large source chunks.
+const sourceBytes = 16384;
+// Maximum decoded STT audio payload, excluding the protobuf envelope.
+const messageBytes = 65536;
+/** One utterance of headerless signed PCM16 little-endian mono 16 kHz output.
+ * Fully consume each feed iterator before feeding the next chunk. Conversion
+ * advances one bounded source portion at a time so callers can apply backpressure.
+ */
+export class InputAudio {
+    converter;
+    frameBytes;
+    pending = Buffer.alloc(0);
+    finished = false;
+    constructor(format) {
+        const resolved = resolvePCMFormat(format);
+        this.converter = new InputConverter(resolved);
+        this.frameBytes = 2 * resolved.channels;
+    }
+    *feed(data) {
+        if (this.finished)
+            throw new RimeInputError("The audio input is finished");
+        if (!(data instanceof Uint8Array))
+            throw new RimeAudioFormatError("The audio source must yield PCM16 Uint8Array chunks");
+        for (let offset = 0; offset < data.length; offset += sourceBytes) {
+            const part = Buffer.concat([
+                this.pending,
+                data.subarray(offset, offset + sourceBytes),
+            ]);
+            const complete = part.length - (part.length % this.frameBytes);
+            this.pending = Buffer.from(part.subarray(complete));
+            if (complete) {
+                const converted = this.converter.process(part.subarray(0, complete));
+                for (let start = 0; start < converted.length; start += messageBytes)
+                    yield converted.subarray(start, start + messageBytes);
+            }
+        }
+    }
+    /** Check EOF before committing; never silently pad or truncate a final frame. */
+    finish() {
+        this.finished = true;
+        if (this.pending.length)
+            throw new RimeAudioFormatError("The audio source ended with an incomplete PCM16 frame");
+    }
+}

package/dist/stt/client.d.ts

@@ -0,0 +1,13 @@
+import { Credentials } from "../auth.js";
+import { TranscriptStream } from "./stream.js";
+import type { AudioSource, TranscriptionOptions } from "./types.js";
+export declare class STT {
+    private readonly credentials;
+    private readonly endpoint;
+    private readonly checkOpen;
+    private connection;
+    private streams;
+    constructor(credentials: Credentials, endpoint: string | null | undefined, checkOpen: () => void);
+    stream(source: AudioSource, options: TranscriptionOptions): TranscriptStream;
+    close(): Promise<void>;
+}

package/dist/stt/client.js

@@ -0,0 +1,73 @@
+import { connect, metadata } from "../grpc.js";
+import { endpointAddress, timeout } from "../validation.js";
+import { RimeInputError } from "../errors.js";
+import { resolvePCMFormat } from "../pcm.js";
+import { transport, requireSchema } from "./transport.js";
+import { TranscriptStream } from "./stream.js";
+import { policy } from "./policy.js";
+export class STT {
+    credentials;
+    endpoint;
+    checkOpen;
+    connection = null;
+    streams = new Set();
+    constructor(credentials, endpoint, checkOpen) {
+        this.credentials = credentials;
+        this.endpoint = endpoint;
+        this.checkOpen = checkOpen;
+    }
+    stream(source, options) {
+        this.checkOpen();
+        if (!source || typeof source[Symbol.asyncIterator] !== "function")
+            throw new RimeInputError("audio must be an async iterable of Uint8Array chunks");
+        if (!options || typeof options.language !== "string")
+            throw new RimeInputError("language must be a string");
+        const mode = options.mode ?? "written";
+        if (mode !== "written" && mode !== "verbatim")
+            throw new RimeInputError("mode must be written or verbatim");
+        const terms = options.contextTerms ?? [];
+        if (!Array.isArray(terms) || terms.some((term) => typeof term !== "string"))
+            throw new RimeInputError("contextTerms must be an array of strings");
+        const format = resolvePCMFormat(options.inputFormat);
+        const target = this.endpoint == null
+            ? policy.target
+            : endpointAddress(this.endpoint).target;
+        const stream = new TranscriptStream({
+            checkOpen: this.checkOpen,
+            forget: (stream) => this.streams.delete(stream),
+            prepare: async (signal) => {
+                this.checkOpen();
+                requireSchema();
+                signal.throwIfAborted();
+                const headers = metadata(this.credentials.authorization());
+                this.connection ??= transport.makeClient(target);
+                await connect(this.connection, signal, policy.connectionTimeout);
+                signal.throwIfAborted();
+                return {
+                    client: this.connection,
+                    metadata: headers,
+                };
+            },
+        }, source, {
+            language: options.language,
+            mode,
+            contextTerms: [...terms],
+            inputFormat: format,
+            timeout: timeout(options.timeout),
+            signal: options.signal,
+        });
+        this.streams.add(stream);
+        return stream;
+    }
+    async close() {
+        try {
+            const results = await Promise.allSettled([...this.streams].map((stream) => stream.cancel()));
+            for (const result of results)
+                if (result.status === "rejected")
+                    throw result.reason;
+        }
+        finally {
+            this.connection?.close();
+        }
+    }
+}

package/dist/stt/policy.d.ts

@@ -0,0 +1,11 @@
+/** Independent resource and transport limits for recognition. Durations are seconds. */
+export declare const policy: {
+    target: string;
+    connectionTimeout: number;
+    acceptanceTimeout: number;
+    completionTimeout: number;
+    cleanupTimeout: number;
+    receiveBytes: number;
+    transcriptBytes: number;
+    queuedUpdates: number;
+};

package/dist/stt/policy.js

@@ -0,0 +1,11 @@
+/** Independent resource and transport limits for recognition. Durations are seconds. */
+export const policy = {
+    target: "stt.api.rime.ai:443",
+    connectionTimeout: 10,
+    acceptanceTimeout: 10,
+    completionTimeout: 120,
+    cleanupTimeout: 2,
+    receiveBytes: 262144,
+    transcriptBytes: 65536,
+    queuedUpdates: 16,
+};

package/dist/stt/protocol.d.ts

@@ -0,0 +1,13 @@
+import * as schema from "@rimelabs/api";
+import type { TranscriptionPartial, TranscriptionFinal } from "./types.js";
+/** Validate the wire sequence before exposing any final result. */
+export declare class TranscriptState {
+    private readonly textLimit;
+    language: string | null;
+    private revision;
+    private text;
+    private final;
+    constructor(textLimit: number);
+    accept(message: schema.StreamingTranscriptionResponse, inputDone: boolean): TranscriptionPartial | null;
+    finish(): TranscriptionFinal;
+}

package/dist/stt/protocol.js

@@ -0,0 +1,61 @@
+import * as schema from "@rimelabs/api";
+import { RimeStreamError, RimeResourceLimitError } from "../errors.js";
+/** Validate the wire sequence before exposing any final result. */
+export class TranscriptState {
+    textLimit;
+    language = null;
+    revision = 0n;
+    text = "";
+    final = null;
+    constructor(textLimit) {
+        this.textLimit = textLimit;
+    }
+    accept(message, inputDone) {
+        const payload = message.payload;
+        if (payload.case === undefined)
+            return null;
+        if (this.final)
+            throw new RimeStreamError("Received a message after transcription completion");
+        if (payload.case === "accepted") {
+            const accepted = payload.value;
+            if (this.language !== null)
+                throw new RimeStreamError("Received duplicate transcription acceptance");
+            if (accepted.outputContract !==
+                schema.StreamingOutputContract.REVISED_HYPOTHESES)
+                throw new RimeStreamError("The service accepted a different transcript contract");
+            if (!accepted.language?.tag || !accepted.language.source)
+                throw new RimeStreamError("The service did not confirm a resolved language");
+            this.language = accepted.language.tag;
+            return null;
+        }
+        if (this.language === null)
+            throw new RimeStreamError("Received a transcript before acceptance");
+        if (payload.case !== "hypothesis" && payload.case !== "done")
+            throw new RimeStreamError("Received an unexpected transcription response");
+        const value = payload.value;
+        if (Buffer.byteLength(value.text, "utf8") > this.textLimit)
+            throw new RimeResourceLimitError("The transcript exceeds the SDK text limit");
+        if (payload.case === "hypothesis") {
+            if (value.revision <= this.revision ||
+                (this.revision === 0n && value.revision !== 1n))
+                throw new RimeStreamError("Transcript revisions are not increasing from one");
+            this.revision = value.revision;
+            this.text = value.text;
+            return { kind: "partial", text: value.text };
+        }
+        if (!inputDone)
+            throw new RimeStreamError("The service completed before input finished");
+        if (value.revision !== this.revision || value.text !== this.text)
+            throw new RimeStreamError("Final transcript does not match the last hypothesis");
+        const language = payload.value.language;
+        if (language?.tag !== this.language || !language.source)
+            throw new RimeStreamError("Final language does not match the accepted language");
+        this.final = { kind: "final", text: value.text, language: language.tag };
+        return null;
+    }
+    finish() {
+        if (!this.final)
+            throw new RimeStreamError("The service ended without a final transcript");
+        return this.final;
+    }
+}

package/dist/stt/queue.d.ts

@@ -0,0 +1,16 @@
+import type { TranscriptionUpdate } from "./types.js";
+/** Bounded, atomic transcript events; failure wakes producers and consumers. */
+export declare class TranscriptQueue {
+    private readonly limit;
+    private items;
+    private closed;
+    private error;
+    private waiters;
+    constructor(limit: number);
+    private wait;
+    private wake;
+    put(value: TranscriptionUpdate): Promise<void>;
+    get(): Promise<IteratorResult<TranscriptionUpdate>>;
+    finish(): void;
+    fail(error: Error): void;
+}

package/dist/stt/queue.js

@@ -0,0 +1,45 @@
+/** Bounded, atomic transcript events; failure wakes producers and consumers. */
+export class TranscriptQueue {
+    limit;
+    items = [];
+    closed = false;
+    error = null;
+    waiters = new Set();
+    constructor(limit) {
+        this.limit = limit;
+    }
+    wait() {
+        return new Promise((resolve) => this.waiters.add(resolve));
+    }
+    wake() {
+        for (const resolve of this.waiters)
+            resolve();
+        this.waiters.clear();
+    }
+    async put(value) {
+        while (!this.closed && this.items.length >= this.limit)
+            await this.wait();
+        if (!this.closed) {
+            this.items.push(value);
+            this.wake();
+        }
+    }
+    async get() {
+        while (!this.closed && !this.items.length)
+            await this.wait();
+        if (this.error)
+            throw this.error;
+        const value = this.items.shift();
+        this.wake();
+        return value ? { done: false, value } : { done: true, value: undefined };
+    }
+    finish() {
+        this.closed = true;
+        this.wake();
+    }
+    fail(error) {
+        this.error ??= error;
+        this.items = [];
+        this.finish();
+    }
+}

package/dist/stt/stream.d.ts

@@ -0,0 +1,48 @@
+import type * as grpc from "@grpc/grpc-js";
+import type { AudioSource, TranscriptionOptions, TranscriptionUpdate } from "./types.js";
+interface StreamOwner {
+    prepare(signal: AbortSignal): Promise<{
+        client: grpc.Client;
+        metadata: grpc.Metadata;
+    }>;
+    checkOpen(): void;
+    forget(stream: TranscriptStream): void;
+}
+/** Lazy, single-consumer stream. Source exhaustion commits input; return/cancel aborts it. */
+export declare class TranscriptStream implements AsyncIterableIterator<TranscriptionUpdate> {
+    private readonly owner;
+    private source;
+    private readonly options;
+    private readonly controller;
+    private readonly queue;
+    private readonly timers;
+    private worker;
+    private workerFinished;
+    private call;
+    private failure;
+    private reading;
+    private finished;
+    private inputDone;
+    private accepted;
+    private readonly acceptance;
+    private admission;
+    private completion;
+    private readonly externalAbort;
+    constructor(owner: StreamOwner, source: AudioSource | null, options: TranscriptionOptions);
+    /** Server request identity, available once response headers arrive. */
+    get requestId(): string | null;
+    private deadline;
+    private start;
+    private fail;
+    private produce;
+    private read;
+    private run;
+    [Symbol.asyncIterator](): this;
+    next(): Promise<IteratorResult<TranscriptionUpdate>>;
+    private cleanup;
+    /** Cancel unfinished work and release owned resources; safe to repeat. */
+    cancel(): Promise<void>;
+    return(): Promise<IteratorResult<TranscriptionUpdate>>;
+    [Symbol.asyncDispose](): Promise<void>;
+}
+export {};

package/dist/stt/stream.js

@@ -0,0 +1,224 @@
+import { abortable, scope } from "../cancellation.js";
+import { RimeError, RimeInputError, RimeCancelledError, RimeTimeoutError, RimeStreamError, } from "../errors.js";
+import { InputAudio } from "./audio.js";
+import { TranscriptQueue } from "./queue.js";
+import { TranscriptState } from "./protocol.js";
+import { TranscriptionCall } from "./transport.js";
+import { policy } from "./policy.js";
+/** Lazy, single-consumer stream. Source exhaustion commits input; return/cancel aborts it. */
+export class TranscriptStream {
+    owner;
+    source;
+    options;
+    controller = new AbortController();
+    queue = new TranscriptQueue(policy.queuedUpdates);
+    timers = new Set();
+    worker = null;
+    workerFinished = false;
+    call = null;
+    failure = null;
+    reading = false;
+    finished = false;
+    inputDone = false;
+    accepted;
+    acceptance = new Promise((resolve) => {
+        this.accepted = resolve;
+    });
+    admission = null;
+    completion = null;
+    externalAbort = () => this.fail(new RimeCancelledError("Transcription cancelled"));
+    constructor(owner, source, options) {
+        this.owner = owner;
+        this.source = source;
+        this.options = options;
+    }
+    /** Server request identity, available once response headers arrive. */
+    get requestId() {
+        return this.call?.requestId ?? null;
+    }
+    deadline(seconds, message) {
+        const timer = scope([], seconds, new RimeTimeoutError(message));
+        timer.signal.addEventListener("abort", () => this.fail(timer.signal.reason), { once: true });
+        if (timer.signal.aborted)
+            this.fail(timer.signal.reason);
+        this.timers.add(timer);
+        return timer;
+    }
+    start() {
+        if (this.failure)
+            throw this.failure;
+        if (this.finished || this.worker)
+            return;
+        this.owner.checkOpen();
+        if (this.options.signal?.aborted) {
+            this.externalAbort();
+            throw this.failure;
+        }
+        this.options.signal?.addEventListener("abort", this.externalAbort, {
+            once: true,
+        });
+        if (this.options.timeout != null)
+            this.deadline(this.options.timeout, "Overall transcription deadline expired");
+        this.worker = this.run();
+    }
+    fail(error) {
+        if (this.failure || this.finished)
+            return;
+        if (error.requestId === null && this.requestId !== null)
+            error = new error.constructor(error.message, this.requestId, { cause: error.cause });
+        this.failure = error;
+        this.queue.fail(error);
+        for (const timer of this.timers)
+            timer.dispose();
+        this.controller.abort(error);
+        this.call?.cancel();
+        this.options.signal?.removeEventListener("abort", this.externalAbort);
+        if (!this.worker || this.workerFinished) {
+            this.source = null;
+            this.owner.forget(this);
+        }
+    }
+    async produce() {
+        const signal = this.controller.signal;
+        await abortable(this.acceptance, signal);
+        const audio = new InputAudio(this.options.inputFormat);
+        let iterator;
+        try {
+            try {
+                iterator = this.source[Symbol.asyncIterator]();
+            }
+            catch (error) {
+                throw new RimeInputError("The audio source failed", this.requestId, {
+                    cause: error,
+                });
+            }
+            for (;;) {
+                let result;
+                try {
+                    result = await abortable(iterator.next(), signal);
+                }
+                catch (error) {
+                    if (signal.aborted)
+                        throw signal.reason;
+                    throw new RimeInputError("The audio source failed", this.requestId, {
+                        cause: error,
+                    });
+                }
+                if (result.done)
+                    break;
+                for (const part of audio.feed(result.value))
+                    await this.call.writeAudio(part);
+            }
+            audio.finish();
+            this.inputDone = true;
+            this.completion = this.deadline(policy.completionTimeout, "Transcription completion timed out");
+            this.call.finishInput();
+        }
+        finally {
+            if (iterator?.return) {
+                const cleanup = scope([], policy.cleanupTimeout, new RimeTimeoutError("Audio source cleanup timed out"));
+                try {
+                    await abortable(Promise.resolve().then(() => iterator.return()), cleanup.signal);
+                }
+                catch {
+                    /* Caller-owned source cleanup cannot replace the operation outcome. */
+                }
+                finally {
+                    cleanup.dispose();
+                }
+            }
+        }
+    }
+    async read() {
+        const state = new TranscriptState(policy.transcriptBytes);
+        for await (const message of this.call.responses()) {
+            const update = state.accept(message, this.inputDone);
+            if (state.language !== null) {
+                this.admission?.dispose();
+                this.accepted();
+            }
+            if (update)
+                await this.queue.put(update);
+        }
+        return state.finish();
+    }
+    async run() {
+        let tasks = [];
+        try {
+            const prepared = await this.owner.prepare(this.controller.signal);
+            this.controller.signal.throwIfAborted();
+            this.call = new TranscriptionCall(prepared.client, prepared.metadata, this.controller.signal);
+            this.admission = this.deadline(policy.acceptanceTimeout, "Transcription acceptance timed out");
+            await this.call.start(this.options.language, this.options.mode ?? "written", this.options.contextTerms ?? []);
+            const producer = this.produce(), reader = this.read();
+            tasks = [producer, reader];
+            const [, final] = await Promise.all([producer, reader]);
+            this.completion?.dispose();
+            await this.queue.put(final);
+            this.queue.finish();
+        }
+        catch (error) {
+            this.fail(error instanceof RimeError
+                ? error
+                : new RimeStreamError("Transcription failed", this.requestId, {
+                    cause: error,
+                }));
+        }
+        finally {
+            if (this.failure)
+                this.controller.abort(this.failure);
+            this.call?.cancel();
+            await Promise.allSettled(tasks);
+            this.source = null;
+            this.workerFinished = true;
+            if (this.failure)
+                this.owner.forget(this);
+        }
+    }
+    [Symbol.asyncIterator]() {
+        return this;
+    }
+    async next() {
+        if (this.reading)
+            throw new RimeInputError("TranscriptStream permits only one concurrent reader");
+        this.reading = true;
+        try {
+            this.start();
+            if (this.finished)
+                return { done: true, value: undefined };
+            const result = await this.queue.get();
+            if (this.failure)
+                throw this.failure;
+            if (!result.done && result.value.kind === "final") {
+                await this.cleanup();
+                if (this.failure)
+                    throw this.failure;
+                this.finished = true;
+            }
+            return result;
+        }
+        finally {
+            this.reading = false;
+        }
+    }
+    async cleanup() {
+        await this.worker;
+        for (const timer of this.timers)
+            timer.dispose();
+        this.options.signal?.removeEventListener("abort", this.externalAbort);
+        this.owner.forget(this);
+    }
+    /** Cancel unfinished work and release owned resources; safe to repeat. */
+    async cancel() {
+        this.fail(new RimeCancelledError("Transcription cancelled"));
+        await this.cleanup();
+        this.source = null;
+    }
+    async return() {
+        await this.cancel();
+        return { done: true, value: undefined };
+    }
+    async [Symbol.asyncDispose]() {
+        await this.cancel();
+    }
+}