@rimelabs/sdk 0.1.0-alpha.5 → 0.1.0-alpha.6

package/dist/client.js

@@ -1,18 +1,15 @@
-import { AudioFormat } from "./audio.js";
-import { AudioStream, constructionKey } from "./stream.js";
 import { Credentials } from "./auth.js";
-import { policy, resolve, timeout, nonempty, abortable } from "./policy.js";
-import { transport, connect, metadata, discover, } from "./transport.js";
-import { RimeAuthenticationError, RimeInputError, RimeAudioFormatError, RimeCancelledError, RimeTimeoutError, RimeUnavailableError, } from "./errors.js";
+import { RimeAuthenticationError, RimeInputError } from "./errors.js";
+import { TTS, } from "./tts/client.js";
+import { resolve, timeout } from "./tts/policy.js";
+import { Realtime } from "./realtime/client.js";
 export class Rime {
-    deployment;
     credentials;
-    client = null;
+    ttsClient;
+    realtimeClient;
+    realtime;
     closed = false;
-    streams = new Set();
-    discoveryControllers = new Set();
     closing = null;
-    defaultTimeout;
     tts;
     voices;
     languages;
@@ -22,99 +19,33 @@ export class Rime {
             : options.apiKey;
         if (typeof key !== "string" || !key.trim())
             throw new RimeAuthenticationError("Provide apiKey or set RIME_API_KEY");
-        this.defaultTimeout = timeout(options.timeout);
-        this.deployment = resolve(options.model ?? "coda", options.endpoint);
-        this.credentials = new Credentials(key, this.deployment);
-        this.tts = { stream: (text, options = {}) => this.stream(text, options) };
-        this.voices = {
-            list: async (options = {}) => {
-                if (options.language !== undefined && options.language !== null)
-                    nonempty(options.language, "language");
-                return this.discover("voices", options.language, options.timeout);
-            },
+        const defaultTimeout = timeout(options.timeout);
+        const deployment = resolve(options.model ?? "coda", options.endpoint);
+        this.credentials = new Credentials(key, deployment);
+        this.realtimeClient = new Realtime(this.credentials, () => this.checkOpen());
+        this.realtime = {
+            connect: (options) => this.realtimeClient.connect(options),
         };
-        this.languages = {
-            list: (options = {}) => this.discover("languages", undefined, options.timeout),
+        this.ttsClient = new TTS(this.credentials, deployment, defaultTimeout, () => this.checkOpen());
+        this.tts = {
+            stream: (text, options) => this.ttsClient.stream(text, options),
         };
+        this.voices = this.ttsClient.voices;
+        this.languages = this.ttsClient.languages;
     }
     checkOpen() {
         if (this.closed)
             throw new RimeInputError("The Rime client is closed");
     }
-    async prepare(signal) {
-        this.checkOpen();
-        const auth = await this.credentials.metadata(signal);
-        this.checkOpen();
-        this.client ??= transport.makeClient(this.deployment.target);
-        await connect(this.client, signal);
-        return { client: this.client, metadata: metadata(auth) };
-    }
-    stream(text, options) {
-        this.checkOpen();
-        if (typeof text === "string") {
-            if (!text.trim())
-                throw new RimeInputError("Text must contain non-whitespace characters");
-        }
-        else if (!text || typeof text[Symbol.asyncIterator] !== "function")
-            throw new RimeInputError("text must be a string or an async iterable of strings");
-        const voice = options.voice === undefined || options.voice === null
-            ? this.deployment.defaultVoice
-            : nonempty(options.voice, "voice");
-        const language = nonempty(options.language ?? "en", "language");
-        const format = options.audioFormat ?? AudioFormat.PCM_24000;
-        if (format !== AudioFormat.PCM_24000 && format !== AudioFormat.MULAW_8000)
-            throw new RimeAudioFormatError("Select a named AudioFormat profile");
-        const audio = new AudioStream(constructionKey, {
-            prepare: (s) => this.prepare(s),
-            checkOpen: () => this.checkOpen(),
-            forget: (s) => this.streams.delete(s),
-        }, text, voice, language, format, timeout(options.timeout, this.defaultTimeout));
-        this.streams.add(audio);
-        return audio;
-    }
-    async discover(kind, language, override) {
-        this.checkOpen();
-        const budget = Math.min(timeout(override, this.defaultTimeout) ?? Infinity, policy.discoveryTimeout);
-        const controller = new AbortController();
-        let requestId = null;
-        this.discoveryControllers.add(controller);
-        const timer = setTimeout(() => controller.abort(new RimeTimeoutError("Discovery deadline expired")), budget * 1000);
-        try {
-            const prepared = await this.prepare(controller.signal);
-            for (let attempt = 0;; attempt++) {
-                requestId = null;
-                try {
-                    return await discover(prepared, kind, language, controller.signal);
-                }
-                catch (error) {
-                    if (!(error instanceof RimeUnavailableError) || attempt === 2)
-                        throw error;
-                    requestId = error.requestId;
-                    await abortable(new Promise((resolve) => setTimeout(resolve, 50 * 2 ** attempt)), controller.signal);
-                }
-            }
-        }
-        catch (error) {
-            if (error instanceof RimeTimeoutError && requestId !== null)
-                throw new RimeTimeoutError(error.message, requestId, {
-                    cause: error.cause,
-                });
-            throw error;
-        }
-        finally {
-            clearTimeout(timer);
-            this.discoveryControllers.delete(controller);
-        }
-    }
     async close() {
         if (!this.closing) {
             this.closed = true;
             this.closing = (async () => {
-                for (const controller of this.discoveryControllers)
-                    controller.abort(new RimeCancelledError("Client closed during discovery"));
-                await Promise.all([...this.streams].map((stream) => stream.cancel()));
+                await Promise.all([
+                    this.ttsClient.close(),
+                    this.realtimeClient.close(),
+                ]);
                 await this.credentials.close();
-                this.client?.close();
             })();
         }
         await this.closing;

package/dist/index.d.ts

@@ -1,4 +1,6 @@
 export { Rime, type RimeOptions, type SynthesisOptions, type DiscoveryOptions, type VoiceListOptions, } from "./client.js";
-export { AudioStream } from "./stream.js";
-export { AudioFormat } from "./audio.js";
+export { AudioStream } from "./tts/stream.js";
+export { AudioFormat } from "./tts/audio.js";
 export * from "./errors.js";
+export { RealtimeSession } from "./realtime/session.js";
+export * from "./realtime/types.js";

package/dist/index.js

@@ -1,4 +1,6 @@
 export { Rime, } from "./client.js";
-export { AudioStream } from "./stream.js";
-export { AudioFormat } from "./audio.js";
+export { AudioStream } from "./tts/stream.js";
+export { AudioFormat } from "./tts/audio.js";
 export * from "./errors.js";
+export { RealtimeSession } from "./realtime/session.js";
+export * from "./realtime/types.js";

package/dist/realtime/async.d.ts

@@ -0,0 +1,27 @@
+export declare class Deferred<T> {
+    readonly promise: Promise<T>;
+    resolve: (value: T) => void;
+    reject: (error: unknown) => void;
+    constructor();
+}
+export declare class Flag {
+    private waiters;
+    isSet: boolean;
+    set(): void;
+    clear(): void;
+    wait(signal: AbortSignal): Promise<void>;
+}
+export declare class Mutex {
+    private locked;
+    private waiters;
+    acquire(signal: AbortSignal): Promise<() => void>;
+}
+/** A disposable deadline avoids retaining timers and caller abort listeners. */
+export declare function scope(signals: (AbortSignal | undefined)[], seconds?: number, error?: Error): {
+    signal: AbortSignal;
+    dispose(): void;
+};
+export declare function cancellation(signal?: AbortSignal): {
+    signal: AbortSignal;
+    dispose: () => void | undefined;
+};

package/dist/realtime/async.js

@@ -0,0 +1,122 @@
+import { abortable } from "../cancellation.js";
+import { RimeCancelledError } from "../errors.js";
+export class Deferred {
+    promise;
+    resolve;
+    reject;
+    constructor() {
+        this.promise = new Promise((resolve, reject) => {
+            this.resolve = resolve;
+            this.reject = reject;
+        });
+        // Session failures can arrive before a caller starts waiting.
+        this.promise.catch(() => { });
+    }
+}
+export class Flag {
+    waiters = new Set();
+    isSet = false;
+    set() {
+        this.isSet = true;
+        for (const wake of this.waiters)
+            wake();
+    }
+    clear() {
+        this.isSet = false;
+    }
+    wait(signal) {
+        if (signal.aborted)
+            return Promise.reject(signal.reason);
+        if (this.isSet)
+            return Promise.resolve();
+        return new Promise((resolve, reject) => {
+            const cleanup = () => {
+                this.waiters.delete(wake);
+                signal.removeEventListener("abort", abort);
+            };
+            const wake = () => {
+                cleanup();
+                resolve();
+            };
+            const abort = () => {
+                cleanup();
+                reject(signal.reason);
+            };
+            this.waiters.add(wake);
+            signal.addEventListener("abort", abort, { once: true });
+        });
+    }
+}
+export class Mutex {
+    locked = false;
+    waiters = new Set();
+    async acquire(signal) {
+        signal.throwIfAborted();
+        while (this.locked) {
+            const wake = new Deferred();
+            const notify = () => wake.resolve();
+            this.waiters.add(notify);
+            try {
+                await abortable(wake.promise, signal);
+            }
+            finally {
+                this.waiters.delete(notify);
+            }
+            signal.throwIfAborted();
+        }
+        this.locked = true;
+        return () => {
+            this.locked = false;
+            for (const wake of this.waiters)
+                wake();
+        };
+    }
+}
+/** A disposable deadline avoids retaining timers and caller abort listeners. */
+export function scope(signals, seconds, error) {
+    const controller = new AbortController();
+    const handlers = new Map();
+    for (const signal of new Set(signals)) {
+        if (!signal)
+            continue;
+        const abort = () => controller.abort(signal.reason);
+        if (signal.aborted)
+            abort();
+        else {
+            signal.addEventListener("abort", abort, { once: true });
+            handlers.set(signal, abort);
+        }
+    }
+    let timer;
+    if (seconds !== undefined) {
+        const end = performance.now() + seconds * 1000;
+        const expire = () => {
+            const remaining = end - performance.now();
+            if (remaining <= 0)
+                controller.abort(error);
+            else
+                timer = setTimeout(expire, Math.min(remaining, 2147483647));
+        };
+        expire();
+    }
+    return {
+        signal: controller.signal,
+        dispose() {
+            clearTimeout(timer);
+            for (const [signal, handler] of handlers)
+                signal.removeEventListener("abort", handler);
+        },
+    };
+}
+export function cancellation(signal) {
+    const controller = new AbortController();
+    const abort = () => controller.abort(new RimeCancelledError("Realtime operation cancelled"));
+    if (signal?.aborted)
+        abort();
+    else
+        signal?.addEventListener("abort", abort, { once: true });
+    return {
+        signal: controller.signal,
+        dispose: () => signal?.removeEventListener("abort", abort),
+    };
+}

package/dist/realtime/audio.d.ts

@@ -0,0 +1,13 @@
+import type { AudioChunk, PCMFormat } from "./types.js";
+export declare function formatOf(chunk: AudioChunk): Required<PCMFormat>;
+/** Streaming linear interpolation, matching Python audioop.ratecv with no filter.
+ * Integer phase remains bounded and preserves samples across chunk boundaries.
+ */
+export declare class InputConverter {
+    readonly format: Required<PCMFormat>;
+    private phase;
+    private previous;
+    constructor(format: Required<PCMFormat>);
+    clone(): InputConverter;
+    process(input: Uint8Array): Buffer;
+}

package/dist/realtime/audio.js

@@ -0,0 +1,54 @@
+import { RimeAudioFormatError } from "../errors.js";
+export function formatOf(chunk) {
+    const format = {
+        sampleRate: chunk.format?.sampleRate ?? 16000,
+        channels: chunk.format?.channels ?? 1,
+        encoding: chunk.format?.encoding ?? "pcm_s16le",
+    };
+    if (![8000, 16000, 24000, 48000].includes(format.sampleRate) ||
+        ![1, 2].includes(format.channels) ||
+        format.encoding !== "pcm_s16le")
+        throw new RimeAudioFormatError("Use PCM16 at 8, 16, 24 or 48 kHz, with one or two channels");
+    if (!(chunk.data instanceof Uint8Array) ||
+        chunk.data.length % (2 * format.channels))
+        throw new RimeAudioFormatError("Audio must contain complete PCM16 frames as Uint8Array");
+    if (chunk.data.length > 192000)
+        throw new RimeAudioFormatError("Send audio in chunks of at most 192000 bytes");
+    return format;
+}
+/** Streaming linear interpolation, matching Python audioop.ratecv with no filter.
+ * Integer phase remains bounded and preserves samples across chunk boundaries.
+ */
+export class InputConverter {
+    format;
+    phase = -16000;
+    previous = 0;
+    constructor(format) {
+        this.format = format;
+    }
+    clone() {
+        const copy = new InputConverter(this.format);
+        copy.phase = this.phase;
+        copy.previous = this.previous;
+        return copy;
+    }
+    process(input) {
+        const data = Buffer.from(input.buffer, input.byteOffset, input.byteLength);
+        const samples = [];
+        for (let offset = 0; offset < data.length; offset += this.format.channels * 2) {
+            const current = this.format.channels === 1
+                ? data.readInt16LE(offset)
+                : Math.floor((data.readInt16LE(offset) + data.readInt16LE(offset + 2)) / 2);
+            this.phase += 16000;
+            while (this.phase >= 0) {
+                samples.push(Math.floor((this.previous * this.phase + current * (16000 - this.phase)) /
+                    16000));
+                this.phase -= this.format.sampleRate;
+            }
+            this.previous = current;
+        }
+        const output = Buffer.alloc(samples.length * 2);
+        samples.forEach((sample, index) => output.writeInt16LE(sample, index * 2));
+        return output;
+    }
+}

package/dist/realtime/client.d.ts

@@ -0,0 +1,12 @@
+import { Credentials } from "../auth.js";
+import { RealtimeSession } from "./session.js";
+import type { RealtimeConnectOptions } from "./types.js";
+export declare class Realtime {
+    private readonly credentials;
+    private readonly checkOpen;
+    private readonly sessions;
+    private readonly opening;
+    constructor(credentials: Credentials, checkOpen: () => void);
+    connect(options: RealtimeConnectOptions): Promise<RealtimeSession>;
+    close(): Promise<void>;
+}

package/dist/realtime/client.js

@@ -0,0 +1,156 @@
+import { abortable } from "../cancellation.js";
+import { RimeCancelledError, RimeInputError, RimeUnavailableError, } from "../errors.js";
+import { cancellation, scope } from "./async.js";
+import { RealtimeSession } from "./session.js";
+import { RealtimeConnection } from "./transport.js";
+function endpoint(value) {
+    let url;
+    try {
+        url = new URL(value);
+    }
+    catch {
+        throw new RimeInputError("endpoint must be a full ws:// or wss:// URL ending in /v1/realtime");
+    }
+    if (!["ws:", "wss:"].includes(url.protocol) ||
+        !url.hostname ||
+        url.username ||
+        url.password ||
+        url.hash ||
+        url.pathname.replace(/\/+$/, "") !== "/v1/realtime")
+        throw new RimeInputError("endpoint must be a full ws:// or wss:// URL ending in /v1/realtime");
+    if (url.protocol === "ws:" &&
+        !["localhost", "127.0.0.1", "[::1]"].includes(url.hostname))
+        throw new RimeInputError("Use wss:// outside localhost to protect credentials");
+    return url.href;
+}
+function timeouts(options = {}) {
+    const result = {
+        connectS: options.connectS ?? 10,
+        readyS: options.readyS ?? 30,
+        requestS: options.requestS ?? 10,
+    };
+    if (Object.values(result).some((value) => typeof value !== "number" || !Number.isFinite(value) || value <= 0))
+        throw new RimeInputError("Timeouts must be finite and positive");
+    return result;
+}
+/** Reject values that JSON.stringify would silently drop or change. */
+function copyJson(value) {
+    if (value === null || typeof value === "string" || typeof value === "boolean")
+        return value;
+    if (typeof value === "number" && Number.isFinite(value))
+        return value;
+    if (Array.isArray(value))
+        return Array.from(value, copyJson);
+    if (value &&
+        typeof value === "object" &&
+        [Object.prototype, null].includes(Object.getPrototypeOf(value)))
+        return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, copyJson(item)]));
+    throw new RimeInputError("Tool schemas must contain finite JSON values");
+}
+export class Realtime {
+    credentials;
+    checkOpen;
+    sessions = new Set();
+    opening = new Map();
+    constructor(credentials, checkOpen) {
+        this.credentials = credentials;
+        this.checkOpen = checkOpen;
+    }
+    async connect(options) {
+        this.checkOpen();
+        const address = endpoint(options.endpoint);
+        const limits = timeouts(options.timeouts);
+        if ((options.model ?? "prism") !== "prism")
+            throw new RimeInputError("The realtime model must be prism");
+        if (options.interruptOnSpeech !== undefined &&
+            typeof options.interruptOnSpeech !== "boolean")
+            throw new RimeInputError("interruptOnSpeech must be a boolean");
+        const names = new Set();
+        const settings = {
+            modalities: ["text", "audio"],
+            input_audio_format: "pcm16",
+            turn_detection: { interrupt_response: options.interruptOnSpeech ?? true },
+            tools: (options.tools ?? []).map((tool) => {
+                if (typeof tool.name !== "string" ||
+                    !tool.name.trim() ||
+                    names.has(tool.name))
+                    throw new RimeInputError("Tool names must be nonblank and unique");
+                names.add(tool.name);
+                if (!tool.parameters ||
+                    Array.isArray(tool.parameters) ||
+                    typeof tool.parameters !== "object" ||
+                    (tool.description !== undefined &&
+                        typeof tool.description !== "string"))
+                    throw new RimeInputError("Tool parameters must be a JSON object and description must be a string");
+                let parameters;
+                try {
+                    parameters = copyJson(tool.parameters);
+                }
+                catch {
+                    throw new RimeInputError("Tool schemas must contain finite JSON values");
+                }
+                if (parameters === null ||
+                    typeof parameters !== "object" ||
+                    Array.isArray(parameters))
+                    throw new RimeInputError("Tool parameters must be an object");
+                return {
+                    type: "function",
+                    function: {
+                        name: tool.name,
+                        description: tool.description ?? "",
+                        parameters,
+                    },
+                };
+            }),
+        };
+        for (const name of ["voice", "instructions"]) {
+            const value = options[name];
+            if (value !== undefined && value !== null) {
+                if (typeof value !== "string" || !value.trim())
+                    throw new RimeInputError(`${name} must be nonblank`);
+                settings[name] = value;
+            }
+        }
+        const owner = new AbortController();
+        const caller = cancellation(options.signal);
+        const openingScope = scope([caller.signal, owner.signal]);
+        const task = (async () => {
+            let session;
+            try {
+                openingScope.signal.throwIfAborted();
+                let socket;
+                try {
+                    socket = new RealtimeConnection(address, this.credentials.realtimeAuthorization());
+                }
+                catch {
+                    throw new RimeUnavailableError("Realtime handshake failed");
+                }
+                session = new RealtimeSession(socket, limits, () => this.sessions.delete(session));
+                this.sessions.add(session);
+                await abortable(session.initialize(settings, openingScope.signal), openingScope.signal);
+                openingScope.signal.throwIfAborted();
+                this.checkOpen();
+                return session;
+            }
+            catch (error) {
+                await session?.close();
+                throw error;
+            }
+        })();
+        this.opening.set(owner, task);
+        try {
+            return await task;
+        }
+        finally {
+            this.opening.delete(owner);
+            openingScope.dispose();
+            caller.dispose();
+        }
+    }
+    async close() {
+        for (const controller of this.opening.keys())
+            controller.abort(new RimeCancelledError("Client closed"));
+        await Promise.allSettled(this.opening.values());
+        await Promise.all([...this.sessions].map((session) => session.close()));
+    }
+}

package/dist/realtime/protocol.d.ts

@@ -0,0 +1,80 @@
+import * as t from "./types.js";
+export interface SessionSettings {
+    modalities: ("text" | "audio")[];
+    input_audio_format: "pcm16";
+    turn_detection: {
+        interrupt_response: boolean;
+    };
+    tools: {
+        type: "function";
+        function: {
+            name: string;
+            description: string;
+            parameters: t.JsonObject;
+        };
+    }[];
+    voice?: string;
+    instructions?: string;
+}
+export interface SessionView {
+    id: string;
+    voice: string;
+    interruptOnSpeech: boolean | null;
+    toolResultTimeoutS: number | null;
+    toolContinuationTimeoutS: number | null;
+}
+export interface SessionCreated {
+    kind: "session.created";
+    session: SessionView;
+}
+export interface SessionUpdated {
+    kind: "session.updated";
+    session: SessionView;
+}
+export interface ItemCreated {
+    kind: "item.created";
+    itemId: string;
+}
+export interface ToolResultCreated {
+    kind: "tool.result.created";
+    callId: string;
+}
+export interface AudioCleared {
+    kind: "audio.cleared";
+}
+export interface ItemTruncated {
+    kind: "item.truncated";
+    itemId: string;
+    contentIndex: number;
+}
+export interface ResponseEnded extends t.ResponseEnded {
+    drainToken: string | null;
+}
+export type Acknowledgment = SessionUpdated | ItemCreated | ToolResultCreated | AudioCleared | ItemTruncated | t.ResponseStarted | ResponseEnded;
+export type ServerPayload = SessionCreated | Acknowledgment | t.InputEvent | t.MessageStarted | t.TextDelta | t.TextDone | t.AudioDelta | t.AudioDone | t.ToolCall | t.FaultEvent;
+export interface ServerEvent {
+    eventId: string;
+    requestId: string | null;
+    payload: ServerPayload;
+}
+export type RequestKind = "session.update" | "conversation.item.create" | "response.create" | "response.cancel" | "input_audio_buffer.clear" | "conversation.item.truncate";
+/** A request carries its result type and checks it before exposing the result. */
+export declare class Request<T extends Acknowledgment> {
+    readonly kind: RequestKind;
+    private readonly accepts;
+    readonly allowsItemCreated: boolean;
+    constructor(kind: RequestKind, accepts: (event: Acknowledgment) => event is T, allowsItemCreated?: boolean);
+    result(event: Acknowledgment): T;
+    check(event: Acknowledgment): void;
+}
+export declare const requests: {
+    update: Request<SessionUpdated>;
+    item: Request<ItemCreated>;
+    toolResult: Request<ToolResultCreated>;
+    create: Request<t.ResponseStarted>;
+    text: Request<t.ResponseStarted>;
+    cancel: Request<ResponseEnded>;
+    clear: Request<AudioCleared>;
+    truncate: Request<ItemTruncated>;
+};
+export declare function decode(raw: string, sessionId: string | null): ServerEvent | null;