@rimelabs/sdk 0.1.0-alpha.2 → 0.1.0-alpha.21

package/dist/realtime/client.js

@@ -0,0 +1,139 @@
+import { abortable } from "../cancellation.js";
+import { RimeCancelledError, RimeInputError, RimeUnavailableError, } from "../errors.js";
+import { cancellation, scope } from "./async.js";
+import { RealtimeSession } from "./session.js";
+import { RealtimeConnection } from "./transport.js";
+import { toolSettings } from "./settings.js";
+import { responseMetadata } from "./protocol.js";
+const DEFAULT_ENDPOINT = "wss://prism.api.rime.ai/v1/realtime";
+function endpoint(value) {
+    let url;
+    try {
+        url = new URL(value);
+    }
+    catch {
+        throw new RimeInputError("endpoint must be a full ws:// or wss:// URL ending in /v1/realtime");
+    }
+    if (!["ws:", "wss:"].includes(url.protocol) ||
+        !url.hostname ||
+        url.username ||
+        url.password ||
+        url.hash ||
+        url.pathname.replace(/\/+$/, "") !== "/v1/realtime")
+        throw new RimeInputError("endpoint must be a full ws:// or wss:// URL ending in /v1/realtime");
+    if (url.protocol === "ws:" &&
+        !["localhost", "127.0.0.1", "[::1]"].includes(url.hostname))
+        throw new RimeInputError("Use wss:// outside localhost to protect credentials");
+    return url.href;
+}
+function timeouts(options = {}) {
+    const result = {
+        connectS: options.connectS ?? 10,
+        readyS: options.readyS ?? 30,
+        requestS: options.requestS ?? 10,
+    };
+    if (Object.values(result).some((value) => typeof value !== "number" || !Number.isFinite(value) || value <= 0))
+        throw new RimeInputError("Timeouts must be finite and positive");
+    return result;
+}
+export class Realtime {
+    credentials;
+    checkOpen;
+    sessions = new Set();
+    opening = new Map();
+    constructor(credentials, checkOpen) {
+        this.credentials = credentials;
+        this.checkOpen = checkOpen;
+    }
+    async connect(options = {}) {
+        this.checkOpen();
+        const address = endpoint(options.endpoint === undefined ? DEFAULT_ENDPOINT : options.endpoint);
+        const limits = timeouts(options.timeouts);
+        if ((options.model ?? "prism") !== "prism")
+            throw new RimeInputError("The realtime model must be prism");
+        if (options.interruptOnSpeech !== undefined &&
+            typeof options.interruptOnSpeech !== "boolean")
+            throw new RimeInputError("interruptOnSpeech must be a boolean");
+        const settings = {
+            modalities: ["text", "audio"],
+            input_audio_format: "pcm16",
+            turn_detection: { interrupt_response: options.interruptOnSpeech ?? true },
+            tools: toolSettings(options.tools ?? []),
+        };
+        if (options.turnDetection !== undefined) {
+            const detection = options.turnDetection;
+            if (!detection ||
+                typeof detection !== "object" ||
+                Array.isArray(detection))
+                throw new RimeInputError("turnDetection must be an object");
+            if (detection.eagerness !== undefined) {
+                if (!["low", "medium", "high", "auto"].includes(detection.eagerness))
+                    throw new RimeInputError("Turn eagerness must be low, medium, high, or auto");
+                settings.turn_detection.eagerness = detection.eagerness;
+            }
+            if (detection.endOfTurnTimeoutMs !== undefined) {
+                const value = detection.endOfTurnTimeoutMs;
+                if (!Number.isInteger(value) || value < 1000 || value > 10000)
+                    throw new RimeInputError("End-of-turn timeout must be an integer from 1000 to 10000 ms");
+                settings.turn_detection.prism_end_of_turn_timeout_ms = value;
+            }
+        }
+        if (options.responseMetadata != null)
+            settings.prism_response_metadata = responseMetadata(options.responseMetadata);
+        const requestId = options.requestId ?? undefined;
+        if (requestId !== undefined &&
+            (typeof requestId !== "string" ||
+                !requestId.trim() ||
+                !/^[\x20-\x7e]+$/.test(requestId)))
+            throw new RimeInputError("requestId must be nonblank printable ASCII");
+        for (const name of ["voice", "instructions"]) {
+            const value = options[name];
+            if (value !== undefined && value !== null) {
+                if (typeof value !== "string" || !value.trim())
+                    throw new RimeInputError(`${name} must be nonblank`);
+                settings[name] = value;
+            }
+        }
+        const owner = new AbortController();
+        const caller = cancellation(options.signal);
+        const openingScope = scope([caller.signal, owner.signal]);
+        const task = (async () => {
+            let session;
+            try {
+                openingScope.signal.throwIfAborted();
+                let socket;
+                try {
+                    socket = new RealtimeConnection(address, this.credentials.realtimeAuthorization(), requestId);
+                }
+                catch {
+                    throw new RimeUnavailableError("Realtime handshake failed");
+                }
+                session = new RealtimeSession(socket, limits, () => this.sessions.delete(session));
+                this.sessions.add(session);
+                await abortable(session.initialize(settings, openingScope.signal), openingScope.signal);
+                openingScope.signal.throwIfAborted();
+                this.checkOpen();
+                return session;
+            }
+            catch (error) {
+                await session?.close();
+                throw error;
+            }
+        })();
+        this.opening.set(owner, task);
+        try {
+            return await task;
+        }
+        finally {
+            this.opening.delete(owner);
+            openingScope.dispose();
+            caller.dispose();
+        }
+    }
+    async close() {
+        for (const controller of this.opening.keys())
+            controller.abort(new RimeCancelledError("Client closed"));
+        await Promise.allSettled(this.opening.values());
+        await Promise.all([...this.sessions].map((session) => session.close()));
+    }
+}

package/dist/realtime/conversation.d.ts

@@ -0,0 +1,47 @@
+import type { RealtimeSession } from "./session.js";
+import type { AudioChunk, NormalizedRealtimeFault, OutputRef, SessionEvent } from "./types.js";
+export interface ConversationSource {
+    /** Return one chunk, or null at EOF. Abort must release a pending read. */
+    read(signal: AbortSignal): Promise<AudioChunk | null>;
+}
+export interface ConversationOutput {
+    /** Queue audio without blocking. Raise on overflow; never silently drop audio. */
+    write(chunk: AudioChunk): void;
+    /** Seal this output; completion still waits for the device to drain. */
+    end(): void;
+    /** Measured played milliseconds. Reject on measurement failure or abort.
+     * Abort stops the wait, without cancelling the device receipt. */
+    completion(signal: AbortSignal): Promise<number>;
+}
+export interface ConversationPlayback {
+    /** Queue an output in open order; audible playback must be FIFO.
+     * Keep completion and interruption state separate for each output.
+     * Do not mix outputs or play a later output ahead of an earlier one. */
+    open(output: OutputRef): ConversationOutput;
+    /** Stop selected outputs as one batch, including outputs after end().
+     * Discard their unplayed audio before starting any remaining output.
+     * Resume remaining outputs in FIFO order without waiting for new audio.
+     * Repeated interruptions are harmless; each receipt remains independent. */
+    interrupt(outputs: readonly OutputRef[]): void;
+    /** Confirm all playback has stopped and queued audio is discarded.
+     * Caller resources stay open. Reject and release the wait on abort.
+     * The runner supplies a separate signal with a five-second deadline and
+     * waits for cancellation cleanup before returning. */
+    stop(signal: AbortSignal): Promise<void>;
+}
+export interface ConversationOptions {
+    readonly source: ConversationSource;
+    readonly playback: ConversationPlayback;
+    /** Called inline for every event. Must be synchronous and nonblocking.
+     * Scheduled application work remains the caller's responsibility. */
+    readonly onEvent?: (event: SessionEvent<NormalizedRealtimeFault>) => void;
+    readonly signal?: AbortSignal;
+}
+/** Run until input EOF, session closure, failure, or abort.
+ * Owns the session's active lifetime and closes it on exit. The caller owns
+ * adapter resource contexts. Do not consume session.events or report playback
+ * elsewhere. Adapter operations must settle on abort. No input or callback
+ * queue is added. Limits: 64 responses, 128 outputs, 256 pending operations.
+ * Overflow fails the run. Callback and adapter failures reach the caller.
+ */
+export declare function runConversation(session: RealtimeSession, { source, playback, onEvent, signal }: ConversationOptions): Promise<void>;

package/dist/realtime/conversation.js

@@ -0,0 +1,321 @@
+import { abortable, scope } from "../cancellation.js";
+import { RimeCancelledError, RimeInputError, RimeResourceLimitError, RimeTimeoutError, } from "../errors.js";
+import { Deferred } from "./async.js";
+import { isSessionCloseError, responseInterruptsSpeech } from "./session.js";
+import { RimeRealtimeError } from "./types.js";
+/** Run until input EOF, session closure, failure, or abort.
+ * Owns the session's active lifetime and closes it on exit. The caller owns
+ * adapter resource contexts. Do not consume session.events or report playback
+ * elsewhere. Adapter operations must settle on abort. No input or callback
+ * queue is added. Limits: 64 responses, 128 outputs, 256 pending operations.
+ * Overflow fails the run. Callback and adapter failures reach the caller.
+ */
+export async function runConversation(session, { source, playback, onEvent, signal }) {
+    const controller = new AbortController();
+    const running = controller.signal;
+    const receipts = new AbortController();
+    const abort = () => controller.abort(signal?.reason);
+    if (signal?.aborted)
+        abort();
+    else
+        signal?.addEventListener("abort", abort, { once: true });
+    const replies = new Map();
+    const parents = new Map();
+    const interrupted = new Set();
+    const tasks = new Set();
+    let fail;
+    let failed = false;
+    let failureReason;
+    const failure = new Promise((_, reject) => {
+        fail = (error) => {
+            if (!failed) {
+                failed = true;
+                failureReason = error;
+                reject(error);
+            }
+        };
+    });
+    void failure.catch(() => { });
+    function start(work, local) {
+        if (tasks.size >= 256)
+            throw new RimeResourceLimitError("Conversation pending operation limit exceeded");
+        const task = work();
+        tasks.add(task);
+        void task.then(() => tasks.delete(task), (error) => {
+            tasks.delete(task);
+            const expected = [local, running, receipts.signal].some((signal) => signal?.aborted &&
+                (error === signal.reason ||
+                    error instanceof RimeCancelledError ||
+                    (error instanceof DOMException && error.name === "AbortError")));
+            if (!expected && !isSessionCloseError(error))
+                fail(error);
+        });
+        return task;
+    }
+    async function measure(stream) {
+        const value = await stream.completion(receipts.signal);
+        if (typeof value !== "number" || !Number.isFinite(value) || value < 0)
+            throw new RimeInputError("Playback measurement must be finite nonnegative milliseconds");
+        return value;
+    }
+    function end(output) {
+        if (!output.sealed) {
+            output.sealed = true;
+            output.stream.end();
+        }
+    }
+    function cut(output) {
+        if (output.report)
+            return;
+        end(output);
+        output.report = start(async () => {
+            const played = await output.receipt;
+            running.throwIfAborted();
+            try {
+                await session.reportPlayback({
+                    kind: "interrupted",
+                    output: output.ref,
+                    audioEndMs: Math.floor(played),
+                }, { signal: running });
+            }
+            catch (error) {
+                // The session also forwards the refusal to the application callback.
+                if (!(error instanceof RimeRealtimeError) ||
+                    error.fault.scope !== "event" ||
+                    error.fault.code !== "truncate_not_current")
+                    throw error;
+            }
+        });
+    }
+    function finish(reply) {
+        const earlier = [];
+        for (const previous of replies.values()) {
+            if (previous === reply)
+                break;
+            earlier.push(previous.drained.promise);
+        }
+        const finishing = new AbortController();
+        reply.finish = finishing;
+        const cancel = () => finishing.abort(running.reason);
+        if (running.aborted)
+            cancel();
+        else
+            running.addEventListener("abort", cancel, { once: true });
+        start(async () => {
+            try {
+                let played = 0;
+                for (const output of reply.outputs.values())
+                    played += await abortable(output.receipt, finishing.signal);
+                reply.drained.resolve();
+                if (reply.interrupted) {
+                    for (const output of reply.outputs.values())
+                        if (output.report)
+                            await abortable(output.report, finishing.signal);
+                }
+                else {
+                    // Prism's latest drain token clears all earlier playback too.
+                    for (const drained of earlier)
+                        await abortable(drained, finishing.signal);
+                    await session.reportPlayback({ kind: "finished", response: reply.ref, playedMs: played }, { signal: finishing.signal });
+                }
+                finishing.signal.throwIfAborted();
+                replies.delete(reply.ref.responseId);
+            }
+            finally {
+                running.removeEventListener("abort", cancel);
+            }
+        }, finishing.signal);
+    }
+    function wasInterrupted(responseId) {
+        let current = responseId;
+        // The session retains at most 128 response references.
+        for (let count = 0; current !== undefined && count < 128; count++) {
+            if (interrupted.has(current))
+                return true;
+            current = parents.get(current);
+        }
+        return false;
+    }
+    function interrupt(selected) {
+        const affected = selected.filter((reply) => !reply.interrupted);
+        const outputs = affected.flatMap((reply) => [...reply.outputs.values()]);
+        for (const reply of affected) {
+            reply.interrupted = true;
+            if (parents.has(reply.ref.responseId))
+                interrupted.add(reply.ref.responseId);
+            reply.finish?.abort();
+        }
+        if (outputs.length)
+            playback.interrupt(outputs.map((output) => output.ref));
+        for (const output of outputs)
+            cut(output);
+        for (const reply of affected)
+            if (reply.ended)
+                finish(reply);
+    }
+    function outputFor(ref) {
+        const reply = replies.get(ref.response.responseId);
+        if (!reply || reply.ended)
+            return;
+        const key = JSON.stringify([ref.itemId, ref.outputIndex, ref.contentIndex]);
+        let output = reply.outputs.get(key);
+        if (!output) {
+            let count = 0;
+            for (const value of replies.values())
+                count += value.outputs.size;
+            if (count >= 128)
+                throw new RimeResourceLimitError("Conversation output limit exceeded");
+            const stream = playback.open(ref);
+            output = {
+                ref,
+                stream,
+                receipt: start(() => measure(stream)),
+                sealed: false,
+                measured: false,
+            };
+            const measuredOutput = output;
+            void output.receipt.then(() => {
+                measuredOutput.measured = true;
+            }, () => { });
+            reply.outputs.set(key, output);
+            if (reply.interrupted) {
+                playback.interrupt([ref]);
+                cut(output);
+            }
+        }
+        return output;
+    }
+    async function send() {
+        while (!running.aborted) {
+            const chunk = await source.read(running);
+            if (chunk === null)
+                return;
+            await session.sendAudio(chunk, { signal: running });
+        }
+        running.throwIfAborted();
+    }
+    async function read() {
+        for await (const event of session.events) {
+            running.throwIfAborted();
+            const payload = event.payload;
+            switch (payload.kind) {
+                case "response.started": {
+                    if (replies.size >= 64)
+                        throw new RimeResourceLimitError("Conversation response limit exceeded");
+                    const id = payload.response.responseId;
+                    parents.set(id, payload.parent?.responseId);
+                    if (parents.size > 128) {
+                        const expired = parents.keys().next().value;
+                        parents.delete(expired);
+                        interrupted.delete(expired);
+                    }
+                    const blocked = wasInterrupted(id);
+                    if (blocked)
+                        interrupted.add(id);
+                    replies.set(id, {
+                        ref: payload.response,
+                        interruptOnSpeech: responseInterruptsSpeech(session, payload.response),
+                        outputs: new Map(),
+                        interrupted: blocked,
+                        ended: false,
+                        drained: new Deferred(),
+                    });
+                    break;
+                }
+                case "message.started":
+                case "audio.delta":
+                case "audio.done": {
+                    const output = outputFor(payload.output);
+                    if (output) {
+                        const reply = replies.get(payload.output.response.responseId);
+                        if (payload.kind === "audio.delta" && !reply.interrupted)
+                            output.stream.write(payload.audio);
+                        else if (payload.kind === "audio.done")
+                            end(output);
+                    }
+                    break;
+                }
+                case "speech.started": {
+                    // The newest eligible reply controls the playback batch.
+                    // A pending finished report is no longer physical playback.
+                    const eligible = [...replies.values()].filter((reply) => !reply.interrupted &&
+                        (!reply.ended ||
+                            [...reply.outputs.values()].some((output) => !output.measured)));
+                    if (eligible.at(-1)?.interruptOnSpeech)
+                        interrupt(eligible);
+                    break;
+                }
+                case "response.abandoned": {
+                    if (parents.has(payload.response.responseId))
+                        interrupted.add(payload.response.responseId);
+                    interrupt([...replies.values()].filter((reply) => wasInterrupted(reply.ref.responseId)));
+                    break;
+                }
+                case "response.ended": {
+                    const reply = replies.get(payload.response.responseId);
+                    if (reply) {
+                        reply.ended = true;
+                        for (const output of reply.outputs.values())
+                            end(output);
+                        finish(reply);
+                    }
+                    break;
+                }
+            }
+            const result = onEvent?.(event);
+            if (result &&
+                typeof result.then === "function") {
+                void Promise.resolve(result).catch(() => { });
+                throw new RimeInputError("onEvent must be synchronous and nonblocking");
+            }
+        }
+    }
+    let original = false;
+    try {
+        running.throwIfAborted();
+        await abortable(Promise.race([start(send), start(read), failure]), running);
+        if (failed)
+            throw failureReason;
+    }
+    catch (error) {
+        if (!isSessionCloseError(error)) {
+            original = true;
+            throw error;
+        }
+    }
+    finally {
+        let cleanupError;
+        // Stop input, event processing and reports, but keep receipt waits alive
+        // until the adapter confirms playback has stopped or the deadline expires.
+        controller.abort();
+        const shutdown = scope([], 5, new RimeTimeoutError("Playback shutdown confirmation timed out"));
+        try {
+            // Await the adapter itself, including cleanup after the deadline aborts.
+            // Racing this promise against abort would leave that cleanup running.
+            await playback.stop(shutdown.signal);
+            shutdown.signal.throwIfAborted();
+        }
+        catch (error) {
+            cleanupError = error;
+        }
+        finally {
+            shutdown.dispose();
+        }
+        receipts.abort();
+        try {
+            await session.close();
+        }
+        catch (error) {
+            cleanupError ??= error;
+        }
+        await Promise.allSettled(tasks);
+        signal?.removeEventListener("abort", abort);
+        if (!original) {
+            if (failed)
+                throw failureReason;
+            signal?.throwIfAborted();
+            if (cleanupError !== undefined)
+                throw cleanupError;
+        }
+    }
+}

package/dist/realtime/faults.d.ts

@@ -0,0 +1,2 @@
+import type { NormalizedRealtimeFault, RealtimeFault } from "./types.js";
+export declare function normalizeFault(fault: RealtimeFault): NormalizedRealtimeFault;

package/dist/realtime/faults.js

@@ -0,0 +1,25 @@
+// Reuse SDK-normalized faults so events and rejected requests share one object.
+const normalizedFaults = new WeakSet();
+export function normalizeFault(fault) {
+    if (normalizedFaults.has(fault))
+        return fault;
+    const snapshot = {
+        code: fault.code,
+        message: fault.message,
+        scope: fault.scope,
+        requestId: fault.requestId,
+        responseId: fault.responseId,
+        itemId: fault.itemId,
+        callIds: Object.freeze([...(fault.callIds ?? [])]),
+        correlationId: fault.correlationId,
+        parameter: fault.parameter,
+    };
+    const normalized = Object.freeze({
+        ...snapshot,
+        retryable: snapshot.code === "capacity_exceeded" ||
+            snapshot.code === "unavailable" ||
+            snapshot.code === "response_timeout",
+    });
+    normalizedFaults.add(normalized);
+    return normalized;
+}

package/dist/realtime/protocol.d.ts

@@ -0,0 +1,99 @@
+import * as t from "./types.js";
+export interface SessionSettings {
+    modalities: ("text" | "audio")[];
+    input_audio_format: "pcm16";
+    turn_detection: {
+        interrupt_response: boolean;
+        eagerness?: t.TurnEagerness;
+        prism_end_of_turn_timeout_ms?: number;
+    };
+    tools: {
+        type: "function";
+        function: {
+            name: string;
+            description: string;
+            parameters: t.JsonObject;
+        };
+        prism_filler?: t.PrismFiller;
+    }[];
+    voice?: string;
+    instructions?: string;
+    prism_response_metadata?: t.Labels;
+}
+/** A copy of the client's labels for every reply: keys to strings. */
+export declare function responseMetadata(value: unknown): t.Labels;
+export interface SessionPatch {
+    voice?: string;
+    instructions?: string;
+    tools?: SessionSettings["tools"];
+    turn_detection?: {
+        interrupt_response: boolean;
+    };
+}
+export interface SessionView {
+    settingsRevision: number | null;
+    id: string;
+    requestId: string | null;
+    responseMetadata: t.Labels | null;
+    voice: string;
+    interruptOnSpeech: boolean | null;
+    toolResultTimeoutS: number | null;
+    toolContinuationTimeoutS: number | null;
+    turnDetection: t.TurnDetectionInfo;
+}
+export interface SessionCreated {
+    kind: "session.created";
+    session: SessionView;
+}
+export interface SessionUpdated {
+    info?: t.SessionInfo;
+    kind: "session.updated";
+    session: SessionView;
+}
+export interface ItemCreated {
+    kind: "item.created";
+    itemId: string;
+}
+export interface ToolResultCreated {
+    kind: "tool.result.created";
+    callId: string;
+}
+export interface AudioCleared {
+    kind: "audio.cleared";
+}
+export interface ItemTruncated {
+    kind: "item.truncated";
+    itemId: string;
+    contentIndex: number;
+}
+export interface ResponseEnded extends t.ResponseEnded {
+    drainToken: string | null;
+}
+export type Acknowledgment = SessionUpdated | ItemCreated | ToolResultCreated | AudioCleared | ItemTruncated | t.ResponseStarted | ResponseEnded;
+export type ServerPayload = SessionCreated | Acknowledgment | t.InputEvent<t.NormalizedRealtimeFault> | t.MessageStarted | t.TextDelta | t.TextDone | t.AudioDelta | t.AudioDone | t.ToolCall | t.FaultEvent<t.NormalizedRealtimeFault>;
+export interface ServerEvent {
+    eventId: string;
+    requestId: string | null;
+    payload: ServerPayload;
+}
+export type RequestKind = "session.update" | "conversation.item.create" | "response.create" | "response.cancel" | "input_audio_buffer.clear" | "conversation.item.truncate";
+/** A request carries its result type and checks it before exposing the result. */
+export declare class Request<T extends Acknowledgment> {
+    readonly kind: RequestKind;
+    private readonly accepts;
+    readonly allowsItemCreated: boolean;
+    constructor(kind: RequestKind, accepts: (event: Acknowledgment) => event is T, allowsItemCreated?: boolean);
+    result(event: Acknowledgment): T;
+    check(event: Acknowledgment): void;
+}
+export declare const requests: {
+    update: Request<SessionUpdated>;
+    item: Request<ItemCreated>;
+    toolResult: Request<ToolResultCreated>;
+    create: Request<t.ResponseStarted>;
+    text: Request<t.ResponseStarted>;
+    cancel: Request<ResponseEnded>;
+    clear: Request<AudioCleared>;
+    truncate: Request<ItemTruncated>;
+};
+export declare function decode(raw: string, sessionId: string | null): ServerEvent | null;