@rimelabs/sdk 0.1.0-alpha.5 → 0.1.0-alpha.7

package/dist/realtime/transport.d.ts

@@ -0,0 +1,16 @@
+import { RimeError } from "../errors.js";
+/** Private transport. The public declarations do not require ws types. */
+export declare class RealtimeConnection {
+    private readonly socket;
+    private readonly opened;
+    private readonly ended;
+    private closing;
+    private connected;
+    readonly ready: Promise<void>;
+    onMessage: (raw: string) => void;
+    onFailure: (error: RimeError) => void;
+    constructor(endpoint: string, authorization: string);
+    private failure;
+    send(encoded: string): Promise<void>;
+    close(): Promise<void>;
+}

package/dist/realtime/transport.js

@@ -0,0 +1,83 @@
+import WebSocket from "ws";
+import { RimeAuthenticationError, RimeStreamError, RimeUnavailableError, } from "../errors.js";
+import { Deferred } from "./async.js";
+/** Private transport. The public declarations do not require ws types. */
+export class RealtimeConnection {
+    socket;
+    opened = new Deferred();
+    ended = new Deferred();
+    closing = null;
+    connected = false;
+    ready = this.opened.promise;
+    onMessage = () => { };
+    onFailure = () => { };
+    constructor(endpoint, authorization) {
+        this.socket = new WebSocket(endpoint, {
+            headers: { Authorization: authorization },
+            followRedirects: false,
+            perMessageDeflate: false,
+            maxPayload: 1024 * 1024,
+        });
+        this.socket.on("open", () => {
+            this.connected = true;
+            this.opened.resolve();
+        });
+        this.socket.on("unexpected-response", (request, response) => {
+            const error = [401, 403].includes(response.statusCode ?? 0)
+                ? new RimeAuthenticationError("Realtime endpoint rejected credentials")
+                : new RimeUnavailableError("Realtime endpoint rejected the connection");
+            this.failure(error);
+            response.destroy();
+            request.destroy();
+            this.socket.terminate();
+        });
+        this.socket.on("error", () => this.failure(!this.connected
+            ? new RimeUnavailableError("Realtime handshake failed")
+            : new RimeStreamError("Invalid event or lost realtime connection")));
+        this.socket.on("message", (data, binary) => {
+            if (binary) {
+                this.failure(new RimeStreamError("Expected a JSON text event"));
+                return;
+            }
+            this.onMessage(data.toString());
+        });
+        this.socket.on("close", () => {
+            this.ended.resolve();
+            this.failure(new RimeStreamError("Realtime connection closed; reconnect is not automatic"));
+        });
+    }
+    failure(error) {
+        this.opened.reject(error);
+        this.onFailure(error);
+    }
+    send(encoded) {
+        return new Promise((resolve, reject) => {
+            this.socket.send(encoded, (error) => {
+                if (error)
+                    reject(new RimeStreamError("Realtime write failed; outcome unknown"));
+                else
+                    resolve();
+            });
+        });
+    }
+    close() {
+        if (!this.closing) {
+            this.closing = (async () => {
+                if (this.socket.readyState === WebSocket.CLOSED)
+                    return;
+                const timer = setTimeout(() => this.socket.terminate(), 2000);
+                try {
+                    if (this.socket.readyState === WebSocket.CONNECTING)
+                        this.socket.terminate();
+                    else
+                        this.socket.close();
+                    await this.ended.promise;
+                }
+                finally {
+                    clearTimeout(timer);
+                }
+            })();
+        }
+        return this.closing;
+    }
+}

package/dist/realtime/types.d.ts

@@ -0,0 +1,190 @@
+import { RimeError, RimeTimeoutError } from "../errors.js";
+export type JsonValue = null | boolean | number | string | JsonValue[] | JsonObject;
+export interface JsonObject {
+    [key: string]: JsonValue;
+}
+/** All durations are positive, finite seconds. */
+export interface RealtimeTimeouts {
+    readonly connectS?: number;
+    readonly readyS?: number;
+    readonly requestS?: number;
+}
+export interface RealtimeOperationOptions {
+    readonly signal?: AbortSignal;
+}
+export interface ToolDefinition {
+    readonly name: string;
+    readonly parameters: JsonObject;
+    readonly description?: string;
+}
+export interface RealtimeConnectOptions extends RealtimeOperationOptions {
+    readonly endpoint: string;
+    readonly model?: "prism";
+    readonly voice?: string | null;
+    readonly instructions?: string | null;
+    readonly tools?: readonly ToolDefinition[];
+    readonly interruptOnSpeech?: boolean;
+    readonly timeouts?: RealtimeTimeouts;
+}
+export interface ReplyOptions extends RealtimeOperationOptions {
+    readonly instruction?: string | null;
+    readonly toolCall?: ToolCallRef | null;
+}
+/** Omitted fields mean 16 kHz, mono, signed little-endian PCM16. */
+export interface PCMFormat {
+    readonly sampleRate?: 8000 | 16000 | 24000 | 48000;
+    readonly channels?: 1 | 2;
+    readonly encoding?: "pcm_s16le";
+}
+export interface AudioChunk {
+    readonly data: Uint8Array;
+    readonly format?: PCMFormat;
+}
+export interface ResponseRef {
+    readonly sessionId: string;
+    readonly responseId: string;
+}
+export interface ItemRef {
+    readonly sessionId: string;
+    readonly itemId: string;
+}
+export interface ToolCallRef extends ResponseRef {
+    readonly callId: string;
+}
+export interface OutputRef {
+    readonly response: ResponseRef;
+    readonly itemId: string;
+    readonly outputIndex: number;
+    readonly contentIndex: number;
+}
+export interface SessionInfo {
+    readonly sessionId: string;
+    readonly model: "prism";
+    readonly voice: string;
+    readonly interruptOnSpeech: boolean;
+    readonly toolResultTimeoutS: number | null;
+    readonly toolContinuationTimeoutS: number | null;
+}
+export interface RealtimeFault {
+    readonly code: string;
+    readonly message: string;
+    readonly scope: "event" | "utterance" | "response" | "tool_roundtrip" | "session" | "unknown";
+    readonly requestId: string | null;
+    readonly responseId: string | null;
+    readonly itemId: string | null;
+    readonly callIds: readonly string[];
+    readonly correlationId: string | null;
+    readonly parameter: string | null;
+}
+export declare class RealtimeAdmissionTimeout extends RimeTimeoutError {
+}
+export declare class RimeRealtimeError extends RimeError {
+    readonly fault: RealtimeFault;
+    constructor(fault: RealtimeFault);
+}
+export interface InputReady {
+    readonly kind: "input.ready";
+}
+export interface SpeechStarted {
+    readonly kind: "speech.started";
+    readonly itemId: string;
+    readonly audioStartMs: number;
+}
+export interface SpeechStopped {
+    readonly kind: "speech.stopped";
+    readonly itemId: string;
+    readonly audioEndMs: number;
+}
+export interface InputCommitted {
+    readonly kind: "input.committed";
+    readonly itemId: string;
+}
+export interface TranscriptDelta {
+    readonly kind: "transcript.delta";
+    readonly itemId: string;
+    readonly delta: string;
+}
+export interface TranscriptFinal {
+    readonly kind: "transcript.final";
+    readonly itemId: string;
+    readonly text: string;
+}
+export interface TranscriptFailed {
+    readonly kind: "transcript.failed";
+    readonly itemId: string;
+    readonly error: RealtimeFault;
+}
+export interface ResponseStarted {
+    readonly kind: "response.started";
+    readonly response: ResponseRef;
+    readonly cause: "speech" | "text" | "proactive" | "tools" | "unknown";
+    readonly parent: ResponseRef | null;
+    readonly inputItemId: string | null;
+}
+export interface MessageStarted {
+    readonly kind: "message.started";
+    readonly output: OutputRef;
+}
+export interface TextDelta {
+    readonly kind: "text.delta";
+    readonly output: OutputRef;
+    readonly delta: string;
+}
+export interface TextDone {
+    readonly kind: "text.done";
+    readonly output: OutputRef;
+    readonly text: string;
+}
+export interface AudioDelta {
+    readonly kind: "audio.delta";
+    readonly output: OutputRef;
+    readonly audio: {
+        readonly data: Uint8Array;
+        readonly format: Required<PCMFormat>;
+    };
+}
+export interface AudioDone {
+    readonly kind: "audio.done";
+    readonly output: OutputRef;
+}
+export interface ToolCall {
+    readonly kind: "tool.call";
+    readonly call: ToolCallRef;
+    readonly itemId: string;
+    readonly name: string;
+    readonly arguments: JsonObject;
+}
+export interface ResponseAbandoned {
+    readonly kind: "response.abandoned";
+    readonly response: ResponseRef;
+}
+export interface ResponseEnded {
+    readonly kind: "response.ended";
+    readonly response: ResponseRef;
+    readonly status: "completed" | "cancelled" | "failed" | "unknown";
+    readonly reason: string | null;
+}
+export interface FaultEvent {
+    readonly kind: "error";
+    readonly error: RealtimeFault;
+}
+export type InputEvent = InputReady | SpeechStarted | SpeechStopped | InputCommitted | TranscriptDelta | TranscriptFinal | TranscriptFailed;
+export type ResponseEvent = ResponseStarted | ResponseAbandoned | MessageStarted | TextDelta | TextDone | AudioDelta | AudioDone | ToolCall | ResponseEnded;
+export interface SessionEvent {
+    readonly sessionId: string;
+    readonly eventId: string;
+    readonly requestId: string | null;
+    readonly payload: InputEvent | ResponseEvent | FaultEvent;
+}
+export interface PlaybackInterrupted {
+    readonly kind: "interrupted";
+    readonly output: OutputRef;
+    readonly audioEndMs: number;
+}
+export interface PlaybackFinished {
+    readonly kind: "finished";
+    readonly response: ResponseRef;
+    readonly playedMs?: number | null;
+    readonly audibleTailMs?: number | null;
+}
+export type PlaybackReport = PlaybackInterrupted | PlaybackFinished;

package/dist/realtime/types.js

@@ -0,0 +1,10 @@
+import { RimeError, RimeTimeoutError } from "../errors.js";
+export class RealtimeAdmissionTimeout extends RimeTimeoutError {
+}
+export class RimeRealtimeError extends RimeError {
+    fault;
+    constructor(fault) {
+        super(`${fault.code}: ${fault.message}`, fault.requestId);
+        this.fault = fault;
+    }
+}

package/dist/{audio.js → tts/audio.js}

@@ -1,4 +1,4 @@
-import { RimeAudioFormatError } from "./errors.js";
+import { RimeAudioFormatError } from "../errors.js";
 export class AudioFormat {
     encoding;
     sampleRate;

package/dist/tts/client.d.ts

@@ -0,0 +1,36 @@
+import { AudioFormat } from "./audio.js";
+import { AudioStream, type TextSource } from "./stream.js";
+import { Credentials } from "../auth.js";
+import { policy } from "./policy.js";
+export interface SynthesisOptions {
+    voice?: string | null;
+    language?: string;
+    audioFormat?: AudioFormat | null;
+    timeout?: number | null;
+}
+export interface DiscoveryOptions {
+    timeout?: number | null;
+}
+export interface VoiceListOptions extends DiscoveryOptions {
+    language?: string | null;
+}
+export declare class TTS {
+    private readonly credentials;
+    private readonly deployment;
+    private readonly defaultTimeout;
+    private readonly checkOpen;
+    private client;
+    private streams;
+    private discoveryOperations;
+    readonly voices: {
+        list: (options?: VoiceListOptions) => Promise<string[]>;
+    };
+    readonly languages: {
+        list: (options?: DiscoveryOptions) => Promise<string[]>;
+    };
+    constructor(credentials: Credentials, deployment: Readonly<typeof policy>, defaultTimeout: number | null, checkOpen: () => void);
+    private prepare;
+    stream(text: TextSource, options?: SynthesisOptions): AudioStream;
+    private discover;
+    close(): Promise<void>;
+}

package/dist/tts/client.js

@@ -0,0 +1,111 @@
+import { AudioFormat } from "./audio.js";
+import { AudioStream, constructionKey } from "./stream.js";
+import { policy, timeout, nonempty } from "./policy.js";
+import { abortable } from "../cancellation.js";
+import { transport, connect, metadata, discover, } from "./transport.js";
+import { RimeInputError, RimeAudioFormatError, RimeCancelledError, RimeTimeoutError, RimeUnavailableError, } from "../errors.js";
+export class TTS {
+    credentials;
+    deployment;
+    defaultTimeout;
+    checkOpen;
+    client = null;
+    streams = new Set();
+    discoveryOperations = new Map();
+    voices;
+    languages;
+    constructor(credentials, deployment, defaultTimeout, checkOpen) {
+        this.credentials = credentials;
+        this.deployment = deployment;
+        this.defaultTimeout = defaultTimeout;
+        this.checkOpen = checkOpen;
+        this.voices = {
+            list: async (options = {}) => {
+                if (options.language !== undefined && options.language !== null)
+                    nonempty(options.language, "language");
+                return this.discover("voices", options.language, options.timeout);
+            },
+        };
+        this.languages = {
+            list: (options = {}) => this.discover("languages", undefined, options.timeout),
+        };
+    }
+    async prepare(signal) {
+        this.checkOpen();
+        const auth = await this.credentials.metadata(signal);
+        this.checkOpen();
+        this.client ??= transport.makeClient(this.deployment.target);
+        await connect(this.client, signal);
+        return { client: this.client, metadata: metadata(auth) };
+    }
+    stream(text, options = {}) {
+        this.checkOpen();
+        if (typeof text === "string") {
+            if (!text.trim())
+                throw new RimeInputError("Text must contain non-whitespace characters");
+        }
+        else if (!text || typeof text[Symbol.asyncIterator] !== "function")
+            throw new RimeInputError("text must be a string or an async iterable of strings");
+        const voice = options.voice === undefined || options.voice === null
+            ? this.deployment.defaultVoice
+            : nonempty(options.voice, "voice");
+        const language = nonempty(options.language ?? "en", "language");
+        const format = options.audioFormat ?? AudioFormat.PCM_24000;
+        if (format !== AudioFormat.PCM_24000 && format !== AudioFormat.MULAW_8000)
+            throw new RimeAudioFormatError("Select a named AudioFormat profile");
+        const audio = new AudioStream(constructionKey, {
+            prepare: (s) => this.prepare(s),
+            checkOpen: () => this.checkOpen(),
+            forget: (s) => this.streams.delete(s),
+        }, text, voice, language, format, timeout(options.timeout, this.defaultTimeout));
+        this.streams.add(audio);
+        return audio;
+    }
+    async discover(kind, language, override) {
+        this.checkOpen();
+        const budget = Math.min(timeout(override, this.defaultTimeout) ?? Infinity, policy.discoveryTimeout);
+        const controller = new AbortController();
+        let requestId = null;
+        const timer = setTimeout(() => controller.abort(new RimeTimeoutError("Discovery deadline expired")), budget * 1000);
+        const operation = (async () => {
+            try {
+                const prepared = await this.prepare(controller.signal);
+                for (let attempt = 0;; attempt++) {
+                    requestId = null;
+                    try {
+                        return await discover(prepared, kind, language, controller.signal);
+                    }
+                    catch (error) {
+                        if (!(error instanceof RimeUnavailableError) || attempt === 2)
+                            throw error;
+                        requestId = error.requestId;
+                        await abortable(new Promise((resolve) => setTimeout(resolve, 50 * 2 ** attempt)), controller.signal);
+                    }
+                }
+            }
+            catch (error) {
+                if (error instanceof RimeTimeoutError && requestId !== null)
+                    throw new RimeTimeoutError(error.message, requestId, {
+                        cause: error.cause,
+                    });
+                throw error;
+            }
+            finally {
+                clearTimeout(timer);
+                this.discoveryOperations.delete(controller);
+            }
+        })();
+        this.discoveryOperations.set(controller, operation);
+        return operation;
+    }
+    async close() {
+        const discoveries = [...this.discoveryOperations.values()];
+        for (const controller of this.discoveryOperations.keys())
+            controller.abort(new RimeCancelledError("Client closed during discovery"));
+        await Promise.all([
+            ...[...this.streams].map((stream) => stream.cancel()),
+            Promise.allSettled(discoveries),
+        ]);
+        this.client?.close();
+    }
+}

package/dist/{policy.d.ts → tts/policy.d.ts}

@@ -18,4 +18,3 @@ export declare const policy: {
 export declare function resolve(model: string, endpoint?: string | null): Readonly<typeof policy>;
 export declare function timeout(value: unknown, inherited?: number | null): number | null;
 export declare function nonempty(value: unknown, name: string): string;
-export declare function abortable<T>(promise: PromiseLike<T>, signal: AbortSignal): Promise<T>;

package/dist/{policy.js → tts/policy.js}

@@ -1,4 +1,4 @@
-import { RimeInputError } from "./errors.js";
+import { RimeInputError } from "../errors.js";
 const codaHostname = "coda.api.rime.ai";
 export const policy = {
     target: `${codaHostname}:443`,
@@ -62,21 +62,3 @@ export function nonempty(value, name) {
         throw new RimeInputError(`${name} must be a non-empty string`);
     return value;
 }
-export function abortable(promise, signal) {
-    if (signal.aborted)
-        return Promise.reject(signal.reason);
-    return new Promise((resolve, reject) => {
-        const abort = () => {
-            signal.removeEventListener("abort", abort);
-            reject(signal.reason);
-        };
-        signal.addEventListener("abort", abort, { once: true });
-        Promise.resolve(promise).then((value) => {
-            signal.removeEventListener("abort", abort);
-            resolve(value);
-        }, (error) => {
-            signal.removeEventListener("abort", abort);
-            reject(error);
-        });
-    });
-}

package/dist/{sentences.js → tts/sentences.js}

@@ -1,7 +1,7 @@
 import { createRequire } from "node:module";
-import { RimeInputError, RimeResourceLimitError } from "./errors.js";
+import { RimeInputError, RimeResourceLimitError } from "../errors.js";
 const require = createRequire(import.meta.url);
-const module = require("../vendor/blingfire.cjs");
+const module = require("../../vendor/blingfire.cjs");
 export const ready = module.ready;
 // These marks schedule scans. Only BlingFire decides sentence boundaries.
 // Keep the set and code-point counts in sync with Python's _sentences.py.

package/dist/{stream.js → tts/stream.js}

@@ -2,11 +2,12 @@ import { randomUUID } from "node:crypto";
 import { debuglog } from "node:util";
 const log = debuglog("rime-sdk");
 import { Converter } from "./audio.js";
-import { policy, abortable } from "./policy.js";
+import { policy } from "./policy.js";
+import { abortable } from "../cancellation.js";
 import { ByteQueue } from "./queue.js";
 import { SentenceBuffer, ready } from "./sentences.js";
 import { SynthesisCall } from "./transport.js";
-import { RimeError, RimeInputError, RimeCancelledError, RimeTimeoutError, RimeStreamError, } from "./errors.js";
+import { RimeError, RimeInputError, RimeCancelledError, RimeTimeoutError, RimeStreamError, } from "../errors.js";
 export const constructionKey = Symbol("private stream constructor");
 export class AudioStream {
     owner;

package/dist/{transport.d.ts → tts/transport.d.ts}

@@ -1,5 +1,5 @@
 import * as grpc from "@grpc/grpc-js";
-import * as errors from "./errors.js";
+import * as errors from "../errors.js";
 export type Connection = grpc.Client;
 export interface PreparedConnection {
     client: Connection;

package/dist/{transport.js → tts/transport.js}

@@ -1,8 +1,9 @@
 import * as grpc from "@grpc/grpc-js";
 import { create, fromBinary, toBinary, } from "@bufbuild/protobuf";
 import * as schema from "@rimelabs/api";
-import { policy, abortable } from "./policy.js";
-import * as errors from "./errors.js";
+import { policy } from "./policy.js";
+import { abortable } from "../cancellation.js";
+import * as errors from "../errors.js";
 export function rpcError(code, requestId = null) {
     const classes = {
         [grpc.status.UNAUTHENTICATED]: errors.RimeAuthenticationError,