@rimelabs/sdk 0.1.0-alpha.2 → 0.1.0-alpha.21

package/dist/realtime/protocol.js

@@ -0,0 +1,432 @@
+// SPDX-License-Identifier: Apache-2.0
+/** Decode only the protocol fields the SDK uses. Unknown events and fields stay open. */
+import { RimeInputError, RimeStreamError } from "../errors.js";
+import { normalizeFault } from "./faults.js";
+/** A copy of the client's labels for every reply: keys to strings. */
+export function responseMetadata(value) {
+    if (!value ||
+        typeof value !== "object" ||
+        ![Object.prototype, null].includes(Object.getPrototypeOf(value)) ||
+        Object.values(value).some((label) => typeof label !== "string"))
+        throw new RimeInputError("responseMetadata must map keys to strings");
+    return Object.freeze({ ...value });
+}
+/** A request carries its result type and checks it before exposing the result. */
+export class Request {
+    kind;
+    accepts;
+    allowsItemCreated;
+    constructor(kind, accepts, allowsItemCreated = false) {
+        this.kind = kind;
+        this.accepts = accepts;
+        this.allowsItemCreated = allowsItemCreated;
+    }
+    result(event) {
+        if (!this.accepts(event))
+            throw new RimeStreamError(`Unexpected ${event.kind} acknowledgment for ${this.kind}`);
+        return event;
+    }
+    check(event) {
+        this.result(event);
+    }
+}
+export const requests = {
+    update: new Request("session.update", (e) => e.kind === "session.updated"),
+    item: new Request("conversation.item.create", (e) => e.kind === "item.created"),
+    toolResult: new Request("conversation.item.create", (e) => e.kind === "tool.result.created"),
+    create: new Request("response.create", (e) => e.kind === "response.started"),
+    text: new Request("response.create", (e) => e.kind === "response.started", true),
+    cancel: new Request("response.cancel", (e) => e.kind === "response.ended"),
+    clear: new Request("input_audio_buffer.clear", (e) => e.kind === "audio.cleared"),
+    truncate: new Request("conversation.item.truncate", (e) => e.kind === "item.truncated"),
+};
+function invalid(field) {
+    throw new RimeStreamError(`Invalid realtime field: ${field}`);
+}
+function object(value, field) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        return invalid(field);
+    return value;
+}
+function string(value, field) {
+    if (typeof value !== "string")
+        return invalid(field);
+    return value;
+}
+function id(value, field) {
+    const result = string(value, field);
+    if (!result)
+        return invalid(field);
+    return result;
+}
+function optionalString(value, field) {
+    return value == null ? null : string(value, field);
+}
+function integer(value, field) {
+    if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0)
+        return invalid(field);
+    return value;
+}
+function revision(value) {
+    return value.prism_settings_revision === undefined
+        ? null
+        : integer(value.prism_settings_revision, "prism_settings_revision");
+}
+function duration(value, field) {
+    if (value == null)
+        return null;
+    if (typeof value !== "number" || !Number.isFinite(value) || value <= 0)
+        return invalid(field);
+    return value;
+}
+function session(value) {
+    const view = object(value, "session");
+    const detection = view.turn_detection === undefined
+        ? {}
+        : object(view.turn_detection, "session.turn_detection");
+    let eagerness = null;
+    if (detection.eagerness !== undefined) {
+        const value = string(detection.eagerness, "session.turn_detection.eagerness");
+        eagerness =
+            value === "low" ||
+                value === "medium" ||
+                value === "high" ||
+                value === "auto"
+                ? value
+                : "unknown";
+    }
+    let endOfTurnTimeoutMs = null;
+    if (detection.prism_end_of_turn_timeout_ms !== undefined) {
+        const value = detection.prism_end_of_turn_timeout_ms;
+        if (typeof value !== "number" ||
+            !Number.isInteger(value) ||
+            value < 1000 ||
+            value > 10000)
+            invalid("session.turn_detection.prism_end_of_turn_timeout_ms");
+        endOfTurnTimeoutMs = value;
+    }
+    const waits = view.prism_tool_waits === undefined
+        ? {}
+        : object(view.prism_tool_waits, "session.prism_tool_waits");
+    if (view.prism_tool_waits !== undefined &&
+        (!("result_timeout_s" in waits) || !("continuation_timeout_s" in waits)))
+        invalid("session.prism_tool_waits");
+    const interrupt = view.prism_effective_interrupt_response;
+    if (interrupt !== undefined && typeof interrupt !== "boolean")
+        invalid("session.prism_effective_interrupt_response");
+    return {
+        settingsRevision: revision(view),
+        id: id(view.id, "session.id"),
+        requestId: optionalString(view.request_id, "session.request_id"),
+        responseMetadata: view.prism_response_metadata === undefined
+            ? null
+            : clientMetadata(object(view.prism_response_metadata, "session.prism_response_metadata")),
+        voice: view.voice === undefined ? "" : string(view.voice, "session.voice"),
+        interruptOnSpeech: interrupt ?? null,
+        turnDetection: Object.freeze({ eagerness, endOfTurnTimeoutMs }),
+        toolResultTimeoutS: duration(waits.result_timeout_s, "result_timeout_s"),
+        toolContinuationTimeoutS: duration(waits.continuation_timeout_s, "continuation_timeout_s"),
+    };
+}
+/** The client's labels in `metadata`: its string values under keys that are
+ * not Prism's `prism_` ones. */
+function clientMetadata(metadata) {
+    return Object.freeze(Object.fromEntries(Object.entries(metadata).filter((entry) => !entry[0].startsWith("prism_") && typeof entry[1] === "string")));
+}
+function fault(value) {
+    const error = object(value, "error");
+    const owner = error.owner == null ? {} : object(error.owner, "error.owner");
+    const scope = error.scope === undefined ? "unknown" : string(error.scope, "error.scope");
+    const calls = owner.call_ids ?? [];
+    if (!Array.isArray(calls))
+        return invalid("error.owner.call_ids");
+    const code = error.code === undefined ? "unknown" : string(error.code, "error.code");
+    return normalizeFault({
+        code,
+        message: error.message === undefined ? "" : string(error.message, "error.message"),
+        scope: scope === "event" ||
+            scope === "utterance" ||
+            scope === "response" ||
+            scope === "tool_roundtrip" ||
+            scope === "session"
+            ? scope
+            : "unknown",
+        requestId: optionalString(owner.event_id, "error.owner.event_id"),
+        responseId: optionalString(owner.response_id, "error.owner.response_id"),
+        itemId: optionalString(owner.item_id, "error.owner.item_id"),
+        callIds: calls.map((value) => id(value, "error.owner.call_ids")),
+        correlationId: optionalString(error.correlation_id, "error.correlation_id"),
+        parameter: optionalString(error.param, "error.param"),
+    });
+}
+function json(value) {
+    if (value === null || typeof value === "string" || typeof value === "boolean")
+        return value;
+    if (typeof value === "number" && Number.isFinite(value))
+        return value;
+    if (Array.isArray(value))
+        return value.map(json);
+    const entries = object(value, "arguments");
+    return Object.fromEntries(Object.entries(entries).map(([key, value]) => [key, json(value)]));
+}
+function argumentsObject(value) {
+    const result = json(JSON.parse(string(value, "arguments")));
+    if (!result || typeof result !== "object" || Array.isArray(result))
+        return invalid("arguments");
+    return result;
+}
+function audio(value) {
+    const encoded = string(value, "delta");
+    if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(encoded))
+        return invalid("audio.delta");
+    const data = Buffer.from(encoded, "base64");
+    if (data.length % 2)
+        return invalid("audio.delta");
+    return Object.freeze({
+        data,
+        format: Object.freeze({
+            sampleRate: 24000,
+            channels: 1,
+            encoding: "pcm_s16le",
+        }),
+    });
+}
+export function decode(raw, sessionId) {
+    try {
+        return decodeValue(JSON.parse(raw), sessionId);
+    }
+    catch (error) {
+        if (error instanceof RimeStreamError)
+            throw error;
+        throw new RimeStreamError("Invalid realtime event");
+    }
+}
+function decodeValue(value, sessionId) {
+    const event = object(value, "event"), kind = string(event.type, "type");
+    const ref = (value) => {
+        if (sessionId === null)
+            throw new RimeStreamError("Event arrived before session.created");
+        return Object.freeze({ sessionId, responseId: id(value, "response.id") });
+    };
+    let payload;
+    switch (kind) {
+        case "session.created":
+            payload = { kind: "session.created", session: session(event.session) };
+            break;
+        case "session.updated":
+            payload = { kind: "session.updated", session: session(event.session) };
+            break;
+        case "conversation.item.created": {
+            const item = object(event.item, "item");
+            if (item.type === "function_call_output")
+                payload = {
+                    kind: "tool.result.created",
+                    callId: id(item.call_id, "item.call_id"),
+                };
+            else {
+                if (item.type !== undefined && item.type !== "message")
+                    invalid("item.type");
+                payload = { kind: "item.created", itemId: id(item.id, "item.id") };
+            }
+            break;
+        }
+        case "input_audio_buffer.cleared":
+            payload = { kind: "audio.cleared" };
+            break;
+        case "conversation.item.truncated":
+            payload = {
+                kind: "item.truncated",
+                itemId: id(event.item_id, "item_id"),
+                contentIndex: integer(event.content_index, "content_index"),
+            };
+            break;
+        case "prism.typed_input.ready":
+            payload = { kind: "input.ready" };
+            break;
+        case "prism.input_audio_buffer.speech_hint":
+            payload = {
+                kind: "speech.hint",
+                audioStartMs: integer(event.audio_start_ms, "audio_start_ms"),
+            };
+            break;
+        case "input_audio_buffer.speech_started":
+            payload = {
+                kind: "speech.started",
+                itemId: id(event.item_id, "item_id"),
+                audioStartMs: integer(event.audio_start_ms, "audio_start_ms"),
+            };
+            break;
+        case "input_audio_buffer.speech_stopped": {
+            const reason = event.prism_end_reason === undefined
+                ? null
+                : string(event.prism_end_reason, "prism_end_reason");
+            payload = {
+                kind: "speech.stopped",
+                itemId: id(event.item_id, "item_id"),
+                audioEndMs: integer(event.audio_end_ms, "audio_end_ms"),
+                endReason: reason === null ||
+                    reason === "semantic" ||
+                    reason === "timeout" ||
+                    reason === "silence" ||
+                    reason === "max_duration" ||
+                    reason === "detector_changed"
+                    ? reason
+                    : "unknown",
+            };
+            break;
+        }
+        case "input_audio_buffer.committed":
+            payload = {
+                kind: "input.committed",
+                itemId: id(event.item_id, "item_id"),
+            };
+            break;
+        case "conversation.item.input_audio_transcription.delta":
+            payload = {
+                kind: "transcript.delta",
+                itemId: id(event.item_id, "item_id"),
+                delta: string(event.delta, "delta"),
+            };
+            break;
+        case "conversation.item.input_audio_transcription.completed":
+            payload = {
+                kind: "transcript.final",
+                itemId: id(event.item_id, "item_id"),
+                text: string(event.transcript, "transcript"),
+            };
+            break;
+        case "conversation.item.input_audio_transcription.failed": {
+            const error = object(event.error, "error"), itemId = id(event.item_id, "item_id");
+            payload = {
+                kind: "transcript.failed",
+                itemId,
+                error: fault({
+                    code: string(error.code, "error.code"),
+                    message: string(error.message, "error.message"),
+                    scope: "utterance",
+                    owner: { item_id: itemId },
+                }),
+            };
+            break;
+        }
+        case "response.created": {
+            const response = object(event.response, "response"), metadata = response.metadata == null
+                ? {}
+                : object(response.metadata, "metadata");
+            const cause = metadata.prism_cause === undefined
+                ? undefined
+                : string(metadata.prism_cause, "prism_cause");
+            const parent = optionalString(metadata.prism_parent_response_id, "prism_parent_response_id");
+            const inputItemId = optionalString(metadata.prism_input_item_id, "prism_input_item_id");
+            const inputItemIds = response.prism_input_item_ids ??
+                (inputItemId === null ? [] : [inputItemId]);
+            if (!Array.isArray(inputItemIds))
+                return invalid("prism_input_item_ids");
+            payload = {
+                kind: "response.started",
+                settingsRevision: revision(response),
+                response: ref(response.id),
+                cause: cause === undefined
+                    ? "speech"
+                    : cause === "user_text"
+                        ? "text"
+                        : cause === "tool_continuation"
+                            ? "tools"
+                            : cause === "proactive"
+                                ? "proactive"
+                                : "unknown",
+                parent: parent === null ? null : ref(parent),
+                inputItemId,
+                metadata: clientMetadata(metadata),
+                inputItemIds: Object.freeze(inputItemIds.map((value) => id(value, "prism_input_item_ids"))),
+            };
+            break;
+        }
+        case "response.done": {
+            const response = object(event.response, "response"), metadata = response.metadata == null
+                ? {}
+                : object(response.metadata, "metadata");
+            const details = response.status_details == null
+                ? {}
+                : object(response.status_details, "status_details");
+            const status = string(response.status, "response.status");
+            payload = {
+                kind: "response.ended",
+                settingsRevision: revision(response),
+                response: ref(response.id),
+                status: status === "completed" ||
+                    status === "cancelled" ||
+                    status === "failed"
+                    ? status
+                    : "unknown",
+                reason: optionalString(details.reason, "reason"),
+                drainToken: optionalString(metadata.prism_drain_token, "prism_drain_token"),
+            };
+            break;
+        }
+        case "response.function_call_arguments.done":
+            payload = {
+                kind: "tool.call",
+                call: Object.freeze({
+                    ...ref(event.response_id),
+                    callId: id(event.call_id, "call_id"),
+                }),
+                itemId: id(event.item_id, "item_id"),
+                name: string(event.name, "name"),
+                arguments: argumentsObject(event.arguments),
+            };
+            break;
+        case "response.output_item.added":
+        case "response.text.delta":
+        case "response.text.done":
+        case "response.audio.delta":
+        case "response.audio.done": {
+            const item = kind === "response.output_item.added"
+                ? object(event.item, "item")
+                : null;
+            if (item && string(item.type, "item.type") !== "message")
+                return null;
+            const contentIndex = kind === "response.output_item.added" &&
+                event.content_index === undefined
+                ? 0
+                : integer(event.content_index, "content_index");
+            if (contentIndex !== 0)
+                throw new RimeStreamError("Unsupported Prism contract: content_index must be zero");
+            const output = Object.freeze({
+                response: ref(event.response_id),
+                itemId: id(item ? item.id : event.item_id, "item_id"),
+                outputIndex: integer(event.output_index, "output_index"),
+                contentIndex,
+            });
+            if (kind === "response.output_item.added")
+                payload = { kind: "message.started", output };
+            else if (kind === "response.text.delta")
+                payload = {
+                    kind: "text.delta",
+                    output,
+                    delta: string(event.delta, "delta"),
+                };
+            else if (kind === "response.text.done")
+                payload = {
+                    kind: "text.done",
+                    output,
+                    text: string(event.text, "text"),
+                };
+            else if (kind === "response.audio.delta")
+                payload = { kind: "audio.delta", output, audio: audio(event.delta) };
+            else
+                payload = { kind: "audio.done", output };
+            break;
+        }
+        case "error":
+            payload = { kind: "error", error: fault(event.error) };
+            break;
+        default:
+            return null;
+    }
+    return {
+        eventId: event.event_id === undefined ? "" : string(event.event_id, "event_id"),
+        requestId: optionalString(event.prism_request_event_id, "prism_request_event_id"),
+        payload,
+    };
+}

package/dist/realtime/session.d.ts

@@ -0,0 +1,79 @@
+import { RealtimeConnection } from "./transport.js";
+import * as t from "./types.js";
+import * as p from "./protocol.js";
+/** @internal Read the policy captured before queuing the response event. */
+export declare function responseInterruptsSpeech(session: RealtimeSession, response: t.ResponseRef): boolean;
+/** @internal Recognize only failures created by intentional session closure. */
+export declare function isSessionCloseError(error: unknown): boolean;
+/** One conversation. Consume events concurrently with session operations. */
+export declare class RealtimeSession {
+    private readonly socket;
+    private readonly timeouts;
+    private readonly forget;
+    private readonly created;
+    private sessionId;
+    private sessionInfo;
+    private readonly lifetime;
+    private failure;
+    private closed;
+    private closing;
+    private consuming;
+    private readonly queue;
+    private readonly queued;
+    private readonly ready;
+    private readyEpoch;
+    private readonly admission;
+    private readonly writeLock;
+    private readonly audioLock;
+    private readonly truncateLock;
+    private readonly toolLock;
+    private readonly pending;
+    private readonly responses;
+    private readonly interruptionPolicies;
+    private latest;
+    private readonly tasks;
+    private converter;
+    /** @internal Use client.realtime.connect(). */
+    constructor(socket: RealtimeConnection, timeouts: Required<t.RealtimeTimeouts>, forget: () => void);
+    /** @internal */
+    initialize(settings: p.SessionSettings, signal: AbortSignal): Promise<void>;
+    get info(): t.SessionInfo;
+    get events(): AsyncIterableIterator<t.SessionEvent<t.NormalizedRealtimeFault>>;
+    private readEvents;
+    private id;
+    private check;
+    private response;
+    private track;
+    private capacity;
+    private operation;
+    private locked;
+    private send;
+    private request;
+    private runRequest;
+    private abandon;
+    private create;
+    /** Apply supplied settings and wait for acknowledgment. Tools replace the
+     * complete list. Replies already started keep their original settings.
+     * Cancellation after submission does not undo an accepted update. */
+    update(options?: t.RealtimeUpdateOptions): Promise<t.SessionInfo>;
+    sendText(value: string, options?: t.RealtimeOperationOptions): Promise<t.ResponseRef>;
+    requestReply(options?: t.ReplyOptions): Promise<t.ResponseRef>;
+    addMessage(role: "user" | "assistant", value: string, options?: t.RealtimeOperationOptions): Promise<t.ItemRef>;
+    submitToolResult(call: t.ToolCallRef, output: string, options?: t.RealtimeOperationOptions): Promise<void>;
+    continueReply(parent: t.ResponseRef, options?: t.RealtimeOperationOptions): Promise<t.ResponseRef>;
+    cancel(response: t.ResponseRef, options?: t.RealtimeOperationOptions): Promise<void>;
+    /** Replaces the labels later replies carry; a reply already started keeps
+     * its own. Resolves once Prism confirms the change. */
+    setResponseMetadata(metadata: t.Labels, options?: t.RealtimeOperationOptions): Promise<void>;
+    clearAudio(options?: t.RealtimeOperationOptions): Promise<void>;
+    sendAudio(chunk: t.AudioChunk, options?: t.RealtimeOperationOptions): Promise<void>;
+    reportPlayback(report: t.PlaybackReport, options?: t.RealtimeOperationOptions): Promise<void>;
+    private settle;
+    private reject;
+    private fail;
+    private dispatch;
+    private wireResponse;
+    private publish;
+    close(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
+}