@nuancelabs/js-sdk 0.0.0-stage → 0.1.0

package/README.md

@@ -1,3 +1,35 @@
-# Temporary Holding Version
+# @nuancelabs/js-sdk
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Put a live, face-to-face conversation with a Nuance avatar in your web page.
+
+> **Preview.** Until 1.0, a minor version (0.2.0) may change the API; patch versions (0.1.1) only fix.
+
+```bash
+npm install @nuancelabs/js-sdk
+```
+
+Your server mints a short-lived session token with your API key, and the page
+starts the conversation with it. The API key never reaches the browser.
+
+```ts
+import { createNuanceConversation } from "@nuancelabs/js-sdk";
+
+const conversation = createNuanceConversation({
+  // Your endpoint, which mints a session token with your API key.
+  token: async () => {
+    const response = await fetch("/api/nuance-token", { method: "POST" });
+    return (await response.json()).token;
+  },
+});
+
+conversation.onChange((event) => {
+  if (event.type === "tracks-changed") {
+    conversation.avatar.videoTrack?.attach(videoElement);
+    conversation.avatar.audioTrack?.attach(audioElement);
+  }
+});
+
+await conversation.connect();
+```
+
+Guides, examples and the full reference: https://docs.nuancelabs.ai

package/index.d.ts

@@ -0,0 +1,480 @@
+/**
+ * Raised by {@link Conversation.connect} when the caller already holds another
+ * active conversation session. The application must end that session before a
+ * new one can be created — retrying immediately will fail the same way.
+ */
+export declare class ActiveSessionError extends Error {
+    constructor();
+}
+
+/**
+ * An audio track that can be attached to or detached from an audio element.
+ */
+export declare interface AudioTrack {
+    /** Attaches the audio track to an HTML audio element. */
+    attach(element: HTMLAudioElement): void;
+    /** Detaches the track from its currently attached element, if any. */
+    detach(): void;
+    /** Whether the audio track is currently enabled. */
+    readonly isEnabled: boolean;
+    /** The underlying media track, when the provider exposes one — feeds
+     *  local visualizations (e.g. an analyser-driven waveform). */
+    readonly mediaStreamTrack?: MediaStreamTrack | null;
+}
+
+/**
+ * The remote avatar participant in a conversation.
+ */
+export declare interface AvatarParticipant {
+    /** Avatar video, when available. */
+    readonly videoTrack: VideoTrack | null;
+    /** Avatar audio, when available. */
+    readonly audioTrack: AudioTrack | null;
+    /** Mutes avatar audio playback locally. */
+    muteAudio(): Promise<void>;
+    /** Unmutes avatar audio playback locally. */
+    unmuteAudio(): Promise<void>;
+    /** Whether avatar audio playback is muted locally. */
+    readonly isMuted: boolean;
+    /** Whether avatar video is currently enabled. */
+    readonly isVideoEnabled: boolean;
+    /**
+     * Whether the avatar is ready to be shown — its first audiovisual output has
+     * reached the client. Audio playback is gated on this automatically (kept muted
+     * while attached so buffers warm, then unmuted on ready), so consumers only need
+     * this to gate visual presentation, e.g. revealing the video once in sync.
+     */
+    readonly isReady: boolean;
+}
+
+/** The direction a camera faces relative to the user. */
+export declare type CameraFacing = "user" | "environment";
+
+/**
+ * The runtime's answer to a command: `"applied"`, or `"rejected"` with the
+ * runtime's reason in `error`.
+ */
+export declare interface CommandResult {
+    status: "applied" | "rejected";
+    error: string | null;
+}
+
+
+/**
+ * A live avatar conversation controlled by the SDK.
+ *
+ * A conversation owns the connection lifecycle and exposes the local user and
+ * remote avatar participants after it is connected.
+ */
+export declare interface Conversation {
+    /**
+     * Starts the conversation.
+     *
+     * This creates or joins the underlying session and requests local camera and
+     * microphone access when required.
+     */
+    connect(): Promise<void>;
+    /**
+     * Ends the conversation and releases local connection resources.
+     */
+    disconnect(): Promise<void>;
+    /**
+     * Current lifecycle state for the conversation.
+     */
+    readonly state: ConversationStateValue;
+    /**
+     * Server-assigned conversation id, available once connected. Remains readable
+     * after disconnect so callers can reference the session (e.g. to submit
+     * feedback). Null before the conversation has been created.
+     */
+    readonly id: string | null;
+    /**
+     * Local user participant.
+     *
+     * @throws If accessed before the conversation is connected.
+     */
+    readonly user: UserParticipant;
+    /**
+     * Remote avatar participant.
+     *
+     * @throws If accessed before the conversation is connected.
+     */
+    readonly avatar: AvatarParticipant;
+    /**
+     * Switches the avatar's background to one the runtime serves, by id
+     * (`"original"` is the avatar's own backdrop). The runtime composites the
+     * video, so nothing changes on the client's side of it.
+     *
+     * Resolves with the runtime's answer: a runtime without a background
+     * library, or one that does not serve the id, rejects it.
+     *
+     * @throws If the id is not a background id, the conversation is not
+     * connected, the runtime takes no session commands, or it does not answer
+     * in time.
+     */
+    setBackground(background: string): Promise<CommandResult>;
+    /**
+     * Subscribes to conversation lifecycle and media events.
+     *
+     * @returns A function that removes the event handler.
+     */
+    onChange(handler: (event: ConversationEvent) => void): () => void;
+}
+
+
+/**
+ * Events emitted by {@link Conversation.onChange}.
+ */
+export declare type ConversationEvent = 
+/** The conversation lifecycle state changed. */
+    {
+    type: "state-changed";
+    state: ConversationStateValue;
+}
+/**
+* The server created the session and assigned {@link Conversation.id}.
+* Fires before any wait for a slot, so a host can key UI on the id
+* (a page for the conversation) while the connection is still underway.
+*/
+| {
+    type: "session-created";
+    id: string;
+}
+/** The avatar participant joined the conversation. */
+| {
+    type: "avatar-connected";
+}
+/** The avatar participant left the conversation. */
+| {
+    type: "avatar-disconnected";
+}
+/**
+* The avatar is ready to be shown ({@link AvatarParticipant.isReady}). Gate
+* presentation of the avatar's video/audio on this so it is revealed in view
+* and in sync.
+*/
+| {
+    type: "avatar-ready";
+}
+/**
+* The avatar is presented as she should be seen: her video has reached its
+* full size and held it, or the wait for it ran out. Follows `avatar-ready`.
+* The session opens on it: her first words, and its recording. Reveal her
+* fully on this; between the two she is in view but not yet speaking.
+*/
+| {
+    type: "avatar-presented";
+}
+/** User or avatar media tracks changed. */
+| {
+    type: "tracks-changed";
+}
+/** The active microphone changed; read {@link UserParticipant.microphoneId}. */
+| {
+    type: "microphone-changed";
+}
+/** The active camera changed; read {@link UserParticipant.cameraId}. */
+| {
+    type: "camera-changed";
+}
+/** Speaking activity changed for the user or avatar. */
+| {
+    type: "speaking-changed";
+    participant: "user" | "avatar";
+    isSpeaking: boolean;
+}
+/**
+* The user's microphone is on in the app but nothing from it reaches the
+* call: the browser sends no audio, or the system stopped or muted the
+* capture. Never reported while the user has muted it in the app. Fires
+* on each change, and clears when the conversation stops sampling.
+*/
+| {
+    type: "mic-silent-changed";
+    silent: boolean;
+}
+/** The connection's {@link NetworkQuality} changed. Starts `"good"`. */
+| {
+    type: "network-changed";
+    quality: NetworkQuality;
+}
+/**
+* The runtime found this client's link to the avatar poor for a sustained
+* span; sent at most once per session, and only by a runtime that offers it.
+* A caller can offer the user to end the call.
+*/
+| {
+    type: "link-poor";
+}
+/**
+* The session's place in the queue moved, or its estimate changed.
+*
+* Emitted only while waiting. `estimatedWaitSeconds` never increases for a
+* given session: a wait that grows reads as the service failing, and arriving
+* early is forgiven. Null when there is no serving capacity to estimate from.
+*/
+/**
+* How long the live session has left, when it is capped.
+*
+* Emitted on each heartbeat, so it steps rather than ticks; a caller showing a
+* countdown should interpolate between them.
+*/
+| {
+    type: "time-remaining";
+    secondsRemaining: number;
+}
+/**
+* The server considers the session over.
+*
+* A runtime that stops at its limit does not necessarily leave the transport,
+* so a caller cannot infer this from the connection alone. `reason` is
+* `"time-limit"` when the session was capped and ran out its clock,
+* `"policy-violation"` when the runtime ended it for content.
+*/
+| {
+    type: "session-ended";
+    reason: SessionEndReason;
+} | {
+    type: "queue-changed";
+    position: number;
+    estimatedWaitSeconds: number | null;
+    poolSlots: number | null;
+};
+
+/**
+ * Lifecycle states emitted by a conversation.
+ */
+export declare enum ConversationState {
+    /** The conversation has been created locally but has not started connecting. */
+    Idle = "idle",
+    /** Waiting for a runtime slot. Distinct from Connecting: nothing is wrong and
+     * there is nothing to retry — the session has a place in line and will start
+     * when one frees. */
+    Queued = "queued",
+    /** The SDK is creating or joining the underlying session. */
+    Connecting = "connecting",
+    /** The conversation is connected and media can flow. */
+    Connected = "connected",
+    /** The conversation was disconnected intentionally or by the remote service. */
+    Disconnected = "disconnected",
+    /** The conversation failed to connect or encountered a terminal error. */
+    Failed = "failed"
+}
+
+/**
+ * String value form of {@link ConversationState}.
+ */
+export declare type ConversationStateValue = `${ConversationState}`;
+
+/**
+ * Creates a Nuance avatar conversation.
+ *
+ * The returned conversation is idle until {@link Conversation.connect} is
+ * called. The SDK creates the backend conversation session, waits until it is
+ * ready, and connects using the connection details returned by the Nuance API.
+ *
+ * @example
+ * ```ts
+ * import { createNuanceConversation } from "@nuancelabs/js-sdk";
+ *
+ * const conversation = createNuanceConversation({
+ *   // Your server mints a session token with its API key.
+ *   token: async () => {
+ *     const response = await fetch("/api/nuance-token", { method: "POST" });
+ *     return (await response.json()).token;
+ *   },
+ * });
+ *
+ * conversation.onChange((event) => {
+ *   if (event.type === "tracks-changed") {
+ *     conversation.avatar.videoTrack?.attach(videoElement);
+ *   }
+ * });
+ *
+ * await conversation.connect();
+ * ```
+ */
+export declare function createNuanceConversation(options?: CreateNuanceConversationOptions): Conversation;
+
+export declare interface CreateNuanceConversationOptions {
+    /**
+     * Fetches a session token, for authenticating from a web page.
+     *
+     * Your server mints the token with its API key (`POST /session-tokens`) and
+     * returns it to the page. The SDK asks for a fresh one each time it starts a
+     * conversation, since a token creates one conversation and then reaches only
+     * that conversation. The conversation's settings are fixed when your
+     * server mints the token.
+     *
+     * @example
+     * ```ts
+     * token: async () => {
+     *   const response = await fetch("/api/nuance-token", { method: "POST" });
+     *   return (await response.json()).token;
+     * }
+     * ```
+     */
+    token?: () => Promise<string>;
+    /**
+     * Capture constraints for the user's camera.
+     *
+     * When omitted, the provider's capture defaults apply (16:9 720p for
+     * LiveKit, the browser's default for WebRTC).
+     */
+    videoCapture?: VideoCaptureConstraints;
+    /** Local capture preferences, resolved once for each connection attempt. */
+    mediaDevices?: MediaDevicePreferences | (() => MediaDevicePreferences);
+}
+
+
+
+
+
+/**
+ * Raised by {@link Conversation.connect} when the server refuses a new session
+ * because the account has not completed onboarding or accepted the current
+ * Terms of Service. The application must send the user through its legal
+ * consent flow; retrying without that will fail the same way.
+ */
+export declare class LegalRequiredError extends Error {
+    constructor();
+}
+
+/**
+ * Raised by {@link Conversation.connect} when the server refuses a new session
+ * because the service is under maintenance. `message` is the notice the
+ * operators wrote, safe to show as is.
+ */
+export declare class MaintenanceError extends Error {
+    constructor(message: string);
+}
+
+/**
+ * A local media failure raised by {@link Conversation.connect}. Exposes the
+ * affected device and a classified reason so the application can render (and
+ * localize) the user-facing copy and choose recovery UI. The `Error` message is
+ * terse and developer-facing — do not show it to users.
+ */
+export declare class MediaAccessError extends Error {
+    readonly device: MediaDevice;
+    readonly reason: MediaErrorReason;
+    constructor(cause: unknown, device?: MediaDevice);
+}
+
+/**
+ * Which local capture device a media failure concerns.
+ */
+export declare type MediaDevice = "camera" | "microphone" | "unknown";
+
+/** Preferred local devices. Unavailable device IDs fall back to browser defaults. */
+export declare interface MediaDevicePreferences {
+    microphoneId?: string;
+    cameraId?: string;
+    /** Whether to capture video on connect. Defaults to true; video can be enabled later. */
+    cameraEnabled?: boolean;
+    /** Required camera direction; takes precedence over cameraId. */
+    cameraFacing?: CameraFacing;
+}
+
+/**
+ * Why a local capture device could not be acquired.
+ */
+export declare type MediaErrorReason = "permission-denied" | "device-not-found" | "device-in-use" | "unknown";
+
+/**
+ * How well media is getting through: the worse of the user's own link and
+ * the avatar's, as the media server scores them. `"poor"`: degraded (loss,
+ * jitter, low bandwidth). `"lost"`: media has stopped arriving.
+ */
+export declare type NetworkQuality = "good" | "poor" | "lost";
+
+
+
+/**
+ * Raised by {@link Conversation.connect} when the server refuses a new session
+ * because the runtime recently ended too many of this account's sessions for
+ * content-policy violations. `retryAfterSeconds` is the server's estimate of
+ * when a new session may be started, when it sent one.
+ */
+export declare class PolicyCurfewError extends Error {
+    readonly retryAfterSeconds: number | null;
+    constructor(retryAfterSeconds: number | null);
+}
+
+
+/**
+ * Why a session is over. `"time-limit"`: the capped session ran out its clock.
+ * `"policy-violation"`: the runtime ended it for content that breaks the usage
+ * policy. `"ended"`: any other server-side end.
+ */
+export declare type SessionEndReason = "time-limit" | "ended" | "policy-violation" | "connection-lost";
+
+
+
+
+
+/**
+ * The local user participant in a conversation.
+ */
+export declare interface UserParticipant {
+    /** Local camera video, when available. */
+    readonly videoTrack: VideoTrack | null;
+    /** Local microphone audio, when available. */
+    readonly audioTrack: AudioTrack | null;
+    /** Current microphone device id, or the selected device while muted. */
+    readonly microphoneId: string | null;
+    /** Changes the microphone without ending the conversation or changing mute. */
+    setMicrophone(deviceId: string): Promise<void>;
+    /** Current camera device id, or the selected device while the camera is off. */
+    readonly cameraId: string | null;
+    /** Changes the camera without ending the conversation or enabling video. */
+    setCamera(deviceId: string): Promise<void>;
+    /** Current camera direction, or the selected direction while video is off. */
+    readonly cameraFacing: CameraFacing | null;
+    /** Selects the front or back camera without enabling video. */
+    setCameraFacing(facing: CameraFacing): Promise<void>;
+    /** Enables the local camera. */
+    enableVideo(): Promise<void>;
+    /** Disables the local camera. */
+    disableVideo(): Promise<void>;
+    /** Mutes the local microphone. */
+    muteAudio(): Promise<void>;
+    /** Unmutes the local microphone. */
+    unmuteAudio(): Promise<void>;
+    /** Whether the local microphone is muted. */
+    readonly isMuted: boolean;
+    /** Whether the local camera is enabled. */
+    readonly isVideoEnabled: boolean;
+    /** Whether the user is currently detected as speaking. */
+    readonly isSpeaking: boolean;
+}
+
+/**
+ * Capture constraints for the user's published camera track.
+ *
+ * Use this to match the capture to what your UI actually displays, so
+ * recordings and other downstream consumers receive the same frame the user
+ * sees in their self-view — e.g. a square self-view should capture square
+ * video. Values are applied as ideal constraints; a browser that cannot
+ * satisfy them falls back to its nearest supported format.
+ */
+export declare interface VideoCaptureConstraints {
+    width: number;
+    height: number;
+    /** @defaultValue `width / height` */
+    aspectRatio?: number;
+}
+
+/**
+ * A video track that can be attached to or detached from a video element.
+ */
+export declare interface VideoTrack {
+    /** Attaches the video track to an HTML video element. */
+    attach(element: HTMLVideoElement): void;
+    /** Detaches the track from its currently attached element, if any. */
+    detach(): void;
+    /** Whether the video track is currently enabled. */
+    readonly isEnabled: boolean;
+}
+
+export { }