@nuancelabs/js-sdk 0.13.0 → 0.14.1

package/README.md

@@ -18,6 +18,7 @@ const conversation = createNuanceConversation({
   // Your endpoint, which mints a session token with your API key.
   token: async () => {
     const response = await fetch("/api/nuance-token", { method: "POST" });
+    if (!response.ok) throw new Error(await response.text());
     return (await response.json()).token;
   },
 });

package/index.d.ts

@@ -86,6 +86,38 @@ export declare interface AgentTurnStartedEvent extends SessionEventBase {
     source: "reply" | "say" | (string & {});
 }
 
+/**
+ * An error answer from the Nuance API that the SDK has no more specific error
+ * for, such as a `401` for an expired or used session token, a `429` when this
+ * month's conversation time is used, or a `503` once the SDK has stopped
+ * retrying a busy service. See the Errors guide for each status and `code`.
+ *
+ * Act on `status` and `code`, never on `message`: the message is the server's
+ * `detail`, written for you, and it can change.
+ *
+ * @example
+ * ```ts
+ * try {
+ *   await conversation.connect();
+ * } catch (error) {
+ *   if (error instanceof ApiError && error.status === 401) {
+ *     // The session token expired or was used: the next connect() fetches a new one.
+ *   }
+ * }
+ * ```
+ */
+export declare class ApiError extends Error {
+    /** The HTTP status, such as `401` or `503`. */
+    readonly status: number;
+    /** The error's `code` when the server sent one, such as `"busy"` or `"unavailable"`. */
+    readonly code: string | null;
+    /** How long to wait before trying again, when the server said. */
+    readonly retryAfterSeconds: number | null;
+    /** The parsed JSON body of the answer, or `null` when it had none. */
+    readonly body: unknown;
+    constructor(status: number, detail: string, body?: unknown);
+}
+
 /**
  * An audio track that can be attached to or detached from an audio element.
  */
@@ -362,7 +394,8 @@ export declare type ConversationEvent =
 * `"time-limit"` when the session was capped and ran out its clock,
 * `"policy-violation"` when the runtime ended it for content, `"refused"`
 * when the runtime would not start it, with its reason in `message` when it
-* gave one.
+* gave one. Call {@link Conversation.disconnect} then: after the runtime's
+* own end the call stays connected, with the camera and microphone on.
 */
 | {
     type: "session-ended";
@@ -416,6 +449,7 @@ export declare type ConversationStateValue = `${ConversationState}`;
  *   // Your server mints a session token with its API key.
  *   token: async () => {
  *     const response = await fetch("/api/nuance-token", { method: "POST" });
+ *     if (!response.ok) throw new Error(await response.text());
  *     return (await response.json()).token;
  *   },
  * });
@@ -448,6 +482,7 @@ export declare interface CreateNuanceConversationOptions {
      * ```ts
      * token: async () => {
      *   const response = await fetch("/api/nuance-token", { method: "POST" });
+     *   if (!response.ok) throw new Error(await response.text());
      *   return (await response.json()).token;
      * }
      * ```
@@ -557,6 +592,7 @@ export declare class GuestLimitError extends Error {
     constructor(reason: string, message: string);
 }
 
+
 /**
  * Raised by {@link Conversation.connect} when the server refuses a new session
  * because the account has not completed onboarding or accepted the current

package/index.js

@@ -4,7 +4,7 @@ function isRecord(value) {
 }
 
 // src/core/sdk-version.ts
-var SDK_HEADER = `js/${"0.13.0"}`;
+var SDK_HEADER = `js/${"0.14.1"}`;
 
 // src/utils.ts
 function normalizeHost(host) {
@@ -14,12 +14,16 @@ function normalizeHost(host) {
 // src/api/client.ts
 class ApiError extends Error {
   status;
+  code;
+  retryAfterSeconds;
   body;
   constructor(status, detail, body = null) {
     super(detail);
     this.name = "ApiError";
     this.status = status;
     this.body = body;
+    this.code = isRecord(body) && typeof body.code === "string" ? body.code : null;
+    this.retryAfterSeconds = isRecord(body) && typeof body.retry_after_seconds === "number" ? body.retry_after_seconds : null;
   }
 }
 
@@ -99,7 +103,6 @@ class NuanceApiClient {
     return this.headersProvider ? await this.headersProvider() : {};
   }
 }
-
 // src/core/active-session-error.ts
 class ActiveSessionError extends Error {
   constructor() {
@@ -1676,6 +1679,12 @@ var COMMAND_TIMEOUT_MS = 5000;
 var STREAM_TIMEOUT_MS = 30000;
 var BACKGROUND_ID = /^[a-z0-9][a-z0-9_-]{0,63}$/;
 var AVATAR_RETURN_TIMEOUT_MS = 8000;
+var RUNTIME_RETURN_TIMEOUT_MS = 20000;
+var RUNTIME_RECHECK_MS = 1000;
+function contextItem(item) {
+  const name = item.name === undefined ? {} : { name: item.name };
+  return item.type === "text" ? { type: "text", text: item.text, ...name } : { type: "image", media_type: item.mediaType, data: item.data, ...name };
+}
 function isReconnecting(lkState) {
   return lkState === ConnectionState2.Reconnecting || lkState === ConnectionState2.SignalReconnecting;
 }
@@ -1734,8 +1743,10 @@ class LiveKitConversation {
   everConnected = false;
   away = false;
   awayTimer = null;
+  runtimeTimer = null;
   presentationPaused = false;
   readySending = false;
+  contextSent = null;
   leaving = null;
   connectGeneration = 0;
   _id;
@@ -1744,6 +1755,8 @@ class LiveKitConversation {
   cameraFacing;
   tokenProvider;
   opensOnReady;
+  presentsAtFullSize;
+  initialContext;
   resilient;
   constructor(config) {
     this.room = new Room({
@@ -1765,6 +1778,8 @@ class LiveKitConversation {
     this.cameraFacing = config.mediaDevices?.cameraFacing;
     this.tokenProvider = config.tokenProvider;
     this.opensOnReady = config.opensOnReady ?? false;
+    this.presentsAtFullSize = config.presentsAtFullSize ?? this.opensOnReady;
+    this.initialContext = config.initialContext ?? [];
     this.resilient = config.resilientReconnect ?? false;
   }
   get id() {
@@ -1780,6 +1795,7 @@ class LiveKitConversation {
     this.sequence.stop();
     this.sequence = this.newSequence();
     this.readyOwed = false;
+    this.contextSent = null;
     this.everConnected = false;
     if (!this.eventsRegistered) {
       this.registerRoomEvents();
@@ -2039,10 +2055,14 @@ class LiveKitConversation {
       queueMicrotask(() => {
         if (this.room.state === ConnectionState2.Disconnected)
           return;
-        if (isReconnecting(this.room.state))
+        if (isReconnecting(this.room.state)) {
           this.avatarAway();
-        else
+        } else if (this.sessionEnded) {
           this.avatarLeft();
+        } else if (this.avatarInRoom()) {} else {
+          this.avatarAway();
+          this.awaitRuntimeReturn();
+        }
       });
     });
     this.room.on(RoomEvent3.Disconnected, () => {
@@ -2150,6 +2170,7 @@ class LiveKitConversation {
     if (packet === null)
       return;
     this.speaksProtocol = true;
+    this.sendContext();
     const { seq, event } = packet;
     if (this.sequence.repeats(seq))
       return;
@@ -2243,6 +2264,10 @@ class LiveKitConversation {
       this.emit({ type: "avatar-presented" });
       return;
     }
+    if (!this.presentsAtFullSize) {
+      this.markAvatarPresented();
+      return;
+    }
     this.startPresentation();
   }
   startPresentation() {
@@ -2253,7 +2278,7 @@ class LiveKitConversation {
   markAvatarPresented() {
     this.presentation = null;
     if (this.speaksProtocol && this.room.state === ConnectionState2.Connected) {
-      this.command({ type: "session.ready" }).catch(() => {
+      this.ready().catch(() => {
         return;
       });
     } else if (this.resilient && this.speaksProtocol && isReconnecting(this.room.state)) {
@@ -2261,11 +2286,28 @@ class LiveKitConversation {
     }
     this.emit({ type: "avatar-presented" });
   }
+  sendContext() {
+    if (this.contextSent !== null || this.initialContext.length === 0)
+      return;
+    this.contextSent = this.command({
+      type: "session.context",
+      items: this.initialContext.map(contextItem)
+    }).then(() => {
+      return;
+    }, () => {
+      return;
+    });
+  }
+  ready() {
+    this.sendContext();
+    const ready = () => this.command({ type: "session.ready" });
+    return this.contextSent === null ? ready() : this.contextSent.then(ready);
+  }
   sendOwedReady() {
     if (!this.readyOwed || !this.speaksProtocol || this.readySending)
       return;
     this.readySending = true;
-    this.command({ type: "session.ready" }).then(() => {
+    this.ready().then(() => {
       this.readyOwed = false;
     }).catch(() => {
       return;
@@ -2307,6 +2349,33 @@ class LiveKitConversation {
       this.avatarLeft();
     }, AVATAR_RETURN_TIMEOUT_MS);
   }
+  awaitRuntimeReturn(delay = RUNTIME_RETURN_TIMEOUT_MS) {
+    if (this.runtimeTimer !== null)
+      return;
+    this.runtimeTimer = setTimeout(() => {
+      this.runtimeTimer = null;
+      if (!this.away)
+        return;
+      if (isReconnecting(this.room.state)) {
+        this.awaitRuntimeReturn(RUNTIME_RECHECK_MS);
+        return;
+      }
+      if (this.room.state === ConnectionState2.Disconnected)
+        return;
+      if (this.avatarInRoom())
+        return;
+      this.clearAway();
+      this.avatarLeft();
+      this.reportEnd("connection-lost");
+    }, delay);
+  }
+  avatarInRoom() {
+    for (const [, participant] of this.room.remoteParticipants) {
+      if (this.isAvatarParticipant(participant))
+        return true;
+    }
+    return false;
+  }
   avatarBack() {
     const resume = this.presentationPaused && this._avatar.isReady;
     this.clearAway();
@@ -2322,6 +2391,10 @@ class LiveKitConversation {
       clearTimeout(this.awayTimer);
       this.awayTimer = null;
     }
+    if (this.runtimeTimer !== null) {
+      clearTimeout(this.runtimeTimer);
+      this.runtimeTimer = null;
+    }
   }
   startLayerFallback() {
     if (this.layerFallback !== null)
@@ -2429,6 +2502,7 @@ class NuanceConversation {
   sessionOverrides;
   personality;
   tools;
+  initialContext;
   variant;
   maxSessionSeconds;
   videoCapture;
@@ -2468,6 +2542,7 @@ class NuanceConversation {
     this.sessionOverrides = options.sessionOverrides ?? {};
     this.personality = options.personality;
     this.tools = options.tools;
+    this.initialContext = options.initialContext ?? [];
     this.variant = options.variant;
     this.maxSessionSeconds = options.maxSessionSeconds;
     this.videoCapture = options.videoCapture;
@@ -2497,7 +2572,7 @@ class NuanceConversation {
       try {
         const request = {
           session_overrides: this.sessionOverrides,
-          client_ready: this.startOn === "full-quality"
+          client_ready: this.startOn === "full-quality" || this.initialContext.length > 0
         };
         const timezone = this.timezone();
         if (timezone)
@@ -2778,7 +2853,7 @@ class NuanceConversation {
       return "time-limit";
     if (reason === "policy_violation")
       return "policy-violation";
-    if (this.cutOff())
+    if (reason === "runtime_silent" || this.cutOff())
       return "connection-lost";
     return "ended";
   }
@@ -2809,6 +2884,8 @@ class NuanceConversation {
       cameraEnabled: !this.sessionOverrides.disable_camera && mediaDevices.cameraEnabled !== false,
       videoCapture: this.videoCapture,
       opensOnReady: conversation.provider_metadata?.client_ready === true,
+      presentsAtFullSize: this.startOn === "full-quality",
+      initialContext: this.initialContext,
       resilientReconnect: this.resilient
     });
   }
@@ -3012,5 +3089,6 @@ export {
   LegalRequiredError,
   GuestLimitError,
   ConversationState,
+  ApiError,
   ActiveSessionError
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nuancelabs/js-sdk",
-  "version": "0.13.0",
+  "version": "0.14.1",
   "description": "JavaScript SDK for Nuance Labs conversation video AI avatar",
   "license": "UNLICENSED",
   "repository": {