@nuancelabs/js-sdk 0.1.1 → 0.2.0

package/index.d.ts

@@ -157,10 +157,10 @@ export declare type ConversationEvent =
     type: "avatar-ready";
 }
 /**
-* The avatar is presented as she should be seen: her video has reached its
-* full size and held it, or the wait for it ran out. Follows `avatar-ready`.
-* The session opens on it: her first words, and its recording. Reveal her
-* fully on this; between the two she is in view but not yet speaking.
+* The avatar starts speaking: her first words, and the session's recording.
+* Follows `avatar-ready`, at once by default. With `startOn: "full-quality"`
+* it waits until her video has reached its full size and held it, or the
+* wait ran out; in between she is in view but not yet speaking.
 */
 | {
     type: "avatar-presented";
@@ -296,6 +296,9 @@ export declare type ConversationStateValue = `${ConversationState}`;
  */
 export declare function createNuanceConversation(options?: CreateNuanceConversationOptions): Conversation;
 
+/**
+ * Options for creating a Nuance conversation.
+ */
 export declare interface CreateNuanceConversationOptions {
     /**
      * Fetches a session token, for authenticating from a web page.
@@ -324,6 +327,20 @@ export declare interface CreateNuanceConversationOptions {
     videoCapture?: VideoCaptureConstraints;
     /** Local capture preferences, resolved once for each connection attempt. */
     mediaDevices?: MediaDevicePreferences | (() => MediaDevicePreferences);
+    /**
+     * When the avatar starts speaking.
+     *
+     * - `"first-frame"`: as soon as her first frame reaches the page.
+     *   `avatar-presented` follows `avatar-ready` at once. Her video may be
+     *   soft for its first few seconds while the connection ramps up.
+     * - `"full-quality"`: once her video has reached its full size and held
+     *   it, or after at most about 12 seconds. She is in view but silent in
+     *   between, so a page using this should cover her until
+     *   `avatar-presented`.
+     *
+     * @defaultValue `"first-frame"`
+     */
+    startOn?: StartOn;
 }
 
 
@@ -412,6 +429,9 @@ export declare type SessionEndReason = "time-limit" | "ended" | "policy-violatio
 
 
 
+/** When the avatar starts speaking: see {@link CreateNuanceConversationOptions.startOn}. */
+export declare type StartOn = "first-frame" | "full-quality";
+
 
 /**
  * The local user participant in a conversation.

package/index.js

@@ -4,7 +4,7 @@ function isRecord(value) {
 }
 
 // src/core/sdk-version.ts
-var SDK_HEADER = `js/${"0.1.1"}`;
+var SDK_HEADER = `js/${"0.2.0"}`;
 
 // src/utils.ts
 function normalizeHost(host) {
@@ -1078,6 +1078,9 @@ function parseSessionEvent(payload) {
       }
     };
   }
+  if (decoded.type === "floor") {
+    return { type: "floor", speaking: decoded.state === "speaking" };
+  }
   if (decoded.type === "session.status" && decoded.status === "ended") {
     return {
       type: "ended",
@@ -1520,6 +1523,7 @@ class LiveKitConversation {
   eventsRegistered = false;
   sessionEnded = false;
   speaksProtocol = false;
+  avatarSpeaking = false;
   answered = null;
   commands = new Map;
   commandCount = 0;
@@ -1665,6 +1669,12 @@ class LiveKitConversation {
     }
     return this.command({ type: "scene.background", background });
   }
+  async sendText(text) {
+    const line = text.trim();
+    if (!line)
+      throw new Error("Nothing to send: the text is blank");
+    return this.command({ type: "input.text", text: line, respond: true });
+  }
   onChange(handler) {
     return this.onEvent((event) => {
       if (event.type !== "data-event")
@@ -1748,6 +1758,8 @@ class LiveKitConversation {
       const mapped = mapConnectionState(lkState, this.resilient && this.everConnected);
       if (mapped === "connected")
         this.everConnected = true;
+      if (lkState === ConnectionState2.Reconnecting)
+        this.setAvatarSpeaking(false);
       if (mapped !== this._state) {
         this.setState(mapped);
       }
@@ -1787,6 +1799,7 @@ class LiveKitConversation {
     this.room.on(RoomEvent3.Disconnected, () => {
       this.stats?.abandon();
       this.stats = null;
+      this.setAvatarSpeaking(false);
       this.stopPresentation();
       this.stopReadyFallback();
       this.stopLayerFallback();
@@ -1902,6 +1915,8 @@ class LiveKitConversation {
         this.commands.delete(id);
         settle({ status: "rejected", error: `unknown command ${type}` });
       }
+    } else if (event.type === "floor") {
+      this.setAvatarSpeaking(event.speaking);
     } else if (event.type === "ended") {
       this.answered?.();
       if (!event.byClient)
@@ -1934,6 +1949,16 @@ class LiveKitConversation {
     this.readyFallback?.stop();
     this.readyFallback = null;
   }
+  setAvatarSpeaking(speaking) {
+    if (speaking === this.avatarSpeaking)
+      return;
+    this.avatarSpeaking = speaking;
+    this.emit({
+      type: "speaking-changed",
+      participant: "avatar",
+      isSpeaking: speaking
+    });
+  }
   markAvatarReady() {
     this.stopReadyFallback();
     if (this._avatar.isReady)
@@ -2116,6 +2141,7 @@ class NuanceConversation {
   personality;
   variant;
   videoCapture;
+  startOn;
   mediaDevices;
   resilientReconnect;
   delegate = null;
@@ -2151,6 +2177,7 @@ class NuanceConversation {
     this.personality = options.personality;
     this.variant = options.variant;
     this.videoCapture = options.videoCapture;
+    this.startOn = options.startOn ?? "first-frame";
     this.mediaDevices = options.mediaDevices;
     this.resilientReconnect = options.resilientReconnect ?? false;
   }
@@ -2176,7 +2203,7 @@ class NuanceConversation {
       try {
         const request = {
           session_overrides: this.sessionOverrides,
-          client_ready: true
+          client_ready: this.startOn === "full-quality"
         };
         const timezone = this.timezone();
         if (timezone)
@@ -2281,6 +2308,12 @@ class NuanceConversation {
     }
     return this.delegate.setBackground(background);
   }
+  async sendText(text) {
+    if (!this.delegate) {
+      throw new Error("Not connected — call connect() first");
+    }
+    return this.delegate.sendText(text);
+  }
   onChange(handler) {
     return this.onEvent((event) => {
       if (event.type !== "data-event")

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nuancelabs/js-sdk",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "JavaScript SDK for Nuance Labs conversation video AI avatar",
   "license": "UNLICENSED",
   "repository": {