@nuancelabs/js-sdk 0.10.0 → 0.11.0

package/index.d.ts

@@ -499,6 +499,13 @@ export declare interface SessionPersonality {
 export declare interface SessionTokenRequest {
     /** Who the avatar is in the conversation. Leave it out for the default personality. */
     personality?: SessionPersonality;
+    /**
+     * Your own id for the person this conversation is for, such as a hash of
+     * your user id. After repeated policy violations, new conversations pause
+     * for this person only, not for your whole app. Letters, digits and
+     * `_ . : -`, up to 128 characters: never an email address or a name.
+     */
+    end_user_id?: string;
 }
 
 /** What `POST /session-tokens` returns. */
@@ -507,11 +514,17 @@ export declare interface SessionTokenResponse {
     token: string;
     /** When the token stops being able to start a conversation (UTC, ISO 8601). */
     expires_at: string;
+    /** What to change in how you mint tokens; empty when nothing. */
+    warnings?: string[];
 }
 
 /** When the avatar starts speaking: see {@link CreateNuanceConversationOptions.startOn}. */
 export declare type StartOn = "first-frame" | "full-quality";
 
+
+
+
+
 /**
  * The local user participant in a conversation.
  */

package/index.js

@@ -4,7 +4,7 @@ function isRecord(value) {
 }
 
 // src/core/sdk-version.ts
-var SDK_HEADER = `js/${"0.10.0"}`;
+var SDK_HEADER = `js/${"0.11.0"}`;
 
 // src/utils.ts
 function normalizeHost(host) {
@@ -1098,6 +1098,17 @@ function parseSessionEvent(payload) {
   if (decoded.type === "floor") {
     return { type: "floor", speaking: decoded.state === "speaking" };
   }
+  if (decoded.type === "tool.call" && typeof decoded.call_id === "string" && typeof decoded.name === "string" && isRecord(decoded.arguments)) {
+    return {
+      type: "tool-call",
+      call: {
+        callId: decoded.call_id,
+        name: decoded.name,
+        label: typeof decoded.label === "string" ? decoded.label : "",
+        arguments: decoded.arguments
+      }
+    };
+  }
   if (decoded.type === "session.status" && decoded.status === "ended") {
     return {
       type: "ended",
@@ -1544,6 +1555,7 @@ class LiveKitUserParticipant {
 // src/providers/livekit/conversation.ts
 var DATA_EVENT_TOPIC = "nuance.data";
 var SESSION_TOPIC = "nuance.v1";
+var PACKET_MAX_BYTES = 14000;
 var RUNTIME_TOPIC = "runtime";
 var CLOCK_SYNC_TOPIC = "clock-sync";
 var STATS_FLUSH_TIMEOUT_MS = 300;
@@ -1551,6 +1563,7 @@ var SESSION_END_TIMEOUT_MS = 300;
 var SESSION_END_ID = "client-session-end";
 var REFUSAL_REASON_WAIT_MS = 500;
 var COMMAND_TIMEOUT_MS = 5000;
+var STREAM_TIMEOUT_MS = 30000;
 var BACKGROUND_ID = /^[a-z0-9][a-z0-9_-]{0,63}$/;
 var AVATAR_RETURN_TIMEOUT_MS = 8000;
 function isReconnecting(lkState) {
@@ -1764,9 +1777,24 @@ class LiveKitConversation {
   async releaseAgent() {
     return this.command({ type: "floor.release" });
   }
+  async sendToolResult(callId, output, options = {}) {
+    return this.command({
+      type: "tool.result",
+      call_id: callId,
+      output,
+      ...options.images?.length && {
+        images: options.images.map(({ mediaType, data }) => ({
+          media_type: mediaType,
+          data
+        }))
+      },
+      is_error: options.isError ?? false,
+      final: options.final ?? true
+    });
+  }
   onChange(handler) {
     return this.onEvent((event) => {
-      if (event.type !== "data-event" && event.type !== "screen-share-changed")
+      if (event.type !== "data-event" && event.type !== "screen-share-changed" && event.type !== "tool-call")
         handler(event);
     });
   }
@@ -1783,6 +1811,12 @@ class LiveKitConversation {
       }
     });
   }
+  onToolCall(handler) {
+    return this.onEvent((event) => {
+      if (event.type === "tool-call")
+        handler(event.call);
+    });
+  }
   async command(command) {
     if (this.room.state !== ConnectionState2.Connected) {
       throw new Error("Not connected — call connect() first");
@@ -1794,11 +1828,18 @@ class LiveKitConversation {
     const answer = Promise.withResolvers();
     this.commands.set(id, { type: command.type, settle: answer.resolve });
     let timer;
+    const text = JSON.stringify({ ...command, id });
+    const payload = new TextEncoder().encode(text);
+    const streamed = payload.byteLength > PACKET_MAX_BYTES;
+    const sent = streamed ? this.room.localParticipant.sendText(text, { topic: SESSION_TOPIC }) : this.room.localParticipant.publishData(payload, {
+      reliable: true,
+      topic: SESSION_TOPIC
+    });
     try {
       return await Promise.race([
-        this.room.localParticipant.publishData(new TextEncoder().encode(JSON.stringify({ ...command, id })), { reliable: true, topic: SESSION_TOPIC }).then(() => answer.promise),
+        sent.then(() => answer.promise),
         new Promise((_, reject) => {
-          timer = setTimeout(() => reject(new Error(`The runtime did not answer ${command.type}`)), COMMAND_TIMEOUT_MS);
+          timer = setTimeout(() => reject(new Error(`The runtime did not answer ${command.type}`)), streamed ? STREAM_TIMEOUT_MS : COMMAND_TIMEOUT_MS);
         })
       ]);
     } finally {
@@ -2006,6 +2047,8 @@ class LiveKitConversation {
       }
     } else if (event.type === "floor") {
       this.setAvatarSpeaking(event.speaking);
+    } else if (event.type === "tool-call") {
+      this.emit({ type: "tool-call", call: event.call });
     } else if (event.type === "ended") {
       this.answered?.();
       if (!event.byClient)
@@ -2446,9 +2489,15 @@ class NuanceConversation {
     }
     return this.delegate.releaseAgent();
   }
+  async sendToolResult(callId, output, options) {
+    if (!this.delegate) {
+      throw new Error("Not connected — call connect() first");
+    }
+    return this.delegate.sendToolResult(callId, output, options);
+  }
   onChange(handler) {
     return this.onEvent((event) => {
-      if (event.type !== "data-event" && event.type !== "screen-share-changed")
+      if (event.type !== "data-event" && event.type !== "screen-share-changed" && event.type !== "tool-call")
         handler(event);
     });
   }
@@ -2465,6 +2514,12 @@ class NuanceConversation {
       }
     });
   }
+  onToolCall(handler) {
+    return this.onEvent((event) => {
+      if (event.type === "tool-call")
+        handler(event.call);
+    });
+  }
   async waitForSlot(conversationId, attempt) {
     const admission = await this.followEvents(conversationId, attempt);
     if (admission !== null) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nuancelabs/js-sdk",
-  "version": "0.10.0",
+  "version": "0.11.0",
   "description": "JavaScript SDK for Nuance Labs conversation video AI avatar",
   "license": "UNLICENSED",
   "repository": {