@nuancelabs/js-sdk 0.12.0 → 0.13.1

package/index.d.ts

@@ -217,6 +217,37 @@ export declare interface Conversation {
      * @returns A function that removes the handler.
      */
     on<K extends keyof SessionEventMap>(type: K, handler: (event: SessionEventMap[K]) => void): () => void;
+    /**
+     * Subscribes to the agent's calls of the conversation's tools, named when
+     * your server mints the session token. Run the tool in your page and answer each call with
+     * {@link Conversation.sendToolResult}. The agent keeps talking while a call
+     * is open, and uses the result when it arrives.
+     *
+     * @example
+     * ```ts
+     * conversation.onToolCall(async (call) => {
+     *   if (call.name === "show_confetti") {
+     *     launchConfetti();
+     *     await conversation.sendToolResult(call.callId, "Confetti is falling.");
+     *   }
+     * });
+     * ```
+     *
+     * @returns A function that removes the handler.
+     */
+    onToolCall(handler: (call: ToolCall) => void): () => void;
+    /**
+     * Answers a tool call: what it produced, or with `isError` what went wrong.
+     * The agent keeps the first 2,000 characters of an output, so return what
+     * it needs, not everything you have.
+     *
+     * @param callId - The call's {@link ToolCall.callId}.
+     * @returns Whether the runtime took the result: `status` is `"rejected"`,
+     * with the reason in `error`, for an unknown or closed call.
+     * @throws If the conversation is not connected, or the runtime does not
+     * answer in time.
+     */
+    sendToolResult(callId: string, output: ToolOutput, options?: ToolResultOptions): Promise<CommandResult>;
 }
 
 
@@ -487,6 +518,33 @@ export declare interface FloorEvent extends SessionEventBase {
     state: "waiting" | "listening" | "speaking" | (string & {});
 }
 
+/**
+ * A tool the agent may call, which your page runs: show something on the
+ * screen, read what the user has open, look something up in your app. The
+ * agent decides when to call it from its `description`, and fills its
+ * arguments to match its `parameters`. A conversation has up to 16 tools.
+ */
+export declare interface FunctionTool {
+    /** Always `"function"`, the default. */
+    type?: "function";
+    /**
+     * The tool's name: letters, digits, `_` and `-`, up to 64 characters,
+     * unique in the conversation. `ask_user`, `get_current_time`, `web_search`
+     * and the agent's other own tools' names are taken.
+     */
+    name: string;
+    /**
+     * What the tool does and when to use it, written for the agent, up to
+     * 2,000 characters. Say when to call it, not only what it does.
+     */
+    description: string;
+    /**
+     * The arguments, as a JSON Schema whose `type` is `"object"`, up to 16,000
+     * characters as compact JSON. `{ type: "object", properties: {} }` takes none.
+     */
+    parameters: Record<string, unknown>;
+}
+
 /**
  * Raised by {@link Conversation.connect} when the server refuses a guest's new
  * session because a guest limit is used up. `reason` says which: `used` (this
@@ -499,6 +557,7 @@ export declare class GuestLimitError extends Error {
     constructor(reason: string, message: string);
 }
 
+
 /**
  * Raised by {@link Conversation.connect} when the server refuses a new session
  * because the account has not completed onboarding or accepted the current
@@ -651,6 +710,23 @@ export declare interface SessionPersonality {
 export declare interface SessionTokenRequest {
     /** Who the avatar is in the conversation. Leave it out for the default personality. */
     personality?: SessionPersonality;
+    /**
+     * Tools the agent may call, which your page runs and answers with
+     * `onToolCall` and `sendToolResult`;
+     * up to 16. Fixed for the conversation: the page can't change them.
+     *
+     * @example
+     * ```ts
+     * tools: [
+     *   {
+     *     name: "show_confetti",
+     *     description: "Plays confetti on the user's screen. Use it to celebrate with them.",
+     *     parameters: { type: "object", properties: {} },
+     *   },
+     * ]
+     * ```
+     */
+    tools?: FunctionTool[];
     /**
      * Your own id for the person this conversation is for, such as a hash of
      * your user id. After repeated policy violations, new conversations pause
@@ -673,9 +749,53 @@ export declare interface SessionTokenResponse {
 /** When the avatar starts speaking: see {@link CreateNuanceConversationOptions.startOn}. */
 export declare type StartOn = "first-frame" | "full-quality";
 
+/**
+ * The agent's call of one of the conversation's tools: run it, then answer
+ * with {@link Conversation.sendToolResult} and its `callId`.
+ */
+export declare interface ToolCall {
+    /** Identifies the call; answer with it. */
+    callId: string;
+    /** The tool, by its `name`. */
+    name: string;
+    /** The agent's own short name for this use of the tool, such as `"confetti"`. */
+    label: string;
+    /** The call's arguments, held to the tool's `parameters` schema. */
+    arguments: Record<string, unknown>;
+}
 
+/**
+ * An image a tool produced, for the agent to look at: a screenshot, say, or
+ * a page of a document. Up to 8,000,000 base64 characters (about 6 MB); the
+ * runtime fits it to 1920 px on its long side.
+ */
+export declare interface ToolImage {
+    mediaType: "image/jpeg" | "image/png";
+    /** The encoded image, base64. */
+    data: string;
+}
 
+/**
+ * What a tool call produced: text, or JSON the agent reads as it is. Up to
+ * 24,000 characters (its JSON, for an object); the agent keeps the first 2,000.
+ */
+export declare type ToolOutput = string | Record<string, unknown> | unknown[];
 
+/**
+ * How {@link Conversation.sendToolResult} reports a result.
+ */
+export declare interface ToolResultOptions {
+    /** Images the tool produced, up to 4; the agent looks at them, in order,
+     *  where the result enters the conversation. */
+    images?: ToolImage[];
+    /** The output says what went wrong rather than what the tool produced. */
+    isError?: boolean;
+    /** Whether this is the call's last part; a part with `final: false` adds to
+     *  what the agent knows and keeps the call open.
+     *
+     * @defaultValue `true` */
+    final?: boolean;
+}
 
 /**
  * The local user participant in a conversation.

package/index.js

@@ -4,7 +4,7 @@ function isRecord(value) {
 }
 
 // src/core/sdk-version.ts
-var SDK_HEADER = `js/${"0.12.0"}`;
+var SDK_HEADER = `js/${"0.13.1"}`;
 
 // src/utils.ts
 function normalizeHost(host) {
@@ -1676,6 +1676,10 @@ var COMMAND_TIMEOUT_MS = 5000;
 var STREAM_TIMEOUT_MS = 30000;
 var BACKGROUND_ID = /^[a-z0-9][a-z0-9_-]{0,63}$/;
 var AVATAR_RETURN_TIMEOUT_MS = 8000;
+function contextItem(item) {
+  const name = item.name === undefined ? {} : { name: item.name };
+  return item.type === "text" ? { type: "text", text: item.text, ...name } : { type: "image", media_type: item.mediaType, data: item.data, ...name };
+}
 function isReconnecting(lkState) {
   return lkState === ConnectionState2.Reconnecting || lkState === ConnectionState2.SignalReconnecting;
 }
@@ -1736,6 +1740,7 @@ class LiveKitConversation {
   awayTimer = null;
   presentationPaused = false;
   readySending = false;
+  contextSent = null;
   leaving = null;
   connectGeneration = 0;
   _id;
@@ -1744,6 +1749,8 @@ class LiveKitConversation {
   cameraFacing;
   tokenProvider;
   opensOnReady;
+  presentsAtFullSize;
+  initialContext;
   resilient;
   constructor(config) {
     this.room = new Room({
@@ -1765,6 +1772,8 @@ class LiveKitConversation {
     this.cameraFacing = config.mediaDevices?.cameraFacing;
     this.tokenProvider = config.tokenProvider;
     this.opensOnReady = config.opensOnReady ?? false;
+    this.presentsAtFullSize = config.presentsAtFullSize ?? this.opensOnReady;
+    this.initialContext = config.initialContext ?? [];
     this.resilient = config.resilientReconnect ?? false;
   }
   get id() {
@@ -1780,6 +1789,7 @@ class LiveKitConversation {
     this.sequence.stop();
     this.sequence = this.newSequence();
     this.readyOwed = false;
+    this.contextSent = null;
     this.everConnected = false;
     if (!this.eventsRegistered) {
       this.registerRoomEvents();
@@ -2150,6 +2160,7 @@ class LiveKitConversation {
     if (packet === null)
       return;
     this.speaksProtocol = true;
+    this.sendContext();
     const { seq, event } = packet;
     if (this.sequence.repeats(seq))
       return;
@@ -2243,6 +2254,10 @@ class LiveKitConversation {
       this.emit({ type: "avatar-presented" });
       return;
     }
+    if (!this.presentsAtFullSize) {
+      this.markAvatarPresented();
+      return;
+    }
     this.startPresentation();
   }
   startPresentation() {
@@ -2253,7 +2268,7 @@ class LiveKitConversation {
   markAvatarPresented() {
     this.presentation = null;
     if (this.speaksProtocol && this.room.state === ConnectionState2.Connected) {
-      this.command({ type: "session.ready" }).catch(() => {
+      this.ready().catch(() => {
         return;
       });
     } else if (this.resilient && this.speaksProtocol && isReconnecting(this.room.state)) {
@@ -2261,11 +2276,28 @@ class LiveKitConversation {
     }
     this.emit({ type: "avatar-presented" });
   }
+  sendContext() {
+    if (this.contextSent !== null || this.initialContext.length === 0)
+      return;
+    this.contextSent = this.command({
+      type: "session.context",
+      items: this.initialContext.map(contextItem)
+    }).then(() => {
+      return;
+    }, () => {
+      return;
+    });
+  }
+  ready() {
+    this.sendContext();
+    const ready = () => this.command({ type: "session.ready" });
+    return this.contextSent === null ? ready() : this.contextSent.then(ready);
+  }
   sendOwedReady() {
     if (!this.readyOwed || !this.speaksProtocol || this.readySending)
       return;
     this.readySending = true;
-    this.command({ type: "session.ready" }).then(() => {
+    this.ready().then(() => {
       this.readyOwed = false;
     }).catch(() => {
       return;
@@ -2428,6 +2460,8 @@ class NuanceConversation {
   ticker;
   sessionOverrides;
   personality;
+  tools;
+  initialContext;
   variant;
   maxSessionSeconds;
   videoCapture;
@@ -2466,6 +2500,8 @@ class NuanceConversation {
     this.ticker = deps.ticker ?? pageTicker;
     this.sessionOverrides = options.sessionOverrides ?? {};
     this.personality = options.personality;
+    this.tools = options.tools;
+    this.initialContext = options.initialContext ?? [];
     this.variant = options.variant;
     this.maxSessionSeconds = options.maxSessionSeconds;
     this.videoCapture = options.videoCapture;
@@ -2495,13 +2531,15 @@ class NuanceConversation {
       try {
         const request = {
           session_overrides: this.sessionOverrides,
-          client_ready: this.startOn === "full-quality"
+          client_ready: this.startOn === "full-quality" || this.initialContext.length > 0
         };
         const timezone = this.timezone();
         if (timezone)
           request.timezone = timezone;
         if (this.personality)
           request.personality = this.personality;
+        if (this.tools)
+          request.tools = this.tools;
         if (this.variant)
           request.variant = this.variant;
         if (this.maxSessionSeconds !== undefined) {
@@ -2805,6 +2843,8 @@ class NuanceConversation {
       cameraEnabled: !this.sessionOverrides.disable_camera && mediaDevices.cameraEnabled !== false,
       videoCapture: this.videoCapture,
       opensOnReady: conversation.provider_metadata?.client_ready === true,
+      presentsAtFullSize: this.startOn === "full-quality",
+      initialContext: this.initialContext,
       resilientReconnect: this.resilient
     });
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nuancelabs/js-sdk",
-  "version": "0.12.0",
+  "version": "0.13.1",
   "description": "JavaScript SDK for Nuance Labs conversation video AI avatar",
   "license": "UNLICENSED",
   "repository": {