@nuancelabs/js-sdk 0.2.2 → 0.5.0

package/index.d.ts

@@ -216,8 +216,8 @@ export declare type ConversationEvent =
 /**
 * How long the live session has left, when it is capped.
 *
-* Emitted on each heartbeat, so it steps rather than ticks; a caller showing a
-* countdown should interpolate between them.
+* Emitted when the time left is learned or corrected, so it steps rather
+* than ticks; a caller showing a countdown should count down from the latest.
 */
 | {
     type: "time-remaining";
@@ -318,6 +318,26 @@ export declare interface CreateNuanceConversationOptions {
      * ```
      */
     token?: () => Promise<string>;
+    /**
+     * Who the avatar is in the conversation, such as your own
+     * {@link SessionPersonality.system_prompt}. Left out, it is the default
+     * personality.
+     *
+     * Only for a conversation the SDK creates with your API key (see
+     * {@link CreateNuanceConversationOptions.headers}). With a
+     * {@link CreateNuanceConversationOptions.token}, set it when your server
+     * mints the token instead: the token fixes it, and a conversation that sets
+     * its own is refused.
+     *
+     * @example
+     * ```ts
+     * createNuanceConversation({
+     *   headers: () => ({ Authorization: `Bearer ${process.env.NUANCE_API_KEY}` }),
+     *   personality: { system_prompt: "You are Sam, at Acme Bank..." },
+     * });
+     * ```
+     */
+    personality?: PersonalityChoice;
     /**
      * Capture constraints for the user's camera.
      *
@@ -344,9 +364,6 @@ export declare interface CreateNuanceConversationOptions {
 }
 
 
-
-
-
 /**
  * Raised by {@link Conversation.connect} when the server refuses a new session
  * because the account has not completed onboarding or accepted the current
@@ -405,7 +422,12 @@ export declare type MediaErrorReason = "permission-denied" | "device-not-found"
  */
 export declare type NetworkQuality = "good" | "poor" | "lost";
 
-
+/**
+ * Who the avatar is in a conversation created with
+ * {@link CreateNuanceConversationOptions.personality}.
+ */
+export declare interface PersonalityChoice extends SessionPersonality {
+}
 
 /**
  * Raised by {@link Conversation.connect} when the server refuses a new session
@@ -428,11 +450,37 @@ export declare type SessionEndReason = "time-limit" | "ended" | "policy-violatio
 
 
 
+/** Who the avatar is in a conversation. */
+export declare interface SessionPersonality {
+    /**
+     * Your own prompt: who the avatar is, where it works, and what it knows
+     * about the person and the situation, written as you would brief a person
+     * before a call, in plain text.
+     */
+    system_prompt?: string;
+}
+
+/**
+ * What your server sends to `POST /session-tokens` with its API key. Every
+ * field is fixed for the conversation the token starts; send `{}` for the
+ * defaults.
+ */
+export declare interface SessionTokenRequest {
+    /** Who the avatar is in the conversation. Leave it out for the default personality. */
+    personality?: SessionPersonality;
+}
+
+/** What `POST /session-tokens` returns. */
+export declare interface SessionTokenResponse {
+    /** The session token, `st_…`: pass it to {@link CreateNuanceConversationOptions.token}. */
+    token: string;
+    /** When the token stops being able to start a conversation (UTC, ISO 8601). */
+    expires_at: string;
+}
 
 /** When the avatar starts speaking: see {@link CreateNuanceConversationOptions.startOn}. */
 export declare type StartOn = "first-frame" | "full-quality";
 
-
 /**
  * The local user participant in a conversation.
  */

package/index.js

@@ -4,7 +4,7 @@ function isRecord(value) {
 }
 
 // src/core/sdk-version.ts
-var SDK_HEADER = `js/${"0.2.2"}`;
+var SDK_HEADER = `js/${"0.5.0"}`;
 
 // src/utils.ts
 function normalizeHost(host) {
@@ -56,6 +56,9 @@ class NuanceApiClient {
   async heartbeatConversation(conversationId) {
     return await this.request(`/conversation/${conversationId}/heartbeat`, { method: "POST", token: this.tokens.get(conversationId) });
   }
+  async conversationEvents(conversationId, after, signal) {
+    return await this.request(`/conversation/${conversationId}/events?after=${after}`, { token: this.tokens.get(conversationId), signal });
+  }
   async abandonConversation(conversationId) {
     return await this.request(`/conversation/${conversationId}/abandon`, { method: "POST", token: this.tokens.get(conversationId) });
   }
@@ -71,7 +74,8 @@ class NuanceApiClient {
     const response = await fetch(`${this.host}${path}`, {
       method: options?.method ?? "GET",
       headers,
-      body: options?.body !== undefined ? JSON.stringify(options.body) : undefined
+      body: options?.body !== undefined ? JSON.stringify(options.body) : undefined,
+      signal: options?.signal
     });
     if (!response.ok) {
       let detail = response.statusText;
@@ -2137,7 +2141,8 @@ class LiveKitConversation {
 var DEFAULT_HOST = "https://api.nuancelabs.ai";
 var QUEUE_HEARTBEAT_INTERVAL_MS = 5000;
 var HEARTBEAT_INTERVAL_MS = 5000;
-var ADMISSION_FAILED_BEATS = 6;
+var EVENTS_RETRY_MS = 5000;
+var ADMISSION_FAILED_REQUESTS = 6;
 var BUSY_RETRY_LIMIT_MS = 60000;
 var BUSY_RETRY_DEFAULT_MS = 3000;
 var LOST_WINDOW_MS = 30000;
@@ -2204,6 +2209,7 @@ class NuanceConversation {
   delegate = null;
   delegateUnsubscribe = null;
   stopBeats = null;
+  stopEvents = null;
   admitted = null;
   failedBeats = 0;
   resilient = false;
@@ -2292,10 +2298,8 @@ class NuanceConversation {
       this._id = conversation.id;
       this.held = conversation.id;
       this.emit({ type: "session-created", id: conversation.id });
-      this.admitted = Promise.withResolvers();
-      this.startHeartbeat(conversation.id, QUEUE_HEARTBEAT_INTERVAL_MS);
-      await this.admitted.promise;
-      this.delegate = await this.createDelegate(conversation, mediaDevices, attempt);
+      const grant = await this.waitForSlot(conversation.id, attempt);
+      this.delegate = await this.createDelegate(conversation, grant, mediaDevices, attempt);
       this.delegateUnsubscribe = this.delegate.onEvent((event) => {
         if (event.type === "state-changed") {
           this.setState(event.state);
@@ -2318,7 +2322,9 @@ class NuanceConversation {
         this.emit(event);
       });
       await this.delegate.connect();
-      this.startHeartbeat(conversation.id, HEARTBEAT_INTERVAL_MS);
+      if (grant === null) {
+        this.startHeartbeat(conversation.id, HEARTBEAT_INTERVAL_MS);
+      }
     } catch (error) {
       if (attempt !== this.attempt) {
         throw error;
@@ -2332,6 +2338,8 @@ class NuanceConversation {
     this.attempt += 1;
     this.clearLimit();
     this.stopHeartbeat();
+    this.stopEvents?.();
+    this.stopEvents = null;
     this.admitted?.reject(new Error("Disconnected while waiting for a slot"));
     this.admitted = null;
     this.delegateUnsubscribe?.();
@@ -2390,19 +2398,146 @@ class NuanceConversation {
       }
     });
   }
-  async createDelegate(conversation, mediaDevices, attempt) {
+  async waitForSlot(conversationId, attempt) {
+    const admission = await this.followEvents(conversationId, attempt);
+    if (admission !== null) {
+      this.followLive(conversationId, admission.after, attempt);
+      return admission.grant;
+    }
+    this.admitted = Promise.withResolvers();
+    this.startHeartbeat(conversationId, QUEUE_HEARTBEAT_INTERVAL_MS);
+    await this.admitted.promise;
+    return null;
+  }
+  async followEvents(conversationId, attempt) {
+    const abort = new AbortController;
+    const stop = () => abort.abort();
+    this.stopEvents = stop;
+    let after = 0;
+    let answered = false;
+    let failures = 0;
+    try {
+      for (;; ) {
+        let events;
+        try {
+          ({ events } = await this.api.conversationEvents(conversationId, after, abort.signal));
+        } catch (error) {
+          if (attempt !== this.attempt)
+            throw new DisconnectedError;
+          if (!answered && error instanceof ApiError && error.status === 404) {
+            return null;
+          }
+          failures += 1;
+          if (failures >= ADMISSION_FAILED_REQUESTS) {
+            throw new Error("Lost contact with the server while waiting", {
+              cause: error
+            });
+          }
+          await this.pause(EVENTS_RETRY_MS, abort.signal);
+          if (attempt !== this.attempt)
+            throw new DisconnectedError;
+          continue;
+        }
+        if (attempt !== this.attempt)
+          throw new DisconnectedError;
+        answered = true;
+        failures = 0;
+        for (const event of events) {
+          if (event.type === "admitted") {
+            const grant = {
+              server_url: event.livekit_url,
+              participant_token: event.token
+            };
+            return { grant, after: event.id };
+          }
+          after = Math.max(after, event.id);
+          if (event.type === "ended") {
+            throw new TerminalStatusError(event.reason);
+          }
+          if (event.type === "queued" && event.position != null) {
+            this.reportPlace(event.position, event.estimated_wait_seconds ?? null, null);
+          }
+        }
+      }
+    } finally {
+      if (this.stopEvents === stop)
+        this.stopEvents = null;
+    }
+  }
+  async followLive(conversationId, after, attempt) {
+    const abort = new AbortController;
+    const stop = () => abort.abort();
+    this.stopEvents = stop;
+    let failures = 0;
+    try {
+      while (attempt === this.attempt && !this.ended) {
+        let events;
+        try {
+          ({ events } = await this.api.conversationEvents(conversationId, after, abort.signal));
+        } catch {
+          if (attempt !== this.attempt)
+            return;
+          failures += 1;
+          await this.pause(EVENTS_RETRY_MS, abort.signal);
+          continue;
+        }
+        if (attempt !== this.attempt)
+          return;
+        const receivedAt = Date.now();
+        this.lostContact = failures > 0;
+        failures = 0;
+        for (const event of events) {
+          after = Math.max(after, event.id);
+          if (event.type === "started" && event.seconds_remaining != null) {
+            this.capped = true;
+            this.emit({
+              type: "time-remaining",
+              secondsRemaining: event.seconds_remaining
+            });
+            this.armLimit(event.seconds_remaining, receivedAt);
+          }
+          if (event.type === "ended") {
+            this.endSession(this.endReason(event.reason));
+            return;
+          }
+        }
+      }
+    } finally {
+      if (this.stopEvents === stop)
+        this.stopEvents = null;
+    }
+  }
+  endReason(reason) {
+    if (reason === "time_limit" || this.atLimit())
+      return "time-limit";
+    if (reason === "policy_violation")
+      return "policy-violation";
+    if (this.cutOff())
+      return "connection-lost";
+    return "ended";
+  }
+  pause(ms, signal) {
+    return new Promise((resolve) => {
+      let stopTicker = () => {};
+      const done = () => {
+        stopTicker();
+        signal.removeEventListener("abort", done);
+        resolve();
+      };
+      stopTicker = this.ticker(done, ms);
+      signal.addEventListener("abort", done);
+    });
+  }
+  async createDelegate(conversation, grant, mediaDevices, attempt) {
     if (conversation.provider !== "livekit") {
       throw new Error(`Unsupported provider ${conversation.provider}`);
     }
-    const response = await this.api.connectConversation(conversation.id);
+    const connection = grant ?? await this.connectRoom(conversation.id);
     if (attempt !== this.attempt) {
       throw new DisconnectedError;
     }
-    if (response.connection.kind !== "livekit") {
-      throw new Error(`Expected LiveKit connection, received ${response.connection.kind}`);
-    }
     this.resilient = typeof this.resilientReconnect === "function" ? this.resilientReconnect() : this.resilientReconnect;
-    return this.provider(response.connection, {
+    return this.provider(connection, {
       conversationId: conversation.id,
       mediaDevices,
       cameraEnabled: !this.sessionOverrides.disable_camera && mediaDevices.cameraEnabled !== false,
@@ -2411,6 +2546,13 @@ class NuanceConversation {
       resilientReconnect: this.resilient
     });
   }
+  async connectRoom(conversationId) {
+    const response = await this.api.connectConversation(conversationId);
+    if (response.connection.kind !== "livekit") {
+      throw new Error(`Expected LiveKit connection, received ${response.connection.kind}`);
+    }
+    return response.connection;
+  }
   async createWhenNotBusy(request, attempt) {
     const giveUpAt = Date.now() + BUSY_RETRY_LIMIT_MS;
     for (;; ) {
@@ -2458,7 +2600,7 @@ class NuanceConversation {
           return;
         if (!(error instanceof TerminalStatusError)) {
           this.failedBeats += 1;
-          if (this.admitted && this.failedBeats >= ADMISSION_FAILED_BEATS) {
+          if (this.admitted && this.failedBeats >= ADMISSION_FAILED_REQUESTS) {
             this.admitted.reject(new Error("Lost contact with the server while waiting", {
               cause: error
             }));
@@ -2497,14 +2639,17 @@ class NuanceConversation {
     if (state.status !== "queued" || state.position == null) {
       return;
     }
+    this.reportPlace(state.position, state.estimated_wait_seconds ?? null, state.pool_slots ?? null);
+  }
+  reportPlace(position, estimate, poolSlots) {
     if (this._state !== "queued") {
       this.setState("queued");
     }
     this.emit({
       type: "queue-changed",
-      position: state.position,
-      estimatedWaitSeconds: this.smoothEstimate(state.position, state.estimated_wait_seconds ?? null),
-      poolSlots: state.pool_slots ?? null
+      position,
+      estimatedWaitSeconds: this.smoothEstimate(position, estimate),
+      poolSlots
     });
   }
   smoothEstimate(position, estimate) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nuancelabs/js-sdk",
-  "version": "0.2.2",
+  "version": "0.5.0",
   "description": "JavaScript SDK for Nuance Labs conversation video AI avatar",
   "license": "UNLICENSED",
   "repository": {