@nuancelabs/js-sdk 0.2.1 → 0.4.0

package/index.d.ts

@@ -216,8 +216,8 @@ export declare type ConversationEvent =
 /**
 * How long the live session has left, when it is capped.
 *
-* Emitted on each heartbeat, so it steps rather than ticks; a caller showing a
-* countdown should interpolate between them.
+* Emitted when the time left is learned or corrected, so it steps rather
+* than ticks; a caller showing a countdown should count down from the latest.
 */
 | {
     type: "time-remaining";
@@ -344,9 +344,6 @@ export declare interface CreateNuanceConversationOptions {
 }
 
 
-
-
-
 /**
  * Raised by {@link Conversation.connect} when the server refuses a new session
  * because the account has not completed onboarding or accepted the current
@@ -406,7 +403,6 @@ export declare type MediaErrorReason = "permission-denied" | "device-not-found"
 export declare type NetworkQuality = "good" | "poor" | "lost";
 
 
-
 /**
  * Raised by {@link Conversation.connect} when the server refuses a new session
  * because the runtime recently ended too many of this account's sessions for
@@ -428,11 +424,9 @@ export declare type SessionEndReason = "time-limit" | "ended" | "policy-violatio
 
 
 
-
 /** When the avatar starts speaking: see {@link CreateNuanceConversationOptions.startOn}. */
 export declare type StartOn = "first-frame" | "full-quality";
 
-
 /**
  * The local user participant in a conversation.
  */

package/index.js

@@ -4,7 +4,7 @@ function isRecord(value) {
 }
 
 // src/core/sdk-version.ts
-var SDK_HEADER = `js/${"0.2.1"}`;
+var SDK_HEADER = `js/${"0.4.0"}`;
 
 // src/utils.ts
 function normalizeHost(host) {
@@ -56,6 +56,9 @@ class NuanceApiClient {
   async heartbeatConversation(conversationId) {
     return await this.request(`/conversation/${conversationId}/heartbeat`, { method: "POST", token: this.tokens.get(conversationId) });
   }
+  async conversationEvents(conversationId, after, signal) {
+    return await this.request(`/conversation/${conversationId}/events?after=${after}`, { token: this.tokens.get(conversationId), signal });
+  }
   async abandonConversation(conversationId) {
     return await this.request(`/conversation/${conversationId}/abandon`, { method: "POST", token: this.tokens.get(conversationId) });
   }
@@ -71,7 +74,8 @@ class NuanceApiClient {
     const response = await fetch(`${this.host}${path}`, {
       method: options?.method ?? "GET",
       headers,
-      body: options?.body !== undefined ? JSON.stringify(options.body) : undefined
+      body: options?.body !== undefined ? JSON.stringify(options.body) : undefined,
+      signal: options?.signal
     });
     if (!response.ok) {
       let detail = response.statusText;
@@ -1167,6 +1171,15 @@ function microphonePublishOptions(resilient) {
     return;
   return { dtx: false, audioPreset: AudioPresets.speech };
 }
+var SCREEN_CAPTURE = {
+  audio: false,
+  resolution: { width: 1920, height: 1080, frameRate: 2 },
+  contentHint: "detail"
+};
+var SCREEN_PUBLISH = {
+  screenShareEncoding: { maxBitrate: 1500000, maxFramerate: 2 },
+  simulcast: false
+};
 
 class LiveKitUserParticipant {
   room;
@@ -1174,6 +1187,8 @@ class LiveKitUserParticipant {
   onSpeakingChanged;
   onMicrophoneChanged;
   onCameraChanged;
+  onScreenShareChanged;
+  screenTrack;
   inputs = {
     audioinput: { changing: false, unavailable: false },
     videoinput: { changing: false, unavailable: false }
@@ -1188,7 +1203,7 @@ class LiveKitUserParticipant {
   lastAudioSource = null;
   _isSpeaking = false;
   speakingHandler = null;
-  constructor(room, onTracksChanged, onSpeakingChanged, onMicrophoneChanged, onCameraChanged = () => {}, initialCameraFacing, resilient = false) {
+  constructor(room, onTracksChanged, onSpeakingChanged, onMicrophoneChanged, onCameraChanged = () => {}, initialCameraFacing, resilient = false, onScreenShareChanged = () => {}) {
     this.room = room;
     this.onTracksChanged = onTracksChanged;
     this.onSpeakingChanged = onSpeakingChanged;
@@ -1196,7 +1211,9 @@ class LiveKitUserParticipant {
     this.onCameraChanged = onCameraChanged;
     this.initialCameraFacing = initialCameraFacing;
     this.microphoneOptions = microphonePublishOptions(resilient);
+    this.onScreenShareChanged = onScreenShareChanged;
     this.room.on(RoomEvent2.ActiveDeviceChanged, this.handleDeviceChanged);
+    this.room.localParticipant.on(ParticipantEvent2.LocalTrackUnpublished, this.handleTrackUnpublished);
   }
   updateTracks() {
     let changed = false;
@@ -1357,9 +1374,14 @@ class LiveKitUserParticipant {
       this.notifyInput(kind);
     }
   };
+  handleTrackUnpublished = (publication) => {
+    if (!this.disposed && publication.source === Track5.Source.ScreenShare)
+      this.onScreenShareChanged();
+  };
   cleanup() {
     this.disposed = true;
     this.room.off(RoomEvent2.ActiveDeviceChanged, this.handleDeviceChanged);
+    this.room.localParticipant.off(ParticipantEvent2.LocalTrackUnpublished, this.handleTrackUnpublished);
     if (this.inputs.audioinput.changing)
       this.inputTrack("audioinput")?.stop();
     if (this.inputs.videoinput.changing)
@@ -1437,10 +1459,44 @@ class LiveKitUserParticipant {
   get isVideoEnabled() {
     return this.room.localParticipant.isCameraEnabled;
   }
+  async startScreenShare() {
+    await this.setScreenShare(true);
+  }
+  async stopScreenShare() {
+    const track = this.screenTrack;
+    this.screenTrack = undefined;
+    try {
+      if (!this.disposed)
+        await this.setScreenShare(false);
+    } finally {
+      track?.stop();
+    }
+  }
+  get isScreenSharing() {
+    return this.room.localParticipant.isScreenShareEnabled;
+  }
+  async setScreenShare(enabled) {
+    if (this.disposed)
+      throw new Error("Participant is disconnected");
+    try {
+      const publication = await this.room.localParticipant.setScreenShareEnabled(enabled, SCREEN_CAPTURE, SCREEN_PUBLISH);
+      if (this.disposed)
+        publication?.track?.stop();
+      else if (enabled)
+        this.screenTrack = publication?.track;
+    } catch (error) {
+      throw new MediaAccessError(error);
+    } finally {
+      if (!this.disposed)
+        this.onScreenShareChanged();
+    }
+  }
   releaseDevices() {
     for (const pub of this.room.localParticipant.trackPublications.values()) {
       pub.track?.stop();
     }
+    this.screenTrack?.stop();
+    this.screenTrack = undefined;
   }
   startSpeakingDetection() {
     this.stopSpeakingDetection();
@@ -1601,7 +1657,7 @@ class LiveKitConversation {
         type: "speaking-changed",
         participant: "user",
         isSpeaking
-      }), () => this.emit({ type: "microphone-changed" }), () => this.emit({ type: "camera-changed" }), this.cameraFacing, this.resilient);
+      }), () => this.emit({ type: "microphone-changed" }), () => this.emit({ type: "camera-changed" }), this.cameraFacing, this.resilient, () => this.emit({ type: "screen-share-changed" }));
       this._user.startSpeakingDetection();
       try {
         await this._user.unmuteAudio();
@@ -1677,7 +1733,7 @@ class LiveKitConversation {
   }
   onChange(handler) {
     return this.onEvent((event) => {
-      if (event.type !== "data-event")
+      if (event.type !== "data-event" && event.type !== "screen-share-changed")
         handler(event);
     });
   }
@@ -2085,7 +2141,8 @@ class LiveKitConversation {
 var DEFAULT_HOST = "https://api.nuancelabs.ai";
 var QUEUE_HEARTBEAT_INTERVAL_MS = 5000;
 var HEARTBEAT_INTERVAL_MS = 5000;
-var ADMISSION_FAILED_BEATS = 6;
+var EVENTS_RETRY_MS = 5000;
+var ADMISSION_FAILED_REQUESTS = 6;
 var BUSY_RETRY_LIMIT_MS = 60000;
 var BUSY_RETRY_DEFAULT_MS = 3000;
 var LOST_WINDOW_MS = 30000;
@@ -2152,6 +2209,7 @@ class NuanceConversation {
   delegate = null;
   delegateUnsubscribe = null;
   stopBeats = null;
+  stopEvents = null;
   admitted = null;
   failedBeats = 0;
   resilient = false;
@@ -2240,10 +2298,8 @@ class NuanceConversation {
       this._id = conversation.id;
       this.held = conversation.id;
       this.emit({ type: "session-created", id: conversation.id });
-      this.admitted = Promise.withResolvers();
-      this.startHeartbeat(conversation.id, QUEUE_HEARTBEAT_INTERVAL_MS);
-      await this.admitted.promise;
-      this.delegate = await this.createDelegate(conversation, mediaDevices, attempt);
+      const grant = await this.waitForSlot(conversation.id, attempt);
+      this.delegate = await this.createDelegate(conversation, grant, mediaDevices, attempt);
       this.delegateUnsubscribe = this.delegate.onEvent((event) => {
         if (event.type === "state-changed") {
           this.setState(event.state);
@@ -2266,7 +2322,9 @@ class NuanceConversation {
         this.emit(event);
       });
       await this.delegate.connect();
-      this.startHeartbeat(conversation.id, HEARTBEAT_INTERVAL_MS);
+      if (grant === null) {
+        this.startHeartbeat(conversation.id, HEARTBEAT_INTERVAL_MS);
+      }
     } catch (error) {
       if (attempt !== this.attempt) {
         throw error;
@@ -2280,6 +2338,8 @@ class NuanceConversation {
     this.attempt += 1;
     this.clearLimit();
     this.stopHeartbeat();
+    this.stopEvents?.();
+    this.stopEvents = null;
     this.admitted?.reject(new Error("Disconnected while waiting for a slot"));
     this.admitted = null;
     this.delegateUnsubscribe?.();
@@ -2321,7 +2381,7 @@ class NuanceConversation {
   }
   onChange(handler) {
     return this.onEvent((event) => {
-      if (event.type !== "data-event")
+      if (event.type !== "data-event" && event.type !== "screen-share-changed")
         handler(event);
     });
   }
@@ -2338,19 +2398,146 @@ class NuanceConversation {
       }
     });
   }
-  async createDelegate(conversation, mediaDevices, attempt) {
+  async waitForSlot(conversationId, attempt) {
+    const admission = await this.followEvents(conversationId, attempt);
+    if (admission !== null) {
+      this.followLive(conversationId, admission.after, attempt);
+      return admission.grant;
+    }
+    this.admitted = Promise.withResolvers();
+    this.startHeartbeat(conversationId, QUEUE_HEARTBEAT_INTERVAL_MS);
+    await this.admitted.promise;
+    return null;
+  }
+  async followEvents(conversationId, attempt) {
+    const abort = new AbortController;
+    const stop = () => abort.abort();
+    this.stopEvents = stop;
+    let after = 0;
+    let answered = false;
+    let failures = 0;
+    try {
+      for (;; ) {
+        let events;
+        try {
+          ({ events } = await this.api.conversationEvents(conversationId, after, abort.signal));
+        } catch (error) {
+          if (attempt !== this.attempt)
+            throw new DisconnectedError;
+          if (!answered && error instanceof ApiError && error.status === 404) {
+            return null;
+          }
+          failures += 1;
+          if (failures >= ADMISSION_FAILED_REQUESTS) {
+            throw new Error("Lost contact with the server while waiting", {
+              cause: error
+            });
+          }
+          await this.pause(EVENTS_RETRY_MS, abort.signal);
+          if (attempt !== this.attempt)
+            throw new DisconnectedError;
+          continue;
+        }
+        if (attempt !== this.attempt)
+          throw new DisconnectedError;
+        answered = true;
+        failures = 0;
+        for (const event of events) {
+          if (event.type === "admitted") {
+            const grant = {
+              server_url: event.livekit_url,
+              participant_token: event.token
+            };
+            return { grant, after: event.id };
+          }
+          after = Math.max(after, event.id);
+          if (event.type === "ended") {
+            throw new TerminalStatusError(event.reason);
+          }
+          if (event.type === "queued" && event.position != null) {
+            this.reportPlace(event.position, event.estimated_wait_seconds ?? null, null);
+          }
+        }
+      }
+    } finally {
+      if (this.stopEvents === stop)
+        this.stopEvents = null;
+    }
+  }
+  async followLive(conversationId, after, attempt) {
+    const abort = new AbortController;
+    const stop = () => abort.abort();
+    this.stopEvents = stop;
+    let failures = 0;
+    try {
+      while (attempt === this.attempt && !this.ended) {
+        let events;
+        try {
+          ({ events } = await this.api.conversationEvents(conversationId, after, abort.signal));
+        } catch {
+          if (attempt !== this.attempt)
+            return;
+          failures += 1;
+          await this.pause(EVENTS_RETRY_MS, abort.signal);
+          continue;
+        }
+        if (attempt !== this.attempt)
+          return;
+        const receivedAt = Date.now();
+        this.lostContact = failures > 0;
+        failures = 0;
+        for (const event of events) {
+          after = Math.max(after, event.id);
+          if (event.type === "started" && event.seconds_remaining != null) {
+            this.capped = true;
+            this.emit({
+              type: "time-remaining",
+              secondsRemaining: event.seconds_remaining
+            });
+            this.armLimit(event.seconds_remaining, receivedAt);
+          }
+          if (event.type === "ended") {
+            this.endSession(this.endReason(event.reason));
+            return;
+          }
+        }
+      }
+    } finally {
+      if (this.stopEvents === stop)
+        this.stopEvents = null;
+    }
+  }
+  endReason(reason) {
+    if (reason === "time_limit" || this.atLimit())
+      return "time-limit";
+    if (reason === "policy_violation")
+      return "policy-violation";
+    if (this.cutOff())
+      return "connection-lost";
+    return "ended";
+  }
+  pause(ms, signal) {
+    return new Promise((resolve) => {
+      let stopTicker = () => {};
+      const done = () => {
+        stopTicker();
+        signal.removeEventListener("abort", done);
+        resolve();
+      };
+      stopTicker = this.ticker(done, ms);
+      signal.addEventListener("abort", done);
+    });
+  }
+  async createDelegate(conversation, grant, mediaDevices, attempt) {
     if (conversation.provider !== "livekit") {
       throw new Error(`Unsupported provider ${conversation.provider}`);
     }
-    const response = await this.api.connectConversation(conversation.id);
+    const connection = grant ?? await this.connectRoom(conversation.id);
     if (attempt !== this.attempt) {
       throw new DisconnectedError;
     }
-    if (response.connection.kind !== "livekit") {
-      throw new Error(`Expected LiveKit connection, received ${response.connection.kind}`);
-    }
     this.resilient = typeof this.resilientReconnect === "function" ? this.resilientReconnect() : this.resilientReconnect;
-    return this.provider(response.connection, {
+    return this.provider(connection, {
       conversationId: conversation.id,
       mediaDevices,
       cameraEnabled: !this.sessionOverrides.disable_camera && mediaDevices.cameraEnabled !== false,
@@ -2359,6 +2546,13 @@ class NuanceConversation {
       resilientReconnect: this.resilient
     });
   }
+  async connectRoom(conversationId) {
+    const response = await this.api.connectConversation(conversationId);
+    if (response.connection.kind !== "livekit") {
+      throw new Error(`Expected LiveKit connection, received ${response.connection.kind}`);
+    }
+    return response.connection;
+  }
   async createWhenNotBusy(request, attempt) {
     const giveUpAt = Date.now() + BUSY_RETRY_LIMIT_MS;
     for (;; ) {
@@ -2406,7 +2600,7 @@ class NuanceConversation {
           return;
         if (!(error instanceof TerminalStatusError)) {
           this.failedBeats += 1;
-          if (this.admitted && this.failedBeats >= ADMISSION_FAILED_BEATS) {
+          if (this.admitted && this.failedBeats >= ADMISSION_FAILED_REQUESTS) {
             this.admitted.reject(new Error("Lost contact with the server while waiting", {
               cause: error
             }));
@@ -2445,14 +2639,17 @@ class NuanceConversation {
     if (state.status !== "queued" || state.position == null) {
       return;
     }
+    this.reportPlace(state.position, state.estimated_wait_seconds ?? null, state.pool_slots ?? null);
+  }
+  reportPlace(position, estimate, poolSlots) {
     if (this._state !== "queued") {
       this.setState("queued");
     }
     this.emit({
       type: "queue-changed",
-      position: state.position,
-      estimatedWaitSeconds: this.smoothEstimate(state.position, state.estimated_wait_seconds ?? null),
-      poolSlots: state.pool_slots ?? null
+      position,
+      estimatedWaitSeconds: this.smoothEstimate(position, estimate),
+      poolSlots
     });
   }
   smoothEstimate(position, estimate) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nuancelabs/js-sdk",
-  "version": "0.2.1",
+  "version": "0.4.0",
   "description": "JavaScript SDK for Nuance Labs conversation video AI avatar",
   "license": "UNLICENSED",
   "repository": {