@nuancelabs/js-sdk 0.11.0 → 0.12.0

package/index.d.ts

@@ -7,6 +7,85 @@ export declare class ActiveSessionError extends Error {
     constructor();
 }
 
+/**
+ * The turn's first audio. `atMs` is when it plays, which can be after the
+ * event arrives.
+ */
+export declare interface AgentSpeechStartedEvent extends SessionEventBase {
+    type: "agent.speech.started";
+    /** The turn, as {@link AgentTurnStartedEvent.turnId} names it. */
+    turnId: string;
+}
+
+/**
+ * The agent's words, in whole words, as they are committed to the
+ * conversation. They arrive ahead of their audio: `atMs` is when the first
+ * of them plays. When the agent's turn is interrupted, words whose `atMs` is
+ * later than the {@link AgentTurnEndedEvent}'s were never heard.
+ */
+export declare interface AgentTextEvent extends SessionEventBase {
+    type: "agent.text";
+    /**
+     * The turn the words belong to, as {@link AgentTurnStartedEvent.turnId}
+     * names it; null for words outside any turn.
+     */
+    turnId: string | null;
+    /** The words, with the spacing to join them to the turn's other words. */
+    text: string;
+}
+
+/**
+ * The agent's turn is over. `atMs` is when the turn's audio ends: for an
+ * interrupted turn, where the agent was cut off.
+ */
+export declare interface AgentTurnEndedEvent extends SessionEventBase {
+    type: "agent.turn.ended";
+    /** The turn, as {@link AgentTurnStartedEvent.turnId} names it. */
+    turnId: string;
+    /**
+     * `"completed"`: the agent said all of it. `"interrupted"`: the user cut
+     * in. `"stopped"`: a command stopped the agent. `"abandoned"`: the turn
+     * gave up, as when nothing of it was heard for a while or it had nothing
+     * to say; `spoken` is then the runtime's estimate of what played. A newer
+     * runtime may send other values.
+     */
+    reason: "completed" | "interrupted" | "stopped" | "abandoned" | (string & {});
+    /**
+     * Every word the turn committed, as its {@link AgentTextEvent}s carried
+     * them. After an interruption it includes words that were cut before they
+     * played; their `agent.text` events have an `atMs` later than this
+     * event's.
+     */
+    spoken: string;
+}
+
+/**
+ * The agent took the floor, before its first audio. Its words follow as
+ * {@link AgentTextEvent}s and the turn closes with an
+ * {@link AgentTurnEndedEvent}, both with the same `turnId`. `atMs` is where
+ * on the timeline the agent took it.
+ */
+export declare interface AgentTurnStartedEvent extends SessionEventBase {
+    type: "agent.turn.started";
+    /** The turn's id, unique within the session. */
+    turnId: string;
+    /**
+     * What opened the turn. `"user_done"`: the agent judged the user had
+     * finished. `"silence"`: the user went quiet long enough. `"check_in"`:
+     * both were quiet and the agent spoke first, as in its opening greeting.
+     * `"command"`: a command asked for the turn. `"tool"`: a tool's result
+     * reached the agent while both were quiet. A newer runtime may send other
+     * values.
+     */
+    openedBy: "user_done" | "silence" | "check_in" | "command" | "tool" | (string & {});
+    /**
+     * `"reply"`: the agent wrote the turn's words itself. `"say"`: the agent
+     * says a line it was given, word for word. A newer runtime may send other
+     * values.
+     */
+    source: "reply" | "say" | (string & {});
+}
+
 /**
  * An audio track that can be attached to or detached from an audio element.
  */
@@ -119,6 +198,25 @@ export declare interface Conversation {
      * @returns A function that removes the event handler.
      */
     onChange(handler: (event: ConversationEvent) => void): () => void;
+    /**
+     * Subscribes to one type of the session's events: what the user and the
+     * agent say, their turns, and who has the floor.
+     *
+     * Each event arrives once, in the session's order (its `seq`). Events the
+     * connection missed while it was down are asked of the runtime again and
+     * delivered in their place. Subscribe before `connect()` to see the
+     * session's first events.
+     *
+     * @example
+     * ```ts
+     * const stop = conversation.on("agent.text", (event) => {
+     *   transcript.textContent += event.text;
+     * });
+     * ```
+     *
+     * @returns A function that removes the handler.
+     */
+    on<K extends keyof SessionEventMap>(type: K, handler: (event: SessionEventMap[K]) => void): () => void;
 }
 
 
@@ -376,6 +474,19 @@ export declare interface CreateNuanceConversationOptions {
 }
 
 
+/**
+ * Who has the floor changed. `atMs` is when the change plays.
+ */
+export declare interface FloorEvent extends SessionEventBase {
+    type: "floor";
+    /**
+     * `"speaking"`: the agent has the floor. `"listening"`: the user has it
+     * and the agent listens. `"waiting"`: nobody has it; the agent waits for
+     * the user. A newer runtime may send other values.
+     */
+    state: "waiting" | "listening" | "speaking" | (string & {});
+}
+
 /**
  * Raised by {@link Conversation.connect} when the server refuses a guest's new
  * session because a guest limit is used up. `reason` says which: `used` (this
@@ -473,6 +584,47 @@ export declare class PolicyCurfewError extends Error {
  */
 export declare type SessionEndReason = "time-limit" | "ended" | "policy-violation" | "refused" | "connection-lost";
 
+/** Any one of the session's events. */
+export declare type SessionEvent = SessionEventMap[keyof SessionEventMap];
+
+/**
+ * What every session event carries: its place in the session's order and on
+ * the session's timeline.
+ */
+export declare interface SessionEventBase {
+    /**
+     * The event's number in the session's sequence: from 1, one more for each
+     * event the runtime sends, of every kind, so the events one handler sees
+     * skip numbers. The SDK delivers events in this order, each once. 0 for an
+     * event the runtime did not number.
+     */
+    seq: number;
+    /**
+     * Where the event falls on the session's timeline, in milliseconds. The
+     * timeline counts the user's audio from the start of the session. A user
+     * event is stamped on the user's audio, as where the user said it; an
+     * agent event where the agent's audio for it plays, which can be ahead of
+     * when the event arrives. Every event of a session is on this one timeline, so
+     * their `atMs` compare. Null when the runtime gives no time.
+     */
+    atMs: number | null;
+}
+
+/**
+ * The session's events by their `type`, for {@link Conversation.on}.
+ */
+export declare interface SessionEventMap {
+    "user.transcript": UserTranscriptEvent;
+    "user.speech.started": UserSpeechStartedEvent;
+    "user.speech.ended": UserSpeechEndedEvent;
+    "user.turn.ended": UserTurnEndedEvent;
+    "agent.turn.started": AgentTurnStartedEvent;
+    "agent.speech.started": AgentSpeechStartedEvent;
+    "agent.text": AgentTextEvent;
+    "agent.turn.ended": AgentTurnEndedEvent;
+    floor: FloorEvent;
+}
+
 
 
 /** Who the avatar is in a conversation. */
@@ -561,6 +713,59 @@ export declare interface UserParticipant {
     readonly isSpeaking: boolean;
 }
 
+/**
+ * The user's voice went off, as the same detection reads it. `atMs` is where
+ * on the timeline it went off.
+ */
+export declare interface UserSpeechEndedEvent extends SessionEventBase {
+    type: "user.speech.ended";
+}
+
+/**
+ * The user's voice came on, as the runtime's voice-activity detection reads
+ * the microphone. `atMs` is where on the timeline it came on.
+ */
+export declare interface UserSpeechStartedEvent extends SessionEventBase {
+    type: "user.speech.started";
+}
+
+/**
+ * The user's words, live: one recognized segment at a time, as the runtime
+ * hears them. A later {@link UserTurnEndedEvent} may read the same speech
+ * differently; its `text` is the record of what the agent answered.
+ */
+export declare interface UserTranscriptEvent extends SessionEventBase {
+    type: "user.transcript";
+    /** The segment's words. */
+    text: string;
+    /**
+     * `"asr"`: recognized from the user's microphone. `"text"`: a line given
+     * as the user's turn in text rather than spoken. `"forced"`: a transcript
+     * supplied for the user's audio in place of recognition. A newer runtime
+     * may send other values.
+     */
+    source: "asr" | "forced" | "text" | (string & {});
+}
+
+/**
+ * The agent is taking the turn to answer the user; sent each time it does.
+ * `atMs` is the point in the user's audio where the agent took the turn: the
+ * same moment as the turn's {@link AgentTurnStartedEvent}, stamped on the
+ * user's side of the timeline, so it is earlier than that event's `atMs` by
+ * the delay before the agent's audio plays.
+ */
+export declare interface UserTurnEndedEvent extends SessionEventBase {
+    type: "user.turn.ended";
+    /**
+     * The user's words since the agent last spoke, as the agent read them: a
+     * second, more careful hearing's words in place of the live
+     * {@link UserTranscriptEvent}s' where the runtime has one. When an agent
+     * turn ends before it said a word, the next `user.turn.ended` covers the
+     * same words again.
+     */
+    text: string;
+}
+
 /**
  * Capture constraints for the user's published camera track.
  *

package/index.js

@@ -4,7 +4,7 @@ function isRecord(value) {
 }
 
 // src/core/sdk-version.ts
-var SDK_HEADER = `js/${"0.11.0"}`;
+var SDK_HEADER = `js/${"0.12.0"}`;
 
 // src/utils.ts
 function normalizeHost(host) {
@@ -185,6 +185,15 @@ class PolicyCurfewError extends Error {
   }
 }
 
+// src/core/session-events.ts
+function sessionEventsOf(type, handler) {
+  return (change) => {
+    if (change.type === "session-event" && change.event.type === type) {
+      handler(change.event);
+    }
+  };
+}
+
 // src/core/ticker.ts
 var WORKER_SOURCE = "onmessage = (e) => { setInterval(() => postMessage(0), e.data); };";
 function backgroundTicker(host) {
@@ -1059,6 +1068,69 @@ async function avatarFramesDecoded(room) {
   return null;
 }
 
+// src/providers/livekit/sequence.ts
+class EventSequence {
+  deliver;
+  replay;
+  last = 0;
+  held = new Map;
+  replaying = false;
+  stopped = false;
+  constructor(deliver, replay) {
+    this.deliver = deliver;
+    this.replay = replay;
+  }
+  repeats(seq) {
+    return seq !== 0 && (seq <= this.last || this.held.has(seq));
+  }
+  accept(seq, item) {
+    if (this.stopped || this.repeats(seq))
+      return;
+    if (seq === 0) {
+      this.deliver(item);
+      return;
+    }
+    if (seq !== this.last + 1) {
+      this.held.set(seq, item);
+      this.requestReplay();
+      return;
+    }
+    this.last = seq;
+    this.deliver(item);
+    this.release(false);
+  }
+  catchUp() {
+    if (!this.stopped)
+      this.requestReplay();
+  }
+  stop() {
+    this.stopped = true;
+    this.held.clear();
+  }
+  requestReplay() {
+    if (this.replaying)
+      return;
+    this.replaying = true;
+    this.replay(this.last).catch(() => {
+      return;
+    }).finally(() => {
+      this.replaying = false;
+      this.release(true);
+    });
+  }
+  release(all) {
+    const pending = [...this.held.keys()].sort((a, b) => a - b);
+    for (const seq of pending) {
+      if (this.stopped || !all && seq !== this.last + 1)
+        return;
+      const item = this.held.get(seq);
+      this.held.delete(seq);
+      this.last = seq;
+      this.deliver(item);
+    }
+  }
+}
+
 // src/providers/livekit/signals.ts
 function decode(payload) {
   try {
@@ -1082,6 +1154,10 @@ function parseSessionEvent(payload) {
   const decoded = decode(payload);
   if (!decoded)
     return null;
+  const seq = Number.isSafeInteger(decoded.seq) && decoded.seq > 0 ? decoded.seq : 0;
+  return { seq, event: protocolEvent(decoded, seq) };
+}
+function protocolEvent(decoded, seq) {
   if (decoded.type === "session.output_ready") {
     return { type: "output-ready" };
   }
@@ -1095,9 +1171,6 @@ function parseSessionEvent(payload) {
       }
     };
   }
-  if (decoded.type === "floor") {
-    return { type: "floor", speaking: decoded.state === "speaking" };
-  }
   if (decoded.type === "tool.call" && typeof decoded.call_id === "string" && typeof decoded.name === "string" && isRecord(decoded.arguments)) {
     return {
       type: "tool-call",
@@ -1130,7 +1203,44 @@ function parseSessionEvent(payload) {
   if (decoded.type === "error" && decoded.code === "bad_command") {
     return { type: "command-unreadable" };
   }
-  return { type: "other" };
+  const event = sessionEvent(decoded, seq);
+  return event ? { type: "session-event", event } : { type: "other" };
+}
+var isString = (value) => typeof value === "string";
+function defaulted(value, absent) {
+  if (value === undefined)
+    return absent;
+  return isString(value) ? value : null;
+}
+var READERS = {
+  "user.transcript": (wire) => {
+    const source = defaulted(wire.source, "asr");
+    return isString(wire.text) && source !== null ? { text: wire.text, source } : null;
+  },
+  "user.speech.started": () => ({}),
+  "user.speech.ended": () => ({}),
+  "user.turn.ended": (wire) => isString(wire.text) ? { text: wire.text } : null,
+  "agent.turn.started": (wire) => {
+    const source = defaulted(wire.source, "reply");
+    return isString(wire.turn_id) && isString(wire.opened_by) && source !== null ? { turnId: wire.turn_id, openedBy: wire.opened_by, source } : null;
+  },
+  "agent.speech.started": (wire) => isString(wire.turn_id) ? { turnId: wire.turn_id } : null,
+  "agent.text": (wire) => {
+    const turnId = wire.turn_id ?? null;
+    return (turnId === null || isString(turnId)) && isString(wire.text) ? { turnId, text: wire.text } : null;
+  },
+  "agent.turn.ended": (wire) => isString(wire.turn_id) && isString(wire.reason) && isString(wire.spoken) ? { turnId: wire.turn_id, reason: wire.reason, spoken: wire.spoken } : null,
+  floor: (wire) => isString(wire.state) ? { state: wire.state } : null
+};
+function sessionEvent(decoded, seq) {
+  const type = decoded.type;
+  if (!isString(type) || !Object.hasOwn(READERS, type))
+    return null;
+  const fields = READERS[type](decoded);
+  if (fields === null)
+    return null;
+  const atMs = Number.isFinite(decoded.at_ms) ? decoded.at_ms : null;
+  return { type, seq, atMs, ...fields };
 }
 function parseRuntimeSignal(payload) {
   const decoded = decode(payload);
@@ -1616,6 +1726,7 @@ class LiveKitConversation {
   refusal = null;
   speaksProtocol = false;
   avatarSpeaking = false;
+  sequence = this.newSequence();
   answered = null;
   commands = new Map;
   commandCount = 0;
@@ -1666,6 +1777,8 @@ class LiveKitConversation {
     this.setState("connecting");
     this.sessionEnded = false;
     this.speaksProtocol = false;
+    this.sequence.stop();
+    this.sequence = this.newSequence();
     this.readyOwed = false;
     this.everConnected = false;
     if (!this.eventsRegistered) {
@@ -1739,6 +1852,7 @@ class LiveKitConversation {
     this._user?.releaseDevices();
     await this.flushStats();
     await this.sendSessionEnd();
+    this.sequence.stop();
     await this.room.disconnect();
     this._avatar.updateParticipant(null);
     this._user = null;
@@ -1794,10 +1908,13 @@ class LiveKitConversation {
   }
   onChange(handler) {
     return this.onEvent((event) => {
-      if (event.type !== "data-event" && event.type !== "screen-share-changed" && event.type !== "tool-call")
+      if (event.type !== "data-event" && event.type !== "screen-share-changed" && event.type !== "tool-call" && event.type !== "session-event")
         handler(event);
     });
   }
+  on(type, handler) {
+    return this.onEvent(sessionEventsOf(type, handler));
+  }
   onEvent(handler) {
     this.changeHandlers.add(handler);
     return () => {
@@ -1899,6 +2016,7 @@ class LiveKitConversation {
           this.awaitAvatarReturn();
       }
     });
+    this.room.on(RoomEvent3.Reconnected, () => this.catchUp());
     this.room.on(RoomEvent3.ParticipantConnected, (participant) => {
       if (this.isAvatarParticipant(participant)) {
         const tracksChanged = this._avatar.updateParticipant(participant);
@@ -1908,6 +2026,7 @@ class LiveKitConversation {
         }
         if (this.away)
           this.avatarBack();
+        this.catchUp();
       }
     });
     this.room.on(RoomEvent3.ParticipantDisconnected, (participant) => {
@@ -1929,6 +2048,7 @@ class LiveKitConversation {
     this.room.on(RoomEvent3.Disconnected, () => {
       this.stats?.abandon();
       this.stats = null;
+      this.sequence.stop();
       this.setAvatarSpeaking(false);
       this.stopPresentation();
       this.stopReadyFallback();
@@ -2026,13 +2146,14 @@ class LiveKitConversation {
     this.answered = null;
   }
   handleSessionEvent(payload) {
-    const event = parseSessionEvent(payload);
-    if (event === null)
+    const packet = parseSessionEvent(payload);
+    if (packet === null)
       return;
     this.speaksProtocol = true;
-    if (event.type === "output-ready") {
-      this.markAvatarReady();
-    } else if (event.type === "command-result") {
+    const { seq, event } = packet;
+    if (this.sequence.repeats(seq))
+      return;
+    if (event.type === "command-result") {
       if (event.id === SESSION_END_ID)
         this.answered?.();
       else if (event.id !== null) {
@@ -2045,16 +2166,32 @@ class LiveKitConversation {
         this.commands.delete(id);
         settle({ status: "rejected", error: `unknown command ${type}` });
       }
-    } else if (event.type === "floor") {
-      this.setAvatarSpeaking(event.speaking);
+    }
+    this.sequence.accept(seq, event);
+  }
+  deliver(event) {
+    if (event.type === "output-ready") {
+      this.markAvatarReady();
     } else if (event.type === "tool-call") {
       this.emit({ type: "tool-call", call: event.call });
     } else if (event.type === "ended") {
       this.answered?.();
       if (!event.byClient)
         this.reportEnd(event.reason, event.message);
+    } else if (event.type === "session-event") {
+      if (event.event.type === "floor") {
+        this.setAvatarSpeaking(event.event.state === "speaking");
+      }
+      this.emit({ type: "session-event", event: event.event });
     }
   }
+  newSequence() {
+    return new EventSequence((event) => this.deliver(event), (afterSeq) => this.command({ type: "session.replay", after_seq: afterSeq }));
+  }
+  catchUp() {
+    if (this.speaksProtocol)
+      this.sequence.catchUp();
+  }
   handleRuntimeSignal(payload) {
     const signal = parseRuntimeSignal(payload);
     if (signal?.type === "session-ended") {
@@ -2497,10 +2634,13 @@ class NuanceConversation {
   }
   onChange(handler) {
     return this.onEvent((event) => {
-      if (event.type !== "data-event" && event.type !== "screen-share-changed" && event.type !== "tool-call")
+      if (event.type !== "data-event" && event.type !== "screen-share-changed" && event.type !== "tool-call" && event.type !== "session-event")
         handler(event);
     });
   }
+  on(type, handler) {
+    return this.onEvent(sessionEventsOf(type, handler));
+  }
   onEvent(handler) {
     this.changeHandlers.add(handler);
     return () => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nuancelabs/js-sdk",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "description": "JavaScript SDK for Nuance Labs conversation video AI avatar",
   "license": "UNLICENSED",
   "repository": {