@kaltura-sdk/avatars 1.15.5 → 1.16.0

package/index.esm.js

@@ -6226,6 +6226,9 @@ const agentRawTextBaseDeltaSchema = z$1.object({
   role: z$1.optional(z$1.string()),
   type: z$1.optional(z$1.string()),
   content: z$1.optional(z$1.union([z$1.string(), z$1.record(z$1.string(), z$1.unknown())])),
+  // present on the bootstrap 'think' segment - the user's turn text, echoed
+  // there instead of a dedicated 'human' segment type.
+  human: z$1.optional(z$1.string()),
   segmentNumber: z$1.optional(z$1.number()),
   et: z$1.optional(z$1.number()),
   segmentStart: z$1.optional(z$1.boolean()),
@@ -7025,13 +7028,13 @@ const agentRecoveryMonitorState = {
       actions: 'resetAgentRecoveryRetryCount'
     }, {
       guard: 'hasMoreRecoveryRetries',
-      actions: ['incrementAgentRecoveryRetryCount', 'sendReprocessTurn']
+      actions: ['removeStaleChatEntriesForRetry', 'incrementAgentRecoveryRetryCount', 'sendReprocessTurn']
     }, {
       guard: 'justExceededRecoveryRetries',
       actions: ['incrementAgentRecoveryRetryCount', 'showAgentRecoveryExhaustedNotification']
     }],
     [AvatarConnectionMachineEventTypes.RequestAgentTurnRecovery]: {
-      actions: 'sendReprocessTurn'
+      actions: ['removeStaleChatEntriesForRetry', 'sendReprocessTurn']
     }
   },
   invoke: {
@@ -20620,9 +20623,51 @@ const appendAvatarChatEntry = assign$1(({
   context,
   event
 }) => {
+  var _a, _b;
   assertEvent(event, AvatarConnectionMachineEventTypes.AvatarChatEntryReceived);
+  // The avatar's FINAL segment responding means the human phase for this
+  // turn is done - a pendingTextInput still anchored to this turnId (see
+  // appendUserChatEntry) has nothing left to merge into; clear it so the
+  // next askViaText starts a fresh entry instead of merging into a turn
+  // that's already been answered. A non-final segment (avatar-filler,
+  // spoken ahead of the main answer during a tool call) must NOT clear it
+  // - the real turn is still open server-side and more sends can still
+  // legitimately merge into it; closing early here caused a real bug: a
+  // later send got a fresh local entry, but the server's own confirmation
+  // for it still carried the still-open turn's turnId, landing two chat
+  // entries on the same id - every later merge (by id) then grew both.
+  //
+  // Compares against `turnId`, NOT `id` - a real capture showed the
+  // server can reuse the same turnId across multiple, already-answered
+  // exchanges (e.g. repeated ambiguous input like "="), so `id` alone is
+  // not a safe proxy for "this is the turn that's still open" once it's
+  // also being used as a collision-safe chatLog entry id (see
+  // on-chat-log.ts / appendUserChatEntry).
+  const isFinalSegment = (_a = event.isFinalSegment) !== null && _a !== void 0 ? _a : true;
+  const pendingTurnId = (_b = context.pendingTextInput) === null || _b === void 0 ? void 0 : _b.turnId;
+  const pendingTextInput = isFinalSegment && pendingTurnId != null && pendingTurnId === event.entry.turnId ? null : context.pendingTextInput;
+  return {
+    chatLog: [...context.chatLog, event.entry],
+    pendingTextInput
+  };
+});
+
+// A retry re-runs the same turn (same userInput -> same turnId) under a new
+// speechId/messageId, re-emitting both the user's `human` segment and the
+// avatar answer — both go stale, not just the avatar one.
+//
+// Matched on messageId, not turnId: the server can reuse the same turnId
+// across distinct, already-answered exchanges (e.g. repeated "="), so
+// turnId would wipe an unrelated earlier exchange too. messageId is
+// constant per exchange and never reused (confirmed against
+// kaltura_agent_executer.py), so it removes exactly the retried exchange.
+const removeStaleChatEntriesForRetry = assign$1(({
+  context,
+  event
+}) => {
+  assertEvent(event, [AvatarConnectionMachineEventTypes.RequestAgentTurnRecovery, AvatarConnectionMachineEventTypes.AgentRecoveryTurnCompleted]);
   return {
-    chatLog: [...context.chatLog, event.entry]
+    chatLog: context.chatLog.filter(entry => entry.messageId !== event.messageId)
   };
 });
 
@@ -20630,18 +20675,39 @@ const appendUserChatEntry = assign$1(({
   context,
   event
 }) => {
+  var _a;
   assertEvent(event, AvatarConnectionMachineEventTypes.UserVoiceTurnReceived);
-  const idx = context.pendingTextInputs.indexOf(event.entry.text);
-  if (idx !== -1) {
-    // Already added optimistically via text input — consume the pending entry and skip
-    const pendingTextInputs = [...context.pendingTextInputs];
-    pendingTextInputs.splice(idx, 1);
+  const pending = context.pendingTextInput;
+  if (!pending) {
     return {
-      pendingTextInputs
+      chatLog: [...context.chatLog, event.entry]
     };
   }
+  // Only one send can be in flight at a time - appendUserTextEntry merges
+  // any further sends into it instead of starting a second one - so this
+  // turn is unambiguously that pending entry's confirmation. The server is
+  // authoritative for what actually gets saved as message history, so its
+  // entry always wins here, whether or not our locally-accumulated text
+  // happens to match it exactly.
+  const chatLog = pending.id ? context.chatLog.map(entry => entry.id === pending.id ? event.entry : entry) : [...context.chatLog, event.entry];
+  // Stay anchored to this turn via its now-confirmed id instead of
+  // clearing to null - the server re-sends this same "human" echo once per
+  // rapid send under one still-open turnId, each carrying the full
+  // cumulative text (see on-chat-log.ts), so a further send before the
+  // avatar responds must keep merging into this same entry. What actually
+  // closes this out is the avatar responding - see appendAvatarChatEntry.
+  // `turnId` is tracked alongside `id` (not derived from it) - a real
+  // capture showed the server can reuse the same turnId across multiple,
+  // already-answered exchanges, so `event.entry.id` must stay collision-safe
+  // (see on-chat-log.ts) while `turnId` is what appendAvatarChatEntry
+  // actually compares against to decide its turn just closed.
   return {
-    chatLog: [...context.chatLog, event.entry]
+    chatLog,
+    pendingTextInput: {
+      id: event.entry.id,
+      turnId: (_a = event.entry.turnId) !== null && _a !== void 0 ? _a : null,
+      text: event.entry.text
+    }
   };
 });
 
@@ -20654,12 +20720,70 @@ const appendUserTextEntry = assign$1(({
     text,
     internal
   } = event;
+  if (context.pendingTextInput) {
+    // A send while the previous turn is still open mirrors the server's
+    // own behavior: it merges rapid back-to-back input into one turn, so
+    // append into the same pending message instead of starting a new one.
+    // The server re-confirms this same turn once per send (see
+    // appendUserChatEntry) rather than clearing pendingTextInput, so
+    // further sends keep landing here until the avatar actually answers
+    // (see appendAvatarChatEntry, which is what closes it out) or the turn
+    // is interrupted.
+    const {
+      id,
+      turnId,
+      text: pendingText
+    } = context.pendingTextInput;
+    const mergedText = `${pendingText} ${text}`;
+    if (id) {
+      return {
+        pendingTextInput: {
+          id,
+          turnId,
+          text: mergedText
+        },
+        chatLog: context.chatLog.map(entry => entry.id === id ? Object.assign(Object.assign({}, entry), {
+          text: mergedText
+        }) : entry)
+      };
+    }
+    if (internal) {
+      // Still an internal-only accumulation - stays out of the chat log.
+      return {
+        pendingTextInput: {
+          id: null,
+          turnId: null,
+          text: mergedText
+        }
+      };
+    }
+    // The pending message so far was internal (invisible) but this
+    // follow-up isn't - it becomes visible now, under a fresh entry.
+    const _entry = {
+      id: crypto.randomUUID(),
+      role: 'user',
+      text: mergedText
+    };
+    return {
+      chatLog: [...context.chatLog, _entry],
+      pendingTextInput: {
+        id: _entry.id,
+        turnId: null,
+        text: mergedText
+      }
+    };
+  }
   // raiseAskViaTextKickoff marks its auto-sent ask `internal` — it's a bootstrap prompt,
-  // not something the user said, so keep it out of the visible chat log. Still record it
-  // in pendingTextInputs so the ASR echo of it (agentTurnToTalk) is deduped the same way.
+  // not something the user said, so keep it out of the visible chat log. Still tracked
+  // in pendingTextInput so the server's turn echo is deduped the same way. No chatLog
+  // entry backs it, so `id: null`.
   if (internal) {
     return {
-      pendingTextInputs: [...context.pendingTextInputs, text]
+      pendingTextInput: {
+        id: null,
+        turnId: null,
+        text
+      }
     };
   }
   const entry = {
@@ -20669,7 +20793,11 @@ const appendUserTextEntry = assign$1(({
   };
   return {
     chatLog: [...context.chatLog, entry],
-    pendingTextInputs: [...context.pendingTextInputs, text]
+    pendingTextInput: {
+      id: entry.id,
+      turnId: null,
+      text
+    }
   };
 });
 
@@ -20720,135 +20848,64 @@ const source$a = createTinyPubsubEventsSource(websocketMessage, {
   agent_raw_text: {
     schema: agentRawTextSchema
   },
-  agentTurnToTalk: {
-    schema: agentTurnToTalkSchema
-  },
   generatingSpeech: {
     schema: generatingSpeechSchema
+  },
+  agentTurnToTalk: {
+    schema: agentTurnToTalkSchema
   }
 }, {
   queueWhilePending: true
 });
-// FLOW OVERVIEW
-// =============
-// Builds the alternating user/avatar chat log from two independent, unordered
-// websocket event streams that share no common id:
-//   - agentTurnToTalk: fires once per USER turn, carries the ASR transcript
-//     (userTranscription). Empty userTranscription means a proactive/opening
-//     turn (avatar speaks first, no preceding user input).
-//   - agent_raw_text: streams the AVATAR's reply for a turn as chunked deltas
-//     keyed by speechId, terminated by a delta with segmentEnd=true.
-//     delta.type === 'avatar' or 'avatar-filler' chunks are chat text (filler
-//     is spoken content too, e.g. "Let me check that for you..." streamed
-//     ahead of the main answer while a tool call runs); 'think'/'tool'/'share'
-//     chunks are filtered out here.
+// Builds alternating user/avatar chat log from agent_raw_text, keyed by turnId.
 //
-// The two streams race: for a given turn, agentTurnToTalk can arrive before,
-// after, or interleaved with that turn's agent_raw_text segmentEnd. Neither
-// event carries a shared id to correlate by (agentTurnToTalk has no speechId;
-// see the pendingUserTurnCount comment below), so pairing is done by arrival
-// order via a queue + counter:
-//   - awaitingUserTurnQueue: avatar answers that finished (segmentEnd) before
-//     their turn's agentTurnToTalk showed up. FIFO — first queued avatar
-//     answer is paired with the next agentTurnToTalk that needs one.
-//   - pendingUserTurnCount: agentTurnToTalk arrivals still waiting on their
-//     avatar answer to finish. Incremented when a talk event arrives with an
-//     empty queue; decremented when an avatar segmentEnd arrives and directly
-//     emits instead of queueing.
-//   - resolvedTurnIds: guards against a single turn producing more than one
-//     avatar segmentEnd — either multiple segments under one speechId (e.g. a
-//     tool call splits the reply into a chunk before and a chunk after the
-//     tool result), or multiple speechIds sharing one turnId (a reprocessTurn
-//     retry regenerates the whole answer under a new speechId but keeps the
-//     same turnId). Only the FIRST resolved segment/attempt for a turnId
-//     goes through the queue/counter above; later ones are known to already
-//     belong to a claimed turn, so they're emitted immediately without
-//     consuming another slot.
+// delta.type:
+//   - 'think' with `human` — user's turn text, one-shot on the bootstrap
+//     "preparing to answer..." segment (precedes the turn's avatar segments).
+//     No `human` field = real thinking tokens, ignored (like 'tool'/'share').
+//     Deduped per turnId:speechId. Entry id is that composite, not bare
+//     turnId — server can reuse one turnId across several already-answered
+//     exchanges (e.g. repeated "="), so speechId keeps entries from colliding.
+//   - 'avatar' / 'avatar-filler' — chat text, chunked by speechId, closed by
+//     segmentEnd=true. Filler is spoken content too (e.g. tool-call "let me
+//     check..." lines).
+//   - 'user-interruption' — turn aborted server-side (no literal 'abort'
+//     type exists). Flushes whatever partial avatar text accumulated and
+//     evicts that turnId:speechId buffer only.
 //
-// Per real turn, the emitted order ends up being: UserVoiceTurnReceived then
-// AvatarChatEntryReceived (see appendUserChatEntry/appendAvatarChatEntry,
-// which just push onto context.chatLog in the order events arrive here).
+// A retry reuses the same turnId under a new speechId; each speechId's
+// segmentEnd/user-interruption is tracked/flushed independently.
 //
-// Separately, generatingSpeech + agentTurnToTalk-with-empty-transcription
-// handle ONE special case: resumed-thread sessions where the avatar speaks an
-// opening phrase with no prior user turn to pair against — that text is
-// accumulated by speechId and flushed as its own avatar-only entry.
+// Every entry carries `messageId` (parsed.messageId) — constant for every
+// segment of one exchange, unique across exchanges even when turnId repeats
+// (confirmed against kaltura_agent_executer.py). remove-stale-chat-entries-
+// for-retry.ts matches on it to drop exactly one exchange on retry.
 //
-// Known unresolved risk (backend-side, not fixable from here — see
-// pendingUserTurnCount's comment): the queue/counter can permanently strand
-// or over-count if either stream's event never fires for a turn.
+// Opening phrase (resumed thread, avatar speaks first): never sent as a
+// normal turnId-keyed segment. Server sends it via `init_response` (too
+// early — fires before this actor's setHandler(), so it's dropped by the
+// pubsub queue reset) and a mirrored `generatingSpeech` event, which we use
+// as the text source instead. `agentTurnToTalk` (userTranscription === '')
+// is the flush trigger — nothing else marks "greeting done, no real user
+// turn happened first". BLANK_TEXT_MARKER is the server's "no opening
+// phrase configured" sentinel, skipped.
 const onChatLog = fromCallback(({
   input,
   sendBack
 }) => {
-  // Accumulate chunks by speechId while avatar is speaking
+  // Accumulate avatar text by `${turnId}:${speechId}` — each speechId
+  // closes independently (e.g. a filler segment vs the main answer).
   const pending = new Map();
-  // speechId of the turn currently accumulating chunks.
-  // When a new speechId is seen, the previous turn's incomplete text is abandoned (interrupted).
-  let currentSpeechId = null;
-  // Avatar answers whose segmentEnd fired before their matching agentTurnToTalk arrived.
-  // Completed avatar answers waiting for their paired agentTurnToTalk.
-  const awaitingUserTurnQueue = [];
-  // agentTurnToTalk events (with transcription) whose matching avatar answer hasn't
-  // completed yet. Talk-vs-avatar-stream ordering isn't fixed — the ASR transcript can
-  // beat or lag the avatar's own text stream on a per-turn basis (races harder when the
-  // answer carries tool content) — so pairing is done via this queue+counter rather than
-  // a single slot, which a later turn's speechId change would otherwise clobber.
-  //
-  // Known risk (backend, not fixable here — agentTurnToTalk carries no turnId/speechId
-  // to correlate by id, only arrival order): both sides can drift out of 1:1 and desync
-  // pairing for a later, unrelated turn instead of the intended one.
-  //   - Queue can strand an entry forever if its own agentTurnToTalk never fires — e.g.
-  //     conversation-manager's ttv.sayTurn throwing on the first chunk, or the
-  //     first-sentence punctuation gate in SpeechBuffer.append swallowing a short
-  //     punctuation-less reply. Raw text still gets forwarded; the talk pipeline doesn't run.
-  //   - Counter can stay inflated forever if agentTurnToTalk fires but its avatar segment
-  //     never reaches segmentEnd — e.g. onUserInterrupted not aborting AgentAdapter's loop,
-  //     dropping buffered avatar text without ever calling finish()/completeStream().
-  // Real fix is backend: always run finish()/completeStream() on error/interrupt, and
-  // populate turnId on agentTurnToTalk (schema already has the field, unused) so pairing
-  // can be done by id instead of arrival order.
-  let pendingUserTurnCount = 0;
-  // turnIds whose first segment already claimed a pairing slot (queued or emitted).
-  // Keyed by turnId, NOT speechId, because a single logical turn can produce more than
-  // one speechId:
-  //   - Genie can stream a reply as multiple avatar segments under one speechId (e.g. a
-  //     tool call splits the answer into a segment before and a segment after the tool
-  //     result) — those segments share both speechId and turnId.
-  //   - A reprocessTurn retry (sendReprocessTurn, triggered by agentRecoveryMonitor or a
-  //     manual "retry" action) makes the backend regenerate the FULL answer under a NEW
-  //     speechId while keeping the SAME turnId — confirmed in logs: two distinct speechIds
-  //     sharing turnId "c83e031d-..." after a reprocessTurn. Guarding on speechId alone
-  //     would treat the retry's answer as an unrelated new turn.
-  // Either way, the extra segment/retry belongs to a turn that's already paired, so it
-  // must bypass the queue/counter entirely; otherwise it would consume the next,
-  // unrelated user turn's pairing slot and desync every pairing for the rest of the
-  // conversation. Intentionally never cleared: a later segment/retry for the same turnId
-  // can arrive arbitrarily long after the first (slow tool call, slow retry, other turns
-  // in between), so "already claimed" must be remembered for the rest of the session,
-  // same lifetime as awaitingUserTurnQueue/pendingUserTurnCount above. Actor-scoped, dies
-  // with it.
-  const resolvedTurnIds = new Set();
-  // Opening phrase accumulation: for resumed-thread sessions where init_response is not sent,
-  // generatingSpeech chunks arrive before the first user turn — accumulate them keyed by speechId
-  // and flush on the first proactive agentTurnToTalk (empty userTranscription).
-  let openingPhraseSpeechId = null;
-  const openingPhraseChunks = new Map();
-  let openingPhraseEmitted = false;
+  const seenHumanTurnKeys = new Set();
   let avatarEntrySequence = 0;
   const createAvatarEntryId = (messageId, speechId) => {
     avatarEntrySequence += 1;
     return `${messageId}:${speechId}:${avatarEntrySequence}`;
   };
-  const emitQueuedAvatarEntry = () => {
-    const entry = awaitingUserTurnQueue.shift();
-    if (entry) {
-      sendBack({
-        type: AvatarConnectionMachineEventTypes.AvatarChatEntryReceived,
-        entry
-      });
-    }
-  };
+  // Opening phrase accumulation — see top-of-file comment.
+  let openingPhraseSpeechId = null;
+  let openingPhraseText = '';
+  let openingPhraseSettled = false;
   return createTinyPubsubEventsHandler({
     source: source$a,
     logger: input.logger,
@@ -20860,115 +20917,89 @@ const onChatLog = fromCallback(({
           const result = agentRawTextDeltaSchema.safeParse(delta);
           if (!result.success) return;
           const parsed = result.data;
-          // 'avatar-filler' is spoken content too (e.g. "Let me check that for you..." while
-          // a tool call runs) — Genie streams it as its own segment ahead of the 'avatar'
-          // segment. Without it here, part of what the avatar actually says never reaches
-          // the chat log.
-          if (parsed.type !== 'avatar' && parsed.type !== 'avatar-filler') return;
-          const speechId = payload.speechId;
-          // New speech started — abandon prior speech's incomplete accumulation.
-          // Does NOT touch awaitingUserTurnQueue/pendingUserTurnCount: those are completed,
-          // already-paired-or-waiting turns and must survive across speechId changes.
-          if (speechId !== currentSpeechId) {
-            pending.delete(currentSpeechId !== null && currentSpeechId !== void 0 ? currentSpeechId : '');
-            currentSpeechId = speechId;
-          }
-          if (typeof parsed.content === 'string' && parsed.content) {
-            pending.set(speechId, ((_a = pending.get(speechId)) !== null && _a !== void 0 ? _a : '') + parsed.content);
-          }
-          if (parsed.segmentEnd) {
-            const text = (_b = pending.get(speechId)) !== null && _b !== void 0 ? _b : '';
-            pending.delete(speechId);
-            if (!text) return;
-            const entry = {
-              id: createAvatarEntryId(parsed.messageId, speechId),
-              role: 'avatar',
-              text,
-              turnId: speechId
-            };
-            if (resolvedTurnIds.has(payload.turnId)) {
-              // Not this turn's first segment/attempt — its pairing slot is already claimed
-              // (multi-segment reply, or a reprocessTurn retry answering the same turnId).
+          const key = `${payload.turnId}:${payload.speechId}`;
+          if (parsed.type === 'think') {
+            if (parsed.human && !seenHumanTurnKeys.has(key)) {
+              seenHumanTurnKeys.add(key);
               sendBack({
-                type: AvatarConnectionMachineEventTypes.AvatarChatEntryReceived,
-                entry
+                type: AvatarConnectionMachineEventTypes.UserVoiceTurnReceived,
+                entry: {
+                  id: key,
+                  role: 'user',
+                  text: parsed.human,
+                  turnId: payload.turnId,
+                  messageId: parsed.messageId
+                }
               });
-              return;
             }
-            resolvedTurnIds.add(payload.turnId);
-            if (pendingUserTurnCount > 0) {
-              // agentTurnToTalk for this turn already fired — emit directly
-              pendingUserTurnCount -= 1;
+            return;
+          }
+          if (parsed.type === 'user-interruption') {
+            const text = pending.get(key);
+            pending.delete(key);
+            if (text) {
               sendBack({
                 type: AvatarConnectionMachineEventTypes.AvatarChatEntryReceived,
-                entry
+                entry: {
+                  id: createAvatarEntryId(parsed.messageId, payload.speechId),
+                  role: 'avatar',
+                  text,
+                  turnId: payload.turnId,
+                  messageId: parsed.messageId
+                }
               });
-            } else {
-              // agentTurnToTalk hasn't arrived yet — queue until it does
-              awaitingUserTurnQueue.push(entry);
             }
+            return;
+          }
+          if (parsed.type !== 'avatar' && parsed.type !== 'avatar-filler') return;
+          if (typeof parsed.content === 'string' && parsed.content) {
+            pending.set(key, ((_a = pending.get(key)) !== null && _a !== void 0 ? _a : '') + parsed.content);
+          }
+          if (parsed.segmentEnd) {
+            const text = (_b = pending.get(key)) !== null && _b !== void 0 ? _b : '';
+            pending.delete(key);
+            if (!text) return;
+            sendBack({
+              type: AvatarConnectionMachineEventTypes.AvatarChatEntryReceived,
+              entry: {
+                id: createAvatarEntryId(parsed.messageId, payload.speechId),
+                role: 'avatar',
+                text,
+                turnId: payload.turnId,
+                messageId: parsed.messageId
+              },
+              // Filler doesn't close an open pendingTextInput (see append-avatar-chat-entry.ts).
+              isFinalSegment: parsed.type !== 'avatar-filler'
+            });
           }
         } catch (_c) {
           // invalid JSON delta — ignore
         }
       },
       generatingSpeech: payload => {
-        var _a;
-        if (openingPhraseEmitted) return;
-        // Server sends this literal sentinel instead of real speech when no opening phrase
-        // is configured — must not leak into the chat log as visible avatar text.
+        if (openingPhraseSettled) return;
+        // No opening phrase configured — must not leak into the chat log.
         if (payload.text === BLANK_TEXT_MARKER) return;
-        if (openingPhraseSpeechId !== null && openingPhraseSpeechId !== payload.speechId) return;
+        if (openingPhraseSpeechId !== null && openingPhraseSpeechId !== payload.speechId) {
+          return;
+        }
         openingPhraseSpeechId = payload.speechId;
-        openingPhraseChunks.set(payload.speechId, ((_a = openingPhraseChunks.get(payload.speechId)) !== null && _a !== void 0 ? _a : '') + payload.text);
+        openingPhraseText += payload.text;
       },
       agentTurnToTalk: payload => {
-        var _a;
-        const {
-          userTranscription,
-          turnId
-        } = payload;
-        if (!userTranscription) {
-          // Proactive turn — flush opening phrase built from generatingSpeech chunks if present
-          if (!openingPhraseEmitted && openingPhraseSpeechId) {
-            const text = (_a = openingPhraseChunks.get(openingPhraseSpeechId)) !== null && _a !== void 0 ? _a : '';
-            openingPhraseChunks.clear();
-            openingPhraseEmitted = true;
-            if (text) {
-              sendBack({
-                type: AvatarConnectionMachineEventTypes.AvatarChatEntryReceived,
-                entry: {
-                  id: openingPhraseSpeechId,
-                  role: 'avatar',
-                  text,
-                  turnId: 'init_response'
-                }
-              });
-            }
-          }
-          // Also emit any queued avatar entry left over from the agent_raw_text path
-          emitQueuedAvatarEntry();
-          return;
-        }
-        openingPhraseEmitted = true;
-        openingPhraseChunks.clear();
-        // Emit user first, then pair with its avatar answer — whichever side finished
-        // streaming first (queue has it) or hasn't yet (bump the counter for agent_raw_text
-        // to consume once its segmentEnd arrives)
+        // Only the first occurrence decides the opening — later turns also fire this.
+        if (openingPhraseSettled) return;
+        openingPhraseSettled = true;
+        if (payload.userTranscription || !openingPhraseText) return;
         sendBack({
-          type: AvatarConnectionMachineEventTypes.UserVoiceTurnReceived,
+          type: AvatarConnectionMachineEventTypes.AvatarChatEntryReceived,
           entry: {
-            id: turnId !== null && turnId !== void 0 ? turnId : crypto.randomUUID(),
-            role: 'user',
-            text: userTranscription,
-            turnId
+            id: openingPhraseSpeechId !== null && openingPhraseSpeechId !== void 0 ? openingPhraseSpeechId : 'init_response',
+            role: 'avatar',
+            text: openingPhraseText,
+            turnId: 'init_response'
           }
         });
-        if (awaitingUserTurnQueue.length > 0) {
-          emitQueuedAvatarEntry();
-        } else {
-          pendingUserTurnCount += 1;
-        }
       }
     }
   });
@@ -23290,6 +23321,7 @@ const avatarConnectionMachine = setup({
     logAgentClientToolCallResult,
     recordClientToolCallLogEntry,
     appendAvatarChatEntry,
+    removeStaleChatEntriesForRetry,
     appendUserChatEntry,
     appendUserTextEntry,
     setScreenShareActive,
@@ -23463,7 +23495,7 @@ const avatarConnectionMachine = setup({
       content: null,
       chatLog: [],
       clientToolCallLog: [],
-      pendingTextInputs: [],
+      pendingTextInput: null,
       userAgent: '',
       userAgentHints: null,
       kalturaAnalyticsService: input.kalturaAnalyticsService,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kaltura-sdk/avatars",
-  "version": "1.15.5",
+  "version": "1.16.0",
   "author": "kaltura",
   "license": "AGPL-3.0",
   "repository": {
@@ -13,7 +13,7 @@
     "@kaltura-sdk/rtc-core": "^1.25.21",
     "@local/genie-shared-utils": "^1.0.0",
     "@unisphere/core": "^1.99.0",
-    "@unisphere/genie-types": "^1.39.1",
+    "@unisphere/genie-types": "^1.39.3",
     "@unisphere/notifications-core": "^1.25.0",
     "@unisphere/runtime": "^1.98.0",
     "@unisphere/runtime-js": "^1.96.0",

package/src/lib/avatar-client-manager/machines/avatar-connection/actions/context/remove-stale-chat-entries-for-retry.d.ts

@@ -0,0 +1,2 @@
+import { AvatarConnectionMachineAction } from '../../types';
+export declare const removeStaleChatEntriesForRetry: AvatarConnectionMachineAction;

package/src/lib/avatar-client-manager/machines/avatar-connection/actions/context/test-utils/as-harness-action.d.ts

@@ -0,0 +1,4 @@
+import type { ActionFunction, MachineContext } from 'xstate';
+export declare function asHarnessAction<TContext extends MachineContext, TEvent extends {
+    type: string;
+}>(action: unknown): ActionFunction<TContext, TEvent, TEvent, undefined, never, never, never, never, never>;

package/src/lib/avatar-client-manager/machines/avatar-connection/avatar-connection-machine.d.ts

@@ -391,6 +391,9 @@ export declare const avatarConnectionMachine: import("xstate").StateMachine<Avat
 } | {
     type: "appendAvatarChatEntry";
     params: unknown;
+} | {
+    type: "removeStaleChatEntriesForRetry";
+    params: unknown;
 } | {
     type: "appendUserChatEntry";
     params: unknown;

package/src/lib/avatar-client-manager/machines/avatar-connection/states/connected-state.d.ts

@@ -728,13 +728,13 @@ export declare const connectedState: {
                     actions: "resetAgentRecoveryRetryCount";
                 } | {
                     guard: "hasMoreRecoveryRetries";
-                    actions: readonly ["incrementAgentRecoveryRetryCount", "sendReprocessTurn"];
+                    actions: readonly ["removeStaleChatEntriesForRetry", "incrementAgentRecoveryRetryCount", "sendReprocessTurn"];
                 } | {
                     guard: "justExceededRecoveryRetries";
                     actions: readonly ["incrementAgentRecoveryRetryCount", "showAgentRecoveryExhaustedNotification"];
                 })[];
                 RequestAgentTurnRecovery: {
-                    actions: "sendReprocessTurn";
+                    actions: readonly ["removeStaleChatEntriesForRetry", "sendReprocessTurn"];
                 };
             };
             invoke: {

package/src/lib/avatar-client-manager/machines/avatar-connection/states/connected-states/agent-recovery-monitor-state.d.ts

@@ -6,13 +6,13 @@ export declare const agentRecoveryMonitorState: {
             actions: "resetAgentRecoveryRetryCount";
         } | {
             guard: "hasMoreRecoveryRetries";
-            actions: readonly ["incrementAgentRecoveryRetryCount", "sendReprocessTurn"];
+            actions: readonly ["removeStaleChatEntriesForRetry", "incrementAgentRecoveryRetryCount", "sendReprocessTurn"];
         } | {
             guard: "justExceededRecoveryRetries";
             actions: readonly ["incrementAgentRecoveryRetryCount", "showAgentRecoveryExhaustedNotification"];
         })[];
         RequestAgentTurnRecovery: {
-            actions: "sendReprocessTurn";
+            actions: readonly ["removeStaleChatEntriesForRetry", "sendReprocessTurn"];
         };
     };
     invoke: {

package/src/lib/avatar-client-manager/machines/avatar-connection/types/context.d.ts

@@ -66,7 +66,11 @@ export interface AvatarConnectionMachineContext {
     content: UnisphereChatBlock | null;
     chatLog: ChatEntry[];
     clientToolCallLog: ClientToolCallLogEntry[];
-    pendingTextInputs: string[];
+    pendingTextInput: {
+        id: string | null;
+        turnId: string | null;
+        text: string;
+    } | null;
     isAgentTalking: boolean;
     userAgent: string;
     userAgentHints: Record<string, string> | null;

package/src/lib/avatar-client-manager/machines/avatar-connection/types/events.d.ts

@@ -164,6 +164,7 @@ export type AgentStartSpeechReceivedEvent = {
 export type AvatarChatEntryReceivedEvent = {
     type: AvatarConnectionMachineEventTypes.AvatarChatEntryReceived;
     entry: ChatEntry;
+    isFinalSegment?: boolean;
 };
 export type UserVoiceTurnReceivedEvent = {
     type: AvatarConnectionMachineEventTypes.UserVoiceTurnReceived;

package/src/lib/avatar-client-manager/machines/avatar-connection/types/schemas.d.ts

@@ -69,6 +69,7 @@ export declare const agentRawTextDeltaSchema: z.ZodMiniObject<{
     role: z.ZodMiniOptional<z.ZodMiniString<string>>;
     type: z.ZodMiniOptional<z.ZodMiniString<string>>;
     content: z.ZodMiniOptional<z.ZodMiniUnion<readonly [z.ZodMiniString<string>, z.ZodMiniRecord<z.ZodMiniString<string>, z.ZodMiniUnknown>]>>;
+    human: z.ZodMiniOptional<z.ZodMiniString<string>>;
     segmentNumber: z.ZodMiniOptional<z.ZodMiniNumber<number>>;
     et: z.ZodMiniOptional<z.ZodMiniNumber<number>>;
     segmentStart: z.ZodMiniOptional<z.ZodMiniBoolean<boolean>>;
@@ -86,6 +87,7 @@ export declare const agentRawTextClientToolDeltaSchema: z.ZodMiniObject<{
     threadId: z.ZodMiniOptional<z.ZodMiniString<string>>;
     role: z.ZodMiniOptional<z.ZodMiniString<string>>;
     content: z.ZodMiniOptional<z.ZodMiniUnion<readonly [z.ZodMiniString<string>, z.ZodMiniRecord<z.ZodMiniString<string>, z.ZodMiniUnknown>]>>;
+    human: z.ZodMiniOptional<z.ZodMiniString<string>>;
     segmentNumber: z.ZodMiniOptional<z.ZodMiniNumber<number>>;
     et: z.ZodMiniOptional<z.ZodMiniNumber<number>>;
     segmentStart: z.ZodMiniOptional<z.ZodMiniBoolean<boolean>>;