@kaltura-sdk/avatars 1.15.5 → 1.16.1

package/index.esm.js

@@ -6226,6 +6226,9 @@ const agentRawTextBaseDeltaSchema = z$1.object({
   role: z$1.optional(z$1.string()),
   type: z$1.optional(z$1.string()),
   content: z$1.optional(z$1.union([z$1.string(), z$1.record(z$1.string(), z$1.unknown())])),
+  // present on the bootstrap 'think' segment - the user's turn text, echoed
+  // there instead of a dedicated 'human' segment type.
+  human: z$1.optional(z$1.string()),
   segmentNumber: z$1.optional(z$1.number()),
   et: z$1.optional(z$1.number()),
   segmentStart: z$1.optional(z$1.boolean()),
@@ -6383,7 +6386,7 @@ const textCommunicationState = {
         }) => ({
           message: event.text
         })
-      }, 'appendUserTextEntry']
+      }, 'trackInternalAskEcho']
     }
   },
   states: {
@@ -7025,13 +7028,13 @@ const agentRecoveryMonitorState = {
       actions: 'resetAgentRecoveryRetryCount'
     }, {
       guard: 'hasMoreRecoveryRetries',
-      actions: ['incrementAgentRecoveryRetryCount', 'sendReprocessTurn']
+      actions: ['removeStaleChatEntriesForRetry', 'incrementAgentRecoveryRetryCount', 'sendReprocessTurn']
     }, {
       guard: 'justExceededRecoveryRetries',
       actions: ['incrementAgentRecoveryRetryCount', 'showAgentRecoveryExhaustedNotification']
     }],
     [AvatarConnectionMachineEventTypes.RequestAgentTurnRecovery]: {
-      actions: 'sendReprocessTurn'
+      actions: ['removeStaleChatEntriesForRetry', 'sendReprocessTurn']
     }
   },
   invoke: {
@@ -20626,50 +20629,63 @@ const appendAvatarChatEntry = assign$1(({
   };
 });
 
+// A retry re-runs the same turn (same userInput -> same turnId) under a new
+// speechId/messageId, re-emitting both the user's `human` segment and the
+// avatar answer — both go stale, not just the avatar one.
+//
+// Matched on messageId, not turnId: the server can reuse the same turnId
+// across distinct, already-answered exchanges (e.g. repeated "="), so
+// turnId would wipe an unrelated earlier exchange too. messageId is
+// constant per exchange and never reused (confirmed against
+// kaltura_agent_executer.py), so it removes exactly the retried exchange.
+const removeStaleChatEntriesForRetry = assign$1(({
+  context,
+  event
+}) => {
+  assertEvent(event, [AvatarConnectionMachineEventTypes.RequestAgentTurnRecovery, AvatarConnectionMachineEventTypes.AgentRecoveryTurnCompleted]);
+  return {
+    chatLog: context.chatLog.filter(entry => entry.messageId !== event.messageId)
+  };
+});
+
+// Upserts by id - a resend superseding a still-open turn keeps that turn's
+// original entry id (see on-chat-log.ts openTurnEntryIds), so this replaces
+// the entry in place instead of duplicating it.
 const appendUserChatEntry = assign$1(({
   context,
   event
 }) => {
   assertEvent(event, AvatarConnectionMachineEventTypes.UserVoiceTurnReceived);
-  const idx = context.pendingTextInputs.indexOf(event.entry.text);
-  if (idx !== -1) {
-    // Already added optimistically via text input — consume the pending entry and skip
-    const pendingTextInputs = [...context.pendingTextInputs];
-    pendingTextInputs.splice(idx, 1);
+  if (context.awaitingInternalAskEcho) {
+    // Invisible kickoff prompt's own echo - consume without surfacing it.
+    return {
+      awaitingInternalAskEcho: false
+    };
+  }
+  const existingIndex = context.chatLog.findIndex(entry => entry.id === event.entry.id);
+  if (existingIndex === -1) {
     return {
-      pendingTextInputs
+      chatLog: [...context.chatLog, event.entry]
     };
   }
+  const chatLog = [...context.chatLog];
+  chatLog[existingIndex] = event.entry;
   return {
-    chatLog: [...context.chatLog, event.entry]
+    chatLog
   };
 });
 
-const appendUserTextEntry = assign$1(({
+// No optimistic entry for askViaText - chat log only reflects the server's
+// echoed `human` text (see on-chat-log.ts). The internal kickoff prompt
+// goes through that same echo, so its next one must be suppressed.
+const trackInternalAskEcho = assign$1(({
   context,
   event
 }) => {
   assertEvent(event, AvatarConnectionMachineEventTypes.AskViaText);
-  const {
-    text,
-    internal
-  } = event;
-  // raiseAskViaTextKickoff marks its auto-sent ask `internal` — it's a bootstrap prompt,
-  // not something the user said, so keep it out of the visible chat log. Still record it
-  // in pendingTextInputs so the ASR echo of it (agentTurnToTalk) is deduped the same way.
-  if (internal) {
-    return {
-      pendingTextInputs: [...context.pendingTextInputs, text]
-    };
-  }
-  const entry = {
-    id: crypto.randomUUID(),
-    role: 'user',
-    text
-  };
+  if (!event.internal) return {};
   return {
-    chatLog: [...context.chatLog, entry],
-    pendingTextInputs: [...context.pendingTextInputs, text]
+    awaitingInternalAskEcho: true
   };
 });
 
@@ -20720,255 +20736,159 @@ const source$a = createTinyPubsubEventsSource(websocketMessage, {
   agent_raw_text: {
     schema: agentRawTextSchema
   },
-  agentTurnToTalk: {
-    schema: agentTurnToTalkSchema
-  },
   generatingSpeech: {
     schema: generatingSpeechSchema
+  },
+  agentTurnToTalk: {
+    schema: agentTurnToTalkSchema
   }
 }, {
   queueWhilePending: true
 });
-// FLOW OVERVIEW
-// =============
-// Builds the alternating user/avatar chat log from two independent, unordered
-// websocket event streams that share no common id:
-//   - agentTurnToTalk: fires once per USER turn, carries the ASR transcript
-//     (userTranscription). Empty userTranscription means a proactive/opening
-//     turn (avatar speaks first, no preceding user input).
-//   - agent_raw_text: streams the AVATAR's reply for a turn as chunked deltas
-//     keyed by speechId, terminated by a delta with segmentEnd=true.
-//     delta.type === 'avatar' or 'avatar-filler' chunks are chat text (filler
-//     is spoken content too, e.g. "Let me check that for you..." streamed
-//     ahead of the main answer while a tool call runs); 'think'/'tool'/'share'
-//     chunks are filtered out here.
+// Builds alternating user/avatar chat log from agent_raw_text, keyed by turnId.
 //
-// The two streams race: for a given turn, agentTurnToTalk can arrive before,
-// after, or interleaved with that turn's agent_raw_text segmentEnd. Neither
-// event carries a shared id to correlate by (agentTurnToTalk has no speechId;
-// see the pendingUserTurnCount comment below), so pairing is done by arrival
-// order via a queue + counter:
-//   - awaitingUserTurnQueue: avatar answers that finished (segmentEnd) before
-//     their turn's agentTurnToTalk showed up. FIFO — first queued avatar
-//     answer is paired with the next agentTurnToTalk that needs one.
-//   - pendingUserTurnCount: agentTurnToTalk arrivals still waiting on their
-//     avatar answer to finish. Incremented when a talk event arrives with an
-//     empty queue; decremented when an avatar segmentEnd arrives and directly
-//     emits instead of queueing.
-//   - resolvedTurnIds: guards against a single turn producing more than one
-//     avatar segmentEnd — either multiple segments under one speechId (e.g. a
-//     tool call splits the reply into a chunk before and a chunk after the
-//     tool result), or multiple speechIds sharing one turnId (a reprocessTurn
-//     retry regenerates the whole answer under a new speechId but keeps the
-//     same turnId). Only the FIRST resolved segment/attempt for a turnId
-//     goes through the queue/counter above; later ones are known to already
-//     belong to a claimed turn, so they're emitted immediately without
-//     consuming another slot.
+// delta.type:
+//   - 'think' with `human` — user's turn text. A resend while the turnId's
+//     exchange is still unanswered cancels it server-side with no signal to
+//     the client (conversation-manager's AgentAdapter.invalidateTask), gets
+//     its own real messageId, and supersedes it in place client-side via
+//     openTurnEntryIds below. A LATER, already-answered exchange reusing
+//     the same turnId is unrelated and gets a fresh entry instead.
+//   - 'avatar' / 'avatar-filler' — chat text, chunked by speechId, closed by
+//     segmentEnd=true. Filler is spoken content too (tool-call ack lines).
+//   - 'user-interruption' — turn aborted server-side. Flushes accumulated
+//     avatar text and evicts that turnId:speechId buffer only.
 //
-// Per real turn, the emitted order ends up being: UserVoiceTurnReceived then
-// AvatarChatEntryReceived (see appendUserChatEntry/appendAvatarChatEntry,
-// which just push onto context.chatLog in the order events arrive here).
+// A retry reuses the turnId under a new speechId, tracked independently.
 //
-// Separately, generatingSpeech + agentTurnToTalk-with-empty-transcription
-// handle ONE special case: resumed-thread sessions where the avatar speaks an
-// opening phrase with no prior user turn to pair against — that text is
-// accumulated by speechId and flushed as its own avatar-only entry.
+// `messageId` is constant per exchange, unique across exchanges even when
+// turnId repeats (confirmed against kaltura_agent_executer.py) —
+// remove-stale-chat-entries-for-retry.ts matches on it for retry cleanup.
 //
-// Known unresolved risk (backend-side, not fixable from here — see
-// pendingUserTurnCount's comment): the queue/counter can permanently strand
-// or over-count if either stream's event never fires for a turn.
+// Opening phrase (resumed thread, avatar speaks first): never sent as a
+// normal turnId-keyed segment — `init_response` fires too early (before
+// this actor's setHandler(), dropped by the pubsub queue reset), so
+// `generatingSpeech` is the text source instead, flushed by
+// `agentTurnToTalk` (userTranscription === ''). BLANK_TEXT_MARKER is the
+// server's "no opening phrase configured" sentinel, skipped.
 const onChatLog = fromCallback(({
   input,
   sendBack
 }) => {
-  // Accumulate chunks by speechId while avatar is speaking
+  // Accumulate avatar text by `${turnId}:${speechId}` — each speechId
+  // closes independently (e.g. a filler segment vs the main answer).
   const pending = new Map();
-  // speechId of the turn currently accumulating chunks.
-  // When a new speechId is seen, the previous turn's incomplete text is abandoned (interrupted).
-  let currentSpeechId = null;
-  // Avatar answers whose segmentEnd fired before their matching agentTurnToTalk arrived.
-  // Completed avatar answers waiting for their paired agentTurnToTalk.
-  const awaitingUserTurnQueue = [];
-  // agentTurnToTalk events (with transcription) whose matching avatar answer hasn't
-  // completed yet. Talk-vs-avatar-stream ordering isn't fixed — the ASR transcript can
-  // beat or lag the avatar's own text stream on a per-turn basis (races harder when the
-  // answer carries tool content) — so pairing is done via this queue+counter rather than
-  // a single slot, which a later turn's speechId change would otherwise clobber.
-  //
-  // Known risk (backend, not fixable here — agentTurnToTalk carries no turnId/speechId
-  // to correlate by id, only arrival order): both sides can drift out of 1:1 and desync
-  // pairing for a later, unrelated turn instead of the intended one.
-  //   - Queue can strand an entry forever if its own agentTurnToTalk never fires — e.g.
-  //     conversation-manager's ttv.sayTurn throwing on the first chunk, or the
-  //     first-sentence punctuation gate in SpeechBuffer.append swallowing a short
-  //     punctuation-less reply. Raw text still gets forwarded; the talk pipeline doesn't run.
-  //   - Counter can stay inflated forever if agentTurnToTalk fires but its avatar segment
-  //     never reaches segmentEnd — e.g. onUserInterrupted not aborting AgentAdapter's loop,
-  //     dropping buffered avatar text without ever calling finish()/completeStream().
-  // Real fix is backend: always run finish()/completeStream() on error/interrupt, and
-  // populate turnId on agentTurnToTalk (schema already has the field, unused) so pairing
-  // can be done by id instead of arrival order.
-  let pendingUserTurnCount = 0;
-  // turnIds whose first segment already claimed a pairing slot (queued or emitted).
-  // Keyed by turnId, NOT speechId, because a single logical turn can produce more than
-  // one speechId:
-  //   - Genie can stream a reply as multiple avatar segments under one speechId (e.g. a
-  //     tool call splits the answer into a segment before and a segment after the tool
-  //     result) — those segments share both speechId and turnId.
-  //   - A reprocessTurn retry (sendReprocessTurn, triggered by agentRecoveryMonitor or a
-  //     manual "retry" action) makes the backend regenerate the FULL answer under a NEW
-  //     speechId while keeping the SAME turnId — confirmed in logs: two distinct speechIds
-  //     sharing turnId "c83e031d-..." after a reprocessTurn. Guarding on speechId alone
-  //     would treat the retry's answer as an unrelated new turn.
-  // Either way, the extra segment/retry belongs to a turn that's already paired, so it
-  // must bypass the queue/counter entirely; otherwise it would consume the next,
-  // unrelated user turn's pairing slot and desync every pairing for the rest of the
-  // conversation. Intentionally never cleared: a later segment/retry for the same turnId
-  // can arrive arbitrarily long after the first (slow tool call, slow retry, other turns
-  // in between), so "already claimed" must be remembered for the rest of the session,
-  // same lifetime as awaitingUserTurnQueue/pendingUserTurnCount above. Actor-scoped, dies
-  // with it.
-  const resolvedTurnIds = new Set();
-  // Opening phrase accumulation: for resumed-thread sessions where init_response is not sent,
-  // generatingSpeech chunks arrive before the first user turn — accumulate them keyed by speechId
-  // and flush on the first proactive agentTurnToTalk (empty userTranscription).
-  let openingPhraseSpeechId = null;
-  const openingPhraseChunks = new Map();
-  let openingPhraseEmitted = false;
   let avatarEntrySequence = 0;
+  // turnId -> entry id of its current unanswered exchange - reused across
+  // superseding resends, cleared on a real answer (see top-of-file comment).
+  const openTurnEntryIds = new Map();
   const createAvatarEntryId = (messageId, speechId) => {
     avatarEntrySequence += 1;
     return `${messageId}:${speechId}:${avatarEntrySequence}`;
   };
-  const emitQueuedAvatarEntry = () => {
-    const entry = awaitingUserTurnQueue.shift();
-    if (entry) {
-      sendBack({
-        type: AvatarConnectionMachineEventTypes.AvatarChatEntryReceived,
-        entry
-      });
-    }
-  };
+  // Opening phrase accumulation — see top-of-file comment.
+  let openingPhraseSpeechId = null;
+  let openingPhraseText = '';
+  let openingPhraseSettled = false;
   return createTinyPubsubEventsHandler({
     source: source$a,
     logger: input.logger,
     handlers: {
       agent_raw_text: payload => {
-        var _a, _b;
+        var _a, _b, _c;
         try {
           const delta = JSON.parse(payload.delta);
           const result = agentRawTextDeltaSchema.safeParse(delta);
           if (!result.success) return;
           const parsed = result.data;
-          // 'avatar-filler' is spoken content too (e.g. "Let me check that for you..." while
-          // a tool call runs) — Genie streams it as its own segment ahead of the 'avatar'
-          // segment. Without it here, part of what the avatar actually says never reaches
-          // the chat log.
-          if (parsed.type !== 'avatar' && parsed.type !== 'avatar-filler') return;
-          const speechId = payload.speechId;
-          // New speech started — abandon prior speech's incomplete accumulation.
-          // Does NOT touch awaitingUserTurnQueue/pendingUserTurnCount: those are completed,
-          // already-paired-or-waiting turns and must survive across speechId changes.
-          if (speechId !== currentSpeechId) {
-            pending.delete(currentSpeechId !== null && currentSpeechId !== void 0 ? currentSpeechId : '');
-            currentSpeechId = speechId;
-          }
-          if (typeof parsed.content === 'string' && parsed.content) {
-            pending.set(speechId, ((_a = pending.get(speechId)) !== null && _a !== void 0 ? _a : '') + parsed.content);
-          }
-          if (parsed.segmentEnd) {
-            const text = (_b = pending.get(speechId)) !== null && _b !== void 0 ? _b : '';
-            pending.delete(speechId);
-            if (!text) return;
-            const entry = {
-              id: createAvatarEntryId(parsed.messageId, speechId),
-              role: 'avatar',
-              text,
-              turnId: speechId
-            };
-            if (resolvedTurnIds.has(payload.turnId)) {
-              // Not this turn's first segment/attempt — its pairing slot is already claimed
-              // (multi-segment reply, or a reprocessTurn retry answering the same turnId).
+          const key = `${payload.turnId}:${payload.speechId}`;
+          if (parsed.type === 'think') {
+            if (parsed.human) {
+              const id = (_a = openTurnEntryIds.get(payload.turnId)) !== null && _a !== void 0 ? _a : parsed.messageId;
+              openTurnEntryIds.set(payload.turnId, id);
               sendBack({
-                type: AvatarConnectionMachineEventTypes.AvatarChatEntryReceived,
-                entry
-              });
-              return;
-            }
-            resolvedTurnIds.add(payload.turnId);
-            if (pendingUserTurnCount > 0) {
-              // agentTurnToTalk for this turn already fired — emit directly
-              pendingUserTurnCount -= 1;
-              sendBack({
-                type: AvatarConnectionMachineEventTypes.AvatarChatEntryReceived,
-                entry
+                type: AvatarConnectionMachineEventTypes.UserVoiceTurnReceived,
+                entry: {
+                  id,
+                  role: 'user',
+                  text: parsed.human,
+                  turnId: payload.turnId,
+                  messageId: parsed.messageId
+                }
               });
-            } else {
-              // agentTurnToTalk hasn't arrived yet — queue until it does
-              awaitingUserTurnQueue.push(entry);
             }
+            return;
           }
-        } catch (_c) {
-          // invalid JSON delta — ignore
-        }
-      },
-      generatingSpeech: payload => {
-        var _a;
-        if (openingPhraseEmitted) return;
-        // Server sends this literal sentinel instead of real speech when no opening phrase
-        // is configured — must not leak into the chat log as visible avatar text.
-        if (payload.text === BLANK_TEXT_MARKER) return;
-        if (openingPhraseSpeechId !== null && openingPhraseSpeechId !== payload.speechId) return;
-        openingPhraseSpeechId = payload.speechId;
-        openingPhraseChunks.set(payload.speechId, ((_a = openingPhraseChunks.get(payload.speechId)) !== null && _a !== void 0 ? _a : '') + payload.text);
-      },
-      agentTurnToTalk: payload => {
-        var _a;
-        const {
-          userTranscription,
-          turnId
-        } = payload;
-        if (!userTranscription) {
-          // Proactive turn — flush opening phrase built from generatingSpeech chunks if present
-          if (!openingPhraseEmitted && openingPhraseSpeechId) {
-            const text = (_a = openingPhraseChunks.get(openingPhraseSpeechId)) !== null && _a !== void 0 ? _a : '';
-            openingPhraseChunks.clear();
-            openingPhraseEmitted = true;
+          if (parsed.type === 'user-interruption') {
+            openTurnEntryIds.delete(payload.turnId);
+            const text = pending.get(key);
+            pending.delete(key);
             if (text) {
               sendBack({
                 type: AvatarConnectionMachineEventTypes.AvatarChatEntryReceived,
                 entry: {
-                  id: openingPhraseSpeechId,
+                  id: createAvatarEntryId(parsed.messageId, payload.speechId),
                   role: 'avatar',
                   text,
-                  turnId: 'init_response'
+                  turnId: payload.turnId,
+                  messageId: parsed.messageId
                 }
               });
             }
+            return;
           }
-          // Also emit any queued avatar entry left over from the agent_raw_text path
-          emitQueuedAvatarEntry();
+          if (parsed.type !== 'avatar' && parsed.type !== 'avatar-filler') return;
+          if (typeof parsed.content === 'string' && parsed.content) {
+            pending.set(key, ((_b = pending.get(key)) !== null && _b !== void 0 ? _b : '') + parsed.content);
+          }
+          if (parsed.segmentEnd) {
+            const text = (_c = pending.get(key)) !== null && _c !== void 0 ? _c : '';
+            pending.delete(key);
+            if (!text) return;
+            // Real answer closes the exchange - a later reuse of this
+            // turnId (e.g. repeated "=") must get a fresh entry, not this one.
+            if (parsed.type === 'avatar') {
+              openTurnEntryIds.delete(payload.turnId);
+            }
+            sendBack({
+              type: AvatarConnectionMachineEventTypes.AvatarChatEntryReceived,
+              entry: {
+                id: createAvatarEntryId(parsed.messageId, payload.speechId),
+                role: 'avatar',
+                text,
+                turnId: payload.turnId,
+                messageId: parsed.messageId
+              }
+            });
+          }
+        } catch (_d) {
+          // invalid JSON delta — ignore
+        }
+      },
+      generatingSpeech: payload => {
+        if (openingPhraseSettled) return;
+        // No opening phrase configured — must not leak into the chat log.
+        if (payload.text === BLANK_TEXT_MARKER) return;
+        if (openingPhraseSpeechId !== null && openingPhraseSpeechId !== payload.speechId) {
           return;
         }
-        openingPhraseEmitted = true;
-        openingPhraseChunks.clear();
-        // Emit user first, then pair with its avatar answer — whichever side finished
-        // streaming first (queue has it) or hasn't yet (bump the counter for agent_raw_text
-        // to consume once its segmentEnd arrives)
+        openingPhraseSpeechId = payload.speechId;
+        openingPhraseText += payload.text;
+      },
+      agentTurnToTalk: payload => {
+        // Only the first occurrence decides the opening — later turns also fire this.
+        if (openingPhraseSettled) return;
+        openingPhraseSettled = true;
+        if (payload.userTranscription || !openingPhraseText) return;
         sendBack({
-          type: AvatarConnectionMachineEventTypes.UserVoiceTurnReceived,
+          type: AvatarConnectionMachineEventTypes.AvatarChatEntryReceived,
           entry: {
-            id: turnId !== null && turnId !== void 0 ? turnId : crypto.randomUUID(),
-            role: 'user',
-            text: userTranscription,
-            turnId
+            id: openingPhraseSpeechId !== null && openingPhraseSpeechId !== void 0 ? openingPhraseSpeechId : 'init_response',
+            role: 'avatar',
+            text: openingPhraseText,
+            turnId: 'init_response'
           }
         });
-        if (awaitingUserTurnQueue.length > 0) {
-          emitQueuedAvatarEntry();
-        } else {
-          pendingUserTurnCount += 1;
-        }
       }
     }
   });
@@ -23290,8 +23210,9 @@ const avatarConnectionMachine = setup({
     logAgentClientToolCallResult,
     recordClientToolCallLogEntry,
     appendAvatarChatEntry,
+    removeStaleChatEntriesForRetry,
     appendUserChatEntry,
-    appendUserTextEntry,
+    trackInternalAskEcho,
     setScreenShareActive,
     setScreenShareStream,
     showScreenShareErrorNotification,
@@ -23463,7 +23384,7 @@ const avatarConnectionMachine = setup({
       content: null,
       chatLog: [],
       clientToolCallLog: [],
-      pendingTextInputs: [],
+      awaitingInternalAskEcho: false,
       userAgent: '',
       userAgentHints: null,
       kalturaAnalyticsService: input.kalturaAnalyticsService,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kaltura-sdk/avatars",
-  "version": "1.15.5",
+  "version": "1.16.1",
   "author": "kaltura",
   "license": "AGPL-3.0",
   "repository": {
@@ -13,7 +13,7 @@
     "@kaltura-sdk/rtc-core": "^1.25.21",
     "@local/genie-shared-utils": "^1.0.0",
     "@unisphere/core": "^1.99.0",
-    "@unisphere/genie-types": "^1.39.1",
+    "@unisphere/genie-types": "^1.39.3",
     "@unisphere/notifications-core": "^1.25.0",
     "@unisphere/runtime": "^1.98.0",
     "@unisphere/runtime-js": "^1.96.0",

package/src/lib/avatar-client-manager/machines/avatar-connection/actions/context/remove-stale-chat-entries-for-retry.d.ts

@@ -0,0 +1,2 @@
+import { AvatarConnectionMachineAction } from '../../types';
+export declare const removeStaleChatEntriesForRetry: AvatarConnectionMachineAction;

package/src/lib/avatar-client-manager/machines/avatar-connection/actions/context/test-utils/as-harness-action.d.ts

@@ -0,0 +1,4 @@
+import type { ActionFunction, MachineContext } from 'xstate';
+export declare function asHarnessAction<TContext extends MachineContext, TEvent extends {
+    type: string;
+}>(action: unknown): ActionFunction<TContext, TEvent, TEvent, undefined, never, never, never, never, never>;

package/src/lib/avatar-client-manager/machines/avatar-connection/actions/context/track-internal-ask-echo.d.ts

@@ -0,0 +1,2 @@
+import { AvatarConnectionMachineAction } from '../../types';
+export declare const trackInternalAskEcho: AvatarConnectionMachineAction;

package/src/lib/avatar-client-manager/machines/avatar-connection/avatar-connection-machine.d.ts

@@ -391,11 +391,14 @@ export declare const avatarConnectionMachine: import("xstate").StateMachine<Avat
 } | {
     type: "appendAvatarChatEntry";
     params: unknown;
+} | {
+    type: "removeStaleChatEntriesForRetry";
+    params: unknown;
 } | {
     type: "appendUserChatEntry";
     params: unknown;
 } | {
-    type: "appendUserTextEntry";
+    type: "trackInternalAskEcho";
     params: unknown;
 } | {
     type: "setScreenShareActive";

package/src/lib/avatar-client-manager/machines/avatar-connection/states/connected-state.d.ts

@@ -96,7 +96,7 @@ export declare const connectedState: {
             initial: string;
             on: {
                 AskViaText: {
-                    actions: ("appendUserTextEntry" | {
+                    actions: ("trackInternalAskEcho" | {
                         type: "sendAskViaText";
                         params: ({ event }: {
                             event: import("../types").AskViaTextEvent;
@@ -728,13 +728,13 @@ export declare const connectedState: {
                     actions: "resetAgentRecoveryRetryCount";
                 } | {
                     guard: "hasMoreRecoveryRetries";
-                    actions: readonly ["incrementAgentRecoveryRetryCount", "sendReprocessTurn"];
+                    actions: readonly ["removeStaleChatEntriesForRetry", "incrementAgentRecoveryRetryCount", "sendReprocessTurn"];
                 } | {
                     guard: "justExceededRecoveryRetries";
                     actions: readonly ["incrementAgentRecoveryRetryCount", "showAgentRecoveryExhaustedNotification"];
                 })[];
                 RequestAgentTurnRecovery: {
-                    actions: "sendReprocessTurn";
+                    actions: readonly ["removeStaleChatEntriesForRetry", "sendReprocessTurn"];
                 };
             };
             invoke: {

package/src/lib/avatar-client-manager/machines/avatar-connection/states/connected-states/agent-recovery-monitor-state.d.ts

@@ -6,13 +6,13 @@ export declare const agentRecoveryMonitorState: {
             actions: "resetAgentRecoveryRetryCount";
         } | {
             guard: "hasMoreRecoveryRetries";
-            actions: readonly ["incrementAgentRecoveryRetryCount", "sendReprocessTurn"];
+            actions: readonly ["removeStaleChatEntriesForRetry", "incrementAgentRecoveryRetryCount", "sendReprocessTurn"];
         } | {
             guard: "justExceededRecoveryRetries";
             actions: readonly ["incrementAgentRecoveryRetryCount", "showAgentRecoveryExhaustedNotification"];
         })[];
         RequestAgentTurnRecovery: {
-            actions: "sendReprocessTurn";
+            actions: readonly ["removeStaleChatEntriesForRetry", "sendReprocessTurn"];
         };
     };
     invoke: {

package/src/lib/avatar-client-manager/machines/avatar-connection/states/connected-states/text-communication-state.d.ts

@@ -4,7 +4,7 @@ export declare const textCommunicationState: {
     initial: string;
     on: {
         AskViaText: {
-            actions: ("appendUserTextEntry" | {
+            actions: ("trackInternalAskEcho" | {
                 type: "sendAskViaText";
                 params: ({ event }: {
                     event: AskViaTextEvent;

package/src/lib/avatar-client-manager/machines/avatar-connection/types/context.d.ts

@@ -66,7 +66,7 @@ export interface AvatarConnectionMachineContext {
     content: UnisphereChatBlock | null;
     chatLog: ChatEntry[];
     clientToolCallLog: ClientToolCallLogEntry[];
-    pendingTextInputs: string[];
+    awaitingInternalAskEcho: boolean;
     isAgentTalking: boolean;
     userAgent: string;
     userAgentHints: Record<string, string> | null;

package/src/lib/avatar-client-manager/machines/avatar-connection/types/schemas.d.ts

@@ -69,6 +69,7 @@ export declare const agentRawTextDeltaSchema: z.ZodMiniObject<{
     role: z.ZodMiniOptional<z.ZodMiniString<string>>;
     type: z.ZodMiniOptional<z.ZodMiniString<string>>;
     content: z.ZodMiniOptional<z.ZodMiniUnion<readonly [z.ZodMiniString<string>, z.ZodMiniRecord<z.ZodMiniString<string>, z.ZodMiniUnknown>]>>;
+    human: z.ZodMiniOptional<z.ZodMiniString<string>>;
     segmentNumber: z.ZodMiniOptional<z.ZodMiniNumber<number>>;
     et: z.ZodMiniOptional<z.ZodMiniNumber<number>>;
     segmentStart: z.ZodMiniOptional<z.ZodMiniBoolean<boolean>>;
@@ -86,6 +87,7 @@ export declare const agentRawTextClientToolDeltaSchema: z.ZodMiniObject<{
     threadId: z.ZodMiniOptional<z.ZodMiniString<string>>;
     role: z.ZodMiniOptional<z.ZodMiniString<string>>;
     content: z.ZodMiniOptional<z.ZodMiniUnion<readonly [z.ZodMiniString<string>, z.ZodMiniRecord<z.ZodMiniString<string>, z.ZodMiniUnknown>]>>;
+    human: z.ZodMiniOptional<z.ZodMiniString<string>>;
     segmentNumber: z.ZodMiniOptional<z.ZodMiniNumber<number>>;
     et: z.ZodMiniOptional<z.ZodMiniNumber<number>>;
     segmentStart: z.ZodMiniOptional<z.ZodMiniBoolean<boolean>>;

package/src/lib/avatar-client-manager/machines/avatar-connection/actions/context/append-user-text-entry.d.ts

@@ -1,2 +0,0 @@
-import { AvatarConnectionMachineAction } from '../../types';
-export declare const appendUserTextEntry: AvatarConnectionMachineAction;