@meshagent/meshagent-agents 0.43.2 → 0.44.0

package/dist/esm/chat-client.js

@@ -1,16 +1,41 @@
-import { Completer, RoomMessageEvent, } from "@meshagent/meshagent";
+import { Completer, } from "@meshagent/meshagent";
 import { v4 as uuidv4 } from "uuid";
-import { AgentMessage as AgentMessageFactory, AgentRealtimeAudioChunk, AgentRealtimeAudioCommit, ChangeModel, CloseThread, DeleteThread, ModelsRequest, OpenThread, RenameThread, StartThread, AgentConnectionStatus, TurnInterrupt, TurnStart, TurnSteer, AgentClientToolCallResponse, agentInputContent, agentRoomMessageType, agentThreadClearedType, agentThreadLoadedType, agentConnectionStatusType, agentThreadStartRejectedType, agentThreadStartedType, agentTurnEndedType, agentTurnStartAcceptedType, agentTurnStartRejectedType, agentTurnStartType, agentTurnStartedType, agentTurnSteerAcceptedType, agentTurnSteerRejectedType, agentTurnSteerType, agentTurnSteeredType, } from "./agent-messages.js";
+import { AgentMessage as AgentMessageFactory, AgentAudioGenerationDelta, AgentAudioTranscriptionDelta, AgentRealtimeAudioChunk, AgentRealtimeAudioCommit, AgentFileContentDelta, AgentModelChanged, AgentReasoningContentDelta, AgentTextContentDelta, AgentThreadStatus, AgentToolCallArgumentsDelta, AgentToolCallLogDelta, ChangeModel, CloseThread, DeleteThread, ModelsResponse, ModelsRequest, OpenThread, RenameThread, StartThread, AgentConnectionStatus, TurnInterrupt, TurnStart, TurnSteer, AgentClientToolCallRequested, AgentClientToolCallResponse, agentInputContent, agentAudioGenerationDeltaType, agentAudioTranscriptionDeltaType, agentFileContentDeltaType, agentImageGenerationCompletedType, agentImageGenerationPartialType, agentModelChangedType, agentModelsResponseType, agentReasoningContentDeltaType, agentRoomMessageType, agentTextContentDeltaType, agentThreadClearedType, agentThreadCreatedType, agentThreadDeletedType, agentThreadLoadedType, agentThreadListedType, agentThreadUpdatedType, agentConnectionStatusType, agentThreadStatusType, agentThreadStartType, agentThreadStartRejectedType, agentThreadStartedType, agentToolCallArgumentsDeltaType, agentToolCallLogDeltaType, agentToolCallEndedType, agentToolCallInProgressType, agentToolCallPendingType, agentToolCallStartedType, agentSecretRequestedType, agentTurnEndedType, agentTurnInterruptAcceptedType, agentTurnInterruptedType, agentTurnStartAcceptedType, agentTurnStartRejectedType, agentTurnStartType, agentTurnStartedType, agentTurnSteerAcceptedType, agentTurnSteerRejectedType, agentTurnSteerType, agentTurnSteeredType, agentUsageUpdatedType, } from "./agent-messages.js";
 import { ChangeEmitter } from "./change-emitter.js";
 export class AgentMessageEvent {
-    constructor({ message, attachment }) {
+    constructor({ message, createdAt, attachment }) {
         this.message = message;
+        this.createdAt = createdAt ?? dateOrNull(message.createdAt) ?? new Date();
         this.attachment = attachment;
     }
     get payload() {
-        return this.message.toJson();
+        return { ...this.message.toJson(), created_at: this.createdAt.toISOString() };
     }
 }
+function agentRoomMessageFromEvent(event) {
+    const candidate = event.message;
+    if (candidate == null || typeof candidate !== "object" || Array.isArray(candidate)) {
+        return undefined;
+    }
+    const roomMessage = candidate;
+    if (roomMessage.type !== agentRoomMessageType) {
+        return undefined;
+    }
+    if (roomMessage.message == null || typeof roomMessage.message !== "object" || Array.isArray(roomMessage.message)) {
+        return undefined;
+    }
+    const rawMessage = roomMessage.message;
+    const payload = typeof rawMessage.type === "string" ? rawMessage : rawMessage.payload;
+    if (payload == null || typeof payload !== "object" || Array.isArray(payload)) {
+        return undefined;
+    }
+    return {
+        fromParticipantId: typeof roomMessage.fromParticipantId === "string" ? roomMessage.fromParticipantId : undefined,
+        message: payload,
+        local: roomMessage.local === true,
+        attachment: roomMessage.attachment instanceof Uint8Array ? roomMessage.attachment : undefined,
+    };
+}
 export class PendingAgentInput {
     constructor({ messageId, messageType, threadPath, payload, createdAt, awaitingAcceptance = false, awaitingApplication = false, awaitingOnline = false, }) {
         this.messageId = messageId;
@@ -69,28 +94,52 @@ export class ChatThreadStartResult {
 }
 class EventQueue {
     constructor() {
-        this.items = [];
-        this.waiters = [];
+        this.subscribers = new Set();
     }
     add(item) {
-        const waiter = this.waiters.shift();
-        if (waiter != null) {
-            waiter({ value: item, done: false });
-            return;
+        for (const subscriber of this.subscribers) {
+            if (subscriber.closed) {
+                continue;
+            }
+            const waiter = subscriber.waiters.shift();
+            if (waiter != null) {
+                waiter({ value: item, done: false });
+            }
+            else {
+                subscriber.items.push(item);
+            }
         }
-        this.items.push(item);
     }
     get stream() {
         return {
-            [Symbol.asyncIterator]: () => ({
-                next: async () => {
-                    const item = this.items.shift();
-                    if (item !== undefined) {
-                        return { value: item, done: false };
-                    }
-                    return new Promise((resolve) => this.waiters.push(resolve));
-                },
-            }),
+            [Symbol.asyncIterator]: () => {
+                const subscriber = {
+                    items: [],
+                    waiters: [],
+                    closed: false,
+                };
+                this.subscribers.add(subscriber);
+                return {
+                    next: async () => {
+                        if (subscriber.closed) {
+                            return { done: true, value: undefined };
+                        }
+                        const item = subscriber.items.shift();
+                        if (item !== undefined) {
+                            return { value: item, done: false };
+                        }
+                        return new Promise((resolve) => subscriber.waiters.push(resolve));
+                    },
+                    return: async () => {
+                        subscriber.closed = true;
+                        this.subscribers.delete(subscriber);
+                        for (const waiter of subscriber.waiters.splice(0)) {
+                            waiter({ done: true, value: undefined });
+                        }
+                        return { done: true, value: undefined };
+                    },
+                };
+            },
         };
     }
 }
@@ -99,6 +148,7 @@ export class BaseChatClient extends ChangeEmitter {
         super(...arguments);
         this.sessionsByPath = new Map();
         this.pendingStartRequests = new Map();
+        this.pendingSessionEventsByPath = new Map();
         this.eventQueue = new EventQueue();
     }
     get events() {
@@ -121,7 +171,7 @@ export class BaseChatClient extends ChangeEmitter {
     cleanParticipantName(value) {
         return nonEmptyTrimmed(value);
     }
-    openThread(threadPath, { load = true, sinceTurn, reloadIfOpen = false } = {}) {
+    openThread(threadPath, { load = true, sinceTurn, reloadIfOpen = false, drainPendingEvents = true, } = {}) {
         const normalized = threadPath.trim();
         if (normalized === "") {
             throw new TypeError("thread path cannot be empty");
@@ -129,23 +179,32 @@ export class BaseChatClient extends ChangeEmitter {
         const existing = this.sessionsByPath.get(normalized);
         if (existing != null) {
             if (!existing.isOpen || reloadIfOpen) {
-                void existing.open({ load, sinceTurn });
+                this.openSession(existing, { load, sinceTurn });
             }
             return existing;
         }
         const created = new ChatThreadSession({ client: this, threadPath: normalized });
         this.sessionsByPath.set(normalized, created);
-        void created.open({ load, sinceTurn });
+        if (drainPendingEvents) {
+            this.drainPendingSessionEvents(normalized, created);
+        }
+        this.openSession(created, { load, sinceTurn });
         this.notifyListeners();
         return created;
     }
-    async startThread({ messageId, message, attachments, name, provider, model, voice, outputModalities, realtimeProtocol, senderName, clientToolkits, omitContent = false, }) {
+    openSession(session, options) {
+        void session.open(options).catch(() => {
+            this.notifyListeners();
+        });
+    }
+    async startThread({ messageId, message, attachments, name, backend, provider, model, voice, outputModalities, realtimeProtocol, senderName, clientToolkits, toolChoice, omitContent = false, }) {
         const resolvedMessageId = messageId == null || messageId.trim() === "" ? uuidv4() : messageId.trim();
         const resolvedSenderName = this.cleanParticipantName(senderName) ?? this.localParticipantName();
         const payload = new StartThread({
             messageId: resolvedMessageId,
             content: omitContent ? undefined : agentInputContent({ text: message, attachments }),
             name: nonEmptyTrimmed(name),
+            backend: nonEmptyTrimmed(backend),
             provider: nonEmptyTrimmed(provider),
             model: nonEmptyTrimmed(model),
             voice: nonEmptyTrimmed(voice),
@@ -153,6 +212,7 @@ export class BaseChatClient extends ChangeEmitter {
             realtimeProtocol: nonEmptyTrimmed(realtimeProtocol),
             senderName: resolvedSenderName,
             clientToolkits: clientToolkits != null && clientToolkits.length > 0 ? clientToolkits : undefined,
+            toolChoice: toolChoice?.toJson(),
         });
         const completer = new Completer();
         this.pendingStartRequests.set(resolvedMessageId, completer);
@@ -165,10 +225,14 @@ export class BaseChatClient extends ChangeEmitter {
             if (threadPath === "") {
                 throw new Error("Agent did not return a thread_id for the new thread.");
             }
-            const session = this.openThread(threadPath, { load: false });
+            const session = this.openThread(threadPath, { load: false, drainPendingEvents: false });
             if (!omitContent) {
-                session.addAgentMessage(new AgentMessageEvent({ message: payload }));
+                session.addAgentMessage(new AgentMessageEvent({
+                    message: this.resolvedStartThreadMessage({ payload, threadPath }),
+                    createdAt: dateOrNull(payload.createdAt) ?? undefined,
+                }));
             }
+            this.drainPendingSessionEvents(threadPath, session);
             return new ChatThreadStartResult({
                 session,
                 threadPath,
@@ -181,16 +245,43 @@ export class BaseChatClient extends ChangeEmitter {
             this.pendingStartRequests.delete(resolvedMessageId);
         }
     }
-    handleAgentMessage(message, { attachment } = {}) {
+    resolvedStartThreadMessage({ payload, threadPath, }) {
+        return new TurnStart({
+            threadId: threadPath,
+            messageId: payload.messageId,
+            senderName: payload.senderName,
+            content: payload.content,
+            backend: payload.backend,
+            provider: payload.provider,
+            model: payload.model,
+            voice: payload.voice,
+            outputModalities: payload.outputModalities,
+            instructions: payload.instructions,
+            mcp: payload.mcp,
+            clientToolkits: payload.clientToolkits,
+            toolkits: payload.toolkits,
+        });
+    }
+    drainPendingSessionEvents(threadPath, session) {
+        const pendingEvents = this.pendingSessionEventsByPath.get(threadPath);
+        if (pendingEvents == null) {
+            return;
+        }
+        this.pendingSessionEventsByPath.delete(threadPath);
+        for (const event of pendingEvents) {
+            session.addAgentMessage(event);
+        }
+    }
+    handleAgentMessage(message, { createdAt, attachment } = {}) {
         if (message.type === agentConnectionStatusType) {
             this._connectionStatus = message;
             for (const session of this.sessionsByPath.values()) {
                 if (session.isOpen) {
-                    session.addAgentMessage(new AgentMessageEvent({ message, attachment }));
+                    session.addAgentMessage(new AgentMessageEvent({ message, createdAt, attachment }));
                 }
             }
         }
-        this.eventQueue.add(new AgentMessageEvent({ message, attachment }));
+        this.eventQueue.add(new AgentMessageEvent({ message, createdAt, attachment }));
         const sourceMessageId = sourceMessageIdFor(message);
         if ((message.type === agentThreadStartedType || message.type === agentThreadStartRejectedType) &&
             sourceMessageId != null &&
@@ -205,7 +296,16 @@ export class BaseChatClient extends ChangeEmitter {
             this.notifyListeners();
             return;
         }
-        this.sessionsByPath.get(threadPath)?.addAgentMessage(new AgentMessageEvent({ message, attachment }));
+        const event = new AgentMessageEvent({ message, createdAt, attachment });
+        const session = this.sessionsByPath.get(threadPath);
+        if (session != null) {
+            session.addAgentMessage(event);
+        }
+        else {
+            const pending = this.pendingSessionEventsByPath.get(threadPath) ?? [];
+            pending.push(event);
+            this.pendingSessionEventsByPath.set(threadPath, pending);
+        }
         this.notifyListeners();
     }
     emitConnectionStatus({ status, message, reason, retryDelay, }) {
@@ -238,7 +338,6 @@ export class MessagingChatClient extends BaseChatClient {
     constructor({ room, agentName }) {
         super();
         this.pendingParticipantWaits = new Map();
-        this.roomAbortController = new AbortController();
         this.started = false;
         this.onMessagingChanged = () => {
             const participant = this.agentParticipant();
@@ -252,10 +351,26 @@ export class MessagingChatClient extends BaseChatClient {
             }
             this.notifyListeners();
         };
+        this.onMessagingMessage = (event) => {
+            this.onMessagingChanged();
+            const roomMessage = agentRoomMessageFromEvent(event);
+            if (roomMessage == null || roomMessage.local) {
+                return;
+            }
+            const participant = this.agentParticipant();
+            const fromSelectedAgent = participant != null && roomMessage.fromParticipantId === participant.id;
+            const fromParticipantlessThreadList = roomMessage.fromParticipantId == null && isThreadListMessage(roomMessage.message);
+            if (!fromSelectedAgent && !fromParticipantlessThreadList) {
+                return;
+            }
+            this.handleAgentMessage(AgentMessageFactory.fromJson(roomMessage.message), {
+                createdAt: dateOrNull(roomMessage.message.created_at),
+                attachment: roomMessage.attachment,
+            });
+        };
         this.room = room;
         this.agentName = agentName;
-        this.room.messaging.addListener("message", this.onMessagingChanged);
-        this.roomPump = this.consumeRoomEvents();
+        this.room.messaging.addListener("message", this.onMessagingMessage);
     }
     localParticipantName() {
         const name = this.room.localParticipant?.getAttribute("name");
@@ -271,15 +386,13 @@ export class MessagingChatClient extends BaseChatClient {
     }
     async stop() {
         this.started = false;
-        this.roomAbortController.abort();
-        this.room.messaging.removeListener("message", this.onMessagingChanged);
+        this.room.messaging.removeListener("message", this.onMessagingMessage);
         for (const wait of this.pendingParticipantWaits.values()) {
             if (!wait.completed) {
                 wait.completeError(new Error("Chat client stopped while waiting for an agent participant."));
             }
         }
         this.pendingParticipantWaits.clear();
-        await this.roomPump?.catch(() => undefined);
     }
     agentParticipant() {
         const normalizedAgentName = this.agentName?.trim();
@@ -320,37 +433,21 @@ export class MessagingChatClient extends BaseChatClient {
             ignoreOffline,
         });
     }
-    async consumeRoomEvents() {
-        try {
-            for await (const event of this.room.listen({ abortSignal: this.roomAbortController.signal })) {
-                this.onRoomEvent(event);
-            }
-        }
-        catch {
-            if (!this.roomAbortController.signal.aborted) {
-                throw new Error("chat client room event listener stopped unexpectedly");
-            }
-        }
-    }
-    onRoomEvent(event) {
-        if (!(event instanceof RoomMessageEvent) || event.message.type !== agentRoomMessageType) {
-            return;
-        }
-        const participant = this.agentParticipant();
-        if (participant == null || event.message.fromParticipantId !== participant.id) {
-            return;
-        }
-        const message = event.message.message;
-        const rawPayload = typeof message.type === "string" ? message : message.payload;
-        if (rawPayload != null && typeof rawPayload === "object" && !Array.isArray(rawPayload)) {
-            this.handleAgentMessage(AgentMessageFactory.fromJson(rawPayload), { attachment: event.message.attachment });
-        }
-    }
+}
+function isThreadListMessage(message) {
+    return message.type === agentThreadListedType
+        || message.type === agentThreadCreatedType
+        || message.type === agentThreadUpdatedType
+        || message.type === agentThreadDeletedType;
 }
 export class ChatThreadSession extends ChangeEmitter {
     constructor({ client, threadPath }) {
         super();
         this._messages = [];
+        this.messageIndexes = new Map();
+        this.localAgentMessageIds = new Set();
+        this.pendingLocalInputMessageIds = new Set();
+        this.mergedDeltaMessageIds = new Set();
         this.pendingInputsById = new Map();
         this.openState = false;
         this.loadStateValue = ChatThreadSessionLoadState.idle();
@@ -375,9 +472,24 @@ export class ChatThreadSession extends ChangeEmitter {
     get lastCompletedTurnId() {
         return this.lastCompletedTurnIdValue;
     }
+    get activeTurnId() {
+        return this.activeTurnIdValue;
+    }
+    get threadStatus() {
+        return this.threadStatusValue;
+    }
+    get currentModel() {
+        return this.currentModelValue;
+    }
+    get modelsResponse() {
+        return this.modelsResponseValue;
+    }
     async open({ load = true, sinceTurn } = {}) {
         if (this.openState) {
             if (load) {
+                if (sinceTurn == null) {
+                    this.resetReplayState();
+                }
                 const openThread = new OpenThread({ threadId: this.threadPath, load: true, sinceTurn });
                 this.setLoadState(this.loadingStateForOpen(openThread, sinceTurn));
                 try {
@@ -408,6 +520,17 @@ export class ChatThreadSession extends ChangeEmitter {
             throw error;
         }
     }
+    resetReplayState() {
+        this._messages.length = 0;
+        this.messageIndexes.clear();
+        this.localAgentMessageIds.clear();
+        this.pendingLocalInputMessageIds.clear();
+        this.mergedDeltaMessageIds.clear();
+        this.pendingInputsById.clear();
+        this.lastCompletedTurnIdValue = undefined;
+        this.lastAgentMessageType = undefined;
+        this.lastAgentMessageAt = undefined;
+    }
     async close() {
         if (!this.openState) {
             return;
@@ -418,10 +541,11 @@ export class ChatThreadSession extends ChangeEmitter {
     requestModels({ ignoreOffline = false } = {}) {
         return this.client.sendAgentMessage(new ModelsRequest({ messageId: uuidv4() }), { ignoreOffline });
     }
-    changeModel({ provider, model, voice }) {
+    changeModel({ backend, provider, model, voice }) {
         return this.client.sendAgentMessage(new ChangeModel({
             threadId: this.threadPath,
             messageId: uuidv4(),
+            backend: nonEmptyTrimmed(backend),
             provider,
             model,
             voice: nonEmptyTrimmed(voice),
@@ -438,8 +562,16 @@ export class ChatThreadSession extends ChangeEmitter {
             response,
         }));
     }
-    async sendText({ messageId, text, attachments, steer = false, turnId, provider, model, voice, outputModalities, senderName, clientToolkits, }) {
+    async sendText({ messageId, text, attachments, steer = false, turnId, backend, provider, model, voice, outputModalities, senderName, clientToolkits, toolChoice, }) {
         const resolvedMessageId = messageId == null || messageId.trim() === "" ? uuidv4() : messageId.trim();
+        const currentModel = this.currentModelValue;
+        const resolvedBackend = nonEmptyTrimmed(backend) ?? currentModel?.backend;
+        const resolvedProvider = nonEmptyTrimmed(provider) ?? currentModel?.provider;
+        const resolvedModel = nonEmptyTrimmed(model) ?? currentModel?.model;
+        const resolvedVoice = nonEmptyTrimmed(voice) ?? currentModel?.voice;
+        const resolvedOutputModalities = outputModalities != null && outputModalities.length > 0
+            ? outputModalities
+            : currentModel?.outputModalities;
         const payload = steer
             ? new TurnSteer({
                 threadId: this.threadPath,
@@ -453,21 +585,16 @@ export class ChatThreadSession extends ChangeEmitter {
                 messageId: resolvedMessageId,
                 senderName: nonEmptyTrimmed(senderName),
                 content: agentInputContent({ text, attachments }),
-                provider: nonEmptyTrimmed(provider),
-                model: nonEmptyTrimmed(model),
-                voice: nonEmptyTrimmed(voice),
-                outputModalities: outputModalities != null && outputModalities.length > 0 ? outputModalities : undefined,
+                backend: resolvedBackend,
+                provider: resolvedProvider,
+                model: resolvedModel,
+                voice: resolvedVoice,
+                outputModalities: resolvedOutputModalities,
                 clientToolkits: clientToolkits != null && clientToolkits.length > 0 ? clientToolkits : undefined,
+                toolChoice: toolChoice?.toJson(),
             });
-        this.markPending(new PendingAgentInput({
-            messageId: resolvedMessageId,
-            messageType: payload.type,
-            threadPath: this.threadPath,
-            payload,
-            createdAt: new Date(),
-            awaitingAcceptance: true,
-            awaitingApplication: true,
-        }));
+        this.localAgentMessageIds.add(resolvedMessageId);
+        this.pendingLocalInputMessageIds.add(resolvedMessageId);
         this.addAgentMessage(new AgentMessageEvent({ message: payload }));
         try {
             await this.client.sendAgentMessage(payload);
@@ -511,58 +638,284 @@ export class ChatThreadSession extends ChangeEmitter {
         return this.client.sendAgentMessage(new RenameThread({ threadId: threadPath, name }));
     }
     addAgentMessage(event) {
-        this._messages.push(event);
         const message = event.message;
         const type = message.type;
         this.lastAgentMessageType = type;
-        this.lastAgentMessageAt = new Date();
+        this.lastAgentMessageAt = event.createdAt;
+        this.updateSessionState(message);
+        this.updatePendingInputsForMessage(message, event.createdAt);
+        this.appendRenderableMessage(event);
+        this.notifyListeners();
+    }
+    updateSessionState(message) {
+        if (message.type === agentTurnEndedType) {
+            const turnId = normalizedString(threadMessageValue(message, "turnId"));
+            if (turnId != null) {
+                this.lastCompletedTurnIdValue = turnId;
+                if (this.activeTurnIdValue === turnId) {
+                    this.activeTurnIdValue = undefined;
+                }
+            }
+            const statusTurnId = normalizedString(this.threadStatusValue?.turnId);
+            if (statusTurnId == null || statusTurnId === turnId) {
+                this.threadStatusValue = undefined;
+            }
+            return;
+        }
+        if (message.type === agentTurnStartAcceptedType || message.type === agentTurnStartedType) {
+            const turnId = normalizedString(threadMessageValue(message, "turnId"));
+            if (turnId != null) {
+                this.activeTurnIdValue = turnId;
+            }
+            if (message.type === agentTurnStartedType) {
+                const statusTurnId = normalizedString(this.threadStatusValue?.turnId);
+                if (statusTurnId == null || statusTurnId === turnId) {
+                    this.threadStatusValue = undefined;
+                }
+            }
+            return;
+        }
+        if (message.type === agentThreadStatusType && message instanceof AgentThreadStatus) {
+            this.threadStatusValue = message;
+            const mode = normalizedString(message.mode);
+            const status = normalizedString(message.status);
+            const turnId = normalizedString(message.turnId);
+            if (status != null && mode === "steerable" && turnId != null) {
+                this.activeTurnIdValue = turnId;
+            }
+            else if (status == null && turnId != null && this.activeTurnIdValue === turnId) {
+                this.activeTurnIdValue = undefined;
+            }
+            return;
+        }
+        if (message.type === agentModelChangedType && message instanceof AgentModelChanged) {
+            this.currentModelValue = message;
+            return;
+        }
+        if (message.type === agentModelsResponseType && message instanceof ModelsResponse) {
+            this.modelsResponseValue = message;
+            this.applyActiveModel(message);
+            return;
+        }
+        if (message.type === agentThreadLoadedType) {
+            this.setLoadState(new ChatThreadSessionLoadState({
+                phase: ChatThreadSessionLoadPhase.Loaded,
+                startedAt: this.loadStateValue.startedAt,
+                completedAt: this.lastAgentMessageAt,
+                requestMessageId: this.loadStateValue.requestMessageId,
+                sinceTurn: this.loadStateValue.sinceTurn,
+                lastMessageType: message.type,
+                lastMessageAt: this.lastAgentMessageAt,
+            }), { notify: false });
+        }
+    }
+    updatePendingInputsForMessage(message, createdAt) {
+        const type = message.type;
         const messageId = message.messageId;
         const sourceMessageId = sourceMessageIdFor(message);
-        if (type === agentTurnStartType || type === agentTurnSteerType) {
+        if (type === agentThreadStartType || type === agentTurnStartType || type === agentTurnSteerType) {
             const normalizedMessageId = messageId.trim();
             if (normalizedMessageId !== "") {
+                this.localAgentMessageIds.add(normalizedMessageId);
+                this.pendingLocalInputMessageIds.add(normalizedMessageId);
                 this.markPending(new PendingAgentInput({
                     messageId: normalizedMessageId,
                     messageType: type,
                     threadPath: this.threadPath,
                     payload: message,
-                    createdAt: new Date(),
+                    createdAt,
                     awaitingAcceptance: true,
                     awaitingApplication: true,
                 }));
             }
         }
+        else if (type === agentThreadStartRejectedType) {
+            if (sourceMessageId != null) {
+                this.pendingInputsById.delete(sourceMessageId);
+            }
+        }
         else if (type === agentTurnStartAcceptedType || type === agentTurnSteerAcceptedType) {
             this.updatePending(sourceMessageId, { awaitingAcceptance: false, awaitingApplication: true });
         }
         else if (type === agentTurnStartedType || type === agentTurnSteeredType) {
             if (sourceMessageId != null) {
                 this.pendingInputsById.delete(sourceMessageId);
+                this.pendingLocalInputMessageIds.delete(sourceMessageId);
             }
         }
         else if (type === agentTurnStartRejectedType || type === agentTurnSteerRejectedType) {
             if (sourceMessageId != null) {
                 this.pendingInputsById.delete(sourceMessageId);
+                this.pendingLocalInputMessageIds.delete(sourceMessageId);
             }
         }
         else if (type === agentTurnEndedType || type === agentThreadClearedType) {
             this.pendingInputsById.clear();
-            if (message.type === agentTurnEndedType && "turnId" in message && typeof message.turnId === "string" && message.turnId.trim() !== "") {
-                this.lastCompletedTurnIdValue = message.turnId.trim();
+            this.pendingLocalInputMessageIds.clear();
+        }
+    }
+    appendRenderableMessage(event) {
+        const message = event.message;
+        if (message.type === agentTurnEndedType) {
+            if (message.error != null) {
+                this.appendMessage(event);
             }
+            return;
         }
-        else if (type === agentThreadLoadedType) {
-            this.setLoadState(new ChatThreadSessionLoadState({
-                phase: ChatThreadSessionLoadPhase.Loaded,
-                startedAt: this.loadStateValue.startedAt,
-                completedAt: this.lastAgentMessageAt,
-                requestMessageId: this.loadStateValue.requestMessageId,
-                sinceTurn: this.loadStateValue.sinceTurn,
-                lastMessageType: type,
-                lastMessageAt: this.lastAgentMessageAt,
-            }), { notify: false });
+        if (message.type === agentToolCallEndedType) {
+            const itemId = normalizedString(message.itemId);
+            if (itemId != null) {
+                this.upsertItemMessage(`tool_call:${itemId}`, event);
+            }
+            else if (message.error != null) {
+                this.appendMessage(event);
+            }
+            return;
+        }
+        if (message.type === agentToolCallPendingType || message.type === agentToolCallInProgressType || message.type === agentToolCallStartedType) {
+            const itemId = normalizedString(message.itemId);
+            if (itemId != null) {
+                this.upsertItemMessage(`tool_call:${itemId}`, event);
+            }
+            return;
+        }
+        if (message.type === agentSecretRequestedType) {
+            this.appendMessage(event);
+            return;
+        }
+        if (message.type === agentModelChangedType) {
+            this.appendMessage(event);
+            return;
+        }
+        if (message.type === agentThreadStartType || message.type === agentTurnStartType || message.type === agentTurnSteerType) {
+            if (agentInputText(message.content).trim() !== "") {
+                this.appendMessage(event);
+            }
+            return;
+        }
+        if (message.type === agentTurnStartAcceptedType || message.type === agentTurnSteerAcceptedType) {
+            const sourceMessageId = normalizedString(sourceMessageIdFor(message));
+            if (sourceMessageId != null && this.localAgentMessageIds.has(sourceMessageId)) {
+                this.pendingLocalInputMessageIds.delete(sourceMessageId);
+                return;
+            }
+            if (agentInputText(message.content).trim() !== "") {
+                this.appendMessage(event, { beforePendingLocalInputs: true });
+            }
+            else {
+                this.appendMessage(event);
+            }
+            return;
+        }
+        if (message.type === agentTurnStartedType ||
+            message.type === agentTurnSteeredType ||
+            message.type === agentTurnStartRejectedType ||
+            message.type === agentTurnSteerRejectedType ||
+            message.type === agentTurnInterruptAcceptedType ||
+            message.type === agentTurnInterruptedType) {
+            this.appendMessage(event);
+            return;
+        }
+        if (isMergeableDelta(message)) {
+            this.appendOrMergeDelta(event);
+            return;
+        }
+        if (message.type === agentImageGenerationPartialType || message.type === agentImageGenerationCompletedType) {
+            const itemId = normalizedString(message.itemId);
+            if (itemId != null) {
+                this.upsertItemMessage(`image_generation:${itemId}`, event);
+            }
+            return;
+        }
+        if (message.type === agentUsageUpdatedType) {
+            this.appendMessage(event);
+            return;
+        }
+        if (message instanceof AgentClientToolCallRequested) {
+            this.appendMessage(event);
+        }
+    }
+    appendMessage(event, { beforePendingLocalInputs = false } = {}) {
+        const normalizedMessageId = normalizedString(event.message.messageId);
+        if (normalizedMessageId == null || this.messageIndexes.has(normalizedMessageId)) {
+            return;
+        }
+        if (beforePendingLocalInputs) {
+            const insertIndex = this._messages.findIndex((existing) => this.pendingLocalInputMessageIds.has(existing.message.messageId));
+            if (insertIndex >= 0) {
+                this._messages.splice(insertIndex, 0, event);
+                this.indexMessagesFrom(insertIndex);
+                return;
+            }
+        }
+        this.messageIndexes.set(normalizedMessageId, this._messages.length);
+        this._messages.push(event);
+    }
+    upsertItemMessage(key, event) {
+        const existingIndex = this.messageIndexes.get(key);
+        if (existingIndex == null) {
+            this.messageIndexes.set(key, this._messages.length);
+            this._messages.push(event);
+            return;
+        }
+        this._messages[existingIndex] = event;
+    }
+    appendOrMergeDelta(event) {
+        const message = event.message;
+        const normalizedMessageId = normalizedString(message.messageId);
+        if (normalizedMessageId != null) {
+            if (this.mergedDeltaMessageIds.has(normalizedMessageId)) {
+                return;
+            }
+            this.mergedDeltaMessageIds.add(normalizedMessageId);
+        }
+        const itemId = normalizedString(message.itemId);
+        if (itemId == null) {
+            return;
+        }
+        const key = `${message.type}:${itemId}`;
+        const existingIndex = this.messageIndexes.get(key);
+        if (existingIndex == null) {
+            this.messageIndexes.set(key, this._messages.length);
+            this._messages.push(event);
+            return;
+        }
+        const existing = this._messages[existingIndex].message;
+        const merged = mergeDeltaMessage(existing, message);
+        if (merged != null) {
+            this._messages[existingIndex] = new AgentMessageEvent({ message: merged, createdAt: event.createdAt, attachment: event.attachment });
+        }
+    }
+    indexMessagesFrom(start) {
+        for (let index = Math.max(0, start); index < this._messages.length; index += 1) {
+            const key = messageIndexKey(this._messages[index].message);
+            if (key != null) {
+                this.messageIndexes.set(key, index);
+            }
+        }
+    }
+    applyActiveModel(response) {
+        for (const provider of response.providers) {
+            for (const model of provider.models) {
+                if (!model.active) {
+                    continue;
+                }
+                this.currentModelValue = new AgentModelChanged({
+                    threadId: this.threadPath,
+                    sourceMessageId: response.sourceMessageId,
+                    backend: provider.backend,
+                    provider: provider.name,
+                    model: model.name,
+                    voice: model.defaultOutputVoice,
+                    inputFormat: model.inputFormat,
+                    outputFormat: model.outputFormat,
+                    turnDetection: model.turnDetection,
+                    realtimeProtocols: model.realtimeProtocols,
+                    outputModalities: model.modalities.length > 0 ? [model.modalities[0]] : ["text"],
+                });
+                return;
+            }
         }
-        this.notifyListeners();
     }
     markClosed({ notify = true } = {}) {
         this.openState = false;
@@ -626,6 +979,84 @@ function sourceMessageIdFor(message) {
     const value = "sourceMessageId" in message ? message.sourceMessageId : undefined;
     return typeof value === "string" ? value : undefined;
 }
+function threadMessageValue(message, key) {
+    const value = message[key];
+    return typeof value === "string" ? value : undefined;
+}
+function agentInputText(content) {
+    if (!Array.isArray(content)) {
+        return "";
+    }
+    return content
+        .filter((entry) => (typeof entry === "object" &&
+        entry !== null &&
+        "type" in entry &&
+        entry.type === "text" &&
+        "text" in entry &&
+        typeof entry.text === "string"))
+        .map((entry) => entry.text)
+        .join("");
+}
+function isMergeableDelta(message) {
+    return message.type === agentAudioGenerationDeltaType ||
+        message.type === agentAudioTranscriptionDeltaType ||
+        message.type === agentFileContentDeltaType ||
+        message.type === agentReasoningContentDeltaType ||
+        message.type === agentTextContentDeltaType ||
+        message.type === agentToolCallArgumentsDeltaType ||
+        message.type === agentToolCallLogDeltaType;
+}
+function messageIndexKey(message) {
+    if (isMergeableDelta(message)) {
+        const itemId = normalizedString(message.itemId);
+        return itemId == null ? undefined : `${message.type}:${itemId}`;
+    }
+    return normalizedString(message.messageId);
+}
+function mergeDeltaMessage(existing, incoming) {
+    if (existing.type !== incoming.type) {
+        return undefined;
+    }
+    if (existing.type === agentAudioGenerationDeltaType && existing instanceof AgentAudioGenerationDelta && incoming instanceof AgentAudioGenerationDelta) {
+        const data = new Uint8Array(existing.data.length + incoming.data.length);
+        data.set(existing.data, 0);
+        data.set(incoming.data, existing.data.length);
+        return new AgentAudioGenerationDelta({ ...existing.toJson(), data });
+    }
+    if (existing.type === agentAudioTranscriptionDeltaType && existing instanceof AgentAudioTranscriptionDelta && incoming instanceof AgentAudioTranscriptionDelta) {
+        return new AgentAudioTranscriptionDelta({ ...existing.toJson(), text: `${existing.text}${incoming.text}` });
+    }
+    if (existing.type === agentFileContentDeltaType && incoming instanceof AgentFileContentDelta) {
+        return incoming;
+    }
+    if (existing.type === agentReasoningContentDeltaType && existing instanceof AgentReasoningContentDelta && incoming instanceof AgentReasoningContentDelta) {
+        return new AgentReasoningContentDelta({ ...existing.toJson(), text: `${existing.text}${incoming.text}` });
+    }
+    if (existing.type === agentTextContentDeltaType && existing instanceof AgentTextContentDelta && incoming instanceof AgentTextContentDelta) {
+        return new AgentTextContentDelta({ ...existing.toJson(), text: `${existing.text}${incoming.text}` });
+    }
+    if (existing.type === agentToolCallArgumentsDeltaType && existing instanceof AgentToolCallArgumentsDelta && incoming instanceof AgentToolCallArgumentsDelta) {
+        return new AgentToolCallArgumentsDelta({ ...existing.toJson(), delta: `${existing.delta}${incoming.delta}` });
+    }
+    if (existing.type === agentToolCallLogDeltaType && existing instanceof AgentToolCallLogDelta && incoming instanceof AgentToolCallLogDelta) {
+        return new AgentToolCallLogDelta({ ...existing.toJson(), lines: [...existing.lines, ...incoming.lines] });
+    }
+    return undefined;
+}
+function normalizedString(value) {
+    const normalized = value?.trim();
+    return normalized == null || normalized === "" ? undefined : normalized;
+}
+function dateOrNull(value) {
+    if (value instanceof Date && !Number.isNaN(value.getTime())) {
+        return value;
+    }
+    if (typeof value !== "string" || value.trim() === "") {
+        return undefined;
+    }
+    const parsed = new Date(value);
+    return Number.isNaN(parsed.getTime()) ? undefined : parsed;
+}
 function nonEmptyTrimmed(value) {
     const normalized = value?.trim();
     return normalized == null || normalized === "" ? undefined : normalized;