@vertesia/ui 1.4.0-dev.20260615.042549Z → 1.4.0-dev.20260629.130134Z

package/src/features/agent/chat/ModernAgentOutput/utils.ts

@@ -1,6 +1,36 @@
 import type { VertesiaClient } from '@vertesia/client';
 import { type AgentMessage, AgentMessageType } from '@vertesia/common';
 import dayjs from 'dayjs';
+import { getWorkstreamActivityDetails, getWorkstreamLaunchDetails } from '../workstreams.js';
+
+function getAgentChatDebugParam(hash: string): string | null {
+    const queryStart = hash.indexOf('?');
+    if (queryStart === -1) return null;
+    return new URLSearchParams(hash.slice(queryStart + 1)).get('agentChatDebug');
+}
+
+export function isAgentChatDebugEnabled(): boolean {
+    if (typeof window === 'undefined') return false;
+
+    const searchValue = new URLSearchParams(window.location.search).get('agentChatDebug');
+    const hashValue = getAgentChatDebugParam(window.location.hash);
+    const urlValue = searchValue ?? hashValue;
+    if (urlValue !== null) {
+        return urlValue === '1' || urlValue === 'true';
+    }
+
+    try {
+        const stored = window.localStorage.getItem('agentChatDebug');
+        return stored === '1' || stored === 'true';
+    } catch {
+        return false;
+    }
+}
+
+export function debugAgentChat(label: string, details?: Record<string, unknown>) {
+    if (!isAgentChatDebugEnabled()) return;
+    console.debug(`[agent-chat] ${label}`, details ?? {});
+}
 
 export function insertMessageInTimeline(arr: AgentMessage[], m: AgentMessage) {
     const t = typeof m.timestamp === 'number' ? m.timestamp : new Date(m.timestamp).getTime();
@@ -28,14 +58,22 @@ export const DONE_STATES = [
 export function isInProgress(messages: AgentMessage[]) {
     if (!messages.length) return true;
 
+    const isTerminalOptimisticFailure = (message: AgentMessage) =>
+        message.details?._optimistic &&
+        message.details?._deliveryStatus === 'failed' &&
+        DONE_STATES.includes(message.type);
+
+    const progressMessages = messages.filter((m) => !isTerminalOptimisticFailure(m));
+
     // Only the main workstream determines whether the conversation is in progress.
     // Child workstream COMPLETE/IDLE messages must not flip this flag.
-    const mainMessages = messages.filter((m) => getWorkstreamId(m) === 'main');
+    const mainMessages = progressMessages.filter((m) => getWorkstreamId(m) === 'main');
 
     // If there are no main workstream messages yet, check if there's exactly one
     // workstream — treat it as main (handles single-workstream conversations).
     if (mainMessages.length === 0) {
-        const lastMessage = messages[messages.length - 1];
+        const lastMessage = progressMessages[progressMessages.length - 1];
+        if (!lastMessage) return true;
         return !DONE_STATES.includes(lastMessage.type);
     }
 
@@ -43,6 +81,19 @@ export function isInProgress(messages: AgentMessage[]) {
     return !DONE_STATES.includes(lastMainMessage.type);
 }
 
+export function isUserStoppedMessage(message: AgentMessage): boolean {
+    if (message.type !== AgentMessageType.IDLE) return false;
+
+    const statusReason = message.details?.status_reason;
+    const displayRole = message.details?.display_role;
+    if (statusReason === 'user_stopped' || displayRole === 'user_stopped') return true;
+
+    return String(message.message ?? '')
+        .trim()
+        .toLowerCase()
+        .startsWith('stopped.');
+}
+
 export const formatRelative = (ts: number | string) =>
     typeof ts === 'number' ? dayjs(ts).fromNow() : dayjs(new Date(ts)).fromNow();
 
@@ -65,6 +116,42 @@ export function getWorkstreamId(message: AgentMessage): string {
     return 'main';
 }
 
+function getDetailsRecord(message: AgentMessage): Record<string, unknown> {
+    const details = message.details;
+    if (!details || typeof details !== 'object' || Array.isArray(details)) return {};
+    return details as Record<string, unknown>;
+}
+
+export function isMainChatTimelineMessage(message: AgentMessage): boolean {
+    if (getWorkstreamId(message) === 'main') return true;
+
+    if (getWorkstreamLaunchDetails(message) || getWorkstreamActivityDetails(message)) return true;
+
+    const details = getDetailsRecord(message);
+    const hasChildWorkflow =
+        typeof details.child_workflow_id === 'string' || typeof details.child_workflow_run_id === 'string';
+    const hasTool = typeof details.tool === 'string' || typeof details.tool_run_id === 'string';
+    const isLegacyPreLaunchFailure =
+        details.event_class === 'activity' &&
+        (message.type === AgentMessageType.ERROR || message.type === AgentMessageType.WARNING) &&
+        !hasChildWorkflow &&
+        !hasTool;
+
+    return isLegacyPreLaunchFailure;
+}
+
+export function filterMessagesForActiveWorkstream(messages: AgentMessage[], activeWorkstream: string): AgentMessage[] {
+    if (activeWorkstream === 'all') {
+        return messages.filter(isMainChatTimelineMessage);
+    }
+
+    return messages.filter((message) => {
+        const workstreamStartDetails = getWorkstreamLaunchDetails(message) ?? getWorkstreamActivityDetails(message);
+        if (workstreamStartDetails?.workstreamId === activeWorkstream) return false;
+        return getWorkstreamId(message) === activeWorkstream;
+    });
+}
+
 export function isSummaryVisibleToolActivity(message: AgentMessage): boolean {
     if (!isToolActivityMessage(message)) return false;
 
@@ -224,6 +311,7 @@ export interface StreamingData {
     isComplete?: boolean;
     startTimestamp: number;
     activityId?: string;
+    streamingId?: string;
 }
 
 function normalizeComparableText(text: unknown): string | undefined {
@@ -247,6 +335,19 @@ export function isStreamReplacedByMessage(streaming: StreamingData, messages: Ag
         if (messageTimestamp < streaming.startTimestamp) return false;
         if (getWorkstreamId(message) !== streamWorkstreamId) return false;
 
+        const messageStreamingId =
+            typeof message.details?.streaming_id === 'string' ? message.details.streaming_id : undefined;
+        if (streaming.streamingId && messageStreamingId) {
+            if (messageStreamingId !== streaming.streamingId) return false;
+            if (message.details?.display_role === 'thinking') {
+                return false;
+            }
+            if (isToolCallMessage(message) || message.details?.tool_status) {
+                return false;
+            }
+            return true;
+        }
+
         if (streaming.activityId && message.details?.activity_id === streaming.activityId) {
             if (message.details?.display_role === 'thinking') {
                 return false;

package/src/features/agent/chat/clipboardFiles.ts

@@ -0,0 +1,33 @@
+/**
+ * Extract pasteable files from a clipboard event's `DataTransferItemList`.
+ *
+ * Used by both `MessageInput` (live chat) and `StartWorkflowView` (start screen)
+ * to handle paste-to-attach. Callers diverge on what they do with the result —
+ * the live chat uploads immediately via a parent callback; the start screen
+ * stages files locally until the workflow starts — so this helper only covers
+ * the shared extraction step.
+ *
+ * Pasted images often arrive without a meaningful filename (most browsers
+ * surface them as `image.png`). Rename those to `pasted-image-<timestamp>.<ext>`
+ * so they sort and display sensibly in the staged-file chip list.
+ */
+export function extractFilesFromClipboard(items: DataTransferItemList | undefined): File[] {
+    if (!items) return [];
+
+    const files: File[] = [];
+    for (let i = 0; i < items.length; i++) {
+        const item = items[i];
+        if (item.kind !== 'file') continue;
+
+        const file = item.getAsFile();
+        if (!file) continue;
+
+        if (item.type.startsWith('image/') && (!file.name || file.name === 'image.png')) {
+            const extension = item.type.split('/')[1] || 'png';
+            files.push(new File([file], `pasted-image-${Date.now()}.${extension}`, { type: file.type }));
+        } else {
+            files.push(file);
+        }
+    }
+    return files;
+}

package/src/features/agent/chat/hooks/useAgentStream.test.tsx

@@ -1,5 +1,5 @@
 import { act, renderHook, waitFor } from '@testing-library/react';
-import type { VertesiaClient } from '@vertesia/client';
+import type { AgentRunStreamMessagesOptions, VertesiaClient } from '@vertesia/client';
 import { type AgentMessage, AgentMessageType, FileProcessingStatus } from '@vertesia/common';
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 import { useAgentStream } from './useAgentStream';
@@ -11,6 +11,7 @@ type StreamMessagesMock = ReturnType<
             onMessage?: (message: AgentMessage, exitFn?: (payload: unknown) => void) => void,
             since?: number,
             signal?: AbortSignal,
+            options?: AgentRunStreamMessagesOptions,
         ) => Promise<unknown>
     >
 >;
@@ -42,6 +43,109 @@ describe('useAgentStream', () => {
         vi.clearAllMocks();
     });
 
+    it('marks initial history as empty when the history fetch returns no messages', async () => {
+        const streamMessages = vi.fn<
+            (
+                id: string,
+                onMessage?: (message: AgentMessage, exitFn?: (payload: unknown) => void) => void,
+                since?: number,
+                signal?: AbortSignal,
+                options?: AgentRunStreamMessagesOptions,
+            ) => Promise<unknown>
+        >(async (_id, _onMessage, _since, _signal, options) => {
+            options?.onHistoryLoaded?.([]);
+            return null;
+        });
+        const client = createClient(streamMessages);
+
+        const { result } = renderHook(() => useAgentStream(client, 'agent-run-1'));
+
+        await waitFor(() => {
+            expect(result.current.initialHistoryStatus).toBe('empty');
+        });
+    });
+
+    it('marks initial history as present when the history fetch returns messages', async () => {
+        const historicalMessage = createMessage(AgentMessageType.ANSWER, 1_000, 'from history');
+        const streamMessages = vi.fn<
+            (
+                id: string,
+                onMessage?: (message: AgentMessage, exitFn?: (payload: unknown) => void) => void,
+                since?: number,
+                signal?: AbortSignal,
+                options?: AgentRunStreamMessagesOptions,
+            ) => Promise<unknown>
+        >(async (_id, onMessage, _since, _signal, options) => {
+            options?.onHistoryLoaded?.([historicalMessage]);
+            onMessage?.(historicalMessage);
+            return null;
+        });
+        const client = createClient(streamMessages);
+
+        const { result } = renderHook(() => useAgentStream(client, 'agent-run-1'));
+
+        await waitFor(() => {
+            expect(result.current.initialHistoryStatus).toBe('has_messages');
+            expect(result.current.messages.map((message) => message.message)).toEqual(['from history']);
+        });
+    });
+
+    it('seeds messages from loaded history even if the stream callback does not replay them', async () => {
+        const historicalMessages = [
+            createMessage(AgentMessageType.QUESTION, 1_000, 'question from history'),
+            createMessage(AgentMessageType.THOUGHT, 1_100, 'thinking from history'),
+            createMessage(AgentMessageType.ANSWER, 1_200, 'answer from history'),
+            createMessage(AgentMessageType.IDLE, 1_300, 'Waiting for your command...'),
+        ];
+        const streamMessages = vi.fn<
+            (
+                id: string,
+                onMessage?: (message: AgentMessage, exitFn?: (payload: unknown) => void) => void,
+                since?: number,
+                signal?: AbortSignal,
+                options?: AgentRunStreamMessagesOptions,
+            ) => Promise<unknown>
+        >(async (_id, _onMessage, _since, _signal, options) => {
+            options?.onHistoryLoaded?.(historicalMessages);
+            return null;
+        });
+        const client = createClient(streamMessages);
+
+        const { result } = renderHook(() => useAgentStream(client, 'agent-run-1'));
+
+        await waitFor(() => {
+            expect(result.current.initialHistoryStatus).toBe('has_messages');
+            expect(result.current.messages.map((message) => message.message)).toEqual([
+                'question from history',
+                'thinking from history',
+                'answer from history',
+                'Waiting for your command...',
+            ]);
+        });
+    });
+
+    it('marks initial history as errored when the history fetch fails', async () => {
+        const streamMessages = vi.fn<
+            (
+                id: string,
+                onMessage?: (message: AgentMessage, exitFn?: (payload: unknown) => void) => void,
+                since?: number,
+                signal?: AbortSignal,
+                options?: AgentRunStreamMessagesOptions,
+            ) => Promise<unknown>
+        >(async (_id, _onMessage, _since, _signal, options) => {
+            options?.onHistoryError?.(new Error('history failed'));
+            return null;
+        });
+        const client = createClient(streamMessages);
+
+        const { result } = renderHook(() => useAgentStream(client, 'agent-run-1'));
+
+        await waitFor(() => {
+            expect(result.current.initialHistoryStatus).toBe('error');
+        });
+    });
+
     it('reconnects the same agent run from the last delivered timestamp without clearing messages', async () => {
         const complete = createMessage(AgentMessageType.COMPLETE, 1_000, 'done');
         const answer = createMessage(AgentMessageType.ANSWER, 1_100, 'continued');
@@ -133,7 +237,7 @@ describe('useAgentStream', () => {
         });
     });
 
-    it('dedupes re-delivered messages by timestamp and replaces optimistic questions', async () => {
+    it('dedupes re-delivered messages by timestamp and replaces ack-matched optimistic questions', async () => {
         let onStreamMessage: ((message: AgentMessage) => void) | undefined;
         const streamMessages = vi.fn<
             (
@@ -157,7 +261,7 @@ describe('useAgentStream', () => {
         act(() => {
             result.current.addOptimisticMessage({
                 ...createMessage(AgentMessageType.QUESTION, 1_000, 'optimistic'),
-                details: { _optimistic: true, _messageId: 'message-1' },
+                details: { _optimistic: true, _messageId: 'message-1', _deliveryStatus: 'sending' },
             });
         });
 
@@ -166,12 +270,13 @@ describe('useAgentStream', () => {
         act(() => {
             onStreamMessage?.({
                 ...createMessage(AgentMessageType.QUESTION, 1_100, 'server question'),
-                details: { _messageId: 'message-1' },
+                details: { ack: 'message-1' },
             });
         });
 
         await waitFor(() => {
             expect(result.current.messages.map((message) => message.message)).toEqual(['server question']);
+            expect(result.current.messages[0]?.details?._deliveryStatus).toBe('consumed');
         });
 
         const answer = createMessage(AgentMessageType.ANSWER, 1_200, 'answer');
@@ -185,6 +290,252 @@ describe('useAgentStream', () => {
         });
     });
 
+    it('replaces an ack-matched optimistic stop marker when the workflow consumes Stop', async () => {
+        let onStreamMessage: ((message: AgentMessage) => void) | undefined;
+        const streamMessages = vi.fn<
+            (
+                id: string,
+                onMessage?: (message: AgentMessage, exitFn?: (payload: unknown) => void) => void,
+                since?: number,
+                signal?: AbortSignal,
+            ) => Promise<unknown>
+        >(async (_id, onMessage) => {
+            onStreamMessage = onMessage;
+            return null;
+        });
+        const client = createClient(streamMessages);
+
+        const { result } = renderHook(() => useAgentStream(client, 'agent-run-1'));
+
+        await waitFor(() => {
+            expect(onStreamMessage).toBeDefined();
+        });
+
+        act(() => {
+            result.current.addOptimisticMessage({
+                ...createMessage(AgentMessageType.IDLE, 1_000, 'Stopped. Waiting for your command...'),
+                details: {
+                    _optimistic: true,
+                    _messageId: 'stop-1',
+                    _deliveryStatus: 'received',
+                    status_reason: 'user_stopped',
+                },
+            });
+        });
+
+        act(() => {
+            onStreamMessage?.({
+                ...createMessage(AgentMessageType.IDLE, 1_100, 'Stopped. Waiting for your command...'),
+                details: { ack: 'stop-1', status_reason: 'user_stopped' },
+            });
+        });
+
+        await waitFor(() => {
+            expect(result.current.messages).toHaveLength(1);
+            expect(result.current.messages[0]?.timestamp).toBe(1_100);
+            expect(result.current.messages[0]?.details?._deliveryStatus).toBe('consumed');
+        });
+    });
+
+    it('keeps non-matching optimistic questions when an acknowledged server question arrives', async () => {
+        let onStreamMessage: ((message: AgentMessage) => void) | undefined;
+        const streamMessages = vi.fn<
+            (
+                id: string,
+                onMessage?: (message: AgentMessage, exitFn?: (payload: unknown) => void) => void,
+                since?: number,
+                signal?: AbortSignal,
+            ) => Promise<unknown>
+        >(async (_id, onMessage) => {
+            onStreamMessage = onMessage;
+            return null;
+        });
+        const client = createClient(streamMessages);
+
+        const { result } = renderHook(() => useAgentStream(client, 'agent-run-1'));
+
+        await waitFor(() => {
+            expect(onStreamMessage).toBeDefined();
+        });
+
+        act(() => {
+            result.current.addOptimisticMessage({
+                ...createMessage(AgentMessageType.QUESTION, 1_000, 'first optimistic'),
+                details: { _optimistic: true, _messageId: 'message-1', _deliveryStatus: 'sending' },
+            });
+            result.current.addOptimisticMessage({
+                ...createMessage(AgentMessageType.QUESTION, 1_050, 'second optimistic'),
+                details: { _optimistic: true, _messageId: 'message-2', _deliveryStatus: 'sending' },
+            });
+        });
+
+        act(() => {
+            onStreamMessage?.({
+                ...createMessage(AgentMessageType.QUESTION, 1_100, 'server second question'),
+                details: { ack: 'message-2' },
+            });
+        });
+
+        await waitFor(() => {
+            expect(result.current.messages.map((message) => message.message)).toEqual([
+                'first optimistic',
+                'server second question',
+            ]);
+        });
+        expect(result.current.messages[0]?.details?._deliveryStatus).toBe('sending');
+        expect(result.current.messages[1]?.details?._deliveryStatus).toBe('consumed');
+    });
+
+    it('updates optimistic message delivery status', async () => {
+        let onStreamMessage: ((message: AgentMessage) => void) | undefined;
+        const streamMessages = vi.fn<
+            (
+                id: string,
+                onMessage?: (message: AgentMessage, exitFn?: (payload: unknown) => void) => void,
+                since?: number,
+                signal?: AbortSignal,
+            ) => Promise<unknown>
+        >(async (_id, onMessage) => {
+            onStreamMessage = onMessage;
+            return null;
+        });
+        const client = createClient(streamMessages);
+
+        const { result } = renderHook(() => useAgentStream(client, 'agent-run-1'));
+
+        await waitFor(() => {
+            expect(onStreamMessage).toBeDefined();
+        });
+
+        act(() => {
+            result.current.addOptimisticMessage({
+                ...createMessage(AgentMessageType.QUESTION, 1_000, 'optimistic'),
+                details: { _optimistic: true, _messageId: 'message-1', _deliveryStatus: 'sending' },
+            });
+        });
+
+        act(() => {
+            result.current.updateOptimisticMessageStatus('message-1', 'received');
+        });
+
+        expect(result.current.messages[0]?.details?._deliveryStatus).toBe('received');
+    });
+
+    it('uses workflow-run scoped streaming ids so reused activity ids do not collide', async () => {
+        let onStreamMessage: ((message: AgentMessage) => void) | undefined;
+        const streamMessages = vi.fn<
+            (
+                id: string,
+                onMessage?: (message: AgentMessage, exitFn?: (payload: unknown) => void) => void,
+                since?: number,
+                signal?: AbortSignal,
+            ) => Promise<unknown>
+        >(async (_id, onMessage) => {
+            onStreamMessage = onMessage;
+            return null;
+        });
+        const client = createClient(streamMessages);
+
+        const { result } = renderHook(() => useAgentStream(client, 'agent-run-1'));
+
+        await waitFor(() => {
+            expect(onStreamMessage).toBeDefined();
+        });
+
+        act(() => {
+            onStreamMessage?.({
+                ...createMessage(AgentMessageType.STREAMING_CHUNK, 1_000, 'first'),
+                details: {
+                    streaming_id: 'run-a-activity-7',
+                    streaming_id_scope: 'workflow_run',
+                    activity_id: 'activity-7',
+                    is_final: false,
+                },
+            });
+            onStreamMessage?.({
+                ...createMessage(AgentMessageType.STREAMING_CHUNK, 1_010, 'second'),
+                details: {
+                    streaming_id: 'run-b-activity-7',
+                    streaming_id_scope: 'workflow_run',
+                    activity_id: 'activity-7',
+                    is_final: false,
+                },
+            });
+        });
+
+        await waitFor(() => {
+            expect([...result.current.streamingMessages.keys()].sort()).toEqual([
+                'run-a-activity-7',
+                'run-b-activity-7',
+            ]);
+        });
+
+        act(() => {
+            onStreamMessage?.({
+                ...createMessage(AgentMessageType.ANSWER, 1_100, 'final second'),
+                details: {
+                    streaming_id: 'run-b-activity-7',
+                    streaming_id_scope: 'workflow_run',
+                    activity_id: 'activity-7',
+                },
+            });
+        });
+
+        await waitFor(() => {
+            expect([...result.current.streamingMessages.keys()]).toEqual(['run-a-activity-7']);
+        });
+    });
+
+    it('keeps legacy activity-id cleanup for streaming chunks without scoped streaming ids', async () => {
+        let onStreamMessage: ((message: AgentMessage) => void) | undefined;
+        const streamMessages = vi.fn<
+            (
+                id: string,
+                onMessage?: (message: AgentMessage, exitFn?: (payload: unknown) => void) => void,
+                since?: number,
+                signal?: AbortSignal,
+            ) => Promise<unknown>
+        >(async (_id, onMessage) => {
+            onStreamMessage = onMessage;
+            return null;
+        });
+        const client = createClient(streamMessages);
+
+        const { result } = renderHook(() => useAgentStream(client, 'agent-run-1'));
+
+        await waitFor(() => {
+            expect(onStreamMessage).toBeDefined();
+        });
+
+        act(() => {
+            onStreamMessage?.({
+                ...createMessage(AgentMessageType.STREAMING_CHUNK, 1_000, 'legacy'),
+                details: {
+                    streaming_id: 'main',
+                    activity_id: 'legacy-activity',
+                    is_final: false,
+                },
+            });
+        });
+
+        await waitFor(() => {
+            expect([...result.current.streamingMessages.keys()]).toEqual(['legacy-activity']);
+        });
+
+        act(() => {
+            onStreamMessage?.({
+                ...createMessage(AgentMessageType.ANSWER, 1_100, 'legacy final'),
+                details: {
+                    activity_id: 'legacy-activity',
+                },
+            });
+        });
+
+        await waitFor(() => {
+            expect(result.current.streamingMessages.size).toBe(0);
+        });
+    });
+
     it('replaces file processing status with the latest server snapshot', async () => {
         let onStreamMessage: ((message: AgentMessage) => void) | undefined;
         const streamMessages = vi.fn<