@vertesia/ui 0.81.1 → 1.0.0-dev.20260203.130115Z

package/src/features/agent/chat/ModernAgentOutput/utils.ts

@@ -200,4 +200,333 @@ export async function getConversationUrl(
     return vertesia.files
         .getDownloadUrl(`agents/${workflowRunId}/conversation.json`)
         .then((r) => r.url);
+}
+
+/**
+ * Represents a group of messages for display purposes
+ * Either a single message or a group of consecutive tool calls
+ */
+export type MessageGroup =
+    | { type: 'single'; message: AgentMessage }
+    | { type: 'tool_group'; messages: AgentMessage[]; firstTimestamp: number | string };
+
+/**
+ * Streaming message data structure
+ */
+export interface StreamingData {
+    text: string;
+    workstreamId?: string;
+    isComplete?: boolean;
+    startTimestamp: number;
+}
+
+/**
+ * Tool execution status for display purposes
+ */
+export type ToolExecutionStatus = "running" | "completed" | "error" | "warning";
+
+/**
+ * Extended group type that includes streaming messages for interleaved rendering
+ */
+export type RenderableGroup =
+    | { type: 'single'; message: AgentMessage }
+    | { type: 'tool_group'; messages: AgentMessage[]; firstTimestamp: number; toolRunId?: string; toolStatus?: ToolExecutionStatus }
+    | { type: 'streaming'; streamingId: string; text: string; workstreamId?: string; startTimestamp: number; isComplete?: boolean };
+
+/**
+ * Check if a message is a tool call (THOUGHT with tool details)
+ */
+export function isToolCallMessage(message: AgentMessage): boolean {
+    return message.type === AgentMessageType.THOUGHT && !!message.details?.tool;
+}
+
+/**
+ * Get the tool_run_id from a message's details, if present
+ */
+export function getToolRunId(message: AgentMessage): string | undefined {
+    return message.details?.tool_run_id;
+}
+
+/**
+ * Get the tool_iteration from a message's details, if present
+ * This is used to group parallel tool calls from the same iteration
+ */
+export function getToolIteration(message: AgentMessage): number | undefined {
+    return message.details?.tool_iteration;
+}
+
+/**
+ * Get the tool execution status from a message's details
+ */
+export function getToolStatus(message: AgentMessage): ToolExecutionStatus | undefined {
+    return message.details?.tool_status;
+}
+
+/**
+ * Group consecutive tool call messages together for a cleaner display
+ * Non-tool messages remain as single items
+ *
+ * @param messages Sorted array of messages
+ * @returns Array of message groups
+ */
+export function groupConsecutiveToolCalls(messages: AgentMessage[]): MessageGroup[] {
+    const groups: MessageGroup[] = [];
+    let currentToolGroup: AgentMessage[] = [];
+
+    const flushToolGroup = () => {
+        if (currentToolGroup.length > 0) {
+            if (currentToolGroup.length === 1) {
+                // Single tool call - no need to group
+                groups.push({ type: 'single', message: currentToolGroup[0] });
+            } else {
+                // Multiple consecutive tool calls - group them
+                groups.push({
+                    type: 'tool_group',
+                    messages: currentToolGroup,
+                    firstTimestamp: currentToolGroup[0].timestamp
+                });
+            }
+            currentToolGroup = [];
+        }
+    };
+
+    for (const message of messages) {
+        if (isToolCallMessage(message)) {
+            currentToolGroup.push(message);
+        } else {
+            // Flush any pending tool group before adding non-tool message
+            flushToolGroup();
+            groups.push({ type: 'single', message });
+        }
+    }
+
+    // Flush any remaining tool group
+    flushToolGroup();
+
+    return groups;
+}
+
+/**
+ * Helper to get timestamp as number
+ */
+function getTimestampMs(timestamp: number | string): number {
+    return typeof timestamp === "number" ? timestamp : new Date(timestamp).getTime();
+}
+
+/**
+ * Group messages with streaming messages interleaved in chronological order
+ * Messages with the same tool_iteration are grouped together (parallel tool calls)
+ * Messages with the same tool_run_id are grouped together (same tool execution)
+ * Messages without either fall back to consecutive grouping
+ *
+ * @param messages Sorted array of messages
+ * @param streamingMessages Map of streaming messages by ID
+ * @param activeWorkstream Optional workstream filter
+ * @returns Array of renderable groups with streaming interleaved
+ */
+export function groupMessagesWithStreaming(
+    messages: AgentMessage[],
+    streamingMessages: Map<string, StreamingData>,
+    activeWorkstream?: string
+): RenderableGroup[] {
+    // First pass: collect messages by tool_iteration (for parallel tool calls in same iteration)
+    const iterationGroups = new Map<number, {
+        messages: AgentMessage[];
+        firstTimestamp: number;
+    }>();
+
+    // Also collect messages by tool_run_id (for sub-messages within same tool)
+    const toolRunGroups = new Map<string, {
+        messages: AgentMessage[];
+        firstTimestamp: number;
+    }>();
+
+    // Messages without tool_iteration or tool_run_id will be processed separately
+    const standaloneMessages: AgentMessage[] = [];
+
+    for (const message of messages) {
+        if (isToolCallMessage(message)) {
+            const toolIteration = getToolIteration(message);
+            const toolRunId = getToolRunId(message);
+
+            if (toolIteration !== undefined) {
+                // Group by tool_iteration - this groups parallel tool calls together
+                if (!iterationGroups.has(toolIteration)) {
+                    iterationGroups.set(toolIteration, {
+                        messages: [],
+                        firstTimestamp: getTimestampMs(message.timestamp)
+                    });
+                }
+                iterationGroups.get(toolIteration)!.messages.push(message);
+            } else if (toolRunId) {
+                // Fallback: group by tool_run_id if no iteration
+                if (!toolRunGroups.has(toolRunId)) {
+                    toolRunGroups.set(toolRunId, {
+                        messages: [],
+                        firstTimestamp: getTimestampMs(message.timestamp)
+                    });
+                }
+                toolRunGroups.get(toolRunId)!.messages.push(message);
+            } else {
+                // No tool_iteration or tool_run_id - will use consecutive grouping
+                standaloneMessages.push(message);
+            }
+        } else {
+            standaloneMessages.push(message);
+        }
+    }
+
+    // Create unified list of items with timestamps
+    type GroupedItem =
+        | { kind: 'message'; message: AgentMessage; timestamp: number }
+        | { kind: 'streaming'; streamingId: string; data: StreamingData; timestamp: number }
+        | { kind: 'iteration_group'; iteration: number; messages: AgentMessage[]; timestamp: number }
+        | { kind: 'tool_run'; toolRunId: string; messages: AgentMessage[]; timestamp: number };
+
+    const items: GroupedItem[] = [];
+
+    // Add iteration groups (parallel tool calls from same iteration)
+    iterationGroups.forEach((group, iteration) => {
+        items.push({
+            kind: 'iteration_group',
+            iteration,
+            messages: group.messages,
+            timestamp: group.firstTimestamp
+        });
+    });
+
+    // Add tool_run groups as single items
+    toolRunGroups.forEach((group, toolRunId) => {
+        items.push({
+            kind: 'tool_run',
+            toolRunId,
+            messages: group.messages,
+            timestamp: group.firstTimestamp
+        });
+    });
+
+    // Add standalone messages
+    for (const message of standaloneMessages) {
+        items.push({
+            kind: 'message',
+            message,
+            timestamp: getTimestampMs(message.timestamp)
+        });
+    }
+
+    // Add streaming messages (filter by workstream if specified)
+    streamingMessages.forEach((data, streamingId) => {
+        // Skip empty streaming messages
+        if (!data.text) return;
+
+        // Filter by workstream if specified
+        if (activeWorkstream && activeWorkstream !== "all") {
+            const streamWorkstream = data.workstreamId || "main";
+            if (activeWorkstream !== streamWorkstream) return;
+        }
+
+        items.push({
+            kind: 'streaming',
+            streamingId,
+            data,
+            timestamp: data.startTimestamp
+        });
+    });
+
+    // Sort by timestamp, but incomplete streaming goes at the end
+    // Complete/final streaming messages stay at their chronological position
+    items.sort((a, b) => {
+        const aIsIncompleteStreaming = a.kind === 'streaming' && !a.data.isComplete;
+        const bIsIncompleteStreaming = b.kind === 'streaming' && !b.data.isComplete;
+
+        // Only incomplete streaming messages go last
+        if (aIsIncompleteStreaming && !bIsIncompleteStreaming) return 1;
+        if (bIsIncompleteStreaming && !aIsIncompleteStreaming) return -1;
+
+        // Both are same type (both incomplete streaming, both complete, or both non-streaming): sort by timestamp
+        return a.timestamp - b.timestamp;
+    });
+
+    // Build final groups with consecutive grouping for standalone tool messages
+    const groups: RenderableGroup[] = [];
+    let currentToolGroup: AgentMessage[] = [];
+
+    const flushToolGroup = () => {
+        if (currentToolGroup.length > 0) {
+            if (currentToolGroup.length === 1) {
+                groups.push({ type: 'single', message: currentToolGroup[0] });
+            } else {
+                groups.push({
+                    type: 'tool_group',
+                    messages: currentToolGroup,
+                    firstTimestamp: getTimestampMs(currentToolGroup[0].timestamp)
+                });
+            }
+            currentToolGroup = [];
+        }
+    };
+
+    for (const item of items) {
+        if (item.kind === 'streaming') {
+            // Streaming breaks consecutive tool grouping
+            flushToolGroup();
+            groups.push({
+                type: 'streaming',
+                streamingId: item.streamingId,
+                text: item.data.text,
+                workstreamId: item.data.workstreamId,
+                startTimestamp: item.data.startTimestamp,
+                isComplete: item.data.isComplete
+            });
+        } else if (item.kind === 'iteration_group') {
+            // Iteration group - parallel tool calls from same iteration
+            flushToolGroup();
+            // Sort messages within the group by timestamp
+            const sortedMessages = [...item.messages].sort(
+                (a, b) => getTimestampMs(a.timestamp) - getTimestampMs(b.timestamp)
+            );
+            // Get the latest status from the group
+            const latestStatus = sortedMessages.reduce<ToolExecutionStatus | undefined>(
+                (status, msg) => getToolStatus(msg) || status,
+                undefined
+            );
+            groups.push({
+                type: 'tool_group',
+                messages: sortedMessages,
+                firstTimestamp: item.timestamp,
+                toolStatus: latestStatus
+            });
+        } else if (item.kind === 'tool_run') {
+            // Tool run group - already grouped by tool_run_id
+            flushToolGroup();
+            // Sort messages within the group by timestamp
+            const sortedMessages = [...item.messages].sort(
+                (a, b) => getTimestampMs(a.timestamp) - getTimestampMs(b.timestamp)
+            );
+            // Get the latest status from the group
+            const latestStatus = sortedMessages.reduce<ToolExecutionStatus | undefined>(
+                (status, msg) => getToolStatus(msg) || status,
+                undefined
+            );
+            groups.push({
+                type: 'tool_group',
+                messages: sortedMessages,
+                firstTimestamp: item.timestamp,
+                toolRunId: item.toolRunId,
+                toolStatus: latestStatus
+            });
+        } else if (isToolCallMessage(item.message)) {
+            // Standalone tool message - use consecutive grouping
+            currentToolGroup.push(item.message);
+        } else {
+            // Non-tool message breaks grouping
+            flushToolGroup();
+            groups.push({ type: 'single', message: item.message });
+        }
+    }
+
+    // Flush any remaining tool group
+    flushToolGroup();
+
+    return groups;
 }

package/src/features/agent/chat/SkillWidgetProvider.tsx

@@ -0,0 +1,151 @@
+import { VertesiaClient } from "@vertesia/client";
+import { normalizeToolCollection } from "@vertesia/common";
+import { FusionFragmentHandler } from "@vertesia/fusion-ux";
+import { useUserSession } from "@vertesia/ui/session";
+import { CodeBlockRendererProps, CodeBlockRendererProvider } from "@vertesia/ui/widgets";
+import { memo, useEffect, useMemo, useState } from "react";
+import { AgentChart, AgentChartSpec } from "./AgentChart";
+import { VegaLiteChart } from "./VegaLiteChart";
+
+interface SkillWidgetProviderProperties {
+    children: React.ReactNode;
+}
+
+/**
+ * Widget for rendering Recharts-based charts (bar, line, pie, etc.)
+ * Used for `chart` code blocks that use Recharts format
+ */
+const RechartsChartWidget = memo(function RechartsChartWidget({ code }: { code: string }) {
+    const spec = useMemo(() => {
+        try {
+            return JSON.parse(code) as AgentChartSpec;
+        } catch {
+            // During streaming, code may be incomplete JSON - return null to skip rendering
+            return null;
+        }
+    }, [code]);
+
+    // Don't render anything while JSON is incomplete (during streaming)
+    if (!spec) {
+        return null;
+    }
+
+    return <AgentChart spec={spec} />
+});
+
+/**
+ * Widget for rendering Vega-Lite charts directly
+ * Used for `vega-lite` and `vegalite` code blocks
+ */
+const VegaLiteChartWidget = memo(function VegaLiteChartWidget({ code }: { code: string }) {
+    const spec = useMemo(() => {
+        try {
+            const parsed = JSON.parse(code);
+            // Wrap native Vega-Lite spec in the expected format
+            return { library: 'vega-lite' as const, spec: parsed };
+        } catch {
+            // During streaming, code may be incomplete JSON - return null to skip rendering
+            return null;
+        }
+    }, [code]);
+
+    // Don't render anything while JSON is incomplete (during streaming)
+    if (!spec) {
+        return null;
+    }
+
+    // Render VegaLiteChart directly - bypass AgentChart routing
+    return <VegaLiteChart spec={spec} />
+});
+
+const defaultComponents: Record<string, React.FunctionComponent<CodeBlockRendererProps>> = {
+    "chart": RechartsChartWidget,
+    "vega-lite": VegaLiteChartWidget,
+    "vegalite": VegaLiteChartWidget,
+    "fusion-fragment": FusionFragmentHandler,
+}
+
+function RemoteWidgetComponent({ url, code }: { url: string, code: string }) {
+    const [Component, setComponent] = useState<React.FunctionComponent<{ code: string }> | null>(null);
+    useEffect(() => {
+        import(/* @vite-ignore */url).then(module => {
+            // register the component
+            // Wrap in arrow function to prevent React from calling it as a state updater
+            setComponent(() => module.default)
+        }).catch(err => {
+            console.error("Failed to load remote widget component from ", url, err);
+        })
+    }, [url]);
+    return Component ? <Component code={code} /> : null;
+}
+
+async function fetchSkillWidgets(client: VertesiaClient): Promise<Record<string, React.FunctionComponent<CodeBlockRendererProps>>> {
+    const installedApps = await client.apps.getInstalledApps("tools");
+    const urls = new Set<string>();
+    for (const app of installedApps) {
+        for (const item of app.manifest.tool_collections || []) {
+            const collection = normalizeToolCollection(item);
+            const collUrl = collection.url;
+            if (collUrl.startsWith("http://") || collUrl.startsWith("https://")) {
+                const i = collUrl.indexOf("/api/");
+                if (i > 0) {
+                    const url = collUrl.substring(0, i);
+                    urls.add(url + '/api/widgets');
+                }
+            }
+        }
+    }
+
+    const allWidgets = await Promise.all(Array.from(urls).map(url => {
+        return fetch(url).then(r => {
+            if (!r.ok) {
+                throw new Error(`Failed to fetch widgets: ${r.status} ${r.statusText}`);
+            }
+            return r.json();
+        }).then(data => {
+            const widgets: { name: string, component: React.FunctionComponent<CodeBlockRendererProps> }[] = [];
+            const widgetsMap = data.widgets as Record<string, { url: string }>;
+            for (const [name, value] of Object.entries(widgetsMap)) {
+                widgets.push({ name, component: (props: CodeBlockRendererProps) => <RemoteWidgetComponent url={value.url} code={props.code} /> });
+            }
+            return widgets;
+        }).catch(() => {
+            console.error("Failed to fetch skill widgets from ", url);
+            return null
+        });
+    }));
+
+    const widgets: Record<string, React.FunctionComponent<CodeBlockRendererProps>> = {};
+
+    for (const widgetSpec of allWidgets.flat()) {
+        if (widgetSpec) {
+            widgets[widgetSpec.name] = widgetSpec.component;
+        }
+    }
+    return widgets;
+}
+
+/**
+ * Provides code block components depending on lagauge form the installed skills
+ * @param param0
+ * @returns
+ */
+export function SkillWidgetProvider({ children }: SkillWidgetProviderProperties) {
+    const { client } = useUserSession();
+    const [components, setComponents] = useState<Record<string, React.FunctionComponent<CodeBlockRendererProps>>>(defaultComponents);
+    useEffect(() => {
+        // fetch all skill components
+        fetchSkillWidgets(client).then(widgets => {
+            setComponents({
+                ...defaultComponents,
+                ...widgets,
+            })
+        })
+    }, []);
+
+
+    return (
+        <CodeBlockRendererProvider components={components}>{children}</CodeBlockRendererProvider>
+    )
+
+}