@adecore/agents-react 0.17.0-beta.2 → 0.18.0

package/src/chat/ui/visual-theme.ts

@@ -0,0 +1,197 @@
+import { useLayoutEffect, useState, type RefObject } from 'react';
+import { cleanVisualVariables, type VisualAppearance, type VisualTheme, type VisualThemeVariable } from '@adecore/agent-contracts/visual';
+import { chatHost } from '../../host';
+
+/* The tokens of the app's theme where a frame stands; each answers '' for a token the theme lacks. */
+export interface VisualTokens {
+    /* As the engine computes the color, which a page's script parses where it would not parse a `color-mix()`. */
+    color(token: string): string;
+    /* In pixels, where the theme may write a length as a calculation. */
+    length(token: string): string;
+    /* As it is declared, such as a font stack. */
+    value(token: string): string;
+}
+
+/*
+ * A page's theme from the app's tokens. The names follow common component themes, where `--accent`
+ * is a quiet hover ground and `--primary` the color that stands out, so the app's accent is the
+ * primary, the ring and the first series of a chart. A variable without a token here, or with a
+ * token the app does not set, keeps the page's default for this appearance.
+ */
+export function visualThemeOf(tokens: VisualTokens, appearance: VisualAppearance, background: string): VisualTheme {
+    const text = tokens.color('--text');
+    const quiet = tokens.color('--surface-hover');
+    const raised = tokens.color('--surface-raised');
+    const accent = tokens.color('--accent');
+    const onAccent = tokens.color('--accent-text');
+    const variables: Partial<Record<VisualThemeVariable, string>> = {
+        '--background': background,
+        '--foreground': text,
+        '--muted': quiet,
+        '--muted-foreground': tokens.color('--text-muted'),
+        '--card': raised,
+        '--card-foreground': text,
+        '--popover': raised,
+        '--popover-foreground': text,
+        '--secondary': quiet,
+        '--secondary-foreground': text,
+        '--border': tokens.color('--border'),
+        '--input': tokens.color('--border-strong'),
+        '--ring': accent,
+        '--primary': accent,
+        '--primary-foreground': onAccent,
+        '--accent': quiet,
+        '--accent-foreground': text,
+        '--destructive': tokens.color('--status-error'),
+        '--destructive-foreground': onAccent,
+        '--warning': tokens.color('--status-needs-you'),
+        '--success': tokens.color('--positive'),
+        '--success-foreground': tokens.color('--positive-text'),
+        '--info': tokens.color('--status-running'),
+        '--code-background': tokens.color('--chat-code-bg'),
+        '--code-foreground': tokens.color('--chat-output'),
+        '--chart-1': tokens.color('--chart-1') || accent,
+        '--chart-2': tokens.color('--chart-2'),
+        '--chart-3': tokens.color('--chart-3'),
+        '--chart-4': tokens.color('--chart-4'),
+        '--chart-5': tokens.color('--chart-5'),
+        '--chart-6': tokens.color('--chart-6'),
+        '--radius': tokens.length('--radius-lg'),
+        '--font-sans': tokens.value('--font-sans'),
+        '--font-mono': tokens.value('--font-mono')
+    };
+    return { appearance, variables: cleanVisualVariables(variables) };
+}
+
+const RGB = /^rgba?\(\s*([\d.]+)[\s,]+([\d.]+)[\s,]+([\d.]+)\s*(?:[,/]\s*([\d.]+)(%?)\s*)?\)$/;
+
+interface Rgba {
+    red: number;
+    green: number;
+    blue: number;
+    alpha: number;
+}
+
+function rgbaOf(color: string): Rgba | null {
+    const match = RGB.exec(color.trim());
+    if (match === null) {
+        return null;
+    }
+    const alpha = match[4] === undefined ? 1 : Number(match[4]) / (match[5] === '%' ? 100 : 1);
+    return { red: Number(match[1]), green: Number(match[2]), blue: Number(match[3]), alpha };
+}
+
+function isTransparent(color: string): boolean {
+    return color === 'transparent' || rgbaOf(color)?.alpha === 0;
+}
+
+/*
+ * What lies behind an element, from the computed backgrounds of it and its ancestors, nearest first:
+ * the first one that covers what is under it, with every translucent one in front of it laid over it.
+ * Null when none of them paints.
+ */
+export function backgroundBehind(layers: readonly string[]): string | null {
+    const translucent: Rgba[] = [];
+    for (const layer of layers) {
+        if (layer === '' || isTransparent(layer)) {
+            continue;
+        }
+        const color = rgbaOf(layer);
+        if (color !== null && color.alpha < 1) {
+            translucent.push(color);
+            continue;
+        }
+        if (color === null || translucent.length === 0) {
+            // A color in a space other than sRGB stays as the engine wrote it, since there is no laying another over it here.
+            return color === null ? layer : `rgb(${color.red}, ${color.green}, ${color.blue})`;
+        }
+        let { red, green, blue } = color;
+        for (const over of translucent.reverse()) {
+            red = over.red * over.alpha + red * (1 - over.alpha);
+            green = over.green * over.alpha + green * (1 - over.alpha);
+            blue = over.blue * over.alpha + blue * (1 - over.alpha);
+        }
+        return `rgb(${Math.round(red)}, ${Math.round(green)}, ${Math.round(blue)})`;
+    }
+    return null;
+}
+
+function sameTheme(left: VisualTheme, right: VisualTheme): boolean {
+    return left.appearance === right.appearance && JSON.stringify(left.variables) === JSON.stringify(right.variables);
+}
+
+/*
+ * The theme where `element` stands. Colors and lengths are read off a probe in that very place, so a
+ * nested theme answers for itself and every value comes out the way the engine computes it.
+ */
+function readTheme(element: HTMLElement, mode: VisualAppearance): VisualTheme {
+    const view = element.ownerDocument.defaultView ?? window;
+    const style = view.getComputedStyle(element);
+    const probe = element.ownerDocument.createElement('span');
+    probe.style.position = 'absolute';
+    probe.style.visibility = 'hidden';
+    element.append(probe);
+    try {
+        const probed = view.getComputedStyle(probe);
+        const declared = (token: string): string => style.getPropertyValue(token).trim();
+        const computed = (property: 'color' | 'borderTopLeftRadius', token: string): string => {
+            if (declared(token) === '') {
+                return '';
+            }
+            probe.style[property] = `var(${token})`;
+            return probed[property];
+        };
+        const tokens: VisualTokens = {
+            color: (token) => computed('color', token),
+            length: (token) => computed('borderTopLeftRadius', token),
+            value: declared
+        };
+        const layers: string[] = [];
+        for (let node: Element | null = element; node !== null; node = node.parentElement) {
+            layers.push(view.getComputedStyle(node).backgroundColor);
+        }
+        const scheme = style.colorScheme.trim();
+        const appearance = scheme === 'light' || scheme === 'dark' ? scheme : mode;
+        return visualThemeOf(tokens, appearance, backgroundBehind(layers) ?? tokens.color('--surface'));
+    } finally {
+        probe.remove();
+    }
+}
+
+/* The attributes an app switches its theme or its accent with. */
+const THEME_ATTRIBUTES = ['class', 'style', 'data-theme'];
+
+/*
+ * The app's theme for a visual drawn in `ref`, null until the element is there. It is read again when
+ * the app turns light or dark, and when the root of the document changes a theme attribute, so an
+ * accent or a font a person picks reaches every frame as well.
+ */
+export function useVisualTheme(ref: RefObject<HTMLElement | null>): VisualTheme | null {
+    const mode = chatHost().code.useMode();
+    const [theme, setTheme] = useState<VisualTheme | null>(null);
+    useLayoutEffect(() => {
+        const element = ref.current;
+        if (element === null) {
+            return;
+        }
+        const read = (): void => {
+            const next = readTheme(element, mode);
+            setTheme((current) => (current !== null && sameTheme(current, next) ? current : next));
+        };
+        read();
+        const owner = element.ownerDocument;
+        const observer = new MutationObserver(read);
+        for (const root of [owner.documentElement, owner.body]) {
+            if (root !== null) {
+                observer.observe(root, { attributes: true, attributeFilter: THEME_ATTRIBUTES });
+            }
+        }
+        const scheme = owner.defaultView?.matchMedia?.('(prefers-color-scheme: dark)') ?? null;
+        scheme?.addEventListener('change', read);
+        return () => {
+            observer.disconnect();
+            scheme?.removeEventListener('change', read);
+        };
+    }, [ref, mode]);
+    return theme;
+}

package/src/chat/visual-bridge.ts

@@ -0,0 +1,176 @@
+import {
+    VISUAL_BRIDGE_METHODS,
+    parseVisualMessage,
+    visualHostContextMessage,
+    visualOpenLinkResult,
+    visualResourceReadyMessage,
+    type ChatVisual,
+    type VisualAppearance,
+    type VisualTheme
+} from '@adecore/agent-contracts/visual';
+import { clampVisualHeight } from './logic/visual-height';
+
+/*
+ * How long a frame that finished loading may stay silent before it counts as failed. The sandbox host
+ * page speaks while it loads, so silence means the address served something else; the margin is for
+ * its message and the load event, which cross between processes and may arrive in either order.
+ */
+const SILENCE_MS = 3000;
+
+/* The window of a visual's frame, as the bridge posts into it. */
+export interface VisualFrameWindow {
+    postMessage(message: unknown, targetOrigin: string): void;
+}
+
+export interface VisualBridgeOptions {
+    /* The frame's window as it is now, or null; read for every message, since only that window may speak for the frame. */
+    frame(): VisualFrameWindow | null;
+    visual: Pick<ChatVisual, 'maxHeight'>;
+    /* The page, as text. It goes to the frame every time the sandbox host page says it listens. */
+    page: Promise<string>;
+    /* The theme the frame's address carries. */
+    theme: VisualTheme;
+    /* Whether a person is acting in the frame right now, so a link the page asks to open is one they followed. */
+    mayOpenLink(): boolean;
+    openLink(url: string): void;
+    onHeight(height: number): void;
+    /* The appearance the page is drawn in, from the first size it reports and after every theme it gets since. */
+    onAppearance(appearance: VisualAppearance): void;
+    /* The page could not be read, or the frame's address answered with something other than the sandbox host page. */
+    onFailure(): void;
+    /* Runs `run` after `ms`, and answers what cancels it; a test hands a clock of its own. */
+    after?(ms: number, run: () => void): () => void;
+}
+
+function timeout(ms: number, run: () => void): () => void {
+    const timer = setTimeout(run, ms);
+    return () => clearTimeout(timer);
+}
+
+/*
+ * Talks to the frame of one visual. It hands the sandbox host page the visual's page when it says it
+ * listens, keeps the page in the app's theme, follows the size the page reports and opens the links a
+ * person follows in it. A message counts only when it comes from the frame's own window, and the
+ * messages that come from the page count only once the page was sent.
+ */
+export class VisualBridge {
+    private readonly options: VisualBridgeOptions;
+    private theme: VisualTheme;
+    private html: string | null = null;
+    /* Whether the sandbox host page ever said it listens, which is what tells it from an error page. */
+    private heard = false;
+    /* The sandbox host page listens and has not been sent the page yet. */
+    private waiting = false;
+    /* The theme the page was last sent, from the moment it was sent; null before that. */
+    private sent: VisualTheme | null = null;
+    /* Whether the page reported a size since it was sent, which says it runs and is drawn. */
+    private live = false;
+    private disposed = false;
+    private cancelSilence: () => void = () => undefined;
+
+    constructor(options: VisualBridgeOptions) {
+        this.options = options;
+        this.theme = options.theme;
+        options.page.then(
+            (html) => {
+                this.html = html;
+                this.sendPage();
+            },
+            () => {
+                if (!this.disposed) {
+                    options.onFailure();
+                }
+            }
+        );
+    }
+
+    /* A `message` event of the window the frame is in. */
+    receive(event: { source: unknown; data: unknown }): void {
+        const frame = this.options.frame();
+        if (this.disposed || frame === null || event.source !== frame) {
+            return;
+        }
+        const message = parseVisualMessage(event.data);
+        switch (message?.method) {
+            case VISUAL_BRIDGE_METHODS.sandboxProxyReady:
+                // A frame that loads again starts over from the sandbox host page.
+                this.heard = true;
+                this.waiting = true;
+                this.sent = null;
+                this.live = false;
+                this.cancelSilence();
+                this.sendPage();
+                return;
+            case VISUAL_BRIDGE_METHODS.sizeChanged:
+                if (this.sent === null) {
+                    return;
+                }
+                if (!this.live) {
+                    this.live = true;
+                    this.options.onAppearance(this.sent.appearance);
+                }
+                this.options.onHeight(clampVisualHeight(this.options.visual, message.height));
+                return;
+            case VISUAL_BRIDGE_METHODS.openLink:
+                if (this.sent === null) {
+                    return;
+                }
+                if (this.options.mayOpenLink()) {
+                    this.options.openLink(message.url);
+                }
+                frame.postMessage(visualOpenLinkResult(message.id), '*');
+                return;
+            default:
+                return;
+        }
+    }
+
+    /* The app's theme changed; a page that was sent follows it at once, without loading again. */
+    setTheme(theme: VisualTheme): void {
+        this.theme = theme;
+        const frame = this.options.frame();
+        if (this.disposed || this.sent === null || frame === null) {
+            return;
+        }
+        this.postTheme(frame);
+        if (this.live) {
+            this.options.onAppearance(theme.appearance);
+        }
+    }
+
+    /* The frame fired `load`. */
+    loaded(): void {
+        if (this.heard || this.disposed) {
+            return;
+        }
+        this.cancelSilence();
+        this.cancelSilence = (this.options.after ?? timeout)(SILENCE_MS, () => {
+            if (!this.heard && !this.disposed) {
+                this.options.onFailure();
+            }
+        });
+    }
+
+    dispose(): void {
+        this.disposed = true;
+        this.cancelSilence();
+    }
+
+    private sendPage(): void {
+        const frame = this.options.frame();
+        if (this.disposed || !this.waiting || this.html === null || frame === null) {
+            return;
+        }
+        this.waiting = false;
+        // Any origin, since the frame's own is opaque; `receive` checks every answer's source instead.
+        frame.postMessage(visualResourceReadyMessage(this.html), '*');
+        // The page writes itself as that message arrives and listens before the next one, so a theme
+        // that changed since the frame's address was set follows right after it.
+        this.postTheme(frame);
+    }
+
+    private postTheme(frame: VisualFrameWindow): void {
+        this.sent = this.theme;
+        frame.postMessage(visualHostContextMessage(this.theme), '*');
+    }
+}

package/src/chat/visuals.ts

@@ -0,0 +1,22 @@
+import { create } from 'zustand';
+
+/*
+ * The one visual in the window that has its large view or the question before its removal up, by the
+ * scope's key of its chat. The thread draws both rather than the row, since the thread only keeps the
+ * rows in view and a row that scrolls out would take an open dialog with it.
+ */
+interface VisualDialog {
+    chatKey: string | null;
+    visualId: string | null;
+    kind: 'expand' | 'remove' | null;
+    open(chatKey: string, visualId: string, kind: 'expand' | 'remove'): void;
+    close(): void;
+}
+
+export const useVisualDialog = create<VisualDialog>((set) => ({
+    chatKey: null,
+    visualId: null,
+    kind: null,
+    open: (chatKey, visualId, kind) => set({ chatKey, visualId, kind }),
+    close: () => set({ chatKey: null, visualId: null, kind: null })
+}));

package/src/host.ts

@@ -154,6 +154,14 @@ export interface ThreadCard {
     render(): ReactNode;
 }
 
+/* Where the visuals agents publish are drawn: a sandboxed frame on an opaque origin, with no rights over the app. */
+export interface VisualHost {
+    /* The address of the sandbox host page (`VISUAL_HOST_PAGE`), on another origin or on the app's own behind a `sandbox` policy. A frame adds the theme to it as a fragment. */
+    frameUrl: string;
+    /* Opens an http or https link a person followed in a visual, outside the app. */
+    openLink(url: string): void;
+}
+
 /* Who writes a chat's replies, as the header over each one names them. */
 export interface ReplyAuthor {
     name: string;
@@ -264,6 +272,8 @@ export interface ChatHost {
     useThreadCards(scopeId: string, chatId: string): readonly ThreadCard[];
     /* A header over every reply with its author and how long ago it began; null names the agent for a screen reader only. */
     useReplyAuthor(scopeId: string, chatId: string): ReplyAuthor | null;
+    /* The pages agents publish, drawn in the thread above their reply; null draws none. */
+    visuals: VisualHost | null;
 }
 
 const NO_PROMPTS: readonly HostPrompt[] = [];
@@ -307,7 +317,8 @@ const DEFAULT_HOST: ChatHost = {
     ComposerSlot: null,
     SubagentSlot: null,
     useThreadCards: () => NO_CARDS,
-    useReplyAuthor: () => null
+    useReplyAuthor: () => null,
+    visuals: null
 };
 
 let host: ChatHost = DEFAULT_HOST;

package/src/locales/en/agent-chat.json

@@ -416,6 +416,18 @@
     "renameFailed": "The bookmark could not be renamed",
     "removeFailed": "The bookmark could not be removed"
   },
+  "visuals": {
+    "actions": "Actions for {{title}}",
+    "expand": "Open large",
+    "more": "More actions",
+    "remove": "Remove visual",
+    "removeTitle": "Remove this visual?",
+    "removeDescription": "{{title}} is deleted from the chat. This cannot be undone.",
+    "removeFailed": "The visual could not be removed",
+    "loading": "Loading {{title}}",
+    "failed": "Could not load {{title}}",
+    "renderFailed": "This visual failed to render"
+  },
   "image": {
     "zoomIn": "Zoom in",
     "zoomOut": "Zoom out",

package/src/locales/nl/agent-chat.json

@@ -416,6 +416,18 @@
     "renameFailed": "De bladwijzer kon niet worden hernoemd",
     "removeFailed": "De bladwijzer kon niet worden verwijderd"
   },
+  "visuals": {
+    "actions": "Acties voor {{title}}",
+    "expand": "Groot openen",
+    "more": "Meer acties",
+    "remove": "Visualisatie verwijderen",
+    "removeTitle": "Deze visualisatie verwijderen?",
+    "removeDescription": "{{title}} wordt uit de chat verwijderd. Dit kan niet ongedaan worden gemaakt.",
+    "removeFailed": "De visualisatie kon niet worden verwijderd",
+    "loading": "{{title}} wordt geladen",
+    "failed": "{{title}} kon niet worden geladen",
+    "renderFailed": "Deze visualisatie kon niet worden getekend"
+  },
   "image": {
     "zoomIn": "Inzoomen",
     "zoomOut": "Uitzoomen",

package/src/state/chats.ts

@@ -1,5 +1,5 @@
 import { create } from 'zustand';
-import type { ChatApprovalItem, ChatBookmark, ChatEvent, ChatHistoryResult, ChatInfo, ChatItem, ChatQuestionItem } from '@adecore/agent-contracts';
+import type { ChatApprovalItem, ChatBookmark, ChatEvent, ChatHistoryResult, ChatInfo, ChatItem, ChatQuestionItem, ChatVisual } from '@adecore/agent-contracts';
 import { useChatScope } from '../scope';
 
 export interface ChatState {
@@ -22,6 +22,8 @@ export interface ChatState {
     waitingBefore?: string[];
     /* The messages marked to come back to, as the host last said; absent until it said anything. */
     bookmarks?: ChatBookmark[];
+    /* The pages agents published in the chat, as the host last said; absent until it said anything. */
+    visuals?: ChatVisual[];
 }
 
 export type ChatHistoryPage = ChatHistoryResult['history'];
@@ -49,6 +51,7 @@ export interface ChatSink {
     /* What a chat is doing, for a thread nobody in this window has open. */
     status(chatId: string, info: ChatInfo): void;
     bookmarks(chatId: string, bookmarks: ChatBookmark[]): void;
+    visuals(chatId: string, visuals: ChatVisual[]): void;
     forget(chatId: string): void;
 }
 
@@ -61,6 +64,7 @@ interface ChatsStore {
     apply(key: string, event: ChatEvent): void;
     status(key: string, info: ChatInfo): void;
     bookmarks(key: string, bookmarks: ChatBookmark[]): void;
+    visuals(key: string, visuals: ChatVisual[]): void;
     forget(key: string): void;
     /* Drops the threads whose key matches, such as every thread of one host. They keep running there; this client is done looking at them. */
     forgetWhere(matches: (key: string) => boolean): void;
@@ -182,11 +186,11 @@ export const useChats = create<ChatsStore>((set) => ({
     statusByKey: {},
     reset(key, info, items, page) {
         set((s) => {
-            // The bookmarks are not part of the thread; they come in on their own and outlive a reset of it.
-            const bookmarks = s.byKey[key]?.bookmarks;
-            const next = stateOf(info, items, page);
+            // The bookmarks and the visuals are not part of the thread; they come in on their own and outlive a reset of it.
+            const { bookmarks, visuals } = s.byKey[key] ?? {};
+            const next = { ...stateOf(info, items, page), ...(bookmarks === undefined ? {} : { bookmarks }), ...(visuals === undefined ? {} : { visuals }) };
             return {
-                byKey: { ...s.byKey, [key]: bookmarks === undefined ? next : { ...next, bookmarks } },
+                byKey: { ...s.byKey, [key]: next },
                 statusByKey: withStatus(s.statusByKey, key, info)
             };
         });
@@ -238,6 +242,15 @@ export const useChats = create<ChatsStore>((set) => ({
             return { byKey: { ...s.byKey, [key]: { ...current, bookmarks } } };
         });
     },
+    visuals(key, visuals) {
+        set((s) => {
+            const current = s.byKey[key];
+            if (!current) {
+                return {};
+            }
+            return { byKey: { ...s.byKey, [key]: { ...current, visuals } } };
+        });
+    },
     forget(key) {
         set((s) => {
             const next = { ...s.byKey };
@@ -260,6 +273,7 @@ export function chatSink(keyOf: (chatId: string) => string): ChatSink {
         apply: (chatId, event) => useChats.getState().apply(keyOf(chatId), event),
         status: (chatId, info) => useChats.getState().status(keyOf(chatId), info),
         bookmarks: (chatId, bookmarks) => useChats.getState().bookmarks(keyOf(chatId), bookmarks),
+        visuals: (chatId, visuals) => useChats.getState().visuals(keyOf(chatId), visuals),
         forget: (chatId) => useChats.getState().forget(keyOf(chatId))
     };
 }

package/src/theme.css

@@ -20,6 +20,13 @@
     --chart-context-system: var(--text-faint);
     /* The current hit of a find, which has to stand out from the accent of a bookmark next to it. */
     --chat-find: #facc15;
+    /* The series of a chart in a visual after the first, which is the accent. A categorical order checked
+       for color vision deficiency on both grounds: orange, aqua, yellow, magenta, green. */
+    --chart-2: #eb6834;
+    --chart-3: #1baf7a;
+    --chart-4: #eda100;
+    --chart-5: #e87ba4;
+    --chart-6: #008300;
 }
 
 [data-theme="dark"] {
@@ -34,6 +41,11 @@
     --chart-context-conversation: #6dd08b;
     --chart-context-system: var(--text-faint);
     --chat-find: #f5c542;
+    --chart-2: #d95926;
+    --chart-3: #199e70;
+    --chart-4: #c98500;
+    --chart-5: #d55181;
+    --chart-6: #008300;
 }
 
 /* Output and diffs take the terminal's colors when the app loads `@adecore/terminal/terminal.css`,
@@ -43,6 +55,7 @@
     --chat-output: var(--term-fg, var(--text));
     --chat-added: var(--term-ansi-green, var(--positive));
     --chat-removed: var(--term-ansi-red, var(--status-error));
+    --chart-1: var(--accent);
 }
 
 @theme inline {
@@ -114,6 +127,8 @@
         @starting-style { opacity: 0; }
     }
     @keyframes chat-fade { from { opacity: 0; } to { opacity: 1; } }
+    /* A visual still on its way says so only once it takes long enough to be noticed. */
+    .chat-visual-wait { animation: chat-fade 200ms ease-out 600ms both; }
 
     /* The row a jump from outside the chat (the card of a change in a file) landed on. */
     .chat-flash { animation: chat-flash 1600ms ease-out both; }