@robota-sdk/agent-ui-web 3.0.0-beta.94 → 3.0.0-beta.96

package/docs/SPEC.md

@@ -34,8 +34,13 @@ does not own session lifecycle, conversation history, or agent runtime state.
   Session and protocol failures are also toasts, beside the conversation rather than in it.
 - A tool call's diff or output is server-computed and shown on demand, preserving its observation
   types, call identity and success or denial/failure through live delivery and replay. Resource references remain inert because
-  displaying a contributor's URI grants no read authority; unsupported media is explained rather than
-  silently discarded. Output is never recomputed here or buried behind a truncated path or argument; an internal signal tool never renders as a call,
+  displaying a contributor's URI grants no read authority; the one exception is a view the call's own
+  server declared for a model call, read on demand through its session-owned view authority. The host
+  admits the document only in an opaque frame with enforced native network isolation; the bridge
+  reaches only that view's declared tools through the session's permission gate. Replacing a view
+  or its session revokes its pending local requests. A surface without a suitable sandbox retains
+  the text result and explains why the app is unavailable.
+  Unsupported media is explained rather than silently discarded. Output is never recomputed here or buried behind a truncated path or argument; an internal signal tool never renders as a call,
   and a projected `/command` tool shows the command it ran, not the provider-facing tool name.
 - A UI-intent opens its GUI screen when there is one — the session picker is the session sidebar
   (unless the host cannot list sessions), settings and the plugin manager both open the Settings
@@ -67,7 +72,8 @@ does not own session lifecycle, conversation history, or agent runtime state.
   host has said one; the desktop window follows the page title.
 - A pending permission or ask prompt is visible whatever view is open, because a gated turn waits
   on it. It never takes focus from a field the person is typing in, and its keys answer it only once
-  it has that focus; a click answers at once.
+  it has that focus; a click answers at once, except on a multi-select ask, whose options toggle
+  until it is submitted.
 - Nothing typed is lost to a lost connection: while the transport is not `connected`, the composer
   keeps its draft and refuses to submit it, and a banner above the conversation — never a full-screen
   replacement — says so: retrying while attempts continue, and once they give up, either the

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@robota-sdk/agent-ui-web",
-  "version": "3.0.0-beta.94",
+  "version": "3.0.0-beta.96",
   "description": "Browser-safe React session presentation, wire reducer, WebSocket client and runtime-host contract for consumer-owned agent renderers.",
   "license": "AGPL-3.0-only OR LicenseRef-Commercial",
   "type": "module",
@@ -58,11 +58,11 @@
     "pretendard": "^1.3.9",
     "react-markdown": "^10.1.0",
     "remark-gfm": "^4.0.1",
-    "@robota-sdk/agent-interface-command": "3.0.0-beta.94",
-    "@robota-sdk/agent-interface-session": "3.0.0-beta.94",
-    "@robota-sdk/agent-interface-transport": "3.0.0-beta.94",
-    "@robota-sdk/agent-interface-execution": "3.0.0-beta.94",
-    "@robota-sdk/agent-transport": "3.0.0-beta.94"
+    "@robota-sdk/agent-interface-command": "3.0.0-beta.96",
+    "@robota-sdk/agent-interface-execution": "3.0.0-beta.96",
+    "@robota-sdk/agent-transport": "3.0.0-beta.96",
+    "@robota-sdk/agent-interface-transport": "3.0.0-beta.96",
+    "@robota-sdk/agent-interface-session": "3.0.0-beta.96"
   },
   "peerDependencies": {
     "react": ">=18"

package/src/app-view/bridge.ts

@@ -0,0 +1,217 @@
+import { outcomeToCallToolResult, type ICallToolResult } from './call-tool-result.js';
+
+import type { TAppToolArguments, TAppToolOutcome, TAppView } from './types.js';
+
+export const APP_VIEW_PROTOCOL_VERSION = '2026-01-26';
+export const APP_VIEW_MIN_HEIGHT_PX = 48;
+export const APP_VIEW_MAX_HEIGHT_PX = 640;
+
+const INVALID_PARAMS = -32602;
+const METHOD_NOT_FOUND = -32601;
+const HOST_ERROR = -32000;
+
+const SANDBOX_PROXY_READY = 'ui/notifications/sandbox-proxy-ready';
+const SANDBOX_RESOURCE_READY = 'ui/notifications/sandbox-resource-ready';
+
+type TJsonRpcId = string | number;
+
+export interface IAppViewMessage {
+  jsonrpc: '2.0';
+  id?: TJsonRpcId;
+  method?: string;
+  params?: unknown;
+  result?: unknown;
+  error?: { code: number; message: string };
+}
+
+export interface IAppViewHostContext {
+  theme: 'light' | 'dark';
+  displayMode: 'inline';
+  availableDisplayModes: readonly ['inline'];
+  containerDimensions: { maxHeight: number };
+  platform: 'desktop' | 'web';
+  locale: string;
+  timeZone: string;
+}
+
+export interface IAppViewBridgeOptions {
+  view: TAppView;
+
+  toolResult: ICallToolResult;
+  hostInfo: { name: string; version: string };
+  hostContext: IAppViewHostContext;
+
+  post: (message: IAppViewMessage) => void;
+
+  source: () => MessageEventSource | null;
+
+  callAppTool: (runtimeName: string, args: TAppToolArguments) => Promise<TAppToolOutcome>;
+
+  onHeight: (px: number) => void;
+}
+
+export interface IAppViewBridge {
+  handleMessage: (event: MessageEvent) => void;
+  dispose: () => void;
+}
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) return false;
+  const prototype: unknown = Object.getPrototypeOf(value);
+  return prototype === null || Object.getPrototypeOf(prototype) === null;
+}
+
+function errorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+export function createAppViewBridge(options: IAppViewBridgeOptions): IAppViewBridge {
+  const { view } = options;
+  let disposed = false;
+  let sandboxAnswered = false;
+  let initialization: 'uninitialized' | 'initializing' | 'ready' = 'uninitialized';
+  let callInFlight = false;
+
+  const respond = (id: TJsonRpcId, result: unknown): void => {
+    options.post({ jsonrpc: '2.0', id, result });
+  };
+  const refuse = (id: TJsonRpcId, code: number, message: string): void => {
+    options.post({ jsonrpc: '2.0', id, error: { code, message } });
+  };
+
+  const callTool = (id: TJsonRpcId, params: unknown): void => {
+    if (initialization !== 'ready') {
+      refuse(id, HOST_ERROR, 'This app view has not initialized.');
+      return;
+    }
+    const name = isPlainObject(params) ? params['name'] : undefined;
+    const runtimeName =
+      typeof name === 'string' && Object.hasOwn(view.callableTools, name)
+        ? view.callableTools[name]
+        : undefined;
+    if (!isPlainObject(params) || runtimeName === undefined) {
+      refuse(id, INVALID_PARAMS, 'This tool is not available to this view.');
+      return;
+    }
+    const rawArguments = params['arguments'] === undefined ? {} : params['arguments'];
+    if (!isPlainObject(rawArguments)) {
+      refuse(id, INVALID_PARAMS, 'Tool arguments must be an object.');
+      return;
+    }
+    if (callInFlight) {
+      refuse(id, HOST_ERROR, 'Another tool call from this view is still running.');
+      return;
+    }
+    callInFlight = true;
+    let outcome: Promise<TAppToolOutcome>;
+    try {
+      outcome = options.callAppTool(runtimeName, rawArguments as TAppToolArguments);
+    } catch (error: unknown) {
+      outcome = Promise.reject(error);
+    }
+    outcome.then(
+      (settled) => {
+        callInFlight = false;
+        if (!disposed) respond(id, outcomeToCallToolResult(settled));
+      },
+      (error: unknown) => {
+        callInFlight = false;
+        if (!disposed) refuse(id, HOST_ERROR, errorMessage(error));
+      },
+    );
+  };
+
+  const handleRequest = (id: TJsonRpcId, method: string, params: unknown): void => {
+    switch (method) {
+      case 'ui/initialize':
+        if (initialization === 'uninitialized') initialization = 'initializing';
+        respond(id, {
+          protocolVersion: APP_VIEW_PROTOCOL_VERSION,
+          hostCapabilities: { serverTools: {} },
+          hostInfo: options.hostInfo,
+          hostContext: options.hostContext,
+        });
+        return;
+      case 'tools/call':
+        callTool(id, params);
+        return;
+      case 'ping':
+        respond(id, {});
+        return;
+      case 'ui/request-display-mode':
+        respond(id, { mode: 'inline' });
+        return;
+      default:
+        refuse(id, METHOD_NOT_FOUND, `This host does not support ${method}.`);
+    }
+  };
+
+  const handleNotification = (method: string, params: unknown): void => {
+    switch (method) {
+      case SANDBOX_PROXY_READY:
+        // The sandbox page is the only sender; the document goes to it once, never again.
+        if (sandboxAnswered) return;
+        sandboxAnswered = true;
+        options.post({
+          jsonrpc: '2.0',
+          method: SANDBOX_RESOURCE_READY,
+          params: { html: view.html },
+        });
+        return;
+      case 'ui/notifications/initialized':
+        if (initialization !== 'initializing') return;
+        initialization = 'ready';
+        options.post({
+          jsonrpc: '2.0',
+          method: 'ui/notifications/tool-input',
+          params: { arguments: view.toolInput ?? {} },
+        });
+        options.post({
+          jsonrpc: '2.0',
+          method: 'ui/notifications/tool-result',
+          params: options.toolResult,
+        });
+        return;
+      case 'ui/notifications/size-changed': {
+        const height = isPlainObject(params) ? params['height'] : undefined;
+        if (typeof height !== 'number' || !Number.isFinite(height)) return;
+        options.onHeight(
+          Math.min(APP_VIEW_MAX_HEIGHT_PX, Math.max(APP_VIEW_MIN_HEIGHT_PX, Math.round(height))),
+        );
+        return;
+      }
+      default:
+        // Anything else a view announces (logging, its own notices) has no consumer here.
+        return;
+    }
+  };
+
+  return {
+    handleMessage(event) {
+      if (disposed) return;
+      const frame = options.source();
+      if (frame === null || event.source !== frame) return;
+      const data: unknown = event.data;
+      if (!isPlainObject(data) || data['jsonrpc'] !== '2.0') return;
+      const { method, id } = data;
+      if (typeof method !== 'string') return;
+      if (id === undefined) {
+        handleNotification(method, data['params']);
+      } else if (typeof id === 'string' || typeof id === 'number') {
+        handleRequest(id, method, data['params']);
+      }
+    },
+    dispose() {
+      if (disposed) return;
+      disposed = true;
+      if (initialization === 'ready') {
+        options.post({
+          jsonrpc: '2.0',
+          id: 'teardown',
+          method: 'ui/resource-teardown',
+          params: { reason: 'closed' },
+        });
+      }
+    },
+  };
+}

package/src/app-view/call-tool-result.ts

@@ -0,0 +1,58 @@
+import type { TAppToolOutcome, TAppToolPart, TAppView } from './types.js';
+import type { IActiveTool } from '../hooks/session-client-types.js';
+
+export type TCallToolContent =
+  | { type: 'text'; text: string }
+  | { type: 'image'; data: string; mimeType: string }
+  | { type: 'audio'; data: string; mimeType: string };
+
+export interface ICallToolResult {
+  content: TCallToolContent[];
+  structuredContent?: Record<string, unknown>;
+  isError: boolean;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+export function contentFromParts(parts: readonly TAppToolPart[] | undefined): TCallToolContent[] {
+  const content: TCallToolContent[] = [];
+  for (const part of parts ?? []) {
+    if (part.type === 'text') {
+      content.push({ type: 'text', text: part.text });
+    } else if (part.type === 'image_inline') {
+      content.push({ type: 'image', data: part.data, mimeType: part.mimeType });
+    } else if (part.type === 'audio_inline') {
+      content.push({ type: 'audio', data: part.data, mimeType: part.mimeType });
+    }
+  }
+  return content;
+}
+
+export function outcomeToCallToolResult(outcome: TAppToolOutcome): ICallToolResult {
+  let content = contentFromParts(outcome.parts);
+  if (content.length === 0 && typeof outcome.data === 'string') {
+    content = [{ type: 'text', text: outcome.data }];
+  }
+  if (!outcome.success && outcome.error !== undefined) {
+    content = [...content, { type: 'text', text: outcome.error }];
+  }
+  return {
+    content,
+    ...(isRecord(outcome.data) ? { structuredContent: outcome.data } : {}),
+    isError: !outcome.success,
+  };
+}
+
+export function toCallToolResult(tool: IActiveTool, view: TAppView): ICallToolResult {
+  let content = contentFromParts(tool.toolResultParts);
+  if (content.length === 0 && tool.toolResultData !== undefined) {
+    content = [{ type: 'text', text: tool.toolResultData }];
+  }
+  return {
+    content,
+    ...(isRecord(view.structuredContent) ? { structuredContent: view.structuredContent } : {}),
+    isError: tool.status === 'error',
+  };
+}

package/src/app-view/host-context.ts

@@ -0,0 +1,37 @@
+import { createContext } from 'react';
+
+import { APP_VIEW_MAX_HEIGHT_PX, type IAppViewHostContext } from './bridge.js';
+
+import type { TAppToolArguments, TAppToolOutcome, TAppView } from './types.js';
+
+export interface IAppViewHost {
+  sandboxUrl?: string;
+
+  readAppView: (viewKey: string, signal?: AbortSignal) => Promise<TAppView>;
+  callAppTool: (
+    viewKey: string,
+    name: string,
+    args: TAppToolArguments,
+    signal?: AbortSignal,
+  ) => Promise<TAppToolOutcome>;
+  hostInfo: { name: string; version: string };
+  platform: 'desktop' | 'web';
+}
+
+export const AppViewHostContext = createContext<IAppViewHost | null>(null);
+
+export function readAppViewHostContext(platform: IAppViewHost['platform']): IAppViewHostContext {
+  const dark =
+    typeof window !== 'undefined' &&
+    typeof window.matchMedia === 'function' &&
+    window.matchMedia('(prefers-color-scheme: dark)').matches;
+  return {
+    theme: dark ? 'dark' : 'light',
+    displayMode: 'inline',
+    availableDisplayModes: ['inline'],
+    containerDimensions: { maxHeight: APP_VIEW_MAX_HEIGHT_PX },
+    platform,
+    locale: typeof navigator === 'undefined' ? 'en-US' : navigator.language,
+    timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone,
+  };
+}

package/src/app-view/types.ts

@@ -0,0 +1,12 @@
+import type { TClientMessage, TServerMessage } from '@robota-sdk/agent-transport';
+
+export type TAppView = Extract<TServerMessage, { type: 'app_view' }>['view'];
+
+export type TAppToolOutcome = Omit<
+  Extract<TServerMessage, { type: 'app_tool_result' }>,
+  'type' | 'requestId'
+>;
+
+export type TAppToolArguments = Extract<TClientMessage, { type: 'call-app-tool' }>['arguments'];
+
+export type TAppToolPart = NonNullable<TAppToolOutcome['parts']>[number];

package/src/components/AppView.tsx

@@ -0,0 +1,147 @@
+'use client';
+
+import React, { useContext, useEffect, useLayoutEffect, useRef, useState } from 'react';
+
+import { createAppViewBridge } from '../app-view/bridge.js';
+import { toCallToolResult } from '../app-view/call-tool-result.js';
+import { AppViewHostContext, readAppViewHostContext } from '../app-view/host-context.js';
+import { appViewKey } from '../hooks/use-app-view-requests.js';
+
+import type { IAppViewHost } from '../app-view/host-context.js';
+import type { TAppView } from '../app-view/types.js';
+import type { IActiveTool } from '../hooks/useSessionClient.js';
+
+const INITIAL_HEIGHT_PX = 160;
+
+export const APP_VIEW_UNAVAILABLE_TEXT =
+  'This tool returned an app view, which this window cannot show.';
+
+type TLoad =
+  | { status: 'loading' }
+  | { status: 'ready'; view: TAppView; host: IAppViewHost; sandboxUrl: string; signal: AbortSignal }
+  | { status: 'failed'; reason: string };
+
+export function AppView({ tool }: { tool: IActiveTool }): React.ReactElement | null {
+  const host = useContext(AppViewHostContext);
+  if (tool.appView === undefined) return null;
+  const viewKey = appViewKey(tool);
+
+  if (host === null || host.sandboxUrl === undefined || viewKey === undefined) {
+    return <p className="text-[13px] text-subtle">{APP_VIEW_UNAVAILABLE_TEXT}</p>;
+  }
+  return (
+    <AppViewFrame
+      key={viewKey}
+      tool={tool}
+      host={host}
+      sandboxUrl={host.sandboxUrl}
+      viewKey={viewKey}
+      sourceId={tool.appView.sourceId}
+    />
+  );
+}
+
+function AppViewFrame({
+  tool,
+  host,
+  sandboxUrl,
+  viewKey,
+  sourceId,
+}: {
+  tool: IActiveTool;
+  host: IAppViewHost;
+  sandboxUrl: string;
+
+  viewKey: string;
+  sourceId: string;
+}): React.ReactElement {
+  const [load, setLoad] = useState<TLoad>({ status: 'loading' });
+  const [height, setHeight] = useState(INITIAL_HEIGHT_PX);
+  const frameRef = useRef<HTMLIFrameElement>(null);
+  const toolRef = useRef(tool);
+  toolRef.current = tool;
+
+  useEffect(() => {
+    const controller = new AbortController();
+    setLoad({ status: 'loading' });
+    host.readAppView(viewKey, controller.signal).then(
+      (view) => {
+        if (!controller.signal.aborted)
+          setLoad({ status: 'ready', view, host, sandboxUrl, signal: controller.signal });
+      },
+      (error: unknown) => {
+        if (!controller.signal.aborted) {
+          setLoad({
+            status: 'failed',
+            reason: error instanceof Error ? error.message : String(error),
+          });
+        }
+      },
+    );
+    return () => {
+      controller.abort();
+    };
+  }, [viewKey, host.readAppView, host.callAppTool, sandboxUrl]);
+
+  const ready =
+    load.status === 'ready' &&
+    load.host.readAppView === host.readAppView &&
+    load.host.callAppTool === host.callAppTool &&
+    load.sandboxUrl === sandboxUrl
+      ? load
+      : null;
+  // A layout effect, so the teardown it posts on unmount still finds the frame in the document.
+  useLayoutEffect(() => {
+    const frame = frameRef.current;
+    if (ready === null || frame === null) return undefined;
+    const current = ready.host;
+    const view = ready.view;
+    const bridge = createAppViewBridge({
+      view,
+      toolResult: toCallToolResult(toolRef.current, view),
+      hostInfo: current.hostInfo,
+      hostContext: readAppViewHostContext(current.platform),
+      // The frame's origin is opaque, so no target origin can name it; the bridge accepts replies
+
+      post: (message) => frame.contentWindow?.postMessage(message, '*'),
+      source: () => frame.contentWindow,
+
+      callAppTool: (name, args) => current.callAppTool(viewKey, name, args, ready.signal),
+      onHeight: setHeight,
+    });
+    const onMessage = (event: MessageEvent): void => bridge.handleMessage(event);
+    window.addEventListener('message', onMessage);
+    return () => {
+      window.removeEventListener('message', onMessage);
+      bridge.dispose();
+    };
+  }, [ready, viewKey]);
+
+  if (load.status === 'failed') {
+    return (
+      <p className="text-[13px] text-subtle">
+        The app from {sourceId} could not be shown: {load.reason}
+      </p>
+    );
+  }
+  if (ready === null) {
+    return <p className="text-[13px] text-subtle">Loading the app from {sourceId}…</p>;
+  }
+  return (
+    <div className="my-1 flex flex-col gap-1">
+      <p className="text-[12px] text-subtle">App from {sourceId}</p>
+      <iframe
+        ref={frameRef}
+
+        // `allow-same-origin` the frame has an opaque origin, so its document cannot read this page.
+        sandbox="allow-scripts"
+        src={sandboxUrl}
+        title={`App from ${sourceId}`}
+        referrerPolicy="no-referrer"
+        data-testid="app-view-frame"
+        style={{ width: '100%', height }}
+        className="block rounded-lg border border-border bg-card"
+      />
+    </div>
+  );
+}

package/src/components/ConversationView.tsx

@@ -21,6 +21,7 @@ import {
 
 import { useCopyFeedback } from '../clipboard.js';
 import { driverAttributionText, isSameSurface } from '../driver-labels.js';
+import { AppView } from './AppView.js';
 import { DiffLines } from './DiffLines.js';
 
 import type {
@@ -584,6 +585,15 @@ function ToolGroup({ tools }: { tools: readonly IActiveTool[] }): React.ReactEle
           className={`flex-shrink-0 text-subtle transition-transform ${open ? 'rotate-90' : ''}`}
         />
       </button>
+      {/* A call's app is the point of the call, so it shows without opening the group — and only
+          here: the rows below never repeat it. */}
+      {visible
+        .filter((tool) => tool.appView !== undefined)
+        .map((tool) => (
+          <div key={tool.id} className="pl-[25px]">
+            <AppView tool={tool} />
+          </div>
+        ))}
       {open && (
         <div className="mt-0.5 flex flex-col pl-[25px]">
           {visible.map((tool) => (
@@ -746,7 +756,14 @@ export function ConversationView({
               {activeTools
                 .filter((tool) => !tool.internal)
                 .map((tool) => (
-                  <ToolCard key={tool.id} tool={tool} />
+                  <React.Fragment key={tool.id}>
+                    <ToolCard tool={tool} />
+                    {tool.appView !== undefined && tool.status !== 'running' && (
+                      <div className="pl-[25px]">
+                        <AppView tool={tool} />
+                      </div>
+                    )}
+                  </React.Fragment>
                 ))}
             </div>
           )}