@assistant-ui/react 0.10.43 → 0.10.45

package/src/model-context/frame/AssistantFrameProvider.ts

@@ -0,0 +1,225 @@
+import {
+  ModelContextProvider,
+  ModelContext,
+} from "../../model-context/ModelContextTypes";
+import { Unsubscribe } from "../../types/Unsubscribe";
+import { Tool } from "assistant-stream";
+import { z } from "zod";
+import {
+  FrameMessage,
+  FRAME_MESSAGE_CHANNEL,
+  SerializedModelContext,
+  SerializedTool,
+} from "./AssistantFrameTypes";
+
+/**
+ * Converts tools to JSON Schema format for serialization
+ */
+const serializeTool = (tool: Tool<any, any>): SerializedTool => ({
+  ...(tool.description && { description: tool.description }),
+  parameters:
+    tool.parameters instanceof z.ZodType
+      ? ((z as any).toJSONSchema?.(tool.parameters) ?? tool.parameters)
+      : tool.parameters,
+  ...(tool.disabled !== undefined && { disabled: tool.disabled }),
+  ...(tool.type && { type: tool.type }),
+});
+
+/**
+ * Serializes a ModelContext for transmission across iframe boundary
+ */
+const serializeModelContext = (
+  context: ModelContext,
+): SerializedModelContext => ({
+  ...(context.system !== undefined && { system: context.system }),
+  ...(context.tools && {
+    tools: Object.fromEntries(
+      Object.entries(context.tools).map(([name, tool]) => [
+        name,
+        serializeTool(tool),
+      ]),
+    ),
+  }),
+});
+
+/**
+ * AssistantFrameProvider - Runs inside an iframe and provides ModelContextProviders
+ * to the parent window's AssistantFrameHost.
+ *
+ * Usage example:
+ * ```typescript
+ * // Inside the iframe
+ * // Add model context providers
+ * const registry = new ModelContextRegistry();
+ * AssistantFrameProvider.addModelContextProvider(registry);
+ *
+ * // Add tools to registry
+ * registry.addTool({
+ *   toolName: "search",
+ *   description: "Search the web",
+ *   parameters: z.object({ query: z.string() }),
+ *   execute: async (args) => {
+ *     // Tool implementation runs in iframe
+ *     return { results: ["..."] };
+ *   }
+ * });
+ * ```
+ */
+export class AssistantFrameProvider {
+  private static _instance: AssistantFrameProvider | null = null;
+
+  private _providers = new Set<ModelContextProvider>();
+  private _providerUnsubscribes = new Map<
+    ModelContextProvider,
+    Unsubscribe | undefined
+  >();
+  private _targetOrigin: string;
+
+  private constructor(targetOrigin: string = "*") {
+    this._targetOrigin = targetOrigin;
+    this.handleMessage = this.handleMessage.bind(this);
+    window.addEventListener("message", this.handleMessage);
+
+    // Send initial update on initialization
+    setTimeout(() => this.broadcastUpdate(), 0);
+  }
+
+  private static getInstance(targetOrigin?: string): AssistantFrameProvider {
+    if (!AssistantFrameProvider._instance) {
+      AssistantFrameProvider._instance = new AssistantFrameProvider(
+        targetOrigin,
+      );
+    }
+    return AssistantFrameProvider._instance;
+  }
+
+  private handleMessage(event: MessageEvent) {
+    // Security: Validate origin if specified
+    if (this._targetOrigin !== "*" && event.origin !== this._targetOrigin)
+      return;
+    if (event.data?.channel !== FRAME_MESSAGE_CHANNEL) return;
+
+    const message = event.data.message as FrameMessage;
+
+    switch (message.type) {
+      case "model-context-request":
+        // Respond with current context
+        this.sendMessage(event, {
+          type: "model-context-update",
+          context: serializeModelContext(this.getModelContext()),
+        });
+        break;
+
+      case "tool-call":
+        this.handleToolCall(message, event);
+        break;
+    }
+  }
+
+  private async handleToolCall(
+    message: Extract<FrameMessage, { type: "tool-call" }>,
+    event: MessageEvent,
+  ) {
+    const tool = this.getModelContext().tools?.[message.toolName];
+
+    let result: any;
+    let error: string | undefined;
+
+    if (!tool) {
+      error = `Tool "${message.toolName}" not found`;
+    } else {
+      try {
+        result = tool.execute
+          ? await tool.execute(message.args, {
+              toolCallId: message.id,
+              abortSignal: new AbortController().signal,
+            })
+          : undefined;
+      } catch (e) {
+        error = e instanceof Error ? e.message : String(e);
+      }
+    }
+
+    this.sendMessage(event, {
+      type: "tool-result",
+      id: message.id,
+      ...(error ? { error } : { result }),
+    });
+  }
+
+  private sendMessage(event: MessageEvent, message: FrameMessage) {
+    event.source?.postMessage(
+      { channel: FRAME_MESSAGE_CHANNEL, message },
+      { targetOrigin: event.origin },
+    );
+  }
+
+  private getModelContext(): ModelContext {
+    const contexts = Array.from(this._providers).map((p) =>
+      p.getModelContext(),
+    );
+
+    return contexts.reduce(
+      (merged, context) => ({
+        system: context.system
+          ? merged.system
+            ? `${merged.system}\n\n${context.system}`
+            : context.system
+          : merged.system,
+        tools: { ...(merged.tools || {}), ...(context.tools || {}) },
+      }),
+      {} as ModelContext,
+    );
+  }
+
+  private broadcastUpdate() {
+    // Always broadcast to parent window
+    if (window.parent && window.parent !== window) {
+      const updateMessage: FrameMessage = {
+        type: "model-context-update",
+        context: serializeModelContext(this.getModelContext()),
+      };
+
+      window.parent.postMessage(
+        { channel: FRAME_MESSAGE_CHANNEL, message: updateMessage },
+        this._targetOrigin,
+      );
+    }
+  }
+
+  static addModelContextProvider(
+    provider: ModelContextProvider,
+    targetOrigin?: string,
+  ): Unsubscribe {
+    const instance = AssistantFrameProvider.getInstance(targetOrigin);
+    instance._providers.add(provider);
+
+    const unsubscribe = provider.subscribe?.(() => instance.broadcastUpdate());
+    if (unsubscribe) {
+      instance._providerUnsubscribes.set(provider, unsubscribe);
+    }
+
+    instance.broadcastUpdate();
+
+    return () => {
+      instance._providers.delete(provider);
+      instance._providerUnsubscribes.get(provider)?.();
+      instance._providerUnsubscribes.delete(provider);
+      instance.broadcastUpdate();
+    };
+  }
+
+  static dispose() {
+    if (AssistantFrameProvider._instance) {
+      const instance = AssistantFrameProvider._instance;
+      window.removeEventListener("message", instance.handleMessage);
+
+      // Unsubscribe from all providers
+      instance._providerUnsubscribes.forEach((unsubscribe) => unsubscribe?.());
+      instance._providerUnsubscribes.clear();
+      instance._providers.clear();
+
+      AssistantFrameProvider._instance = null;
+    }
+  }
+}

package/src/model-context/frame/AssistantFrameTypes.ts

@@ -0,0 +1,40 @@
+export type SerializedTool = {
+  description?: string;
+  parameters: any; // JSON Schema
+  disabled?: boolean;
+  type?: string;
+};
+
+export type SerializedModelContext = {
+  system?: string;
+  tools?: Record<string, SerializedTool>;
+};
+
+export type FrameMessageType =
+  | "model-context-request"
+  | "model-context-update"
+  | "tool-call"
+  | "tool-result";
+
+export type FrameMessage =
+  | {
+      type: "model-context-request";
+    }
+  | {
+      type: "model-context-update";
+      context: SerializedModelContext;
+    }
+  | {
+      type: "tool-call";
+      id: string;
+      toolName: string;
+      args: unknown;
+    }
+  | {
+      type: "tool-result";
+      id: string;
+      result?: unknown;
+      error?: string;
+    };
+
+export const FRAME_MESSAGE_CHANNEL = "assistant-ui-frame";

package/src/model-context/frame/SPEC_AssistantFrame.md

@@ -0,0 +1,104 @@
+## Assistant Frames
+
+Assistant frames allow an iframe to provide model context (tools, instructions) to a parent window's assistant.
+
+### Scope
+
+Supported features are:
+
+- ModelContextProvider API
+- support for tools (defining tool name, description, parameters, execute)
+- support for instructions (system instructions)
+
+Out of scope for now:
+
+- model configuration (temprature, etc.)
+- ToolCallReader API (incremental reading support)
+
+### API design
+
+[SPEC_ModelContextRegistry](../registry/SPEC_ModelContextRegistry.md)
+
+### Inside the iframe (provides context)
+
+```typescript
+// Add model context providers
+const registry = new ModelContextRegistry();
+AssistantFrameProvider.addModelContextProvider(registry);
+
+// Add tools/instructions to registry
+registry.addTool({
+  toolName: "search",
+  description: "Search the web",
+  parameters: z.object({ query: z.string() }),
+  execute: async (args) => {
+    // Tool implementation runs in iframe
+    return { results: ["..."] };
+  },
+});
+```
+
+### In the parent window (consumes context)
+
+```typescript
+// The parent window hosts the assistant that needs the context
+const frameHost = new AssistantFrameHost(iframeWindow);
+
+// Register with assistant runtime
+const runtime = useAssistantRuntime();
+runtime.registerModelContextProvider(frameHost);
+
+// The assistant now has access to tools from the iframe
+```
+
+### Communication Channel Design
+
+The communication between `AssistantFrameProvider` (iframe) and `AssistantFrameHost` (parent window) uses the `window.postMessage` API with a structured protocol. The iframe provides model context to the parent window's assistant.
+
+#### ModelContextProvider API
+
+AssistantFrameHost implements the ModelContextProvider API. It immediately subscribes to the iframe for updates. This is necssary because ModelContextProvider.getModelContext() is synchronous.
+
+#### Message Channel
+
+All messages are wrapped with a channel identifier to avoid conflicts with other postMessage usage:
+
+```typescript
+{
+  channel: "assistant-ui-frame",
+  message: FrameMessage
+}
+```
+
+#### Message Types
+
+1. **Context Discovery**
+   - `model-context-request`: Parent (Host) requests current context from iframe (Provider)
+   - `model-context-update`: Iframe pushes context changes to parent
+2. **Tool Execution**
+   - `tool-call`: Parent requests tool execution in iframe (where tools are defined)
+   - `tool-result`: Iframe returns execution result or error to parent
+
+#### Serialization
+
+- **Tools**: Zod schemas are converted to JSON Schema format using `z.toJSONSchema()`
+- **Parameters**: Tool parameters are serialized as JSON
+- **System messages**: Passed as strings
+- **Unsupported features**: Model config, call settings, and priority are not transmitted
+
+#### Security Considerations
+
+1. **Origin Validation**: Both sides can specify `targetOrigin` to restrict message sources
+2. **Window Reference**: Host (parent) only accepts messages from the specific iframe window it's connected to
+3. **Message Channel**: Using a unique channel identifier prevents cross-talk with other postMessage users
+
+#### Connection Lifecycle
+
+1. **Initialization**: Parent (Host) sends `model-context-request` to iframe on creation
+2. **Updates**: Iframe (Provider) notifies parent whenever any registered ModelContextProvider changes
+
+#### Error Handling
+
+- Tool execution errors are serialized and sent back as error messages
+- Connection failures (timeout, no response) are silently handled - the Host continues to work as an empty ModelContextProvider
+- If the iframe doesn't register any providers, the AssistantFrameHost acts as a no-op empty ModelContextProvider returning `{}` from `getModelContext()`

package/src/model-context/frame/index.ts

@@ -0,0 +1,4 @@
+export * from "./AssistantFrameHost";
+export * from "./AssistantFrameProvider";
+export * from "./AssistantFrameTypes";
+export * from "./useAssistantFrameHost";

package/src/model-context/frame/useAssistantFrameHost.ts

@@ -0,0 +1,48 @@
+"use client";
+
+import { useEffect, RefObject } from "react";
+import { AssistantFrameHost } from "./AssistantFrameHost";
+import { Unsubscribe } from "../../types";
+
+type UseAssistantFrameHostOptions = {
+  iframeRef: Readonly<RefObject<HTMLIFrameElement | null | undefined>>;
+  targetOrigin?: string;
+  register: (frameHost: AssistantFrameHost) => Unsubscribe;
+};
+
+/**
+ * React hook that manages the lifecycle of an AssistantFrameHost and its binding to the current AssistantRuntime.
+ *
+ * Usage example:
+ * ```typescript
+ * function MyComponent() {
+ *   const iframeRef = useRef<HTMLIFrameElement>(null);
+ *
+ *   useAssistantFrameHost({
+ *     iframeRef,
+ *     targetOrigin: "https://trusted-domain.com", // optional
+ *   });
+ *
+ *   return <iframe ref={iframeRef} src="..." />;
+ * }
+ * ```
+ */
+export const useAssistantFrameHost = ({
+  iframeRef,
+  targetOrigin = "*",
+  register,
+}: UseAssistantFrameHostOptions): void => {
+  useEffect(() => {
+    const iframeWindow = iframeRef.current?.contentWindow;
+    if (!iframeWindow) return;
+
+    const frameHost = new AssistantFrameHost(iframeWindow, targetOrigin);
+
+    const unsubscribe = register(frameHost);
+
+    return () => {
+      frameHost.dispose();
+      unsubscribe();
+    };
+  }, [iframeRef, targetOrigin, register]);
+};

package/src/model-context/index.ts

@@ -33,3 +33,6 @@ export { tool } from "./tool";
  */
 export { makeAssistantVisible as makeAssistantReadable } from "./makeAssistantVisible";
 export { makeAssistantVisible } from "./makeAssistantVisible";
+
+export * from "./registry";
+export * from "./frame";

package/src/model-context/registry/ModelContextRegistry.ts

@@ -0,0 +1,165 @@
+import { Tool } from "assistant-stream";
+import {
+  ModelContext,
+  ModelContextProvider,
+  mergeModelContexts,
+} from "../../model-context/ModelContextTypes";
+import { Unsubscribe } from "../../types/Unsubscribe";
+import {
+  ModelContextRegistryToolHandle,
+  ModelContextRegistryInstructionHandle,
+  ModelContextRegistryProviderHandle,
+} from "./ModelContextRegistryHandles";
+import type { AssistantToolProps } from "../../model-context/useAssistantTool";
+import type { AssistantInstructionsConfig } from "../../model-context/useAssistantInstructions";
+
+export class ModelContextRegistry implements ModelContextProvider {
+  private _tools = new Map<symbol, AssistantToolProps<any, any>>();
+  private _instructions = new Map<symbol, string>();
+  private _providers = new Map<symbol, ModelContextProvider>();
+  private _subscribers = new Set<() => void>();
+  private _providerUnsubscribes = new Map<symbol, Unsubscribe | undefined>();
+
+  getModelContext(): ModelContext {
+    // Merge instructions
+    const instructions = Array.from(this._instructions.values()).filter(
+      Boolean,
+    );
+
+    const system =
+      instructions.length > 0 ? instructions.join("\n\n") : undefined;
+
+    // Collect tools
+    const tools: Record<string, Tool<any, any>> = {};
+    for (const toolProps of this._tools.values()) {
+      const { toolName, render, ...tool } = toolProps;
+      tools[toolName] = tool;
+    }
+
+    // Merge provider contexts
+    const providerContexts = mergeModelContexts(
+      new Set(this._providers.values()),
+    );
+
+    // Combine everything
+    const context: ModelContext = {
+      system,
+      tools: Object.keys(tools).length > 0 ? tools : undefined,
+    };
+
+    // Merge with provider contexts
+    if (providerContexts.system) {
+      context.system = context.system
+        ? `${context.system}\n\n${providerContexts.system}`
+        : providerContexts.system;
+    }
+
+    if (providerContexts.tools) {
+      context.tools = { ...(context.tools || {}), ...providerContexts.tools };
+    }
+
+    if (providerContexts.callSettings) {
+      context.callSettings = providerContexts.callSettings;
+    }
+
+    if (providerContexts.config) {
+      context.config = providerContexts.config;
+    }
+
+    return context;
+  }
+
+  subscribe(callback: () => void): Unsubscribe {
+    this._subscribers.add(callback);
+    return () => this._subscribers.delete(callback);
+  }
+
+  private notifySubscribers(): void {
+    for (const callback of this._subscribers) {
+      callback();
+    }
+  }
+
+  addTool<TArgs extends Record<string, unknown>, TResult>(
+    tool: AssistantToolProps<TArgs, TResult>,
+  ): ModelContextRegistryToolHandle<TArgs, TResult> {
+    const id = Symbol();
+
+    this._tools.set(id, tool);
+    this.notifySubscribers();
+
+    return {
+      update: (newTool: AssistantToolProps<TArgs, TResult>) => {
+        if (this._tools.has(id)) {
+          this._tools.set(id, newTool);
+          this.notifySubscribers();
+        }
+      },
+      remove: () => {
+        this._tools.delete(id);
+        this.notifySubscribers();
+      },
+    };
+  }
+
+  addInstruction(
+    config: string | AssistantInstructionsConfig,
+  ): ModelContextRegistryInstructionHandle {
+    const id = Symbol();
+
+    const instruction =
+      typeof config === "string" ? config : config.instruction;
+    const disabled = typeof config === "object" ? config.disabled : false;
+
+    if (!disabled) {
+      this._instructions.set(id, instruction);
+      this.notifySubscribers();
+    }
+
+    return {
+      update: (newConfig: string | AssistantInstructionsConfig) => {
+        const newInstruction =
+          typeof newConfig === "string" ? newConfig : newConfig.instruction;
+        const newDisabled =
+          typeof newConfig === "object" ? newConfig.disabled : false;
+
+        if (newDisabled) {
+          this._instructions.delete(id);
+        } else {
+          this._instructions.set(id, newInstruction);
+        }
+        this.notifySubscribers();
+      },
+      remove: () => {
+        this._instructions.delete(id);
+        this.notifySubscribers();
+      },
+    };
+  }
+
+  addProvider(
+    provider: ModelContextProvider,
+  ): ModelContextRegistryProviderHandle {
+    const id = Symbol();
+
+    this._providers.set(id, provider);
+
+    // Subscribe to provider changes
+    const unsubscribe = provider.subscribe?.(() => {
+      this.notifySubscribers();
+    });
+    this._providerUnsubscribes.set(id, unsubscribe);
+
+    this.notifySubscribers();
+
+    return {
+      remove: () => {
+        this._providers.delete(id);
+        const unsubscribe = this._providerUnsubscribes.get(id);
+        unsubscribe?.();
+        this._providerUnsubscribes.delete(id);
+        this.notifySubscribers();
+      },
+    };
+  }
+}

package/src/model-context/registry/ModelContextRegistryHandles.ts

@@ -0,0 +1,19 @@
+import type { AssistantToolProps } from "../../model-context/useAssistantTool";
+import type { AssistantInstructionsConfig } from "../../model-context/useAssistantInstructions";
+
+export interface ModelContextRegistryToolHandle<
+  TArgs extends Record<string, unknown> = any,
+  TResult = any,
+> {
+  update(tool: AssistantToolProps<TArgs, TResult>): void;
+  remove(): void;
+}
+
+export interface ModelContextRegistryInstructionHandle {
+  update(config: string | AssistantInstructionsConfig): void;
+  remove(): void;
+}
+
+export interface ModelContextRegistryProviderHandle {
+  remove(): void;
+}

package/src/model-context/registry/SPEC_ModelContextRegistry.md

@@ -0,0 +1,40 @@
+# ModelContextRegistry
+
+An imperative API for registering tools and instructions to a model context provider.
+
+```typescript
+const registry = new ModelContextRegistry();
+
+const handle = registry.addTool({
+  toolName: "search",
+  description: "Search the web",
+  parameters: z.object({ query: z.string() }),
+  execute: async (args) => {
+    return { results: ["..."] };
+  },
+});
+```
+
+## API
+
+- addTool(tool: Tool & { toolName: string }): ModelContextRegistryToolHandle
+- addInstruction(instruction: string): ModelContextRegistryInstructionHandle
+- addProvider(provider: ModelContextProvider): ModelContextRegistryProviderHandle
+
+## ModelContextRegistryToolHandle
+
+- update(tool: Tool & { toolName: string }): void;
+- remove(): void;
+
+## ModelContextRegistryInstructionHandle
+
+- update(instruction: string): void;
+- remove(): void;
+
+## ModelContextRegistryProviderHandle
+
+- remove(): void;
+
+## ModelContextProvider
+
+The registry is a ModelContextProvider.

package/src/model-context/registry/index.ts

@@ -0,0 +1,2 @@
+export * from "./ModelContextRegistry";
+export * from "./ModelContextRegistryHandles";

package/src/model-context/useAssistantInstructions.tsx

@@ -3,7 +3,7 @@
 import { useEffect } from "react";
 import { useAssistantRuntime } from "../context";
 
-type AssistantInstructionsConfig = {
+export type AssistantInstructionsConfig = {
   disabled?: boolean | undefined;
   instruction: string;
 };

package/src/runtimes/composer/BaseComposerRuntimeCore.tsx

@@ -124,7 +124,7 @@ export abstract class BaseComposerRuntimeCore
     const adapter = this.getAttachmentAdapter();
     const attachments =
       adapter && this.attachments.length > 0
-        ? await Promise.all(
+        ? Promise.all(
             this.attachments.map(async (a) => {
               if (isAttachmentComplete(a)) return a;
               const result = await adapter.send(a);
@@ -133,15 +133,16 @@ export abstract class BaseComposerRuntimeCore
           )
         : [];
 
+    const text = this.text;
+    this._emptyTextAndAttachments();
     const message: Omit<AppendMessage, "parentId" | "sourceId"> = {
       createdAt: new Date(),
       role: this.role,
-      content: this.text ? [{ type: "text", text: this.text }] : [],
-      attachments,
+      content: text ? [{ type: "text", text }] : [],
+      attachments: await attachments,
       runConfig: this.runConfig,
       metadata: { custom: {} },
     };
-    this._emptyTextAndAttachments();
 
     this.handleSend(message);
     this._notifyEventSubscribers("send");