@frebreco/canvas 0.4.0 → 0.5.0-next.2

package/README.md

@@ -1,7 +1,7 @@
 # canvas
 
-**A multiplayer canvas whose frames are coding-agent sessions, files, browser previews and
-terminals.** The agents run on one person's machine; everyone else joins peer to peer from the
+**A multiplayer canvas whose frames are coding-agent sessions, files, browser previews,
+terminals and drawings.** The agents run on one person's machine; everyone else joins peer to peer from the
 browser.
 
 ```bash

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@frebreco/canvas",
-  "version": "0.4.0",
+  "version": "0.5.0-next.2",
   "description": "A multiplayer canvas for coding agents that run on your machine.",
   "license": "MIT",
   "type": "module",

package/src/server/agents.ts

@@ -94,8 +94,8 @@ export function detectAgents(): ReadonlyArray<AgentDefinition> {
         OPENCODE_CONFIG_CONTENT: JSON.stringify({
           // The starting model; people pick another in the frame.
           model: opencodeModel,
-          // Shell commands ask, so a person approves anything the agent runs.
-          permission: { bash: "ask" },
+          // No permission policy: opencode's defaults (ask outside the
+          // project) and the host's own opencode config decide.
           // canvas's own skills; the host's stay available beside them.
           skills: { paths: [SKILLS_DIR] },
         }),

package/src/server/board-mcp.ts

@@ -17,6 +17,9 @@
  * shared set: the browser keeps it to find the lines again as the file
  * changes.
  *
+ * A drawing (ADR 0009) comes back as an image too, which goes to the agent
+ * as MCP image content.
+ *
  * Scratch files (ADR 0005) never reach the browser as content: a call's
  * `content` becomes a file in `.canvas/scratch/` here, and the call goes on
  * with its path. Reading and writing them needs no board, so those tools are
@@ -27,7 +30,7 @@ import { randomBytes } from "node:crypto";
 import type { McpServer } from "@agentclientprotocol/sdk";
 import { BOARD_SERVER_NAME, BOARD_TOOLS, boardInstructions } from "../shared/board-tools";
 import { linesOf, quoteOf } from "../shared/comments";
-import type { FileContent } from "../shared/protocol";
+import type { FileContent, ToolImage } from "../shared/protocol";
 import type { Scratch } from "./scratch";
 import type { Skill } from "./skills";
 
@@ -56,7 +59,8 @@ interface Rpc {
   readonly params?: Record<string, unknown>;
 }
 
-type ToolResult = { content: Array<{ type: "text"; text: string }>; isError?: true };
+type Content = { type: "text"; text: string } | { type: "image"; data: string; mimeType: string };
+type ToolResult = { content: Content[]; isError?: true };
 
 const LATEST_PROTOCOL = "2025-06-18";
 
@@ -89,12 +93,12 @@ export class BoardMcp {
   }
 
   /** The host's browser answered a call. */
-  result(callId: string, ok: boolean, text: string): void {
+  result(callId: string, ok: boolean, text: string, images: ReadonlyArray<ToolImage> = []): void {
     const waiting = this.pending.get(callId);
     if (!waiting) return;
     this.pending.delete(callId);
     clearTimeout(waiting.timer);
-    waiting.resolve(ok ? done(text) : failed(text));
+    waiting.resolve(ok ? done(text, images) : failed(text));
   }
 
   stop(): void {
@@ -227,7 +231,10 @@ export class BoardMcp {
       undo();
       return answer;
     }
-    return notes.length ? done([answer.content[0]!.text, ...notes].join("\n")) : answer;
+    if (!notes.length) return answer;
+    const [first, ...rest] = answer.content;
+    const text = first?.type === "text" ? first.text : "";
+    return { content: [{ type: "text", text: [text, ...notes].join("\n") }, ...rest] };
   }
 
   /** A named file must be in the shared set (or a scratch file), and have the lines asked for. */
@@ -318,5 +325,10 @@ export class BoardMcp {
   }
 }
 
-const done = (text: string): ToolResult => ({ content: [{ type: "text", text }] });
+const done = (text: string, images: ReadonlyArray<ToolImage> = []): ToolResult => ({
+  content: [
+    { type: "text", text },
+    ...images.map((image) => ({ type: "image" as const, ...image })),
+  ],
+});
 const failed = (text: string): ToolResult => ({ content: [{ type: "text", text }], isError: true });

package/src/server/server.ts

@@ -154,7 +154,7 @@ export async function serve(options: ServeOptions) {
       case "term-resize":
         return terminals.resize(message.id, message.cols, message.rows);
       case "board-result":
-        return boardMcp.result(message.callId, message.ok, message.text);
+        return boardMcp.result(message.callId, message.ok, message.text, message.images);
     }
   };
 

package/src/server/trim-event.ts

@@ -5,8 +5,9 @@
  *
  *  - a text chunk repeats the whole message so far in `content`; the thread
  *    is folded from `delta` alone, so the repeat goes;
- *  - images a tool returns (a screenshot the agent read) stay as a
- *    placeholder with their size;
+ *  - images a tool returns (a screenshot the agent read, a drawing) stay as
+ *    a placeholder with their size, as image content or as a data URL
+ *    (opencode's attachments);
  *  - tool output and arguments are cut to their start and end.
  */
 
@@ -48,7 +49,7 @@ function cut(text: string): string {
 
 /** Tool output is a string, JSON when it has more than text: drop image data from it. */
 function withoutImages(content: string): string {
-  if (!content.includes('"image"')) return content;
+  if (!content.includes('"image"') && !content.includes("data:image/")) return content;
   let parsed: unknown;
   try {
     parsed = JSON.parse(content);
@@ -64,6 +65,10 @@ function withoutImages(content: string): string {
       const size = JSON.stringify(value).length;
       return { type: "image", omitted: `${Math.round(size / 1024)} KB` };
     }
+    if (typeof value.url === "string" && value.url.startsWith("data:image/")) {
+      found = true;
+      return { ...value, url: `data:… (${Math.round(value.url.length / 1024)} KB omitted)` };
+    }
     return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, walk(v)]));
   };
   const next = walk(parsed);

package/src/shared/board-tools.ts

@@ -20,13 +20,14 @@ export const BOARD_TOOL_NAMES = [
   "add_comment",
   "edit_comment",
   "delete_comment",
+  "draw",
 ] as const;
 export type BoardToolName = (typeof BOARD_TOOL_NAMES)[number];
 
 /** The MCP server's name, as agents prefix its tools (`mcp__canvas__…`, `canvas_…`). */
 export const BOARD_SERVER_NAME = "canvas";
 
-export type FrameKind = "file" | "browser" | "terminal" | "agent";
+export type FrameKind = "file" | "browser" | "terminal" | "agent" | "drawing";
 export type PlaceSide = "left" | "right" | "above" | "below";
 
 export interface ViewBoardArgs {
@@ -63,7 +64,15 @@ interface FileList {
   readonly files?: ReadonlyArray<FileListEntry>;
 }
 
-export interface OpenFrameArgs extends Placement, FileTarget, ScratchContent, FileList {
+/** What to draw (ADR 0009): the host's browser turns it into Excalidraw elements. */
+interface DrawContent {
+  /** Excalidraw element skeletons (`web/lib/drawing.ts` checks them). */
+  readonly elements?: ReadonlyArray<Record<string, unknown>>;
+  readonly mermaid?: string;
+}
+
+export interface OpenFrameArgs
+  extends Placement, FileTarget, ScratchContent, FileList, DrawContent {
   readonly type: FrameKind;
   readonly title?: string;
   readonly url?: string;
@@ -105,6 +114,13 @@ export interface DeleteCommentArgs {
   readonly comment: string;
 }
 
+export interface DrawArgs extends DrawContent {
+  readonly frame: string;
+  /** Ids of elements to remove. */
+  readonly delete?: ReadonlyArray<string>;
+  readonly clear?: boolean;
+}
+
 export interface CloseFrameArgs {
   readonly frame: string;
 }
@@ -190,6 +206,68 @@ const fileList = {
   },
 };
 
+const idRef = { type: "object", properties: { id: { type: "string" } }, required: ["id"] };
+
+const drawContent = {
+  elements: {
+    type: "array",
+    description:
+      "Excalidraw elements to add, in the drawing's coordinates (view_frame gives the extent of " +
+      "what is there; y grows downwards). Shapes: rectangle, ellipse, diamond with x, y, width, " +
+      "height and a label. Text: x, y, text. Arrows and lines: x, y and points relative to them; " +
+      "an arrow between two shapes needs only start and end — the ids of shapes of this call or " +
+      "already drawn — and is routed between them. Give an id of your own to connect to a shape " +
+      "of the same call; an existing element's id replaces that element.",
+    items: {
+      type: "object",
+      properties: {
+        type: {
+          type: "string",
+          enum: ["rectangle", "ellipse", "diamond", "text", "arrow", "line"],
+        },
+        id: { type: "string" },
+        x: { type: "number" },
+        y: { type: "number" },
+        width: { type: "number" },
+        height: { type: "number" },
+        label: {
+          type: "object",
+          properties: { text: { type: "string" } },
+          required: ["text"],
+          description: "A shape's or an arrow's text.",
+        },
+        text: { type: "string", description: "text: its text." },
+        fontSize: { type: "number", description: "text: default 20." },
+        points: {
+          type: "array",
+          items: { type: "array", items: { type: "number" }, minItems: 2, maxItems: 2 },
+          description: "arrow, line: [[0,0],[dx,dy],…], relative to x, y.",
+        },
+        start: { ...idRef, description: "arrow: the shape it starts at." },
+        end: { ...idRef, description: "arrow: the shape it points to." },
+        strokeColor: { type: "string", description: "Hex, e.g. #e03131." },
+        backgroundColor: { type: "string", description: "Hex; default transparent." },
+        fillStyle: { type: "string", enum: ["solid", "hachure", "cross-hatch"] },
+        strokeWidth: { type: "number", description: "1, 2 (default) or 4." },
+        strokeStyle: { type: "string", enum: ["solid", "dashed", "dotted"] },
+        roundness: {
+          type: "object",
+          properties: { type: { type: "number" } },
+          description: "{type: 3}: rounded corners.",
+        },
+        link: { type: "string", description: "A URL, or a board link." },
+      },
+      required: ["type"],
+    },
+  },
+  mermaid: {
+    type: "string",
+    description:
+      "A mermaid flowchart, sequence or class diagram, turned into shapes and placed below what " +
+      "is drawn. The easiest way to draw anything with more than a few boxes.",
+  },
+};
+
 export const BOARD_TOOLS: ReadonlyArray<{
   readonly name: BoardToolName;
   readonly description: string;
@@ -216,7 +294,8 @@ export const BOARD_TOOLS: ReadonlyArray<{
     description:
       "Look at one frame in full. For a file frame: what it shows, its list of files, and its " +
       "comments — people's and agents' notes on lines of its files, with ids, authors and lines. " +
-      "view_board says which frames have comments.",
+      "view_board says which frames have comments. For a drawing: an image of it, and its " +
+      "elements with ids, labels, places and what arrows connect.",
     inputSchema: {
       type: "object",
       properties: { frame: { type: "string", description: "Id of the frame." } },
@@ -231,17 +310,20 @@ export const BOARD_TOOLS: ReadonlyArray<{
       "or a scratch file: content you pass, kept by canvas outside the project. A file frame can " +
       "also carry a list of files (files): one frame to click through, instead of many. " +
       "browser: a URL, loaded by each viewer's own browser. terminal: an idle shell people can " +
-      "type into. agent: another agent session; `draft` pre-fills its prompt, a person sends it.",
+      "type into. agent: another agent session; `draft` pre-fills its prompt, a person sends it. " +
+      "drawing: a whiteboard people sketch on together, optionally with elements or mermaid " +
+      "to start it (see draw).",
     inputSchema: {
       type: "object",
       properties: {
-        type: { type: "string", enum: ["file", "browser", "terminal", "agent"] },
+        type: { type: "string", enum: ["file", "browser", "terminal", "agent", "drawing"] },
         ...fileTarget,
         ...scratchContent,
         ...fileList,
         url: { type: "string", description: "browser: an http(s) URL." },
         agent: { type: "string", description: "agent: which agent runs it (see view_board)." },
         draft: { type: "string", description: "agent: a prompt draft for people to send." },
+        ...drawContent,
         title: { type: "string", description: "Frame title. Default: from what it shows." },
         ...placement,
       },
@@ -356,6 +438,27 @@ export const BOARD_TOOLS: ReadonlyArray<{
       required: ["frame", "comment"],
     },
   },
+  {
+    name: "draw",
+    description:
+      "Change a drawing frame: add elements, a mermaid diagram, replace elements (an element with " +
+      "an existing id), remove elements (delete) or everything (clear). Look at it first " +
+      "(view_frame): people may have drawn there. Everyone sees the change live.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        frame: { type: "string", description: "Id of the drawing frame." },
+        ...drawContent,
+        delete: {
+          type: "array",
+          items: { type: "string" },
+          description: "Ids of elements to remove (view_frame lists them); labels go with shapes.",
+        },
+        clear: { type: "boolean", description: "Remove everything first." },
+      },
+      required: ["frame"],
+    },
+  },
 ];
 
 /** The agent's standing context, sent as MCP server instructions. */
@@ -363,7 +466,7 @@ export function boardInstructions(
   frameId: string,
   skills: ReadonlyArray<{ readonly name: string; readonly description: string }> = [],
 ): string {
-  return `You are running inside canvas: a shared, multiplayer board that people are looking at together, live. Its frames are coding-agent sessions, files of this project, browser previews and terminals. You are the agent in frame ${frameId}; people write prompts into it and read your replies there. Several people may prompt you.
+  return `You are running inside canvas: a shared, multiplayer board that people are looking at together, live. Its frames are coding-agent sessions, files of this project, browser previews, terminals and drawings. You are the agent in frame ${frameId}; people write prompts into it and read your replies there. Several people may prompt you.
 
 Frames that sit close together form a cluster: people keep related work together that way, and your own cluster is your workspace. Within a cluster frames sit in rows; frames in a row share a height.
 
@@ -380,6 +483,7 @@ The ${BOARD_SERVER_NAME} tools let you see and change the board: ${BOARD_TOOL_NA
 - Any agent may read (read_board_file) and overwrite (write_board_file) any scratch file; view_board lists them.
 - To show several files for one topic, prefer one file frame with a list (files) over a frame per file: people click through it at their own pace. You decide the tree: display paths, folders, order, line ranges. A write-up (a scratch file) at the top and a visualisation at the bottom fit in the same list.
 - People (and agents) comment on lines of files in a file frame. view_board says which frames have comments; view_frame lists them, with ids. You aren't told when someone comments: look when asked to, e.g. to address feedback. add_comment leaves one of yours; edit_comment and delete_comment change agents' comments, never people's. update_frame with a comment's id shows its file at its lines. A comment is "outdated" when its lines changed since; it shows what they were.
+- A drawing frame is an Excalidraw whiteboard people sketch on together. view_frame shows it to you as an image, with its elements as text; draw adds shapes, text, arrows or a mermaid diagram, changes or removes elements. Use one to sketch an architecture or a flow with people, or when asked to draw; a mermaid flowchart is quickest for more than a few boxes. People's sketches in it are theirs: add next to them, don't redraw them unless asked.
 - Markdown and HTML files render. HTML runs its scripts, but relative assets (CSS, images, other scripts) don't load, so inline them. Links do work (see below).
 
 Your replies, comments, markdown and HTML can link to places on the board with ordinary markdown links (or <a href> in HTML). A click takes that person there, opening a file frame if no frame shows the file:

package/src/shared/protocol.ts

@@ -213,14 +213,22 @@ export type ClientToServer =
       readonly cols: number;
       readonly rows: number;
     }
-  /** The answer to a `board-call`: the tool's text, for the agent. */
+  /** The answer to a `board-call`: the tool's text, for the agent, and any images. */
   | {
       readonly t: "board-result";
       readonly callId: string;
       readonly ok: boolean;
       readonly text: string;
+      readonly images?: ReadonlyArray<ToolImage>;
     };
 
+/** An image a board tool shows the agent, e.g. a drawing (ADR 0009). */
+export interface ToolImage {
+  /** base64, no data: prefix. */
+  readonly data: string;
+  readonly mimeType: "image/png";
+}
+
 export type ServerToClient =
   | {
       readonly t: "welcome";