@a-t-h-i/bot-lobby 0.6.8 → 0.6.9

package/README.md

@@ -260,6 +260,7 @@ footer.
 | **5 Metrics** | Run time, success rate, tokens and cost per model and agent |
 | **6 Git** | The repository's open pull requests; review one with an agent, or have Jev read it |
 | **7 Knowledge** | Everything each agent knows about the project; edit an entry, or leave a note every agent reads |
+| **8 Excalidraw** | Up to five shared Excalidraw sessions, each assigned to one agent or several, who read the board and draw on it with you |
 
 ### Lobby
 
@@ -343,6 +344,51 @@ prompt says how many sections it left out and where the whole file is. Nothing
 is looked up on demand: what a step needs has to be in the file and near the
 top of its relevance, so keep entries short, one topic under one heading.
 
+### Excalidraw
+
+A live [Excalidraw](https://excalidraw.com) room that you and your agents draw
+in together. Add up to **five sessions**, assign each to **one agent or several**
+(the oracle, Designer, Backend, QA, scouts, the researcher, quick fix, the
+planner), and those agents can look at what you drew and add to it.
+
+- `a` **adds a session**: in Excalidraw, *Share → Live collaboration → Start
+  session*, copy the link, paste it into the prompt (a name may follow it).
+  `n` **makes a new room** instead and shows its link, for you to open in
+  Excalidraw. Neither works past five sessions: `d d` removes one.
+- `enter` moves into the checklist of agents; `enter` or `space` assigns the
+  picked agent (or takes the session back), `*` assigns every agent.
+- `w` lets agents **draw** in the session or **only look** at it. An agent whose
+  sessions are all look-only gets no drawing tool.
+- `t` **checks** the session: joins the room for a moment and says whether the
+  server can be reached, who is in it, and how much is on the board. `r` renames it.
+
+**What an assigned agent can do.** It gets two tools and the room link:
+`excalidraw_read` describes the board in words (shapes with their labels, arrows
+as *from → to*, free text, each with its id and place), and `excalidraw_draw`
+adds labelled rectangles, ellipses and diamonds, arrows between them (with
+labels), free text and lines, or changes and deletes what is there by id. It
+joins the room as its own collaborator (`Backend · bot-lobby`) with a cursor
+where it drew, so you watch it work. The board stays yours: agents may move,
+recolour and relabel what you drew, but delete only what agents drew (elements
+they draw are marked, and Excalidraw cannot undo another collaborator's
+deletion), and one call draws at most 100 shapes and removes at most 50.
+
+- **You must have the session open in Excalidraw.** A room's board lives in the
+  browsers that are in it; an agent that joins an empty room can read nothing,
+  and is not allowed to draw, since nothing would keep it. It says so and asks
+  you to open the link.
+- **Boards are other people's writing.** What an agent reads from a board comes
+  fenced as untrusted data, like a web page, and it is told never to follow
+  instructions written on it.
+- **The link is a key.** Anyone with a room link can read and draw in the room, so
+  bot-lobby keeps your sessions with your own settings
+  (`~/.pi/bot-lobby/excalidraw/`, one file for each project) and never in the
+  project, where a commit could publish them. An agent's process is handed only the
+  links of the sessions assigned to it.
+- Rooms are Excalidraw's own collaboration protocol, end-to-end encrypted with the
+  key in the link, over `oss-collab.excalidraw.com`. A self-hosted collaboration
+  server is used when `BOT_LOBBY_EXCALIDRAW_SERVER` names it.
+
 Common keys: `tab` switches tabs, `esc` browses (arrows, single-key
 commands), `ctrl+f` searches, `ctrl+s` saves the plan, `alt+o` browses
 sessions, `alt+n` starts a task in a new session, `alt+s` opens settings.
@@ -629,12 +675,15 @@ Live checks (spend tokens or need a key and network):
 BOT_LOBBY_E2E=1 node --test test/e2e.test.ts
 BOT_LOBBY_LIVE_WEB=1 node --test test/web.test.ts
 BOT_LOBBY_JEV_E2E=1 OPENCODE_API_KEY=… node --test test/jev-e2e.test.ts
+BOT_LOBBY_LIVE_EXCALIDRAW=1 node --test test/excalidraw-live.test.ts
 ```
 
 Source layout: `src/workflow` (engine), `src/master` (delegation),
 `src/execution` (subagent processes), `src/lobby` (the UI),
 `src/classifier` (Jev), `src/state` (persistence), `src/ask` (the
-questionnaire), `src/web` (the web tools), `prompts/` (agent prompts).
+questionnaire), `src/web` (the web tools), `src/excalidraw` (shared
+Excalidraw sessions: the room protocol, the sessions, the agents' tools),
+`prompts/` (agent prompts).
 
 ## Publishing
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@a-t-h-i/bot-lobby",
-  "version": "0.6.8",
+  "version": "0.6.9",
   "description": "Structured multi-agent software engineering orchestrator for Pi",
   "type": "module",
   "license": "Apache-2.0",
@@ -51,7 +51,11 @@
     "@earendil-works/pi-coding-agent": "0.87.0",
     "@earendil-works/pi-tui": "0.87.0",
     "@types/node": "^22.10.0",
+    "socket.io": "4.8.4",
     "typebox": "1.3.27",
     "typescript": "^5.7.0"
+  },
+  "dependencies": {
+    "socket.io-client": "4.8.4"
   }
 }

package/src/excalidraw/check.ts

@@ -0,0 +1,31 @@
+/**
+ * "Does this session work?": join the room for a moment, as a seat named
+ * after the lobby, and say what is there — the server reached, who is in the
+ * room, how much is on the board, whether the link's key fits. The seat leaves
+ * again at once, so a check never leaves an extra collaborator in the room.
+ */
+import { ExcalidrawRoom, type RoomOptions } from "./client.ts";
+import { parseRoomLink } from "./room.ts";
+
+export interface CheckResult {
+  ok: boolean;
+  text: string;
+}
+
+export async function checkSession(link: string, name: string, options: Pick<RoomOptions, "server" | "sceneWaitMs" | "connectTimeoutMs" | "connect"> = {}): Promise<CheckResult> {
+  const parsed = parseRoomLink(link);
+  if (!parsed) return { ok: false, text: "not a valid room link" };
+  const room = new ExcalidrawRoom({ link: parsed, username: "bot-lobby · check", name, ...options });
+  try {
+    const status = await room.ready();
+    // Give an answer to the newcomer's arrival a moment more when someone is there but the board has not come yet.
+    if (status.undecryptable > 0 && status.elements === 0) return { ok: false, text: "reached the room, but its messages would not open: the link's key does not match, so it may have been copied incompletely" };
+    if (status.peers === 0) return { ok: true, text: "reached the server; nobody has this session open yet — open the link in Excalidraw, and agents can read and draw" };
+    const who = status.people.length > 0 ? status.people.join(", ") : `${status.peers} other${status.peers === 1 ? "" : "s"}`;
+    return { ok: true, text: `reached the room with ${who}; ${status.synced ? `${status.elements} element${status.elements === 1 ? "" : "s"} on the board` : "the board has not been sent yet (it comes when someone in the room answers)"}` };
+  } catch (error) {
+    return { ok: false, text: (error as Error).message };
+  } finally {
+    room.close();
+  }
+}

package/src/excalidraw/client.ts

@@ -0,0 +1,266 @@
+/**
+ * One agent's seat in an Excalidraw shared session. It joins the room the way
+ * a second browser tab would: over Excalidraw's collaboration socket, sending
+ * and receiving the encrypted scene messages every client in the room sends.
+ * Others in the room answer a newcomer with the whole scene, so a seat learns
+ * the board from whoever is there; what the agent draws goes out as an update
+ * that every browser merges in place, under the agent's name.
+ *
+ * A seat never answers a newcomer with a scene it does not have: a browser
+ * that took an empty scene from an agent would skip loading the board it saved
+ * itself. And it does not draw while nobody else is in the room, because
+ * nothing keeps a scene for a room but the browsers in it.
+ */
+import { io, type Socket } from "socket.io-client";
+import { DEFAULT_SERVER, seal, unseal, type RoomLink } from "./room.ts";
+import { describeScene, mergeElements, plainText, planDraw, sceneBounds, visible, type DrawOutcome, type DrawRequest, type Scene, type SceneElement } from "./scene.ts";
+
+/** The part of a socket.io client a seat uses; tests and other transports can stand in for it. */
+export type SocketLike = Pick<Socket, "on" | "emit" | "close" | "connected"> & { id?: string | undefined };
+
+export interface RoomOptions {
+  link: Pick<RoomLink, "roomId" | "roomKey">;
+  /** The name other people see next to this seat's cursor. */
+  username: string;
+  /** The session's name in the lobby, for readings and messages. */
+  name: string;
+  server?: string;
+  connect?: (server: string) => SocketLike;
+  /** Give up on reaching the server after this long. */
+  connectTimeoutMs?: number;
+  /** How long to wait for a collaborator to send the scene (Excalidraw's own client waits five seconds). */
+  sceneWaitMs?: number;
+}
+
+export interface RoomStatus {
+  connected: boolean;
+  /** Others in the room, people and agents, not counting this seat. */
+  peers: number;
+  /** A collaborator has sent the whole scene. */
+  synced: boolean;
+  elements: number;
+  /** Messages that would not open: the link's key does not match the room's. */
+  undecryptable: number;
+  /** Names the collaborators have given themselves. */
+  people: string[];
+}
+
+const CONNECT_TIMEOUT_MS = 10_000;
+const SCENE_WAIT_MS = 5000;
+/** The collaboration server relays at most a megabyte a message; a scene near that is not sent whole. */
+const MAX_MESSAGE_CHARS = 900_000;
+
+function defaultConnect(server: string): SocketLike {
+  return io(server, { transports: ["websocket", "polling"], autoUnref: true, timeout: 8000 });
+}
+
+export function collaborationServer(env: NodeJS.ProcessEnv = process.env): string {
+  return env.BOT_LOBBY_EXCALIDRAW_SERVER?.trim() || DEFAULT_SERVER;
+}
+
+export class ExcalidrawRoom {
+  readonly scene: Scene = new Map();
+  private readonly options: RoomOptions;
+  private socket: SocketLike | undefined;
+  private opening: Promise<RoomStatus> | undefined;
+  private peerIds: string[] = [];
+  private readonly names = new Map<string, string>();
+  private synced = false;
+  private undecryptable = 0;
+  private closed = false;
+  private timers: Array<ReturnType<typeof setTimeout>> = [];
+
+  constructor(options: RoomOptions) {
+    this.options = options;
+  }
+
+  get name(): string {
+    return this.options.name;
+  }
+
+  /** Which room this seat is in; a seat is replaced when its session's link changes. */
+  get linkKey(): string {
+    return `${this.options.link.roomId},${this.options.link.roomKey}`;
+  }
+
+  status(): RoomStatus {
+    return {
+      connected: this.socket?.connected ?? false,
+      peers: this.peerIds.length,
+      synced: this.synced,
+      elements: visible(this.scene).length,
+      undecryptable: this.undecryptable,
+      people: [...new Set(this.peerIds.map((id) => this.names.get(id)).filter((name): name is string => Boolean(name)))],
+    };
+  }
+
+  /** Join the room (once) and wait until a collaborator has sent the scene, or the wait for it is over. */
+  ready(): Promise<RoomStatus> {
+    this.opening ??= this.open();
+    return this.opening;
+  }
+
+  private open(): Promise<RoomStatus> {
+    return new Promise((resolve, reject) => {
+      // A seat that failed to connect once may try again.
+      this.closed = false;
+      const { roomId } = this.options.link;
+      const server = this.options.server ?? collaborationServer();
+      const socket = (this.options.connect ?? defaultConnect)(server);
+      this.socket = socket;
+      let done = false;
+      let lastError = "";
+      const settle = () => {
+        if (done) return;
+        done = true;
+        resolve(this.status());
+        this.announce();
+      };
+      this.timers.push(
+        setTimeout(() => {
+          if (done) return;
+          done = true;
+          this.opening = undefined;
+          this.close();
+          reject(new Error(`could not reach the Excalidraw collaboration server ${server}${lastError ? ` (${lastError})` : ""}`));
+        }, this.options.connectTimeoutMs ?? CONNECT_TIMEOUT_MS),
+      );
+      socket.on("connect_error", (error: Error) => {
+        lastError = error.message;
+      });
+      // Every (re)connection is greeted with init-room; the seat answers by joining.
+      socket.on("init-room", () => {
+        socket.emit("join-room", roomId);
+        this.timers.push(setTimeout(settle, this.options.sceneWaitMs ?? SCENE_WAIT_MS));
+      });
+      socket.on("first-in-room", () => settle());
+      socket.on("room-user-change", (clients: string[]) => {
+        this.peerIds = clients.filter((id) => id !== socket.id);
+        for (const id of [...this.names.keys()]) if (!clients.includes(id)) this.names.delete(id);
+      });
+      socket.on("new-user", () => {
+        // What this seat does not know, it does not claim to.
+        if (this.synced || visible(this.scene).length > 0) void this.send("SCENE_INIT", [...this.scene.values()]);
+      });
+      socket.on("client-broadcast", (data: unknown, iv: unknown) => {
+        void this.receive(data, iv).then((initial) => {
+          if (initial) settle();
+        });
+      });
+    });
+  }
+
+  /** A received message; true when it was the scene a newcomer is waiting for. */
+  private async receive(data: unknown, iv: unknown): Promise<boolean> {
+    const message = (await unseal(this.options.link.roomKey, data, iv)) as { type?: string; payload?: Record<string, unknown> } | undefined;
+    if (!message || typeof message !== "object") {
+      this.undecryptable += 1;
+      return false;
+    }
+    const payload = message.payload ?? {};
+    switch (message.type) {
+      case "SCENE_INIT":
+      case "SCENE_UPDATE":
+        if (Array.isArray(payload.elements)) mergeElements(this.scene, payload.elements);
+        if (message.type === "SCENE_INIT") this.synced = true;
+        return message.type === "SCENE_INIT";
+      case "MOUSE_LOCATION":
+      case "IDLE_STATUS":
+        if (typeof payload.socketId === "string" && typeof payload.username === "string" && plainText(payload.username)) this.names.set(payload.socketId, plainText(payload.username).slice(0, 40));
+        return false;
+      default:
+        return false;
+    }
+  }
+
+  /** Send a scene message to the room; false when the seat is not connected. */
+  private async send(type: "SCENE_INIT" | "SCENE_UPDATE" | "IDLE_STATUS" | "MOUSE_LOCATION", body: unknown): Promise<boolean> {
+    const socket = this.socket;
+    if (!socket?.connected || this.closed) return false;
+    const payload = type === "SCENE_INIT" || type === "SCENE_UPDATE" ? { elements: body } : body;
+    if (JSON.stringify(payload).length > MAX_MESSAGE_CHARS) return false;
+    const sealed = await seal(this.options.link.roomKey, { type, payload });
+    socket.emit("server-broadcast", this.options.link.roomId, sealed.data, sealed.iv);
+    return true;
+  }
+
+  /**
+   * Tell the room who this seat is (its collaborator name), and where it just
+   * drew. Excalidraw sends these as volatile messages because it sends them
+   * constantly; a seat says it once, so it does not let the server drop it.
+   */
+  private announce(near?: { x: number; y: number }): void {
+    const socketId = this.socket?.id;
+    if (!socketId) return;
+    void this.send("IDLE_STATUS", { socketId, userState: "active", username: this.options.username });
+    if (near) void this.send("MOUSE_LOCATION", { socketId, pointer: { x: near.x, y: near.y, tool: "pointer" }, button: "up", selectedElementIds: {}, username: this.options.username });
+  }
+
+  /** Why nothing may be drawn right now, or undefined when it may. */
+  cannotDraw(): string | undefined {
+    if (!this.socket?.connected) return "this seat is not connected to the room";
+    if (this.peerIds.length === 0) return "nobody else is in the room, and only a browser keeps a room's board; ask the user to open the session's link in Excalidraw, then try again";
+    return undefined;
+  }
+
+  /** Draw: the elements go out to the room and into this seat's own scene. */
+  async draw(request: DrawRequest): Promise<DrawOutcome> {
+    const outcome = planDraw(this.scene, request);
+    if (outcome.changed.length === 0) return outcome;
+    const reason = this.cannotDraw();
+    if (reason) return { ...outcome, changed: [], drawn: [], removed: [], problems: [...outcome.problems, reason] };
+    for (const element of outcome.changed) this.scene.set(element.id, element);
+    await this.send("SCENE_UPDATE", outcome.changed);
+    const around = sceneBounds(outcome.changed.filter((element) => !element.isDeleted));
+    this.announce(around ? { x: around.x1, y: around.y1 } : undefined);
+    return outcome;
+  }
+
+  /** The board in words, for a model, with what is known about the room around it. */
+  read(): string {
+    const status = this.status();
+    const notes: string[] = [];
+    if (!status.connected) notes.push("Not connected to the room right now; this is the board as last seen.");
+    else if (status.undecryptable > 0 && status.elements === 0) notes.push("Messages in this room would not open: the session link's key does not match the room's, so it may have been copied incompletely.");
+    else if (status.peers === 0) notes.push("Nobody else is in the room, so the board above is not known: ask the user to open the session's link in Excalidraw, then read again.");
+    else if (!status.synced) notes.push("No collaborator has sent the whole board yet, so this may be incomplete; read again in a few seconds.");
+    if (status.peers > 0) notes.push(`In the room with you: ${status.people.length > 0 ? status.people.join(", ") : `${status.peers} other${status.peers === 1 ? "" : "s"}`}.`);
+    return [describeScene(this.scene, this.options.name), ...notes].join("\n");
+  }
+
+  elements(): SceneElement[] {
+    return visible(this.scene);
+  }
+
+  close(): void {
+    this.closed = true;
+    for (const timer of this.timers) clearTimeout(timer);
+    this.timers = [];
+    this.socket?.close();
+  }
+}
+
+/** The seats an agent has open, one per session, kept for the length of its run. */
+export class RoomPool {
+  private readonly rooms = new Map<string, ExcalidrawRoom>();
+  private readonly make: (options: RoomOptions) => ExcalidrawRoom;
+
+  constructor(make: (options: RoomOptions) => ExcalidrawRoom = (options) => new ExcalidrawRoom(options)) {
+    this.make = make;
+  }
+
+  /** The seat for a session, opened on first use; a session whose link changed gets a new seat. */
+  room(key: string, options: RoomOptions): ExcalidrawRoom {
+    const existing = this.rooms.get(key);
+    if (existing && existing.linkKey === `${options.link.roomId},${options.link.roomKey}`) return existing;
+    existing?.close();
+    const room = this.make(options);
+    this.rooms.set(key, room);
+    return room;
+  }
+
+  closeAll(): void {
+    for (const room of this.rooms.values()) room.close();
+    this.rooms.clear();
+  }
+}

package/src/excalidraw/room.ts

@@ -0,0 +1,118 @@
+/**
+ * Excalidraw's shared-session ("live collaboration") rooms, the parts that do
+ * not touch the network: reading and making room links, and the AES-GCM
+ * envelope every message in a room travels in. Excalidraw's collaboration
+ * server only relays opaque bytes; the key sits in the link's `#room=id,key`
+ * fragment, which a browser never sends to any server, so whoever holds the
+ * link can read and draw in the room and nobody else can.
+ */
+import { randomBytes, webcrypto } from "node:crypto";
+
+/** Excalidraw's own collaboration server; `BOT_LOBBY_EXCALIDRAW_SERVER` points at a self-hosted one. */
+export const DEFAULT_SERVER = "https://oss-collab.excalidraw.com";
+
+/** The site a made-up room link opens on. */
+export const DEFAULT_ORIGIN = "https://excalidraw.com";
+
+const ROOM_ID = /^[a-zA-Z0-9_-]+$/;
+/** A 128-bit key in base64url is exactly 22 characters. */
+const ROOM_KEY = /^[a-zA-Z0-9_-]{22}$/;
+const FRAGMENT = /#room=([a-zA-Z0-9_-]+),([a-zA-Z0-9_-]+)$/;
+const IV_BYTES = 12;
+
+export interface RoomLink {
+  roomId: string;
+  roomKey: string;
+  /** The link as Excalidraw writes it: `https://excalidraw.com/#room=<id>,<key>`. */
+  url: string;
+}
+
+/**
+ * A room link as people paste it: the full URL (any Excalidraw host, so a
+ * self-hosted one works), `#room=id,key`, or the bare `id,key`. Undefined when
+ * it is none of those or the key is not the 22 characters Excalidraw makes.
+ */
+export function parseRoomLink(input: string): RoomLink | undefined {
+  const text = input.trim();
+  if (!text) return undefined;
+  let roomId: string | undefined;
+  let roomKey: string | undefined;
+  let origin = DEFAULT_ORIGIN;
+  const fragment = text.match(FRAGMENT);
+  if (fragment) {
+    [, roomId, roomKey] = fragment;
+    const before = text.slice(0, fragment.index);
+    if (before) {
+      try {
+        const parsed = new URL(before);
+        if (parsed.protocol === "https:" || parsed.protocol === "http:") origin = `${parsed.origin}${parsed.pathname === "/" ? "" : parsed.pathname}`;
+        else return undefined;
+      } catch {
+        return undefined;
+      }
+    }
+  } else {
+    const bare = text.match(/^([a-zA-Z0-9_-]+),([a-zA-Z0-9_-]+)$/);
+    if (!bare) return undefined;
+    [, roomId, roomKey] = bare;
+  }
+  if (!roomId || !roomKey || !ROOM_ID.test(roomId) || !ROOM_KEY.test(roomKey)) return undefined;
+  return { roomId, roomKey, url: `${origin.replace(/\/$/, "")}/#room=${roomId},${roomKey}` };
+}
+
+/** A fresh room: opening its link in Excalidraw starts the session, and the room is the first thing in it. */
+export function newRoomLink(origin = DEFAULT_ORIGIN): RoomLink {
+  const roomId = randomBytes(10).toString("hex");
+  const roomKey = randomBytes(16).toString("base64url");
+  return { roomId, roomKey, url: `${origin.replace(/\/$/, "")}/#room=${roomId},${roomKey}` };
+}
+
+/** The link with its key hidden, for logs and lists. */
+export function maskedLink(link: RoomLink): string {
+  return `room ${link.roomId}`;
+}
+
+const keys = new Map<string, Promise<webcrypto.CryptoKey>>();
+
+function cryptoKey(roomKey: string, usage: "encrypt" | "decrypt"): Promise<webcrypto.CryptoKey> {
+  const id = `${usage}:${roomKey}`;
+  let key = keys.get(id);
+  if (!key) {
+    key = webcrypto.subtle.importKey("jwk", { alg: "A128GCM", ext: true, k: roomKey, key_ops: ["encrypt", "decrypt"], kty: "oct" }, { name: "AES-GCM", length: 128 }, false, [usage]);
+    keys.set(id, key);
+    if (keys.size > 32) keys.delete(keys.keys().next().value!);
+  }
+  return key;
+}
+
+export interface Sealed {
+  data: Buffer;
+  iv: Buffer;
+}
+
+/** A message as Excalidraw sends it: the JSON of `payload`, encrypted with a fresh IV. */
+export async function seal(roomKey: string, payload: unknown): Promise<Sealed> {
+  const iv = randomBytes(IV_BYTES);
+  const encrypted = await webcrypto.subtle.encrypt({ name: "AES-GCM", iv }, await cryptoKey(roomKey, "encrypt"), new TextEncoder().encode(JSON.stringify(payload)));
+  return { data: Buffer.from(encrypted), iv };
+}
+
+function bytes(value: unknown): Uint8Array | undefined {
+  if (value instanceof Uint8Array) return value;
+  if (value instanceof ArrayBuffer) return new Uint8Array(value);
+  if (ArrayBuffer.isView(value)) return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+  return undefined;
+}
+
+/** What `seal` made, opened; undefined when it is not ours to read (a wrong key, a damaged message). */
+export async function unseal(roomKey: string, data: unknown, iv: unknown): Promise<unknown> {
+  const body = bytes(data);
+  const nonce = bytes(iv);
+  if (!body || !nonce) return undefined;
+  try {
+    const opened = await webcrypto.subtle.decrypt({ name: "AES-GCM", iv: nonce }, await cryptoKey(roomKey, "decrypt"), body);
+    return JSON.parse(new TextDecoder().decode(opened));
+  } catch {
+    return undefined;
+  }
+}