agello 0.3.0 → 0.4.1

package/src/server.ts

@@ -4,24 +4,27 @@
 // GET  /agent-bridge.js  <agent-bridge> web component
 // GET  /status           session liveness (herdr agent get + pane get)
 // GET  /events           SSE: status, chat messages, tool start/end (from transcript JSONL)
-// POST /send             {action, text} -> herdr agent prompt
-//                        (action "annotate": + target {selector, label, rect, text?, url?} from inspect)
+// POST /send             {action, text, images?: [{type, data(base64)}]} -> herdr agent prompt
+//                        (images are saved to $TMPDIR/agello-uploads and sent as [image: <path>])
 // GET  /screen           SSE: terminal-browser screen frames (CDP screencast)
-// WS   /input            user control: CDP Input.* commands forwarded to the page;
-//                        {id, inspect: {x, y} | {selector}, full?} -> {id, result}: element at a point (annotation)
+// POST /upload           {images: [{type, data(base64)}]} -> {paths}: terminal image paste
+// WS   /terminal         interactive herdr terminal frames, keyboard input, resize
+// GET  /panes            herdr workspace -> tab -> pane tree
+// POST /panes/connect    {pane} -> url of that pane's server (started if needed)
+// POST /panes/create     {kind: workspace|tab|pane, cwd?, label?, workspace?, pane?, direction?}
+// WS   /input            user control: CDP Input.* commands forwarded to the page
 // GET  /present          presentation state
 // POST /present          {rect?: {x,y,w,h}} start (or move the crop), {stop: true} end
 //                        ({stop: true, from: "viewer"}: ended from the page, the agent is told
 //                         with "[browser] action=present-stop")
-// GET  /script           presentation script + player position
-// POST /script           {script: {rect?, steps: [{go?, say: string[], hold?}]}} load (keeps position)
-// POST /player           {cmd: "resume" | "pause" | "goto", at?: "3" | "3.2"}
 
-import { homedir } from "node:os";
+import { createTerminal, type SocketData } from "./terminal.ts";
+import { panesRoute } from "./panes.ts";
+import { mkdir } from "node:fs/promises";
+import { homedir, tmpdir } from "node:os";
 import { join } from "node:path";
 import { runJson } from "./run.ts";
-import { createPlayer, parsePos, parseScript, type PlayerInfo } from "./player.ts";
-import { createScreencast, type Inspected, type Rect } from "./screencast.ts";
+import { createScreencast, type Rect } from "./screencast.ts";
 
 export type ServerOptions = {
   port: number;
@@ -32,6 +35,30 @@ export type ServerOptions = {
 };
 
 export const BROWSER_PREFIX = "[browser]";
+const UPLOAD_DIR = join(tmpdir(), "agello-uploads");
+const IMAGE_EXT: Record<string, string> = { "image/png": "png", "image/jpeg": "jpg", "image/gif": "gif", "image/webp": "webp" };
+const MAX_IMAGES = 5;
+const MAX_IMAGE_BYTES = 10 * 1024 * 1024;
+
+// Save pasted/dropped chat images so the agent can open them by path
+// (herdr agent prompt carries text only). null: invalid input.
+export async function saveImages(list: unknown): Promise<string[] | null> {
+  if (list === undefined) return [];
+  if (!Array.isArray(list) || list.length > MAX_IMAGES) return null;
+  const files = list.map((img: any) => {
+    const ext = IMAGE_EXT[img?.type];
+    if (!ext || typeof img.data !== "string") return null;
+    const bytes = Buffer.from(img.data, "base64");
+    return bytes.length && bytes.length <= MAX_IMAGE_BYTES ? { ext, bytes } : null;
+  });
+  if (files.includes(null)) return null;
+  await mkdir(UPLOAD_DIR, { recursive: true, mode: 0o700 });
+  return Promise.all(files.map(async (f) => {
+    const path = join(UPLOAD_DIR, `${crypto.randomUUID()}.${f!.ext}`);
+    await Bun.write(path, f!.bytes);
+    return path;
+  }));
+}
 const WEB_DIR = join(import.meta.dir, "..", "web");
 
 // ---------- pure helpers ----------
@@ -96,22 +123,7 @@ export function parseRect(r: any): Rect | null {
   return { x, y, w, h };
 }
 
-// Annotation prompt: the viewer's request plus the element it points at, in
-// 3 lines (request / selector / element summary).
-export function formatAnnotation(text: string, t: any): string | null {
-  if (!t || typeof t.selector !== "string" || !t.selector || t.selector.length > 1000) return null;
-  const clean = (v: unknown, n: number) => String(v ?? "").replace(/\s+/g, " ").trim().slice(0, n);
-  const request = text.split(/\r?\n/).map((l) => l.trim()).filter(Boolean).join(" / ");
-  const r = t.rect ?? {};
-  const [x, y, w, h] = [r.x, r.y, r.w, r.h].map((v) => Math.round(Number(v) || 0));
-  const label = clean(t.label, 120);
-  const snippet = clean(t.text, 80);
-  const url = clean(t.url, 300);
-  const info = [`요소: ${label}${snippet ? ` "${snippet}"` : ""}`, `위치 ${x},${y} 크기 ${w}x${h}`, url].filter(Boolean);
-  return [request, `대상: ${clean(t.selector, 1000)}`, info.join(" · ")].join("\n");
-}
-
-export type Present = { on: boolean; rect?: Rect; since?: string; player?: PlayerInfo };
+export type Present = { on: boolean; rect?: Rect; since?: string };
 
 type Status =
   | { alive: false; reason: string; pane: string }
@@ -125,6 +137,12 @@ export async function startServer(opts: ServerOptions) {
   const HTML = Bun.file(join(WEB_DIR, "index.html"));
   const COMPONENT = Bun.file(join(WEB_DIR, "agent-bridge.js"));
 
+  const terminal = createTerminal(PANE);
+  const bundle = await Bun.build({ entrypoints: [join(WEB_DIR, "terminal.js")], target: "browser", minify: true });
+  if (!bundle.success) throw new Error(`Terminal bundle failed: ${bundle.logs.join("\n")}`);
+  const terminalJS = await bundle.outputs[0].text();
+  const terminalCSS = Bun.file(new URL(import.meta.resolve("@xterm/xterm/css/xterm.css")));
+
   const agentInfo = async (): Promise<any | null> =>
     (await herdr(3000, "agent", "get", PANE))?.result?.agent ?? null;
 
@@ -324,7 +342,8 @@ export async function startServer(opts: ServerOptions) {
 
   const screen = createScreencast({
     browserKey: opts.browser,
-    herdrTab: async () => (await agentInfo())?.tab_id,
+    // shell panes have no agent: fall back to the pane's own tab
+    herdrTab: async () => (await agentInfo())?.tab_id ?? (await herdr(3000, "pane", "get", PANE))?.result?.pane?.tab_id,
   });
 
   // ----- presentation mode -----
@@ -332,67 +351,10 @@ export async function startServer(opts: ServerOptions) {
   // short-lived bubbles. rect: crop of the visible viewport (unset = whole screen).
 
   let present: Present = { on: false };
-  const presentView = (): Present => ({ ...present, player: player.info() });
   function setPresent(next: Present) {
     present = next;
     screen.setClip(next.on ? (next.rect ?? null) : null);
-    if (!next.on) player.stop();
-    broadcast("present", presentView());
-  }
-
-  // Script player: agello owns order and timing, the agent owns the deck
-  // (each step's `go`) and decides when to resume after a question.
-  const player = createPlayer({
-    async go(expr) {
-      const js = expr.startsWith("#") ? `location.hash = ${JSON.stringify(expr)}` : expr;
-      if (!(await screen.evaluate(js))) console.error(`present: go failed: ${expr}`);
-    },
-    say(text, at, hold) {
-      broadcast("message", { role: "assistant", text, ts: new Date().toISOString(), script: at, hold });
-    },
-    done() {
-      const n = player.info().steps;
-      promptAgent("present-done", `대본 끝까지 재생함 (${n}장)`);
-    },
-    changed() {
-      broadcast("present", presentView());
-    },
-  });
-
-  // "마지막 표시 3.2 · 다음 3.3" for agent notices
-  const where = () => {
-    const i = player.info();
-    if (i.state === "empty") return "";
-    return [i.last && `마지막 표시 ${i.last}`, i.next && `다음 ${i.next}`].filter(Boolean).join(" · ");
-  };
-
-  async function scriptRequest(req: Request): Promise<Response> {
-    if (req.method !== "POST") return Response.json({ script: player.script(), player: player.info() });
-    const data = (await req.json().catch(() => ({}))) as { script?: unknown };
-    const script = parseScript(data.script);
-    if (typeof script === "string") return Response.json({ ok: false, error: script }, { status: 400 });
-    if (script.rect && !parseRect(script.rect))
-      return Response.json({ ok: false, error: "invalid_rect" }, { status: 400 });
-    player.load(script);
-    return Response.json({ ok: true, player: player.info() });
-  }
-
-  async function playerRequest(req: Request): Promise<Response> {
-    const data = (await req.json().catch(() => ({}))) as { cmd?: string; at?: string };
-    let error: string | null = null;
-    if (data.cmd === "resume") {
-      if (!present.on) {
-        const rect = player.script()?.rect;
-        setPresent({ on: true, rect: rect ? parseRect(rect)! : undefined, since: new Date().toISOString() });
-      }
-      error = player.resume();
-    } else if (data.cmd === "pause") player.pause();
-    else if (data.cmd === "goto") {
-      const pos = parsePos(data.at ?? "");
-      error = pos ? await player.goto(pos) : "invalid_position";
-    } else error = "unknown_cmd";
-    if (error) return Response.json({ ok: false, error, player: player.info() }, { status: 400 });
-    return Response.json({ ok: true, player: player.info() });
+    broadcast("present", present);
   }
 
   async function presentRequest(req: Request): Promise<Response> {
@@ -404,11 +366,10 @@ export async function startServer(opts: ServerOptions) {
       // (The CLI stop comes from the agent itself and needs no notice.)
       let notified: string | undefined;
       if (was && data.from === "viewer") {
-        const at = where();
-        const r = await promptAgent("present-stop", `사용자가 페이지에서 발표를 종료함${at ? ` (대본 정지: ${at})` : ""}`);
+        const r = await promptAgent("present-stop", "사용자가 페이지에서 발표를 종료함");
         notified = r.ok ? "ok" : r.error;
       }
-      return Response.json({ ok: true, present: presentView(), notified });
+      return Response.json({ ok: true, present, notified });
     }
     let rect: Rect | undefined;
     if (data.rect != null) {
@@ -416,7 +377,7 @@ export async function startServer(opts: ServerOptions) {
       if (!rect) return Response.json({ ok: false, error: "invalid_rect" }, { status: 400 });
     }
     setPresent({ on: true, rect, since: present.on ? present.since : new Date().toISOString() });
-    return Response.json({ ok: true, present: presentView() });
+    return Response.json({ ok: true, present });
   }
 
   // ----- HTTP -----
@@ -457,7 +418,7 @@ export async function startServer(opts: ServerOptions) {
           tools: [...pendingTools.keys()],
         });
         if (lastStatus) send(c, "status", lastStatus);
-        send(c, "present", presentView());
+        send(c, "present", present);
         for (const t of pendingTools.values()) send(c, "tool_start", t);
         ping = setInterval(() => send(c, "ping", {}), 15000);
       },
@@ -482,38 +443,36 @@ export async function startServer(opts: ServerOptions) {
   }
 
   async function sendPrompt(req: Request): Promise<Response> {
-    const data = (await req.json().catch(() => ({}))) as { action?: string; text?: string; target?: Inspected };
+    const data = (await req.json().catch(() => ({}))) as { action?: string; text?: string; images?: unknown };
     const action = /^[\w-]{1,32}$/.test(data.action ?? "") ? data.action! : "message";
-    let text = data.text ?? "";
-    if (action === "annotate") {
-      const t = formatAnnotation(text, data.target);
-      if (!t || !text.trim()) return Response.json({ ok: false, error: "invalid_target" }, { status: 400 });
-      text = t;
-    }
-    // Raised hand: stop the script at once (no agent round trip) and tell the
-    // agent where it stopped.
-    if (action === "hand-raise") {
-      player.pause();
-      const at = where();
-      if (at) text += ` (대본 정지: ${at})`;
-    }
+    const images = await saveImages(data.images);
+    if (!images) return Response.json({ ok: false, error: "invalid_images" }, { status: 400 });
+    const text = [data.text ?? "", ...images.map((p) => `[image: ${p}]`)].join("\n");
     const r = await promptAgent(action, text);
     return r.ok ? Response.json({ ok: true }) : Response.json({ ok: false, error: r.error }, { status: r.status });
   }
 
   async function route(req: Request, url: URL): Promise<Response> {
+    if (url.pathname.startsWith("/panes")) {
+      const res = await panesRoute(req, url.pathname, PANE);
+      if (res) return res;
+    }
     switch (url.pathname) {
       case "/status":
         return Response.json({
           ...(await checkStatus()),
           browser: opts.browser,
           screen: await screen.describe(),
-          present: presentView(),
+          present,
         });
       case "/events":
         return events();
       case "/screen":
         return screen.handle();
+      case "/terminal.js":
+        return new Response(terminalJS, { headers: { "Content-Type": "text/javascript; charset=utf-8" } });
+      case "/terminal.css":
+        return new Response(terminalCSS, { headers: { "Content-Type": "text/css; charset=utf-8" } });
       case "/agent-bridge.js":
         return new Response(COMPONENT, {
           headers: { "Content-Type": "text/javascript; charset=utf-8", "Cache-Control": "no-store" },
@@ -521,46 +480,36 @@ export async function startServer(opts: ServerOptions) {
       case "/send":
         if (req.method === "POST") return sendPrompt(req);
         break;
-      case "/present":
-        return req.method === "POST" ? presentRequest(req) : Response.json(presentView());
-      case "/script":
-        return scriptRequest(req);
-      case "/player":
-        if (req.method === "POST") return playerRequest(req);
+      case "/upload":
+        if (req.method === "POST") {
+          const data = (await req.json().catch(() => ({}))) as { images?: unknown };
+          const paths = data.images === undefined ? null : await saveImages(data.images);
+          return paths ? Response.json({ ok: true, paths }) : Response.json({ ok: false, error: "invalid_images" }, { status: 400 });
+        }
         break;
+      case "/present":
+        return req.method === "POST" ? presentRequest(req) : Response.json(present);
     }
     return new Response(HTML, { headers: { "Content-Type": "text/html; charset=utf-8", "Cache-Control": "no-store" } });
   }
 
-  const server = Bun.serve({
+  const server = Bun.serve<SocketData>({
     hostname: "127.0.0.1",
     port: opts.port,
     idleTimeout: 0, // keep SSE open
     websocket: {
-      async message(ws, msg) {
-        const raw = String(msg);
-        let m: any;
-        try {
-          m = JSON.parse(raw);
-        } catch {
-          return;
-        }
-        if (m?.inspect) {
-          const q = m.inspect;
-          const query =
-            typeof q.selector === "string"
-              ? { selector: q.selector.slice(0, 1000), full: !!q.full }
-              : { x: Number(q.x) || 0, y: Number(q.y) || 0, full: !!q.full };
-          ws.send(JSON.stringify({ id: m.id, result: await screen.inspect(query) }));
-          return;
-        }
-        screen.input(raw);
+      maxPayloadLength: 128 * 1024,
+      open(ws) { if (ws.data.kind === "terminal") terminal.open(ws); },
+      message(ws, msg) {
+        if (ws.data.kind === "terminal") terminal.message(ws, String(msg));
+        else screen.input(String(msg));
       },
+      close(ws) { if (ws.data.kind === "terminal") terminal.close(ws); },
     },
     async fetch(req, server) {
       const url = new URL(req.url);
       const origin = req.headers.get("Origin");
-      const isScript = url.pathname === "/agent-bridge.js";
+      const isScript = ["/agent-bridge.js", "/terminal.js", "/terminal.css"].includes(url.pathname);
 
       // The component script itself is public; data and input endpoints are origin-checked.
       if (!isScript && !originAllowed(origin, url.origin))
@@ -579,8 +528,9 @@ export async function startServer(opts: ServerOptions) {
           origin,
         );
 
-      if (url.pathname === "/input")
-        return server.upgrade(req) ? undefined : new Response("websocket required", { status: 426 });
+      if (url.pathname === "/input" || url.pathname === "/terminal")
+        return server.upgrade(req, { data: { kind: url.pathname === "/terminal" ? "terminal" : "screen" } })
+          ? undefined : new Response("websocket required", { status: 426 });
 
       const res = await route(req, url);
       return isScript ? (res.headers.set("Access-Control-Allow-Origin", "*"), res) : withCors(res, origin);
@@ -593,6 +543,7 @@ export async function startServer(opts: ServerOptions) {
     session: SESSION,
     transcript,
     stop() {
+      terminal.stop();
       timers.forEach(clearInterval);
       server.stop(true);
     },

package/src/terminal.ts

@@ -0,0 +1,122 @@
+import type { ServerWebSocket } from "bun";
+
+export type SocketData = { kind: "screen" | "terminal" };
+type Socket = ServerWebSocket<SocketData>;
+const MAX_MESSAGE = 128 * 1024;
+
+// The CLI owns Herdr's socket handshake and emits ordered ANSI frame envelopes.
+// Never use --takeover: another controller must release its session first.
+export function createTerminal(pane: string) {
+  let owner: Socket | null = null;
+  let childProcess: ReturnType<typeof Bun.spawn> | null = null;
+  let timer: ReturnType<typeof setTimeout> | undefined;
+
+  function close(ws: Socket) {
+    if (owner !== ws) return;
+    owner = null;
+    clearTimeout(timer);
+    const child = childProcess;
+    childProcess = null;
+    if (child) {
+      try { (child.stdin as any).end(); } catch {}
+      child.kill(); // Disconnect only; Herdr keeps the underlying pane alive.
+    }
+  }
+
+  function fail(ws: Socket, message: string) {
+    ws.send(JSON.stringify({ type: "terminal.error", message }));
+    close(ws);
+    ws.close(1011, "Terminal disconnected");
+  }
+
+  function send(ws: Socket, frame: unknown) {
+    if (owner !== ws) return;
+    if (ws.getBufferedAmount() > 4 * 1024 * 1024) {
+      fail(ws, "연결이 느려 터미널을 닫았습니다. 다시 연결해 주세요.");
+      return;
+    }
+    ws.send(JSON.stringify(frame));
+  }
+
+  async function pump(ws: Socket, child: ReturnType<typeof Bun.spawn>) {
+    let pending = "";
+    let stderr = "";
+    let received = false;
+    const readError = (async () => {
+      for await (const chunk of child.stderr as ReadableStream<Uint8Array>) {
+        stderr = (stderr + new TextDecoder().decode(chunk)).slice(-2000);
+      }
+    })();
+    try {
+      const decoder = new TextDecoder();
+      for await (const chunk of child.stdout as ReadableStream<Uint8Array>) {
+        if (owner !== ws) break;
+        pending += decoder.decode(chunk, { stream: true });
+        if (pending.length > 8 * 1024 * 1024) throw new Error("Terminal frame too large");
+        let end: number;
+        while ((end = pending.indexOf("\n")) >= 0) {
+          const line = pending.slice(0, end);
+          pending = pending.slice(end + 1);
+          if (!line.trim()) continue;
+          const frame = JSON.parse(line);
+          if (frame.type === "terminal.frame" && frame.encoding === "ansi") {
+            if (!received) { received = true; clearTimeout(timer); }
+            send(ws, frame);
+          } else if (frame.type === "terminal.closed") {
+            fail(ws, frame.reason || "터미널 연결이 종료되었습니다.");
+          }
+        }
+      }
+      await child.exited;
+      await readError;
+      if (owner === ws) fail(ws, stderr.trim() || "터미널 연결이 종료되었습니다. 다시 연결해 주세요.");
+    } catch {
+      if (owner === ws) fail(ws, "터미널 스트림을 읽을 수 없습니다.");
+    }
+  }
+
+  return {
+    open(ws: Socket) {
+      if (owner) {
+        ws.send(JSON.stringify({ type: "terminal.error", message: "다른 브라우저가 터미널을 사용 중입니다." }));
+        ws.close(1008, "Terminal busy");
+        return;
+      }
+      owner = ws;
+      timer = setTimeout(() => fail(ws, "터미널 연결 시간이 초과되었습니다."), 10000);
+    },
+    message(ws: Socket, raw: string) {
+      if (owner !== ws) return;
+      try {
+        if (raw.length > MAX_MESSAGE) throw new Error("Message too large");
+        const msg = JSON.parse(raw);
+        const size = () => Number.isInteger(msg.cols) && Number.isInteger(msg.rows)
+          && msg.cols >= 2 && msg.cols <= 500 && msg.rows >= 2 && msg.rows <= 300;
+        if (msg.type === "terminal.start" && !childProcess && size()) {
+          childProcess = Bun.spawn([Bun.which("herdr", { PATH: process.env.PATH }) ?? "herdr", "terminal", "session", "control", pane,
+            "--cols", String(msg.cols), "--rows", String(msg.rows)],
+          { stdin: "pipe", stdout: "pipe", stderr: "pipe" });
+          void pump(ws, childProcess);
+          return;
+        }
+        if (!childProcess) throw new Error("Terminal not started");
+        let command: object;
+        if (msg.type === "terminal.input" && typeof msg.text === "string" && msg.text.length <= 65536) {
+          command = { type: msg.type, text: msg.text };
+        } else if (msg.type === "terminal.resize" && size()) {
+          command = { type: msg.type, cols: msg.cols, rows: msg.rows };
+        } else if (msg.type === "terminal.scroll" && ["up", "down"].includes(msg.direction)
+            && Number.isInteger(msg.lines) && msg.lines > 0 && msg.lines <= 100) {
+          command = { type: msg.type, direction: msg.direction, lines: msg.lines };
+        } else throw new Error("Invalid terminal message");
+        const input = childProcess.stdin as Bun.FileSink;
+        if (input.write(JSON.stringify(command) + "\n") > 1024 * 1024) throw new Error("Input overflow");
+        input.flush();
+      } catch {
+        fail(ws, "터미널 요청을 처리할 수 없습니다.");
+      }
+    },
+    close,
+    stop() { if (owner) close(owner); },
+  };
+}