agello 0.2.1 → 0.3.0

package/README.md

@@ -6,6 +6,7 @@
 - Live session status, tool activity, and prompts queued while the agent is busy (same order as the terminal)
 - Conversation saved per pane in the browser, kept across reloads
 - Optional live view of a `terminal-browser` screen, with an on/off switch to control it yourself
+- Annotate: point at an element on that screen and send a request about it, with its CSS selector
 - Embeddable `<agent-bridge>` web component
 
 Currently supports **Claude Code** sessions.
@@ -59,6 +60,18 @@ Messages sent from the page arrive in the agent as:
 
 The approve / reject buttons send `action=approve` / `action=reject`.
 
+### Annotate
+
+The `주석` button on the screen panel turns on annotate mode: hovering tints the element under the pointer (with its tag and size), clicking outlines it and opens an input next to it. The wheel still scrolls the page, and the outline follows the element as the page changes. The request arrives as:
+
+```
+[browser] action=annotate make this red / and bolder
+대상: #app > main > button:nth-of-type(2)
+요소: button.primary "Save" · 위치 120,340 크기 96x32 · http://localhost:3000/
+```
+
+Selector: nearest unique `id` / `data-testid` ancestor, then `tag:nth-of-type` steps (light DOM only). Coordinates are CSS px of the viewport. Off while presenting and while "내 조작" is on.
+
 ### Commands
 
 | Command | Description |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agello",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "description": "Talk to a coding agent in a herdr pane from the browser: chat, live status, and a shared browser screen.",
   "keywords": [
     "herdr",

package/src/screencast.ts

@@ -275,5 +275,68 @@ export function createScreencast(opts: { browserKey?: string; herdrTab?: () => P
     return !!r && !r.exceptionDetails;
   }
 
-  return { handle, input, describe, setClip, evaluate };
+  // Annotation: which element is at a point (hover / click) or where a
+  // selected element is now (by selector). Coordinates are CSS px of the
+  // visible viewport, same as the full screencast frame. Not while cropped.
+  async function inspect(q: InspectQuery): Promise<Inspected | null> {
+    if (clip) return null;
+    const r = await request("Runtime.evaluate", {
+      expression: `(${INSPECT_JS})(${JSON.stringify(q)})`,
+      returnByValue: true,
+    });
+    return r && !r.exceptionDetails ? (r.result?.value ?? null) : null;
+  }
+
+  return { handle, input, describe, setClip, evaluate, inspect };
 }
+
+export type InspectQuery = { x?: number; y?: number; selector?: string; full?: boolean };
+export type Inspected = {
+  selector: string;
+  label: string; // tag#id.class, for the hover tooltip
+  rect: { x: number; y: number; w: number; h: number };
+  text?: string; // full only
+  url?: string; // full only
+};
+
+// Runs in the page. Selector: nearest unique id / data-testid ancestor, then
+// tag:nth-of-type steps down to the element (light DOM only).
+const INSPECT_JS = String((q: InspectQuery) => {
+  let el: Element | null = null;
+  if (q.selector) {
+    try {
+      el = document.querySelector(q.selector);
+    } catch {}
+  } else el = document.elementFromPoint(q.x ?? 0, q.y ?? 0);
+  if (!el || el === document.documentElement) return null;
+  const esc = CSS.escape;
+  const unique = (s: string) => document.querySelectorAll(s).length === 1;
+  const step = (e: Element): [string, boolean] => {
+    if (e.id && unique(`#${esc(e.id)}`)) return [`#${esc(e.id)}`, true];
+    for (const a of ["data-testid", "data-test", "data-cy"]) {
+      const v = e.getAttribute(a);
+      if (v && unique(`[${a}="${esc(v)}"]`)) return [`[${a}="${esc(v)}"]`, true];
+    }
+    const same = e.parentElement ? [...e.parentElement.children].filter((c) => c.localName === e.localName) : [];
+    return [same.length > 1 ? `${e.localName}:nth-of-type(${same.indexOf(e) + 1})` : e.localName, false];
+  };
+  const parts: string[] = [];
+  for (let e: Element | null = el; e && e !== document.documentElement; e = e.parentElement) {
+    const [s, anchor] = step(e);
+    parts.unshift(s);
+    if (anchor || e === document.body) break;
+  }
+  const cls = [...el.classList].slice(0, 2).map((c) => `.${c}`).join("");
+  const r = el.getBoundingClientRect();
+  const out: any = {
+    selector: parts.join(" > "),
+    label: `${el.localName}${el.id ? `#${el.id}` : ""}${cls}`,
+    rect: { x: r.x, y: r.y, w: r.width, h: r.height },
+  };
+  if (q.full) {
+    const t = (el as HTMLInputElement).value || el.getAttribute("aria-label") || el.getAttribute("alt") || el.textContent || "";
+    out.text = t.replace(/\s+/g, " ").trim().slice(0, 80);
+    out.url = location.href;
+  }
+  return out;
+});

package/src/server.ts

@@ -5,8 +5,10 @@
 // GET  /status           session liveness (herdr agent get + pane get)
 // GET  /events           SSE: status, chat messages, tool start/end (from transcript JSONL)
 // POST /send             {action, text} -> herdr agent prompt
+//                        (action "annotate": + target {selector, label, rect, text?, url?} from inspect)
 // GET  /screen           SSE: terminal-browser screen frames (CDP screencast)
-// WS   /input            user control: CDP Input.* commands forwarded to the page
+// WS   /input            user control: CDP Input.* commands forwarded to the page;
+//                        {id, inspect: {x, y} | {selector}, full?} -> {id, result}: element at a point (annotation)
 // GET  /present          presentation state
 // POST /present          {rect?: {x,y,w,h}} start (or move the crop), {stop: true} end
 //                        ({stop: true, from: "viewer"}: ended from the page, the agent is told
@@ -19,7 +21,7 @@ import { homedir } from "node:os";
 import { join } from "node:path";
 import { runJson } from "./run.ts";
 import { createPlayer, parsePos, parseScript, type PlayerInfo } from "./player.ts";
-import { createScreencast, type Rect } from "./screencast.ts";
+import { createScreencast, type Inspected, type Rect } from "./screencast.ts";
 
 export type ServerOptions = {
   port: number;
@@ -94,6 +96,21 @@ export function parseRect(r: any): Rect | null {
   return { x, y, w, h };
 }
 
+// Annotation prompt: the viewer's request plus the element it points at, in
+// 3 lines (request / selector / element summary).
+export function formatAnnotation(text: string, t: any): string | null {
+  if (!t || typeof t.selector !== "string" || !t.selector || t.selector.length > 1000) return null;
+  const clean = (v: unknown, n: number) => String(v ?? "").replace(/\s+/g, " ").trim().slice(0, n);
+  const request = text.split(/\r?\n/).map((l) => l.trim()).filter(Boolean).join(" / ");
+  const r = t.rect ?? {};
+  const [x, y, w, h] = [r.x, r.y, r.w, r.h].map((v) => Math.round(Number(v) || 0));
+  const label = clean(t.label, 120);
+  const snippet = clean(t.text, 80);
+  const url = clean(t.url, 300);
+  const info = [`요소: ${label}${snippet ? ` "${snippet}"` : ""}`, `위치 ${x},${y} 크기 ${w}x${h}`, url].filter(Boolean);
+  return [request, `대상: ${clean(t.selector, 1000)}`, info.join(" · ")].join("\n");
+}
+
 export type Present = { on: boolean; rect?: Rect; since?: string; player?: PlayerInfo };
 
 type Status =
@@ -465,9 +482,14 @@ export async function startServer(opts: ServerOptions) {
   }
 
   async function sendPrompt(req: Request): Promise<Response> {
-    const data = (await req.json().catch(() => ({}))) as { action?: string; text?: string };
+    const data = (await req.json().catch(() => ({}))) as { action?: string; text?: string; target?: Inspected };
     const action = /^[\w-]{1,32}$/.test(data.action ?? "") ? data.action! : "message";
     let text = data.text ?? "";
+    if (action === "annotate") {
+      const t = formatAnnotation(text, data.target);
+      if (!t || !text.trim()) return Response.json({ ok: false, error: "invalid_target" }, { status: 400 });
+      text = t;
+    }
     // Raised hand: stop the script at once (no agent round trip) and tell the
     // agent where it stopped.
     if (action === "hand-raise") {
@@ -515,8 +537,24 @@ export async function startServer(opts: ServerOptions) {
     port: opts.port,
     idleTimeout: 0, // keep SSE open
     websocket: {
-      message(_ws, msg) {
-        screen.input(String(msg));
+      async message(ws, msg) {
+        const raw = String(msg);
+        let m: any;
+        try {
+          m = JSON.parse(raw);
+        } catch {
+          return;
+        }
+        if (m?.inspect) {
+          const q = m.inspect;
+          const query =
+            typeof q.selector === "string"
+              ? { selector: q.selector.slice(0, 1000), full: !!q.full }
+              : { x: Number(q.x) || 0, y: Number(q.y) || 0, full: !!q.full };
+          ws.send(JSON.stringify({ id: m.id, result: await screen.inspect(query) }));
+          return;
+        }
+        screen.input(raw);
       },
     },
     async fetch(req, server) {

package/web/agent-bridge.js

@@ -12,6 +12,9 @@
 //   no-history   do not persist chat in localStorage (default: persisted per server + pane)
 //   screen       show live terminal-browser screen panel (side by side, stacked when narrow)
 //                with a toggle that lets the viewer control the browser (mouse, wheel, keyboard)
+//                and an annotate toggle: hovering tints the element under the pointer, clicking
+//                outlines it and opens an input; the request goes to the agent with the element's
+//                CSS selector (action=annotate). The wheel still scrolls the page.
 //
 // Theming (CSS custom properties on the element or an ancestor)
 //   --ab-bg --ab-panel --ab-text --ab-muted --ab-line --ab-accent --ab-user --ab-font
@@ -41,7 +44,7 @@
 //   is working (e.g. answering a question), a small loader is shown.
 //
 // Methods
-//   el.send(text, action = 'message') -> Promise<{ok, error?}>
+//   el.send(text, action = 'message', extra) -> Promise<{ok, error?}>  (extra: e.g. {target} for annotate)
 //   el.clearHistory()                  clear saved chat for the current pane
 //   el.stopPresent()                   end presentation mode, tell the agent -> Promise<{ok, error?, notified?}>
 
@@ -72,7 +75,7 @@ const SCREEN_REASONS = {
 };
 const ACTIONS = {
   message: "메시지", approve: "승인", reject: "거절",
-  "present-stop": "발표 종료", "hand-raise": "손들기", "hand-lower": "손 내림",
+  "present-stop": "발표 종료", "hand-raise": "손들기", "hand-lower": "손 내림", annotate: "주석",
 };
 
 const CSS = `
@@ -175,6 +178,30 @@ h1 { font-size: 14px; margin: 0; font-weight: 600; }
 @media (prefers-reduced-motion: reduce) { .control-btn .track, .control-btn .knob { transition: none; } }
 .placeholder { font-size: 13px; color: var(--_muted); text-align: center; }
 
+/* annotate: hover tint, selected outline, request input next to the element */
+.annot-btn { flex: none; font-size: 12px; padding: 2px 10px; border-radius: 999px; }
+.annot-btn[aria-pressed="true"] { background: var(--_accent); border-color: var(--_accent); color: #fff; }
+.viewport.annot img { cursor: crosshair; }
+.hl { position: absolute; z-index: 3; pointer-events: none; display: none; border-radius: 2px; }
+.hl.on { display: block; }
+.hl.hover { background: color-mix(in srgb, var(--_accent) 22%, transparent);
+            box-shadow: inset 0 0 0 1px color-mix(in srgb, var(--_accent) 55%, transparent); }
+.hl.sel { box-shadow: 0 0 0 2px var(--_accent), 0 0 0 5px color-mix(in srgb, var(--_accent) 25%, transparent); }
+.hl-label { position: absolute; z-index: 4; pointer-events: none; display: none; max-width: 60%;
+  padding: 2px 7px; border-radius: 5px; font: 11px/1.5 ui-monospace, Menlo, monospace; color: #fff;
+  background: rgba(20, 20, 22, .85); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
+.hl-label.on { display: block; }
+.note { position: absolute; z-index: 5; width: min(340px, calc(100% - 16px)); padding: 10px; border-radius: 12px;
+  background: var(--_panel); border: 1px solid var(--_line); box-shadow: 0 8px 28px rgba(0,0,0,.22);
+  animation: capIn .18s ease; }
+.note[hidden] { display: none; }
+.note .note-target { font: 11px/1.4 ui-monospace, Menlo, monospace; color: var(--_muted); margin-bottom: 6px;
+  overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+.note textarea { min-height: 52px; font-size: 14px; }
+.note .note-row { display: flex; align-items: center; gap: 8px; margin-top: 6px; }
+.note .note-hint { flex: 1; font-size: 12px; color: var(--_muted); }
+@media (prefers-reduced-motion: reduce) { .note { animation: none; } }
+
 /* presentation mode: the screen panel covers the window, chat stays hidden underneath.
    container-type would make the host the containing block of the fixed layer, so drop it. */
 :host([presenting]) { container-type: normal; }
@@ -336,9 +363,12 @@ const TEMPLATE = `
 </section>
 <aside class="screen" part="screen">
   <div class="screen-bar"><span class="live" hidden>에이전트 조작 중</span><span class="url"></span>
+    <button class="annot-btn" type="button" aria-pressed="false" title="화면의 요소를 골라 에이전트에게 요청 (선택자와 함께 전달)">주석</button>
     <button class="control-btn" role="switch" aria-checked="false" title="켜면 마우스와 키보드 입력이 브라우저로 전달돼요 (끄기: Shift+Esc)">
       <span>내 조작</span><span class="track"><span class="knob"></span></span><span class="state-label">OFF</span></button></div>
-  <div class="viewport"><div class="ring"></div><div class="glow"></div><div class="agent-badge">에이전트 조작 중</div><img alt="에이전트 브라우저 화면" draggable="false"><textarea class="kbd" aria-label="브라우저 키보드 입력"></textarea><div class="placeholder">열린 terminal-browser 없음</div><div class="captions" part="captions" aria-live="polite"></div>
+  <div class="viewport"><div class="ring"></div><div class="glow"></div><div class="agent-badge">에이전트 조작 중</div><img alt="에이전트 브라우저 화면" draggable="false"><textarea class="kbd" aria-label="브라우저 키보드 입력"></textarea><div class="placeholder">열린 terminal-browser 없음</div><div class="hl hover"></div><div class="hl sel"></div><div class="hl-label"></div>
+    <form class="note" part="note" hidden><div class="note-target"></div><textarea aria-label="선택한 요소에 대한 요청" placeholder="이 요소에 대한 요청 (Enter 전송, Esc 취소)"></textarea>
+      <div class="note-row"><span class="note-hint"></span><button type="submit" class="primary">보내기</button></div></form><div class="captions" part="captions" aria-live="polite"></div>
     <div class="busy" part="busy" role="status"><span class="spinner"></span><span>답변 준비 중</span></div>
     <div class="hand-wrap">
       <form class="ask" part="ask" hidden><textarea aria-label="에이전트에게 질문" placeholder="에이전트에게 질문 (Enter 전송, Esc 닫기)"></textarea>
@@ -358,7 +388,14 @@ class AgentBridge extends HTMLElement {
   #root = this.attachShadow({ mode: "open" });
   #es = null;
   #screenEs = null;
-  #controlWs = null;
+  #ws = null; // /input: control input and annotate inspect queries
+  #control = false;
+  #annot = false;
+  #hover = null; // inspected element under the pointer (annotate)
+  #hoverPt = null;
+  #sel = null; // selected element (annotate)
+  #inspectId = 0;
+  #inspectWait = new Map();
   #frame = { w: 0, h: 0 };
   #moveQueued = null;
   #status = null;
@@ -383,6 +420,8 @@ class AgentBridge extends HTMLElement {
       controlBtn: q(".control-btn"), kbd: q(".kbd"), clearBtn: q(".clear-btn"),
       body: q(".body"), screen: q(".screen"), captions: q(".captions"),
       busy: q(".busy"), hand: q(".hand"), end: q(".end"), ask: q(".ask"), askBox: q(".ask textarea"), askHint: q(".ask-hint"),
+      annotBtn: q(".annot-btn"), hlHover: q(".hl.hover"), hlSel: q(".hl.sel"), hlLabel: q(".hl-label"),
+      note: q(".note"), noteBox: q(".note textarea"), noteHint: q(".note-hint"), noteTarget: q(".note-target"),
     };
     this.$.buttons.forEach((b) => b.addEventListener("click", () => this.#submit(b.dataset.action)));
     this.#bindGrip();
@@ -394,6 +433,7 @@ class AgentBridge extends HTMLElement {
     });
     loadMarkdown().then((md) => (this.#md = md));
     this.#bindControl();
+    this.#bindAnnotate();
     this.#bindHand();
     this.$.clearBtn.addEventListener("click", () => this.clearHistory());
   }
@@ -413,6 +453,7 @@ class AgentBridge extends HTMLElement {
     this.#screenEs?.close();
     this.#screenEs = null;
     this.#setControl(false);
+    this.#setAnnotate(false);
   }
   attributeChangedCallback(name) {
     if (name === "heading") this.$.heading.textContent = this.getAttribute("heading") || "agello";
@@ -420,12 +461,12 @@ class AgentBridge extends HTMLElement {
     if (name === "screen" && this.isConnected) this.#syncScreen();
   }
 
-  async send(text, action = "message") {
+  async send(text, action = "message", extra = {}) {
     try {
       const res = await fetch(`${this.server}/send`, {
         method: "POST",
         headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({ action, text }),
+        body: JSON.stringify({ ...extra, action, text }),
       });
       return await res.json();
     } catch {
@@ -491,7 +532,7 @@ class AgentBridge extends HTMLElement {
 
   #syncScreen() {
     const want = this.hasAttribute("screen") || this.#present.on;
-    if (!want) { this.#screenEs?.close(); this.#screenEs = null; this.#setControl(false); return; }
+    if (!want) { this.#screenEs?.close(); this.#screenEs = null; this.#setControl(false); this.#setAnnotate(false); return; }
     if (this.#screenEs) return;
     const es = (this.#screenEs = new EventSource(`${this.server}/screen`));
     es.addEventListener("meta", (e) => this.#renderScreenMeta(JSON.parse(e.data)));
@@ -500,6 +541,7 @@ class AgentBridge extends HTMLElement {
       this.#frame = { w: f.w, h: f.h };
       this.$.img.src = `data:image/jpeg;base64,${f.data}`;
       this.$.viewport.classList.add("has");
+      if (this.#annot) this.#annotFrame();
     });
     es.onerror = () => this.#renderScreenMeta({ connected: false });
   }
@@ -513,7 +555,7 @@ class AgentBridge extends HTMLElement {
     this.#emit("agent-present", this.#present);
     this.#renderBusy();
     if (on === was) return;
-    if (on) this.#setControl(false);
+    if (on) { this.#setControl(false); this.#setAnnotate(false); }
     else { this.$.captions.replaceChildren(); this.#openAsk(false); this.#handUp = false; }
     this.#syncScreen();
     // FLIP: grow from (or shrink back to) the side panel
@@ -652,24 +694,43 @@ class AgentBridge extends HTMLElement {
 
   // ---------- user control ----------
 
-  #setControl(on) {
-    if (on && !this.#controlWs) {
+  // One socket for both modes; open while control or annotate is on.
+  #syncWs() {
+    const want = this.#control || this.#annot;
+    if (want && !this.#ws) {
       const ws = new WebSocket(`${this.server.replace(/^http/, "ws")}/input`);
-      ws.onclose = () => { if (this.#controlWs === ws) this.#setControl(false); };
-      this.#controlWs = ws;
-    } else if (!on && this.#controlWs) {
-      const ws = this.#controlWs;
-      this.#controlWs = null;
+      ws.onmessage = (e) => {
+        let m;
+        try { m = JSON.parse(e.data); } catch { return; }
+        this.#inspectWait.get(m.id)?.(m.result ?? null);
+      };
+      ws.onclose = () => {
+        if (this.#ws !== ws) return;
+        this.#ws = null;
+        this.#setControl(false);
+        this.#setAnnotate(false);
+      };
+      this.#ws = ws;
+    } else if (!want && this.#ws) {
+      const ws = this.#ws;
+      this.#ws = null;
       ws.close();
     }
-    this.$.controlBtn.setAttribute("aria-checked", String(!!this.#controlWs));
-    this.$.controlBtn.querySelector(".state-label").textContent = this.#controlWs ? "ON" : "OFF";
-    this.$.viewport.classList.toggle("control", !!this.#controlWs);
-    if (this.#controlWs) this.$.kbd.focus();
+    if (!this.#ws) { this.#inspectWait.forEach((r) => r(null)); this.#inspectWait.clear(); }
+  }
+
+  #setControl(on) {
+    this.#control = !!on && !this.#present.on;
+    if (this.#control) this.#setAnnotate(false);
+    this.#syncWs();
+    this.$.controlBtn.setAttribute("aria-checked", String(this.#control));
+    this.$.controlBtn.querySelector(".state-label").textContent = this.#control ? "ON" : "OFF";
+    this.$.viewport.classList.toggle("control", this.#control);
+    if (this.#control) this.$.kbd.focus();
   }
 
   #cdp(method, params) {
-    const ws = this.#controlWs;
+    const ws = this.#ws;
     if (ws?.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ method, params }));
   }
 
@@ -683,11 +744,11 @@ class AgentBridge extends HTMLElement {
 
   #bindControl() {
     const { img, kbd, controlBtn } = this.$;
-    const on = () => !!this.#controlWs && this.#frame.w > 0;
+    const on = () => this.#control && this.#frame.w > 0;
     const mods = (e) => (e.altKey ? 1 : 0) | (e.ctrlKey ? 2 : 0) | (e.metaKey ? 4 : 0) | (e.shiftKey ? 8 : 0);
     const BUTTONS = ["left", "middle", "right"];
 
-    controlBtn.addEventListener("click", () => { if (!this.#present.on) this.#setControl(!this.#controlWs); });
+    controlBtn.addEventListener("click", () => this.#setControl(!this.#control));
 
     img.addEventListener("mousedown", (e) => {
       if (!on()) return;
@@ -715,7 +776,7 @@ class AgentBridge extends HTMLElement {
       });
     });
     img.addEventListener("wheel", (e) => {
-      if (!on()) return;
+      if (!on() && !(this.#annot && this.#frame.w > 0)) return; // annotate: scroll to reach the element
       e.preventDefault();
       this.#cdp("Input.dispatchMouseEvent", { type: "mouseWheel", ...this.#point(e), deltaX: e.deltaX, deltaY: e.deltaY,
         modifiers: mods(e) });
@@ -753,6 +814,163 @@ class AgentBridge extends HTMLElement {
     });
   }
 
+  // ---------- annotate ----------
+
+  #setAnnotate(on) {
+    on = !!on && !this.#present.on;
+    if (on === this.#annot) return;
+    this.#annot = on;
+    if (on) this.#setControl(false);
+    else { this.#hover = null; this.#hoverPt = null; this.#select(null); }
+    this.#syncWs();
+    this.$.annotBtn.setAttribute("aria-pressed", String(on));
+    this.$.viewport.classList.toggle("annot", on);
+    this.#drawHl();
+  }
+
+  // Element at a frame point ({x, y}) or by selector; null if unavailable.
+  #inspect(q, full = false) {
+    const ws = this.#ws;
+    if (ws?.readyState !== WebSocket.OPEN) return Promise.resolve(null);
+    const id = ++this.#inspectId;
+    return new Promise((resolve) => {
+      const done = (r) => { clearTimeout(t); this.#inspectWait.delete(id); resolve(r); };
+      const t = setTimeout(() => done(null), 3000);
+      this.#inspectWait.set(id, done);
+      ws.send(JSON.stringify({ id, inspect: { ...q, full } }));
+    });
+  }
+
+  // Latest pointer position wins; at most one hover query in flight.
+  #probing = false;
+  #probeAgain = false;
+  async #probeHover() {
+    if (this.#probing) { this.#probeAgain = true; return; }
+    this.#probing = true;
+    try {
+      do {
+        this.#probeAgain = false;
+        const p = this.#hoverPt;
+        if (!p || !this.#annot) break;
+        const r = await this.#inspect(p);
+        if (this.#hoverPt && this.#annot) { this.#hover = r; this.#drawHl(); }
+      } while (this.#probeAgain);
+    } finally {
+      this.#probing = false;
+    }
+  }
+
+  // The page changed (scroll, layout, HMR): follow the selected element and
+  // re-check what is under the pointer.
+  #tracking = false;
+  async #annotFrame() {
+    if (this.#hoverPt) this.#probeHover();
+    const sel = this.#sel;
+    if (!sel || this.#tracking) { this.#drawHl(); return; }
+    this.#tracking = true;
+    try {
+      const r = await this.#inspect({ selector: sel.selector });
+      if (r && this.#sel === sel) sel.rect = r.rect; // gone from the page: keep the last box
+    } finally {
+      this.#tracking = false;
+    }
+    this.#drawHl();
+  }
+
+  #select(t) {
+    const { note, noteBox, noteHint, noteTarget } = this.$;
+    this.#sel = t;
+    note.hidden = !t;
+    noteHint.textContent = "";
+    if (t) {
+      noteTarget.textContent = t.selector;
+      noteTarget.title = t.selector;
+      noteBox.focus();
+    }
+    this.#drawHl();
+  }
+
+  // Frame (CSS px of the page viewport) -> position over the displayed image.
+  #drawHl() {
+    const { viewport, img, hlHover, hlSel, hlLabel, note } = this.$;
+    const show = this.#annot && viewport.classList.contains("has") && this.#frame.w > 0;
+    const vr = viewport.getBoundingClientRect();
+    const ir = img.getBoundingClientRect();
+    const sx = ir.width / this.#frame.w, sy = ir.height / this.#frame.h;
+    const box = (r) => {
+      if (!show || !r) return null;
+      const x0 = Math.max(r.x, 0), y0 = Math.max(r.y, 0);
+      const x1 = Math.min(r.x + r.w, this.#frame.w), y1 = Math.min(r.y + r.h, this.#frame.h);
+      if (x1 <= x0 || y1 <= y0) return null;
+      return { left: ir.left - vr.left + x0 * sx, top: ir.top - vr.top + y0 * sy, width: (x1 - x0) * sx, height: (y1 - y0) * sy };
+    };
+    const place = (el, b) => {
+      el.classList.toggle("on", !!b);
+      if (b) Object.assign(el.style, { left: `${b.left}px`, top: `${b.top}px`, width: `${b.width}px`, height: `${b.height}px` });
+      return b;
+    };
+    const hb = place(hlHover, box(this.#hover?.rect));
+    const sb = place(hlSel, box(this.#sel?.rect));
+
+    hlLabel.classList.toggle("on", !!hb);
+    if (hb) {
+      const r = this.#hover.rect;
+      hlLabel.textContent = `${this.#hover.label}  ${Math.round(r.w)}×${Math.round(r.h)}`;
+      // above the box, else below it, else inside its top edge
+      const top = hb.top >= 22 ? hb.top - 22 : hb.top + hb.height + 26 <= vr.height ? hb.top + hb.height + 4 : hb.top + 4;
+      Object.assign(hlLabel.style, { left: `${hb.left + (top === hb.top + 4 ? 4 : 0)}px`, top: `${top}px` });
+    }
+
+    // input below the selected element, above it if there is no room
+    if (!note.hidden) {
+      const W = note.offsetWidth, H = note.offsetHeight, pad = 8;
+      const b = sb ?? { left: pad, top: pad, width: 0, height: 0 };
+      let top = b.top + b.height + pad;
+      if (top + H > vr.height - pad) top = b.top - H - pad;
+      top = Math.max(pad, Math.min(top, vr.height - H - pad));
+      const left = Math.max(pad, Math.min(b.left, vr.width - W - pad));
+      Object.assign(note.style, { left: `${left}px`, top: `${top}px` });
+    }
+  }
+
+  #bindAnnotate() {
+    const { img, annotBtn, note, noteBox, noteHint } = this.$;
+    annotBtn.addEventListener("click", () => this.#setAnnotate(!this.#annot));
+    img.addEventListener("mousemove", (e) => {
+      if (!this.#annot || !this.#frame.w) return;
+      this.#hoverPt = this.#point(e);
+      this.#probeHover();
+    });
+    img.addEventListener("mouseleave", () => {
+      if (!this.#annot) return;
+      this.#hoverPt = null;
+      this.#hover = null;
+      this.#drawHl();
+    });
+    img.addEventListener("click", async (e) => {
+      if (!this.#annot || !this.#frame.w) return;
+      e.preventDefault();
+      const t = await this.#inspect(this.#point(e), true);
+      if (t && this.#annot) this.#select(t);
+    });
+    new ResizeObserver(() => this.#drawHl()).observe(img);
+    note.addEventListener("submit", async (e) => {
+      e.preventDefault();
+      const text = noteBox.value.trim();
+      const target = this.#sel;
+      if (!text || !target) return;
+      noteHint.textContent = "보내는 중…";
+      const res = await this.send(text, "annotate", { target });
+      if (!res.ok) { noteHint.textContent = `전송 실패: ${res.error}`; return; }
+      noteBox.value = "";
+      if (this.#sel === target) this.#select(null);
+    });
+    noteBox.addEventListener("keydown", (e) => {
+      if (e.key === "Enter" && !e.shiftKey && !e.isComposing) { e.preventDefault(); note.requestSubmit(); }
+      if (e.key === "Escape") { e.preventDefault(); this.#select(null); }
+    });
+  }
+
   #renderScreenMeta(m) {
     const { live, url, viewport } = this.$;
     if (!m.connected) {
@@ -768,6 +986,7 @@ class AgentBridge extends HTMLElement {
       url.textContent = m.title ? `${m.title} · ${m.url}` : m.url || "";
       url.title = m.url || "";
     }
+    this.#drawHl();
     this.#emit("agent-screen", m);
   }