agello 0.4.1 → 0.4.3

package/README.md

@@ -7,6 +7,7 @@
 - Conversation saved per pane in the browser, kept across reloads
 - Interactive terminal view of the existing herdr pane: live cursor, keyboard input, and automatic resize
 - Optional live view of a `terminal-browser` screen, with an on/off switch to control it yourself
+- Annotate: point at an element on that screen and send a request about it, with its CSS selector
 - Embeddable `<agent-bridge>` web component
 
 Chat currently supports **Claude Code** sessions. The terminal view connects directly to the pane, including ordinary shells and other terminal applications.
@@ -60,6 +61,18 @@ Messages sent from the page arrive in the agent as:
 
 The approve / reject buttons send `action=approve` / `action=reject`.
 
+### Annotate
+
+The `주석` button on the screen panel turns on annotate mode: hovering tints the element under the pointer (with its tag and size), clicking outlines it and opens an input next to it. The wheel still scrolls the page, and the outline follows the element as the page changes. The request arrives as:
+
+```
+[browser] action=annotate make this red / and bolder
+대상: #app > main > button:nth-of-type(2)
+요소: button.primary "Save" · 위치 120,340 크기 96x32 · http://localhost:3000/
+```
+
+Selector: nearest unique `id` / `data-testid` ancestor, then `tag:nth-of-type` steps (light DOM only). Coordinates are CSS px of the viewport. Off while presenting and while "내 조작" is on.
+
 ### Commands
 
 | Command | Description |
@@ -70,6 +83,8 @@ The approve / reject buttons send `action=approve` / `action=reject`.
 | `agello open [--port N \| --pane ID]` | Open the page (default: this pane's server) |
 | `agello present [x,y,w,h] [--port N \| --pane ID]` | Presentation mode (see below) |
 | `agello present stop` | End presentation mode |
+| `agello script load <file> \| show` | Load / show a presentation script |
+| `agello present resume \| pause \| goto <n[.m]>` | Play the script from where it stopped / stop it / move to a step |
 
 `start` options: `--pane <id>`, `--port <n>` (default: first free from 8765), `--session <id>`, `--browser <terminal-browser key>`, `--allow-origin <origin>` (repeatable), `--open`, `--foreground`.
 
@@ -98,10 +113,29 @@ The xterm.js renderer and its styles are served locally; no terminal CDN is requ
 - agent replies appear over it as bubbles that fade after 10s
 - `agello present stop` returns to the normal layout; the replies are in the chat as usual
 - the viewer can ask a question with the raise-hand button (its bubble fades after 10s too) or end the presentation with the end button; the agent then receives `[browser] action=present-stop`
-- raising the hand tells the agent at once (`action=hand-raise`) so it can pause; closing the input without asking sends `action=hand-lower`
+- raising the hand tells the agent at once (`action=hand-raise`) so it can pause; closing it with X (or Esc) before asking sends `action=hand-lower`
+- the input stays open for the whole question: after the agent answers, follow-up questions go in the same input (no need to raise the hand again); X (or Esc) then sends `action=hand-done`, and the agent resumes
 
 Run `present` again to move the crop. User control is off while presenting.
 
+#### Scripts
+
+Write the talk ahead so each line appears the moment its screen does:
+
+```json
+{ "rect": "28,157,688,477",
+  "steps": [
+    { "go": "#1", "say": ["First line", "Second line"] },
+    { "go": "#2", "say": ["..."], "hold": 6 } ] }
+```
+
+- step: `go` brings the screen there (`#n` sets `location.hash`, anything else is JS run in the page; agello knows nothing about the deck), `say` lines are one bubble each, `hold` seconds a line stays on screen (default from length, 10 to 20s; never below 10s). Pacing: screen change, 1s, line, fade out, 0.3s, next line; after a step's last line 0.7s before the next screen
+- `present resume` plays from where it stopped (starts presentation mode with the script's `rect`); it first re-runs the current step's `go`, so the agent may move the screen freely while answering
+- raising a hand pauses at once and rewinds one line, so `present resume` replays the interrupted line; the agent gets `action=hand-raise` with the position (`마지막 표시 2.1 · 다음 2.1`)
+- raising, cancelling (X before asking) and finishing a question (X after an answer) show a small light event bubble for 5s right away, apart from the conversation bubbles; sending a question shows none
+- `script load` again after editing keeps the position; `present goto 2.2` then `present resume` replays from a fixed line
+- at the end the agent gets `action=present-done`; while the script is paused and the agent is working, the page shows a small loader
+
 ## Embed
 
 ```html

package/bin/agello.ts

@@ -27,6 +27,13 @@ Usage:
                          whole screen), with agent replies as bubbles that fade after 10s.
                          Run again to move the crop.
   ${NAME} present stop [--port N|--pane ID]       End presentation mode (bubbles stay in the chat)
+  ${NAME} script load <file.json>   Load a presentation script (keeps the current position):
+                         {"rect": "x,y,w,h", "steps": [{"go": "#1", "say": ["line", ...], "hold": 6}]}
+                         go: "#n" sets location.hash, otherwise JS run in the page; one line = one bubble
+  ${NAME} script show               Script with positions (step.line) and player state
+  ${NAME} present resume            Play the script from where it stopped (starts presentation mode)
+  ${NAME} present pause             Stop the script (raising a hand on the page also pauses)
+  ${NAME} present goto <n[.m]>      Move to step n (line m) and show its screen
   ${NAME} help | --version
 
 start options:
@@ -284,6 +291,56 @@ async function cmdOpen(argv: string[]) {
   console.log(inst.url);
 }
 
+async function api(url: string, path: string, body?: object): Promise<any> {
+  try {
+    const r = await fetch(`${url}${path}`, {
+      method: body ? "POST" : "GET",
+      headers: { "Content-Type": "application/json" },
+      body: body ? JSON.stringify(body) : undefined,
+      signal: AbortSignal.timeout(8000),
+    });
+    return await r.json();
+  } catch {
+    return fail(`server unreachable: ${url}`);
+  }
+}
+
+const playerLine = (p: any) =>
+  `script: ${p.state}${p.last ? ` last=${p.last}` : ""}${p.next ? ` next=${p.next}` : ""} steps=${p.steps}`;
+
+async function cmdScript(argv: string[]) {
+  const { values: o, positionals } = parseArgs({
+    args: argv,
+    options: { port: { type: "string" }, pane: { type: "string" } },
+    allowPositionals: true,
+  });
+  const [sub, file] = positionals;
+  const inst = await resolveTarget(o);
+  if (sub === "load") {
+    if (!file) fail("usage: script load <file.json>");
+    let script: unknown;
+    try {
+      script = await Bun.file(file!).json();
+    } catch (e) {
+      fail(`cannot read ${file}: ${(e as Error).message}`);
+    }
+    const res = await api(inst.url, "/script", { script });
+    if (!res.ok) fail(res.error);
+    console.log(playerLine(res.player));
+  } else if (sub === "show" || !sub) {
+    const res = await api(inst.url, "/script");
+    console.log(playerLine(res.player));
+    for (const [i, st] of (res.script?.steps ?? []).entries()) {
+      console.log(`${i + 1}${st.go ? `  go=${st.go}` : ""}${st.hold ? `  hold=${st.hold}s` : ""}`);
+      st.say.forEach((l: string, j: number) => {
+        const at = `${i + 1}.${j + 1}`;
+        const mark = at === res.player.next ? ">" : at === res.player.last ? "*" : " ";
+        console.log(` ${mark} ${at}  ${l}`);
+      });
+    }
+  } else fail(`unknown script command: ${sub}`);
+}
+
 async function cmdPresent(argv: string[]) {
   const { values: o, positionals } = parseArgs({
     args: argv,
@@ -291,6 +348,14 @@ async function cmdPresent(argv: string[]) {
     allowPositionals: true,
   });
   const inst = await resolveTarget(o);
+  const [sub, at] = positionals;
+  if (sub === "resume" || sub === "pause" || sub === "goto") {
+    if (sub === "goto" && !at) fail("usage: present goto <n[.m]>");
+    const res = await api(inst.url, "/player", { cmd: sub, at });
+    if (!res.ok) fail(`${res.error}${res.player ? ` (${playerLine(res.player)})` : ""}`);
+    console.log(playerLine(res.player));
+    return;
+  }
   const arg = positionals.join(",");
   let body: object;
   if (arg === "stop") body = { stop: true };
@@ -337,6 +402,9 @@ switch (cmd) {
   case "present":
     await cmdPresent(rest);
     break;
+  case "script":
+    await cmdScript(rest);
+    break;
   case "--version":
   case "-v":
     console.log(pkg.version);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agello",
-  "version": "0.4.1",
+  "version": "0.4.3",
   "description": "Talk to a coding agent in a herdr pane from the browser: chat, live status, and a shared browser screen.",
   "keywords": [
     "herdr",

package/src/player.ts

@@ -0,0 +1,244 @@
+// Presentation script player.
+//
+// A script is a list of steps. Each step has an optional `go` (how to bring the
+// screen to that step: "#3" sets location.hash, anything else is evaluated as
+// JS in the page; written by the agent, agello knows nothing about the deck)
+// and lines (`say`), each shown as one bubble.
+//
+// The player owns order and timing: entering a step runs its `go`, then every
+// line is sent and held for its reading time (see PACE). pause() stops at once and keeps
+// the position, resume() re-runs the current step's `go` (the agent may have
+// moved the screen meanwhile) and continues from the next unshown line.
+// rewind() (after a raised hand) moves back so the last shown line plays again.
+// goto() moves the position and shows that step without playing.
+//
+// Positions are "step.line", 1-based, in everything that leaves this module.
+
+import type { Rect } from "./screencast.ts";
+
+export type Step = { go?: string; say: string[]; hold?: number };
+export type Script = { rect?: Rect; steps: Step[] };
+export type PlayerInfo = {
+  state: "empty" | "ready" | "playing" | "paused" | "done";
+  next?: string; // next line to show ("3.2")
+  last?: string; // last line shown
+  steps: number;
+};
+
+// Pacing: the audience looks at a new screen first, then reads and thinks.
+//   go -> 1s -> line (shown >= 10s) -> fades out -> 0.3s -> next line
+//                                     last line -> fades out -> 0.7s -> next step's go
+const MIN_HOLD = 10;
+const MAX_HOLD = 20;
+export const PACE = { afterGo: 1000, fade: 500, betweenLines: 300, afterStep: 700 };
+
+// How long one line stays on screen (s): at least 10s, longer text a bit more
+// (about 8 chars/s past the first ~50 chars), at most 20s. `hold` can only
+// lengthen it.
+export function lineHold(text: string, hold?: number): number {
+  const auto = Math.min(MAX_HOLD, Math.max(MIN_HOLD, 4 + [...text].length / 8));
+  return hold != null && Number.isFinite(hold) && hold > 0 ? Math.max(MIN_HOLD, hold) : auto;
+}
+
+export function parseRectString(v: unknown): Rect | undefined {
+  if (v && typeof v === "object") {
+    const r = v as any;
+    return { x: r.x, y: r.y, w: r.w, h: r.h };
+  }
+  if (typeof v !== "string") return undefined;
+  const n = v.split(/[\s,]+/).filter(Boolean).map(Number);
+  return n.length === 4 ? { x: n[0], y: n[1], w: n[2], h: n[3] } : undefined;
+}
+
+// Validate a script from JSON. Returns the script or an error message.
+export function parseScript(raw: any): Script | string {
+  if (!raw || typeof raw !== "object") return "script must be an object";
+  if (!Array.isArray(raw.steps) || !raw.steps.length) return "steps must be a non-empty array";
+  const steps: Step[] = [];
+  for (const [i, s] of raw.steps.entries()) {
+    const at = `steps[${i}]`;
+    if (!s || typeof s !== "object") return `${at} must be an object`;
+    if (s.go != null && typeof s.go !== "string") return `${at}.go must be a string`;
+    const say = s.say == null ? [] : Array.isArray(s.say) ? s.say : [s.say];
+    if (!say.every((l: unknown) => typeof l === "string" && l.trim())) return `${at}.say must be non-empty strings`;
+    if (s.hold != null && !(Number.isFinite(s.hold) && s.hold > 0)) return `${at}.hold must be a positive number`;
+    steps.push({ go: s.go || undefined, say: say.map((l: string) => l.trim()), hold: s.hold });
+  }
+  const rect = raw.rect == null ? undefined : parseRectString(raw.rect);
+  if (raw.rect != null && !rect) return "rect must be \"x,y,w,h\" or {x,y,w,h}";
+  return { rect, steps };
+}
+
+// "3" or "3.2" (1-based) -> 0-based position
+export function parsePos(v: string): { step: number; line: number } | null {
+  const m = String(v).trim().match(/^(\d+)(?:\.(\d+))?$/);
+  if (!m) return null;
+  const step = Number(m[1]) - 1;
+  const line = m[2] ? Number(m[2]) - 1 : 0;
+  return step >= 0 && line >= 0 ? { step, line } : null;
+}
+
+export function createPlayer(deps: {
+  go: (expr: string) => Promise<void>;
+  say: (text: string, at: string, hold: number) => void;
+  done: () => void;
+  changed: () => void;
+}) {
+  let script: Script | null = null;
+  let state: PlayerInfo["state"] = "empty";
+  let cur = { step: 0, line: 0 }; // next line to show
+  let last: { step: number; line: number } | null = null;
+  let entered = false; // `go` of cur.step already run in this play run
+  let run = 0; // bumps on every pause/goto/load, stale timers check it
+  let timer: Timer | null = null;
+
+  const fmt = (p: { step: number; line: number }) => `${p.step + 1}.${p.line + 1}`;
+  const set = (s: PlayerInfo["state"]) => {
+    state = s;
+    deps.changed();
+  };
+  const cancel = () => {
+    run++;
+    if (timer) clearTimeout(timer);
+    timer = null;
+  };
+  const wait = (ms: number, id: number, fn: () => void) => {
+    timer = setTimeout(() => id === run && fn(), ms);
+  };
+
+  function info(): PlayerInfo {
+    const steps = script?.steps.length ?? 0;
+    // next line as it will be played (a finished step points at the next step)
+    let p = cur;
+    while (script && p.step < steps && p.line >= Math.max(1, script.steps[p.step].say.length))
+      p = { step: p.step + 1, line: 0 };
+    return {
+      state,
+      steps,
+      next: script && p.step < steps ? fmt(p) : undefined,
+      last: last ? fmt(last) : undefined,
+    };
+  }
+
+  // Skip past finished steps (cur.line beyond the step's lines).
+  function normalize() {
+    if (!script) return;
+    while (cur.step < script.steps.length && cur.line >= Math.max(1, script.steps[cur.step].say.length)) {
+      cur = { step: cur.step + 1, line: 0 };
+      entered = false;
+    }
+  }
+
+  async function tick(id: number) {
+    if (id !== run || state !== "playing" || !script) return;
+    normalize();
+    if (cur.step >= script.steps.length) {
+      set("done");
+      deps.done();
+      return;
+    }
+    const step = script.steps[cur.step];
+    if (!entered) {
+      entered = true;
+      if (step.go) {
+        await deps.go(step.go);
+        if (id !== run) return;
+        wait(PACE.afterGo, id, () => tick(id));
+        return;
+      }
+    }
+    if (!step.say.length) {
+      // silent step: just show the screen for `hold` seconds
+      cur = { step: cur.step, line: 1 };
+      deps.changed();
+      wait(lineHold("", step.hold) * 1000 + PACE.afterStep, id, () => tick(id));
+      return;
+    }
+    const text = step.say[cur.line];
+    const hold = lineHold(text, step.hold);
+    const lastLine = cur.line === step.say.length - 1;
+    last = { ...cur };
+    deps.say(text, fmt(cur), hold);
+    cur = { step: cur.step, line: cur.line + 1 };
+    deps.changed();
+    const gap = lastLine ? PACE.afterStep : PACE.betweenLines;
+    wait(hold * 1000 + PACE.fade + gap, id, () => tick(id));
+  }
+
+  return {
+    info,
+    script: () => script,
+
+    // Replace the script, keeping the position (clamped) and the state.
+    load(next: Script) {
+      const playing = state === "playing";
+      cancel();
+      script = next;
+      if (cur.step > next.steps.length) cur = { step: next.steps.length, line: 0 };
+      normalize();
+      if (state === "empty") state = "ready";
+      if (state === "done" && cur.step < next.steps.length) state = "paused";
+      if (playing) {
+        entered = true; // screen is already on this step
+        tick(run);
+      }
+      deps.changed();
+    },
+
+    resume(): string | null {
+      if (!script) return "no_script";
+      if (state === "playing") return null;
+      if (state === "done") return "done";
+      cancel();
+      entered = false; // bring the screen back to the current step first
+      set("playing");
+      tick(run);
+      return null;
+    },
+
+    pause() {
+      if (state !== "playing") return;
+      cancel();
+      set("paused");
+    },
+
+    // Paused by an interruption (raised hand): the next resume replays the
+    // last shown line, since the viewer may have missed it.
+    rewind() {
+      if (state !== "paused" || !last) return;
+      cur = { ...last };
+      entered = false;
+      deps.changed();
+    },
+
+    // Move to a step (and line) and show its screen. Does not play; if it was
+    // playing, it keeps playing from there.
+    async goto(pos: { step: number; line: number }): Promise<string | null> {
+      if (!script) return "no_script";
+      const step = script.steps[pos.step];
+      if (!step) return "no_such_step";
+      if (pos.line > 0 && pos.line >= step.say.length) return "no_such_line";
+      const playing = state === "playing";
+      cancel();
+      cur = { ...pos };
+      entered = false;
+      if (playing) {
+        tick(run);
+        return null;
+      }
+      if (state !== "ready") state = "paused";
+      if (step.go) await deps.go(step.go);
+      entered = true;
+      deps.changed();
+      return null;
+    },
+
+    // Presentation ended: stop where it is.
+    stop() {
+      if (state === "playing") {
+        cancel();
+        set("paused");
+      }
+    },
+  };
+}

package/src/screencast.ts

@@ -268,5 +268,75 @@ export function createScreencast(opts: { browserKey?: string; herdrTab?: () => P
     return browser ? { connected: false, browser: browser.key } : { connected: false, reason };
   }
 
-  return { handle, input, describe, setClip };
+  // Run JS in the relayed tab (presentation script `go`). Needs the relay to be
+  // connected, i.e. at least one viewer. Returns false if it could not run.
+  async function evaluate(expression: string): Promise<boolean> {
+    const r = await request("Runtime.evaluate", { expression, awaitPromise: true });
+    return !!r && !r.exceptionDetails;
+  }
+
+  // Annotation: which element is at a point (hover / click) or where a
+  // selected element is now (by selector). Coordinates are CSS px of the
+  // visible viewport, same as the full screencast frame. Not while cropped.
+  async function inspect(q: InspectQuery): Promise<Inspected | null> {
+    if (clip) return null;
+    const r = await request("Runtime.evaluate", {
+      expression: `(${INSPECT_JS})(${JSON.stringify(q)})`,
+      returnByValue: true,
+    });
+    return r && !r.exceptionDetails ? (r.result?.value ?? null) : null;
+  }
+
+  return { handle, input, describe, setClip, evaluate, inspect };
 }
+
+export type InspectQuery = { x?: number; y?: number; selector?: string; full?: boolean };
+export type Inspected = {
+  selector: string;
+  label: string; // tag#id.class, for the hover tooltip
+  rect: { x: number; y: number; w: number; h: number };
+  text?: string; // full only
+  url?: string; // full only
+};
+
+// Runs in the page. Selector: nearest unique id / data-testid ancestor, then
+// tag:nth-of-type steps down to the element (light DOM only).
+const INSPECT_JS = String((q: InspectQuery) => {
+  let el: Element | null = null;
+  if (q.selector) {
+    try {
+      el = document.querySelector(q.selector);
+    } catch {}
+  } else el = document.elementFromPoint(q.x ?? 0, q.y ?? 0);
+  if (!el || el === document.documentElement) return null;
+  const esc = CSS.escape;
+  const unique = (s: string) => document.querySelectorAll(s).length === 1;
+  const step = (e: Element): [string, boolean] => {
+    if (e.id && unique(`#${esc(e.id)}`)) return [`#${esc(e.id)}`, true];
+    for (const a of ["data-testid", "data-test", "data-cy"]) {
+      const v = e.getAttribute(a);
+      if (v && unique(`[${a}="${esc(v)}"]`)) return [`[${a}="${esc(v)}"]`, true];
+    }
+    const same = e.parentElement ? [...e.parentElement.children].filter((c) => c.localName === e.localName) : [];
+    return [same.length > 1 ? `${e.localName}:nth-of-type(${same.indexOf(e) + 1})` : e.localName, false];
+  };
+  const parts: string[] = [];
+  for (let e: Element | null = el; e && e !== document.documentElement; e = e.parentElement) {
+    const [s, anchor] = step(e);
+    parts.unshift(s);
+    if (anchor || e === document.body) break;
+  }
+  const cls = [...el.classList].slice(0, 2).map((c) => `.${c}`).join("");
+  const r = el.getBoundingClientRect();
+  const out: any = {
+    selector: parts.join(" > "),
+    label: `${el.localName}${el.id ? `#${el.id}` : ""}${cls}`,
+    rect: { x: r.x, y: r.y, w: r.width, h: r.height },
+  };
+  if (q.full) {
+    const t = (el as HTMLInputElement).value || el.getAttribute("aria-label") || el.getAttribute("alt") || el.textContent || "";
+    out.text = t.replace(/\s+/g, " ").trim().slice(0, 80);
+    out.url = location.href;
+  }
+  return out;
+});