@chalky/chalky 0.0.0-stage → 0.1.4

package/app/extensions/point/browser.js

@@ -0,0 +1,247 @@
+// Browser half of the point plugin (experiment). A pencil button in the tool palette (top) turns on a clear
+// layer over the page; the cursor is a pen, blue over something a click would point at.
+//   Click: a note box opens there.  Hold the button and draw: on release the note box opens, and the note
+//   covers the drawn area (its bounding box, in main-space px: where ui_render's place puts a surface).
+// The user sees only the box; the target goes to the agent with the note: surface and component (from
+// ui.js's data-surface / data-cid), the element itself (also inside App frames, which are same-origin),
+// its text, the source file to edit, the spot (main-space px; on empty space the spot is the whole target),
+// and the area if drawn. Enter sends it (point.py passes it to the surface's owner, gives it an id and
+// tells every open page to pin it, with its drawing). The pin fades when the agent marks the note
+// done (event "point_done") or the user closes it with its ✕; open notes are redrawn after a reload.
+// Esc or the button turns it off.
+// The same note box opens for two more things (Enter on an empty box sends its faded suggestion):
+//   combine  ui.js fires "ui-combine" {a, b, x, y} when a surface is dropped onto another: the note
+//            says how to combine them and goes to b's owner as point_combine (b changes; a stays)
+//   files    the files plugin fires "files-dropped" {x, y, files} when files land on the page (from the
+//            computer, or a row from its panel): a note with their paths, on what they were dropped on
+// Parts: an element (or an App page's window) with partAt(x, y) is asked what is at the spot (an enemy in a
+// canvas game); its answer goes with the note as "part". For a drawing, the thing under its middle is asked
+// partIn(points) instead, if it has it: what is inside the drawn shape (the circled cells, paragraphs, an
+// image of the area). points: [[x, y], ...] in window px (an App's window: px in its frame); it may be async.
+export default async function (page) {
+  let names = {};  // "ui-table" -> "Table" (tags are lower case; catalog names are not)
+  try { for (const n of Object.keys(await (await fetch("/ui-api/catalog")).json())) names[`ui-${n.toLowerCase()}`] = n; } catch {}
+
+  const layer = document.createElement("div");  // catches clicks, so the page underneath doesn't react
+  layer.style.cssText = "position:fixed;inset:0;z-index:9998;display:none";
+  // The cursor: the button's pen (white edge, so it shows on dark too), its tip (4, 20) is the spot pointed at.
+  const pen = (color) => `url("data:image/svg+xml,${encodeURIComponent(
+    `<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke-linecap="round" stroke-linejoin="round">` +
+    `<g stroke="#fff" stroke-width="4"><path d="M4 20h4L19 9l-4-4L4 16z"/><path d="m13 7 4 4"/></g>` +
+    `<g stroke="${color}" stroke-width="2"><path d="M4 20h4L19 9l-4-4L4 16z"/><path d="m13 7 4 4"/></g></svg>`)}") 4 20, crosshair`;
+  const PEN = pen("#6e6e73"), PEN_ON = pen("#0071e3");  // grey: empty space; blue: over a thing
+  layer.style.cursor = PEN;
+  const input = document.createElement("input");  // the note box, opened where the user clicked
+  input.id = "point-note";
+  input.placeholder = "Write a note, Enter to send";
+  input.style.cssText = "position:fixed;z-index:9999;width:280px;box-sizing:border-box;font:inherit;font-size:14px;" +
+    "padding:8px 12px;border-radius:12px;outline:none;color:#1d1d1f;background:var(--glass-strong,#fff);" +  // the page's glass
+    "backdrop-filter:var(--blur);-webkit-backdrop-filter:var(--blur);border:1px solid var(--edge,#eee);" +
+    "box-shadow:0 8px 30px rgba(0,0,0,.1);display:none";
+  const SVG = "http://www.w3.org/2000/svg";
+  function strokeSvg(points) {  // a freehand stroke, drawn over the whole window (fixed, like the pins)
+    const svg = document.createElementNS(SVG, "svg");
+    svg.style.cssText = "position:fixed;inset:0;width:100vw;height:100vh;pointer-events:none";
+    const line = document.createElementNS(SVG, "polyline");
+    line.style.cssText = "fill:none;stroke:#f5b700;stroke-width:3;stroke-linecap:round;stroke-linejoin:round;opacity:.85";
+    line.setAttribute("points", points.map((p) => p.join(",")).join(" "));
+    svg.append(line);
+    return svg;
+  }
+  const ink = strokeSvg([]);  // the stroke being drawn
+  ink.style.zIndex = 9999;
+  ink.style.display = "none";
+  let current = null;  // {target, x, y, stroke} of the open note
+  let stroke = null;   // [[x, y], ...] while the mouse button is held down
+  document.body.append(layer, ink, input);
+
+  const button = document.createElement("button");
+  button.innerHTML = `<svg viewBox="0 0 24 24" aria-hidden="true"><path d="M4 20h4L19 9l-4-4L4 16z"/><path d="m13 7 4 4"/></svg>`;
+  button.title = "Point: click anything, or hold and draw around an area, then write a note (Esc to stop)";
+  page.slot("status").prepend(button);  // the first tool in the palette, whichever script loads first
+  const setOn = (on) => {
+    layer.style.display = on ? "block" : "none";
+    button.classList.toggle("on", on);
+    if (!on) input.style.display = ink.style.display = "none";
+    if (!on) current = stroke = null;
+    if (on) document.dispatchEvent(new CustomEvent("tool-on", { detail: "point" }));  // the other tools turn off
+  };
+  button.onclick = () => setOn(layer.style.display === "none");
+  document.addEventListener("tool-on", (e) => { if (e.detail !== "point" && layer.style.display !== "none") setOn(false); });  // one tool at a time
+  document.addEventListener("keydown", (e) => { if (e.key === "Escape") setOn(false); });
+
+  const describe = (e) => e.localName + (e.id ? `#${e.id}` : "") + (e.dataset?.name ? `[data-name="${e.dataset.name}"]` : "") +
+    [...e.classList].slice(0, 2).map((c) => `.${c}`).join("");
+
+  function inMain(x, y) {  // window px -> main-space px (the main space scrolls; ui_render's place uses these)
+    const main = page.slot("top"), m = main.getBoundingClientRect();
+    return { x: Math.round(x - m.left + main.scrollLeft), y: Math.round(y - m.top + main.scrollTop) };
+  }
+
+  function target(x, y) {
+    const el = document.elementsFromPoint(x, y).find((e) => e !== layer);
+    if (!el) return null;
+    let inner = null, frame = null, box = el.getBoundingClientRect();
+    if (el.localName === "iframe") {  // look inside the frame (an App): same origin, so its document is readable
+      try {
+        const r = el.getBoundingClientRect(), dx = r.left + el.clientLeft, dy = r.top + el.clientTop;
+        inner = el.contentDocument?.elementFromPoint(x - dx, y - dy);
+        if (inner) { frame = el; const b = inner.getBoundingClientRect(); box = new DOMRect(b.x + dx, b.y + dy, b.width, b.height); }
+      } catch {}
+    }
+    const hitEl = inner || el;
+    const position = inMain(x, y);
+    if (el === page.slot("top")) return { owner: null, surface: null, components: [], element: "empty space in the main space",
+                              text: "", file: "", position, box: new DOMRect(x - 12, y - 12, 24, 24) };
+    const components = [];  // outermost first, e.g. ["root (Card)", "chart1 (Table)"]
+    for (let e = el.closest("[data-cid]"); e; e = e.parentElement?.closest("[data-cid]"))
+      components.unshift(`${e.dataset.cid} (${names[e.localName] || e.localName})`);
+    const path = [];  // a short selector: up to the nearest element with an id, at most 4 steps
+    for (let e = hitEl; e && path.length < 4 && e.localName !== "body"; e = e.parentElement) {
+      path.unshift(describe(e));
+      if (e.id) break;
+    }
+    const innermost = el.closest("[data-cid]");
+    let part = null;  // what the thing itself says is at this spot (partAt), e.g. {"name": "enemy-2"}
+    try {
+      if (frame?.contentWindow?.partAt) {
+        const r = frame.getBoundingClientRect();
+        part = frame.contentWindow.partAt(x - r.left - frame.clientLeft, y - r.top - frame.clientTop);
+      } else {
+        const owner = [hitEl, ...[...document.elementsFromPoint(x, y)]].find((e) => typeof e?.partAt === "function");
+        if (owner) { const r = owner.getBoundingClientRect(); part = owner.partAt(x - r.left, y - r.top); }
+      }
+    } catch (e) { console.warn("partAt:", e); }
+    const file = frame ? "/app" + new URL(frame.contentWindow.location.href).pathname  // /ui/apps/x/... -> /app/ui/apps/x/...
+      : innermost ? `/app/extensions/ui/components/${names[innermost.localName] || innermost.localName}.js`
+      : "/app/extensions/page/page.html (the page itself)";  // /app/: the code folder (the server puts in the real one)
+    return {
+      owner: el.closest("[data-owner]")?.dataset.owner || null,
+      surface: el.closest("[data-surface]")?.dataset.surface || null,
+      components,
+      element: path.join(" > "),
+      part,
+      text: (hitEl.innerText || hitEl.value || hitEl.alt || "").trim().replace(/\s+/g, " ").slice(0, 60),
+      file,
+      at: { x: Math.round(((x - box.x) / box.width) * 100) + "%", y: Math.round(((y - box.y) / box.height) * 100) + "%" },
+      position,
+      box,
+    };
+  }
+
+  // The note box at (x, y). `note`: what Enter sends besides the text (target, combine, files);
+  // `suggestion`: the faded text, sent as it is if the box is left empty (else the box needs words).
+  function openBox(x, y, note, suggestion = "") {
+    input.style.display = "block";
+    input.style.left = Math.min(x + 12, innerWidth - input.offsetWidth - 8) + "px";
+    input.style.top = Math.min(y + 12, innerHeight - input.offsetHeight - 8) + "px";
+    input.placeholder = suggestion || "Write a note, Enter to send";
+    current = { x, y, suggestion, ...note };
+    input.value = "";
+    input.focus();
+  }
+
+  function openNote(t, x, y, drawn = null) {  // a note on target t (and the drawn stroke)
+    if (!t) return;
+    const { box, ...rest } = t;
+    openBox(x, y, { type: "point_note", target: rest, stroke: drawn, to: rest.owner });
+  }
+
+  // Combine: a surface was dropped onto another (ui.js). The note goes to the target's owner.
+  document.addEventListener("ui-combine", ({ detail: { a, b, x, y } }) => {
+    const to = document.querySelector(`[data-surface="${CSS.escape(b)}"]`)?.dataset.owner;
+    openBox(x, y, { type: "point_combine", a, b, to }, `Add ${a} to ${b}`);
+  });
+
+  // Files dropped on the page (the files plugin: from the computer, or a row from its panel; already
+  // stored in /workspace/drops): a note on what they landed on. Esc closes the box; the files stay stored.
+  document.addEventListener("files-dropped", ({ detail: { x, y, files } }) => {
+    const t = target(x, y) || { owner: null, surface: null, components: [], element: "", text: "", file: "", position: inMain(x, y) };
+    const { box, ...rest } = t;
+    const paths = files.map((f) => `${f.path} (${Math.ceil(f.size / 1024)} KB)`);
+    openBox(x, y, { type: "point_note", target: rest, files: paths, to: rest.owner }, `Use ${files.map((f) => f.name).join(", ")} here`);
+  });
+
+  // Press, move, release: a small movement is a click on one thing; a bigger one is a drawing.
+  layer.onmousedown = (e) => {
+    stroke = [[e.clientX, e.clientY]];
+    input.style.display = ink.style.display = "none";
+  };
+  layer.onmousemove = (e) => {
+    if (!stroke) {
+      const t = target(e.clientX, e.clientY);
+      layer.style.cursor = t && t.surface ? PEN_ON : PEN;
+      return;
+    }
+    const [lx, ly] = stroke.at(-1);
+    if (Math.hypot(e.clientX - lx, e.clientY - ly) < 3) return;  // keep the stroke light
+    stroke.push([e.clientX, e.clientY]);
+    ink.style.display = "block";
+    ink.firstChild.setAttribute("points", stroke.map((p) => p.join(",")).join(" "));
+  };
+  async function partIn(x, y, points) {  // what the thing at (x, y) says is inside the drawing, or null
+    const el = document.elementsFromPoint(x, y).find((e) => e !== layer && e !== ink);
+    if (el?.localName === "iframe") {
+      try {
+        const r = el.getBoundingClientRect(), dx = r.left + el.clientLeft, dy = r.top + el.clientTop;
+        if (typeof el.contentWindow.partIn === "function") return await el.contentWindow.partIn(points.map(([px, py]) => [px - dx, py - dy]));
+      } catch (err) { console.warn("partIn:", err); }
+    }
+    const owner = document.elementsFromPoint(x, y).find((e) => typeof e.partIn === "function");
+    try { return owner ? await owner.partIn(points) : null; } catch (err) { console.warn("partIn:", err); return null; }
+  }
+
+  layer.onmouseup = async (e) => {
+    const points = stroke;
+    stroke = null;
+    if (!points) return;
+    const xs = points.map((p) => p[0]), ys = points.map((p) => p[1]);
+    const left = Math.min(...xs), top = Math.min(...ys), width = Math.max(...xs) - left, height = Math.max(...ys) - top;
+    if (width < 12 && height < 12) return openNote(target(e.clientX, e.clientY), e.clientX, e.clientY);
+    const t = target(left + width / 2, top + height / 2);  // what is under the middle of the drawing
+    if (t) t.area = { ...inMain(left, top), width: Math.round(width), height: Math.round(height) };
+    const inside = t && (await partIn(left + width / 2, top + height / 2, points));
+    if (inside) t.part = inside;
+    openNote(t, e.clientX, e.clientY, points);
+  };
+
+  input.onkeydown = (e) => {
+    const text = input.value.trim() || current?.suggestion;
+    if (e.key !== "Enter" || !text || !current) return;
+    const { suggestion, ...note } = current;
+    page.send({ ...note, text, to: note.to || document.getElementById("to")?.value,  // else the agent picked in the chat
+                board: Number(document.body.dataset.board) || 1 });  // the board shown (boards/): what it points at is on it
+    setOn(false);
+  };
+
+  const pins = {};  // note id -> pin element: its label, and its drawing if it has one
+  function pin({ id, text, to, x, y, stroke, board }) {  // stays where the note was written (doesn't follow scrolling yet)
+    if (pins[id]) return;
+    const el = (pins[id] = document.createElement("div"));
+    el.dataset.board = board ?? 1;  // shown only on its board (boards/)
+    el.style.cssText = "position:fixed;inset:0;z-index:9997;pointer-events:none;transition:opacity 1.5s";
+    if (stroke) el.append(strokeSvg(stroke));
+    const label = document.createElement("div");
+    label.style.cssText = `position:absolute;left:${x}px;top:${y}px;max-width:240px;` +
+      "background:#fff8c4;border:1px solid #e6c200;border-radius:4px;padding:3px 20px 3px 6px;font:12px sans-serif;" +
+      "box-shadow:0 1px 3px #0003";
+    label.textContent = `✎ ${text} · sent to ${to}`;
+    const close = document.createElement("button");
+    close.textContent = "✕";
+    close.title = "Close this note (its agent is told)";
+    close.style.cssText = "position:absolute;top:1px;right:3px;line-height:1;pointer-events:auto;border:0;background:none;cursor:pointer;padding:0 2px;font:inherit;color:#8a7400";
+    close.onclick = () => page.send({ type: "point_dismiss", id });
+    label.append(close);
+    el.append(label);
+    document.body.append(el);
+  }
+  page.on("point_pin", pin);
+  page.on("point_done", ({ id }) => {
+    const el = pins[id];
+    if (!el) return;
+    delete pins[id];
+    el.style.opacity = "0";
+    setTimeout(() => el.remove(), 1500);
+  });
+  try { for (const [id, n] of Object.entries(await (await fetch("/point/notes")).json())) pin({ id, ...n }); } catch {}
+}

package/app/extensions/point/guide.md

@@ -0,0 +1,6 @@
+# Notes on the UI
+The user clicks anything on the page (or draws around it, or drops files there) and writes a note: you get
+"[note on the UI] <their words>" with exactly what it is on.
+- Change that exact thing. If its text is data (a row, a score), change the surface's data, not the component.
+- When you have dealt with it, call note_done(id).
+Before you handle your first note, read README.md.

package/app/extensions/point/index.ts

@@ -0,0 +1,112 @@
+// Notes on the UI (port of the Python point plugin; the browser half, browser.js, is its point.js): the user points at
+// things and writes a note there; the note and its target go to an agent, so it knows exactly what it is about.
+//   point_note     click anything on the page (or draw around an area) and write: the target is the surface,
+//                  component, element, part (partAt), its text and source file, the spot and the area.
+//                  Files dropped on the page come as a note too ("files": paths in <workspace>/drops).
+//   point_combine  the user dragged surface a onto surface b and wrote how to combine them. The agent changes b to
+//                  take a in, reading a's data live ({"ref"} props); a stays as it is.
+// Each note goes to the surface's owner (else the agent picked in the chat), gets an id (n1, n2, ...: never reused)
+// and stays pinned (state/notes.json: {"last", "open": {id: note}}, redrawn after a reload) until the agent calls
+// note_done(id). The user can also close a pin (its ✕): the note is dropped, and taken out of its agent's inbox if
+// its turn hasn't started (the agent never saw it); else its agent is told (a note).
+// guide.md: the short prompt section; README.md (copied from point.py): the whole of it, read before a first note.
+import fs from "node:fs";
+import path from "node:path";
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+
+const host = (globalThis as any).chalky;
+const FILE = path.join(host.state, "notes.json");
+if (!host.shared.has("point"))
+  host.shared.set("point", fs.existsSync(FILE) ? JSON.parse(fs.readFileSync(FILE, "utf8")) : { last: 0, open: {} });
+const notes = host.shared.get("point");  // {last, open: {id: {text, to, x, y, stroke, board}}}
+if (!host.shared.has("point-items")) host.shared.set("point-items", new Map());
+const items: Map<string, any> = host.shared.get("point-items");  // note id -> its swarm inbox item (not saved)
+const save = () => fs.writeFileSync(FILE, JSON.stringify(notes));
+
+function pin(text: string, to: string, msg: any) {  // a new open note: saved, pinned on every page
+  const id = `n${++notes.last}`;
+  notes.open[id] = { text, to, x: msg.x ?? 0, y: msg.y ?? 0, stroke: msg.stroke ?? null,  // with its drawing, if any
+                     board: Number(msg.board) || 1 };  // the board it was written on (boards/): its pin shows there
+  save();
+  host.broadcast("point_pin", { id, ...notes.open[id] });
+  return id;
+}
+
+function describe(sid: string) {  // a source of a combine, for the agent: cut short if long
+  const r = host.shared.get("ui")?.surfaces[sid] ?? {};
+  const text = JSON.stringify({ owner: r.owner, about: r.about, components: r.components, data: r.data });
+  return text.length < 3000 ? text : text.slice(0, 3000) + ` … (more in ${host.app}/ui/surfaces.json and ${host.state}/ui-data.json)`;
+}
+
+export default function (pi: ExtensionAPI) {
+  const guide = fs.readFileSync(path.join(import.meta.dirname, "guide.md"), "utf8")  // short; README.md: the whole of it, read when it says
+    .replace("README.md", path.join(import.meta.dirname, "README.md"));
+  pi.on("before_agent_start", (event) => { event.systemPromptOptions.sections.point = guide; });
+  // a removed board (boards/) takes its open notes with it; a note its agent hasn't seen yet is just taken back
+  pi.on("session_start", () => host.api.boards?.onRemove("point", (n: number) => {
+    const gone: { owner: string; what: string }[] = [];
+    for (const id of Object.keys(notes.open).filter((id) => (notes.open[id].board ?? 1) === n)) {
+      const note = notes.open[id], item = items.get(id);
+      delete notes.open[id];
+      items.delete(id);
+      host.broadcast("point_done", { id });
+      if (!(item && host.api.swarm?.withdraw(item))) gone.push({ owner: note.to, what: `note ${id} ("${note.text}")` });
+    }
+    save();
+    return gone;
+  }));
+
+  pi.registerTool({
+    name: "note_done", label: "Note done", description: "Mark a note the user pinned on the UI as dealt with; its pin fades.",
+    parameters: Type.Object({ id: Type.String({ description: "e.g. n3" }) }),
+    async execute(_id, a) {
+      if (!notes.open[a.id]) return { content: [{ type: "text", text: `error: no open note '${a.id}'. Open notes: ${Object.keys(notes.open).join(", ")}` }], details: undefined };
+      delete notes.open[a.id];
+      items.delete(a.id);
+      save();
+      host.broadcast("point_done", { id: a.id });
+      return { content: [{ type: "text", text: `note ${a.id} is done; its pin fades from the page` }], details: undefined };
+    },
+  });
+
+  host.routes.set("/point/notes", (_req: any, res: any) => res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify(notes.open)));
+
+  host.pageHandlers.set("point", (msg: any) => {
+    const swarm = host.api.swarm;
+    if (msg.type === "point_dismiss" && notes.open[msg.id]) {
+      const note = notes.open[msg.id];
+      delete notes.open[msg.id];
+      save();
+      host.broadcast("point_done", { id: msg.id });
+      const item = items.get(msg.id);
+      items.delete(msg.id);
+      if (!(item && swarm?.withdraw(item)))
+        swarm?.send("user", note.to, `[note on the UI] the user closed note ${msg.id} ("${note.text}"): drop it if you haven't finished it.`, true);
+    }
+    if (!["point_note", "point_combine"].includes(msg.type) || !swarm) return;
+    const to = host.agents.has(msg.to) ? msg.to : "agent1";
+    const text = msg.text ?? "";
+    if (msg.type === "point_combine") {
+      const { a, b } = msg;
+      const madeFrom = [...(host.shared.get("ui")?.surfaces[b]?.made_from ?? []).filter((m: string) => m !== a), a];
+      const id = pin(`combine ${a} into ${b}: ${text}`, to, msg);
+      return items.set(id, swarm.send("user", to, [`[combine on the UI] ${text}`, `  id: ${id} (call note_done("${id}") when dealt with)`,
+        `  board: ${notes.open[id].board}`,
+        `  dragged: ${a} (keep it as it is)`, `  onto: ${b} (change this one)`,
+        `  so: ui_render(surface="${b}", ..., made_from=${JSON.stringify(madeFrom)}, instruction=<their words>), reading ${a}'s data with {"ref": "${a}", "path": ...}`,
+        `  ${a}: ${describe(a)}`, `  ${b}: ${describe(b)}`].join("\n")));
+    }
+    const t = msg.target ?? {};
+    const at = t.at && `${t.at.x} across, ${t.at.y} down the element`;
+    const pos = t.position && `x=${t.position.x}, y=${t.position.y} px in the main space (for ui_render's place)`;
+    const area = t.area && `x=${t.area.x}, y=${t.area.y}, width=${t.area.width}, height=${t.area.height} px in the main space `
+      + (t.surface ? "(the user drew around this part of the surface: the note is about it)" : "(the user drew around empty space: build inside it)");
+    const id = pin(text, to, msg);
+    const lines = [`[note on the UI] ${text}`, `  id: ${id} (call note_done("${id}") when dealt with)`,
+      ...([["board", notes.open[id].board], ["surface", t.surface], ["component", (t.components ?? []).join(" → ")], ["element", t.element],
+           ["part", t.part && JSON.stringify(t.part)], ["at", at], ["text", t.text], ["file", t.file?.replace(/^\/app\//, host.app + "/")], ["position", pos],
+           ["area", area], ["files", msg.files?.join(", ")]] as [string, any][]).filter(([, v]) => v).map(([k, v]) => `  ${k}: ${v}`)];
+    items.set(id, swarm.send("user", to, lines.join("\n")));
+  });
+}

package/app/extensions/prompt/index.ts

@@ -0,0 +1,19 @@
+// Where things are: the paths the prompt needs that depend on this machine, as the <paths> section. The prompt's
+// words are app/SYSTEM.md (pi's documented way to replace its default prompt), which is plain text and can't hold
+// these. pi adds the skills list and the working folder (<cwd>) itself.
+import { getDocsPath, getExamplesPath, getReadmePath, type ExtensionAPI } from "@earendil-works/pi-coding-agent";
+
+export default function (pi: ExtensionAPI) {
+  pi.on("before_agent_start", (event) => {
+    const host = (globalThis as any).chalky;
+    event.systemPromptOptions.sections.paths = [
+      `{code} = ${host.app}: your code (versioned: each turn that changes it is saved, and the user can roll back)`,
+      `{work} = ${host.work}: your working folder (<cwd>), for project work and the user's files`,
+      `{data} = ${host.state}: what the agents and apps keep (conversations, app data, surfaces' data; not versioned)`,
+      `{page} = http://127.0.0.1:${host.port}: the user's page, served by the host`,
+      `The host: ${host.root}/server.ts`,
+      `pi's docs: ${getDocsPath()}/extensions.md for extensions, examples in ${getExamplesPath()}/extensions/; the rest in `
+        + `${getReadmePath()} and ${getDocsPath()}/. Read pi's .md files completely and follow their links.`,
+    ].join("\n");
+  });
+}

package/app/extensions/reload/index.ts

@@ -0,0 +1,21 @@
+// Self-edits take effect: the Python agent's restart plugin, the pi way. The tool only asks; when this agent's turn
+// has ended, the host calls pi's session.reload() (extensions, skills and prompt load again; the conversation stays).
+// Not pi's reload-runtime.ts example: its pi.sendUserMessage("/reload-runtime") reaches the model as text in 1.4.0
+// (expandPromptTemplates defaults to false), and the model kept calling the tool again.
+// No process restart, so no start.sh (a rollback: the versions plugin, which reloads every agent).
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+
+export default function (pi: ExtensionAPI) {
+  pi.registerTool({
+    name: "reload", label: "Reload", description: "Load your edited extensions again, when your turn ends.",
+    parameters: Type.Object({ reason: Type.String() }),
+    async execute(_id, a) {
+      const host = (globalThis as any).chalky, name = pi.getSessionName();
+      host.reloadAfterTurn.set(name, false);  // not quiet: the page notes the reload
+      host.shared.set(`reload-reason ${name}`, a.reason);  // the versions plugin's commit message for this turn
+      return { content: [{ type: "text", text: "Reload scheduled for when your turn ends. End your turn now; don't call reload again." }],
+               details: undefined };
+    },
+  });
+}

package/app/extensions/screen/browser.js

@@ -0,0 +1,76 @@
+// Browser half of the screen extension (Python's screen.js, unchanged but for this header): this tab, shared once (getDisplayMedia), for every plugin that needs a picture
+// of what the user sees. Offered as page.provide("screen", {...}); a plugin gets it with await page.use("screen"):
+//   share()            ask the browser to share this tab (only works right after a click: transient activation);
+//                      resolves true if shared. The user can stop sharing any time (the browser's bar).
+//   shared()           whether the tab is shared now
+//   frame()            the screen right now, as a canvas (null when not shared)
+//   picture(full, box) crop a frame to box (window px {x, y, w, h}), upload it (POST /screen/shot): {path, url}
+// An agent can ask to see the screen (its tool see_user_screen, event screen_ask): if the tab is shared, a picture
+// of the whole window goes back at once; else a card asks "Share / No" first, since the agent's ask is no click
+// (screen_reply {id, shot, window, main}).
+export default async function (page) {
+  const area = page.slot("top");
+  const style = document.createElement("style");
+  style.textContent = `
+    .screen-ask { position: fixed; top: 76px; left: 50%; transform: translateX(-50%); z-index: 100; display: flex; align-items: center; gap: 10px;
+                  padding: 8px 8px 8px 14px; border-radius: 14px; background: var(--glass-strong, #fff); border: 1px solid var(--edge, #eee);
+                  box-shadow: 0 8px 30px rgba(0, 0, 0, .12); font-size: 13px; color: var(--text); }
+    .screen-ask button { border: 0; border-radius: 9px; padding: 5px 12px; font: inherit; cursor: pointer; background: var(--line, #eee); color: var(--text); }
+    .screen-ask button.yes { background: var(--accent, #0071e3); color: #fff; }`;
+  document.head.append(style);
+
+  let stream = null, video = null;
+  async function share() {
+    if (stream) return true;
+    try {
+      const full = { width: { ideal: innerWidth * devicePixelRatio }, height: { ideal: innerHeight * devicePixelRatio } };  // sharp: the page's own pixels
+      stream = await navigator.mediaDevices.getDisplayMedia({ video: full, preferCurrentTab: true, selfBrowserSurface: "include" });
+      stream.getVideoTracks()[0].onended = () => { stream = null; video?.remove(); video = null; };  // the user stopped sharing
+      video = Object.assign(document.createElement("video"), { muted: true, playsInline: true, srcObject: stream });
+      video.style.cssText = "position:fixed;width:1px;height:1px;opacity:0;pointer-events:none";
+      document.body.append(video);
+      await video.play();
+      return true;
+    } catch { stream = null; return false; }  // not shared
+  }
+  const shared = () => !!video?.videoWidth;
+  function frame() {
+    if (!shared()) return null;
+    const c = Object.assign(document.createElement("canvas"), { width: video.videoWidth, height: video.videoHeight });
+    c.getContext("2d").drawImage(video, 0, 0);
+    return c;
+  }
+  async function picture(full, box) {
+    const kx = full.width / innerWidth, ky = full.height / innerHeight;  // frame px per window px (the frame is the tab)
+    const c = Object.assign(document.createElement("canvas"), { width: Math.round(box.w * kx), height: Math.round(box.h * ky) });
+    c.getContext("2d").drawImage(full, box.x * kx, box.y * ky, box.w * kx, box.h * ky, 0, 0, c.width, c.height);
+    const png = await new Promise((ok) => c.toBlob(ok, "image/png"));
+    const out = await (await fetch("/screen/shot", { method: "POST", body: png })).json();
+    return { path: out.path || null, url: URL.createObjectURL(png) };
+  }
+  page.provide("screen", { share, shared, frame, picture });
+
+  // ---- an agent asks to see the screen ----
+  const asks = {};  // id -> its card, while the agent waits
+  async function answer(id) {
+    for (const card of Object.values(asks)) card.remove();  // not in the picture
+    await new Promise((ok) => setTimeout(ok, 300));         // a fresh frame from the shared tab
+    const full = frame();
+    const s = full && await picture(full, { x: 0, y: 0, w: innerWidth, h: innerHeight }).catch(() => null);
+    const m = area.getBoundingClientRect();  // window (0, 0) in main-space px (as point.js)
+    page.send({ type: "screen_reply", id, shot: s?.path || null, window: { width: innerWidth, height: innerHeight },
+                main: { x: area.scrollLeft - m.left, y: area.scrollTop - m.top } });
+  }
+  page.on("screen_ask", ({ id, agent }) => {
+    if (shared()) return answer(id);
+    const card = Object.assign(document.createElement("div"), { className: "screen-ask" });
+    const yes = Object.assign(document.createElement("button"), { className: "yes", textContent: "Share" });
+    const no = Object.assign(document.createElement("button"), { textContent: "No" });
+    card.append(`${agent} wants to see your screen`, yes, no);
+    yes.onclick = async () => { await share(); answer(id); };
+    no.onclick = () => page.send({ type: "screen_reply", id, shot: null });
+    asks[id] = card;
+    document.body.append(card);
+  });
+  page.on("screen_ask_done", ({ id }) => { asks[id]?.remove(); delete asks[id]; });  // answered (here or in another tab), or timed out
+}

package/app/extensions/screen/index.ts

@@ -0,0 +1,80 @@
+// Screen: what the user sees, as pictures (port of the Python screen plugin; browser.js is its screen.js). The
+// browser half shares the tab once (for every extension: doodle's steps, an agent's look) and uploads pictures
+// cropped from it. This half keeps them and gives agents the tool see_user_screen: the user's window as it is now
+// (their size, scroll, panels, running apps; ui-check's headless Chrome is a fresh window of its own).
+//   POST /screen/shot       a png (at most MAX_BYTES): saved in state/screens, {path} back
+//   event "screen_ask"      {id, agent} to every open page, when an agent calls see_user_screen
+//   page message            screen_reply {id, shot, window: {width, height}, main: {x, y}}: the first answers
+//                           (shot null: the user said no; main: window (0, 0) in main-space px)
+//   event "screen_ask_done" {id} to every page when answered or after ASK_WAIT s, so the asks go away
+//   host.api.screen         isShot(path): whether a path is a picture saved here (others attach pictures by path)
+import fs from "node:fs";
+import path from "node:path";
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+
+const host = (globalThis as any).chalky;
+const SHOTS = path.join(host.state, "screens");  // outside app/: not code
+const MAX_BYTES = 8 * 1024 * 1024;
+const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
+const ASK_WAIT = 90;  // s for the user to answer see_user_screen
+
+const isShot = (p: unknown) => typeof p === "string" && path.dirname(p) === SHOTS && fs.existsSync(p);
+host.api.screen = { isShot };
+// id -> resolve, while see_user_screen waits for a page (on the host: the tool is per agent, the page handler one)
+const asks: Map<string, (msg: any) => void> = host.shared.get("screen-asks") ?? new Map();
+host.shared.set("screen-asks", asks);
+
+host.routes.set("/screen/shot", (req: any, res: any) => {
+  if (req.method !== "POST") return res.writeHead(405).end();
+  const chunks: Buffer[] = [];
+  let size = 0;
+  req.on("data", (c: Buffer) => { size += c.length; if (size <= MAX_BYTES) chunks.push(c); });
+  req.on("end", () => {
+    const png = Buffer.concat(chunks);
+    if (size > MAX_BYTES || !png.subarray(0, 8).equals(PNG))
+      return res.writeHead(400, { "content-type": "application/json" }).end(JSON.stringify({ error: `a png of at most ${MAX_BYTES / 2 ** 20} MB` }));
+    fs.mkdirSync(SHOTS, { recursive: true });
+    const p = path.join(SHOTS, `${new Date().toISOString().replace(/[-:]/g, "").replace("T", "-").slice(0, 15)}-${process.hrtime.bigint() % 1000000n}.png`);
+    fs.writeFileSync(p, png);
+    res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({ path: p }));
+  });
+});
+
+host.pageHandlers.set("screen", (msg: any) => {
+  if (msg.type === "screen_reply") asks.get(msg.id)?.(msg);
+});
+
+const result = (text: string, png?: string) => ({ details: undefined, content: [{ type: "text" as const, text },
+  ...(png ? [{ type: "image" as const, data: fs.readFileSync(png).toString("base64"), mimeType: "image/png" }] : [])] });
+
+export default function (pi: ExtensionAPI) {
+  pi.registerTool({
+    name: "see_user_screen", label: "See the user's screen",
+    description: "See the user's screen as they see it right now: a picture of their whole window (their size, scroll, "
+      + "open panels, apps as they are running). ui-check's headless Chrome is a fresh window of its own, not theirs: when "
+      + "the user talks about what they see, or before you say something looks right to them, look here. The first "
+      + "time, the user is asked to share the tab (they may say no); then it's instant.",
+    parameters: Type.Object({}),
+    async execute(_id, _args, signal) {
+      const id = `${pi.getSessionName()}-${Date.now()}`;
+      const reply: any = await new Promise((resolve) => {
+        const timer = setTimeout(() => resolve(null), ASK_WAIT * 1000);
+        const done = (msg: any) => { clearTimeout(timer); resolve(msg); };
+        asks.set(id, done);
+        signal?.addEventListener("abort", () => done(null));
+        host.broadcast("screen_ask", { id, agent: pi.getSessionName() });
+      });
+      asks.delete(id);
+      host.broadcast("screen_ask_done", { id });
+      if (!reply) return result(`no answer within ${ASK_WAIT} s (the page may not be open, or the user is away): ask them what they see`);
+      if (!isShot(reply.shot)) return result("the user didn't share their screen: ask them what they see");
+      const png = fs.readFileSync(reply.shot), pw = png.readUInt32BE(16), ph = png.readUInt32BE(20);  // the png's own size (IHDR)
+      const w = reply.window ?? {}, m = reply.main ?? {}, k = w.width ? pw / w.width : 1;
+      const mx = Math.round(m.x ?? 0), my = Math.round(m.y ?? 0);
+      return result(`the user's screen right now: their window is ${w.width}x${w.height} px; the picture is the whole window, `
+        + `${pw}x${ph} px (${+k.toFixed(3)} picture px per window px). Window (0, 0) is main-space (${mx}, ${my}): a spot in `
+        + `the picture is main-space x = ${mx} + picture x / ${+k.toFixed(3)}, the same for y (what ui_render's place uses).`, reply.shot);
+    },
+  });
+}

package/app/extensions/steps/index.ts

@@ -0,0 +1,17 @@
+// A limit on steps (model calls) per input, as the Python loop's MAX_STEPS: pi's loop has none, and a model that
+// keeps calling tools would run on (it did once: 55 rounds). At the limit the run is stopped and the agent says so.
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+
+const MAX_STEPS = Number(process.env.MAX_STEPS || 30);
+
+export default function (pi: ExtensionAPI) {
+  let steps = 0;
+  pi.on("before_agent_start", () => { steps = 0; });  // once per input (retries and steering don't reset it)
+  pi.on("turn_end", (_event, ctx) => {
+    if (++steps !== MAX_STEPS) return;  // once: the stopped reply is one more turn_end
+    const host = (globalThis as any).chalky, agent = host.agents.get(pi.getSessionName());
+    if (agent) agent.stopped = true;  // on purpose: not shown as an error (host.abort can't be awaited from here)
+    ctx.abort();
+    host.broadcast("error", { agent: pi.getSessionName(), text: `stopped after ${MAX_STEPS} steps` });
+  });
+}

package/app/extensions/swarm/README.md

@@ -0,0 +1,47 @@
+# Your team (the swarm extension)
+Your name and role are in your team section.
+You work with other AI agents as peers; there is no manager.
+
+# Talking
+- What you receive starts with a header: "[<kind> from <name> · <time>]". <name> is "user" (the human),
+  another agent, or a harness part ("system"). Kinds:
+  message (from the user or the harness), question (the sender is waiting for your answer),
+  note (the sender went on working), reply (the answer to a note you sent),
+  fyi (an update from an agent working for you; its final answer still comes).
+- Your final answer in a turn is delivered for you: to the sender of a message, question or note; after a
+  reply, to whoever you were working for (the header says where). Say it once, there; don't repeat it.
+- tell(to, text): send a note and keep working. If the agent is busy it sees the note between its tool
+  calls; if idle it starts on it. Its answer comes back to you later as a reply.
+  Notes that arrive while you work show up between your tool calls: if one needs an answer other than
+  your final one, answer it with tell.
+  A tell to the agent you are working for is information only (it gets your final answer anyway and won't
+  reply to the tell). To ask it something, use ask_agent.
+- Waiting: when all that is left is waiting for others, end your turn. Replies and notes wake
+  you. Never sleep or poll (no sleep loops, no repeated status checks).
+- Don't use tell to deliver your result: your final answer already gets there. Don't send thanks or
+  acknowledgements.
+- ask_agent(to, text): ask and wait for the answer (it comes back as the result). Use it for quick
+  questions you can't continue without; use tell for work that takes a while. Several ask_agent calls
+  in one step run in parallel. You can't ask an agent that is waiting on you (tell it instead).
+- list_agents shows the team and what each agent is doing right now (idle, working on what, waiting on
+  whom). Check it instead of asking agents for status.
+- spawn_agent adds a member (numbered agent1, agent2, ... in creation order; spawn_agent says when the team is full).
+  An agent that is part of something you build (a simulation's member, a character) is an app agent,
+  not a worker: load the agent-apps skill. spawn_agent(..., app=<its surface>) keeps its talk out of the
+  user's messages panel (the app shows it).
+  reset_agent(name, or app=<surface> for all of an app's agents) empties their memory, keeping their
+  roles: for a new round that should start fresh.
+
+# Working as a team
+- Don't spawn agents or delegate for simple work; do it yourself. Split only work that really is separate.
+- One owner per piece of work and per file area. Don't edit files another agent owns; tell it instead.
+- Decisions others depend on (interfaces, formats, conventions, ports) go in CONTRACTS.md in your working
+  folder before you build on them; read it before you start. Give exact paths. Don't change a recorded decision on your own:
+  tell its owner.
+- Done means checked: run it or test it, and say what you ran and what it showed.
+- Found a defect in someone else's work? Tell its owner what is wrong and how you found it; don't fix it
+  in their files yourself.
+- When others' code will import yours, create the file (a stub) first so imports work while you build.
+- Before saying something is missing or not done, check again (files, list_agents): it may have changed.
+- Say so before installing anything or making a decision that affects others.
+- Other agents may change the same files (in {code} too). Check list_agents and coordinate with tell.