@chalky/chalky 0.1.3 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/README.md +10 -6
  2. package/{app → agent}/SYSTEM.md +16 -11
  3. package/agent/extensions/app/README.md +68 -0
  4. package/{app/extensions/ui → agent/extensions/app}/agents.js +22 -16
  5. package/agent/extensions/app/guide.md +5 -0
  6. package/agent/extensions/app/index.ts +66 -0
  7. package/{app/extensions/ui/skills/agent-apps → agent/extensions/app/skills/app-agents}/SKILL.md +17 -18
  8. package/agent/extensions/board/browser.js +359 -0
  9. package/agent/extensions/board/guide.md +4 -0
  10. package/agent/extensions/board/index.ts +100 -0
  11. package/agent/extensions/card/README.md +88 -0
  12. package/agent/extensions/card/blocks/Button.js +34 -0
  13. package/{app/extensions/ui/components → agent/extensions/card/blocks}/Column.js +2 -2
  14. package/agent/extensions/card/blocks/Markdown.js +37 -0
  15. package/{app/extensions/ui/components/Card.js → agent/extensions/card/blocks/Panel.js} +3 -3
  16. package/{app/extensions/ui/components → agent/extensions/card/blocks}/Row.js +2 -2
  17. package/{app/extensions/ui/components → agent/extensions/card/blocks}/TextField.js +3 -3
  18. package/{app/extensions/ui → agent/extensions/card}/browser.js +155 -128
  19. package/agent/extensions/card/guide.md +13 -0
  20. package/agent/extensions/card/index.ts +370 -0
  21. package/agent/extensions/command/browser.js +13 -0
  22. package/agent/extensions/command/guide.md +2 -0
  23. package/agent/extensions/command/index.ts +94 -0
  24. package/agent/extensions/doodle/README.md +57 -0
  25. package/agent/extensions/doodle/browser.js +1013 -0
  26. package/agent/extensions/doodle/guide.md +22 -0
  27. package/agent/extensions/doodle/index.ts +274 -0
  28. package/agent/extensions/file/browser.js +143 -0
  29. package/agent/extensions/file/guide.md +5 -0
  30. package/{app/extensions/files → agent/extensions/file}/index.ts +34 -16
  31. package/{app/extensions/frames → agent/extensions/frame}/README.md +13 -13
  32. package/{app/extensions/frames → agent/extensions/frame}/browser.js +24 -19
  33. package/agent/extensions/frame/guide.md +3 -0
  34. package/agent/extensions/frame/index.ts +81 -0
  35. package/{app/extensions/ui_history → agent/extensions/history}/browser.js +18 -18
  36. package/{app/extensions/ui_history → agent/extensions/history}/index.ts +32 -33
  37. package/{app → agent}/extensions/mcp/browser.js +1 -1
  38. package/agent/extensions/mcp/guide.md +3 -0
  39. package/{app → agent}/extensions/mcp/index.ts +2 -4
  40. package/agent/extensions/model/browser.js +363 -0
  41. package/{app → agent}/extensions/model/index.ts +8 -4
  42. package/agent/extensions/model/login.ts +215 -0
  43. package/agent/extensions/note/README.md +35 -0
  44. package/agent/extensions/note/browser.js +356 -0
  45. package/agent/extensions/note/guide.md +8 -0
  46. package/agent/extensions/note/index.ts +192 -0
  47. package/{app → agent}/extensions/page/index.ts +2 -2
  48. package/{app → agent}/extensions/page/page.html +55 -49
  49. package/agent/extensions/prompt/index.ts +38 -0
  50. package/{app → agent}/extensions/reload/index.ts +3 -3
  51. package/{app → agent}/extensions/screen/browser.js +4 -4
  52. package/{app → agent}/extensions/screen/index.ts +11 -11
  53. package/{app/extensions/steps → agent/extensions/step}/index.ts +1 -1
  54. package/{app/extensions/swarm → agent/extensions/team}/README.md +19 -19
  55. package/{app/extensions/swarm → agent/extensions/team}/guide.md +4 -4
  56. package/{app/extensions/swarm → agent/extensions/team}/index.ts +41 -66
  57. package/{app/extensions/versions → agent/extensions/version}/browser.js +10 -10
  58. package/{app/extensions/versions → agent/extensions/version}/index.ts +39 -54
  59. package/{app → agent}/extensions/viewer/README.md +4 -4
  60. package/{app → agent}/extensions/viewer/browser.js +8 -8
  61. package/agent/extensions/viewer/guide.md +3 -0
  62. package/{app → agent}/extensions/viewer/index.ts +22 -17
  63. package/agent/extensions/voice/browser.js +72 -0
  64. package/agent/extensions/voice/index.ts +39 -0
  65. package/{app/extensions/ui → agent/extensions/voice}/voice.js +5 -5
  66. package/agent/settings.json +8 -0
  67. package/{app/extensions/ui/skills/ui-check → agent/skills/browser-check}/SKILL.md +4 -4
  68. package/{app/extensions/ui/skills/ui-check → agent/skills/browser-check}/scripts/check.mjs +1 -1
  69. package/{app/extensions/ui/skills/ui-design → agent/skills/design}/SKILL.md +14 -14
  70. package/agent/skills/extension-map/SKILL.md +56 -0
  71. package/{app → agent}/skills/write-skill/SKILL.md +1 -1
  72. package/bin/chalky.mjs +3 -3
  73. package/package.json +8 -8
  74. package/server.ts +162 -63
  75. package/app/extensions/appdata/README.md +0 -10
  76. package/app/extensions/appdata/guide.md +0 -4
  77. package/app/extensions/appdata/index.ts +0 -47
  78. package/app/extensions/doodle/README.md +0 -15
  79. package/app/extensions/doodle/browser.js +0 -388
  80. package/app/extensions/doodle/guide.md +0 -6
  81. package/app/extensions/doodle/index.ts +0 -54
  82. package/app/extensions/files/browser.js +0 -96
  83. package/app/extensions/files/guide.md +0 -7
  84. package/app/extensions/frames/guide.md +0 -7
  85. package/app/extensions/frames/index.ts +0 -76
  86. package/app/extensions/mcp/guide.md +0 -5
  87. package/app/extensions/point/README.md +0 -22
  88. package/app/extensions/point/browser.js +0 -245
  89. package/app/extensions/point/guide.md +0 -6
  90. package/app/extensions/point/index.ts +0 -97
  91. package/app/extensions/prompt/index.ts +0 -19
  92. package/app/extensions/ui/README.md +0 -69
  93. package/app/extensions/ui/components/App.js +0 -28
  94. package/app/extensions/ui/components/Button.js +0 -28
  95. package/app/extensions/ui/components/Markdown.js +0 -37
  96. package/app/extensions/ui/guide.md +0 -7
  97. package/app/extensions/ui/index.ts +0 -271
  98. package/app/extensions/viewer/guide.md +0 -5
  99. package/app/mcp.json +0 -3
  100. package/app/skills/plugin-map/SKILL.md +0 -49
  101. package/app/ui/apps/.keep +0 -0
  102. /package/{app/extensions/ui/components → agent/extensions/card/blocks}/Table.js +0 -0
  103. /package/{app/extensions/ui/components → agent/extensions/card/blocks}/Text.js +0 -0
@@ -1,5 +0,0 @@
1
- # MCP servers
2
- Servers are in {code}/mcp.json (pi's built-in MCP: pi's docs/mcp.md); their tools are named mcp__<server>__<tool>.
3
- The user ticks which servers and tools are on with /mcp on the page.
4
- - To add a server when asked: add its entry to that file (secrets as ${ENV_VAR} in headers or env, never written
5
- out), then call reload.
@@ -1,22 +0,0 @@
1
- # Notes on the UI (the point extension)
2
- The user can click anything on the page and write a note there. You get "[note on the UI] <their words>"
3
- with the target: surface and component (from ui_render), the element they clicked (inside App frames
4
- too), the part (if the thing has partAt), its text, and the file it comes from. Act on that exact thing.
5
- - If the text is data (a table row, a score), change the surface's data, not the component's code.
6
- - "here" means the spot: to add something there, ui_render(..., place={"x", "y"}) with its position, and keep
7
- it small (as wide as its content). On empty space the spot is all there is to the target.
8
- - A note with an "area" came with a drawing: the user circled that box. On empty space, build what they ask
9
- for inside it: ui_render(..., place={"x", "y", "width", "height"}) from the area. On a surface, the drawing
10
- marks the part of it the note is about: change that part, and leave the surface's place and size alone.
11
- If the thing drawn on can say what is inside the drawing (partIn), the part lists exactly what was circled
12
- (cells and values, paragraphs, lines, a picture of the area): the note is about that.
13
- - A note with "files": the user dropped those files there, from their computer or their file panel (all
14
- in {work}/drops/). Look at them (read shows you pictures) and do what the note says with them, at
15
- that spot or on that surface.
16
- - "[combine on the UI]": the user dragged surface A onto surface B and said how to combine them. Change B
17
- so it takes A in, as they said: ui_render(surface=B, ..., made_from=<B's made_from plus A>,
18
- instruction=<their words>, about=<updated>). Read A's data with {"ref": "A", "path": "/..."} props, so it
19
- stays live; copy only what has to change shape. Never change or delete A: the user deletes it when they
20
- want (B then gets a copy of what it read, automatically). Keep what B already showed unless they said otherwise.
21
- - Pointing is only as precise as your names: give elements you build a clear id or data-name.
22
- - Each note has an id. When you have dealt with it, call note_done(id): its pin fades from the page.
@@ -1,245 +0,0 @@
1
- // Browser half of the point plugin (experiment). A pencil button in the tool palette (top) turns on a clear
2
- // layer over the page; the cursor is a pen, blue over something a click would point at.
3
- // Click: a note box opens there. Hold the button and draw: on release the note box opens, and the note
4
- // covers the drawn area (its bounding box, in main-space px: where ui_render's place puts a surface).
5
- // The user sees only the box; the target goes to the agent with the note: surface and component (from
6
- // ui.js's data-surface / data-cid), the element itself (also inside App frames, which are same-origin),
7
- // its text, the source file to edit, the spot (main-space px; on empty space the spot is the whole target),
8
- // and the area if drawn. Enter sends it (point.py passes it to the surface's owner, gives it an id and
9
- // tells every open page to pin it, with its drawing). The pin fades when the agent marks the note
10
- // done (event "point_done") or the user closes it with its ✕; open notes are redrawn after a reload.
11
- // Esc or the button turns it off.
12
- // The same note box opens for two more things (Enter on an empty box sends its faded suggestion):
13
- // combine ui.js fires "ui-combine" {a, b, x, y} when a surface is dropped onto another: the note
14
- // says how to combine them and goes to b's owner as point_combine (b changes; a stays)
15
- // files the files plugin fires "files-dropped" {x, y, files} when files land on the page (from the
16
- // computer, or a row from its panel): a note with their paths, on what they were dropped on
17
- // Parts: an element (or an App page's window) with partAt(x, y) is asked what is at the spot (an enemy in a
18
- // canvas game); its answer goes with the note as "part". For a drawing, the thing under its middle is asked
19
- // partIn(points) instead, if it has it: what is inside the drawn shape (the circled cells, paragraphs, an
20
- // image of the area). points: [[x, y], ...] in window px (an App's window: px in its frame); it may be async.
21
- export default async function (page) {
22
- let names = {}; // "ui-table" -> "Table" (tags are lower case; catalog names are not)
23
- try { for (const n of Object.keys(await (await fetch("/ui-api/catalog")).json())) names[`ui-${n.toLowerCase()}`] = n; } catch {}
24
-
25
- const layer = document.createElement("div"); // catches clicks, so the page underneath doesn't react
26
- layer.style.cssText = "position:fixed;inset:0;z-index:9998;display:none";
27
- // The cursor: the button's pen (white edge, so it shows on dark too), its tip (4, 20) is the spot pointed at.
28
- const pen = (color) => `url("data:image/svg+xml,${encodeURIComponent(
29
- `<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke-linecap="round" stroke-linejoin="round">` +
30
- `<g stroke="#fff" stroke-width="4"><path d="M4 20h4L19 9l-4-4L4 16z"/><path d="m13 7 4 4"/></g>` +
31
- `<g stroke="${color}" stroke-width="2"><path d="M4 20h4L19 9l-4-4L4 16z"/><path d="m13 7 4 4"/></g></svg>`)}") 4 20, crosshair`;
32
- const PEN = pen("#6e6e73"), PEN_ON = pen("#0071e3"); // grey: empty space; blue: over a thing
33
- layer.style.cursor = PEN;
34
- const input = document.createElement("input"); // the note box, opened where the user clicked
35
- input.id = "point-note";
36
- input.placeholder = "Write a note, Enter to send";
37
- input.style.cssText = "position:fixed;z-index:9999;width:280px;box-sizing:border-box;font:inherit;font-size:14px;" +
38
- "padding:8px 12px;border-radius:12px;outline:none;color:#1d1d1f;background:var(--glass-strong,#fff);" + // the page's glass
39
- "backdrop-filter:var(--blur);-webkit-backdrop-filter:var(--blur);border:1px solid var(--edge,#eee);" +
40
- "box-shadow:0 8px 30px rgba(0,0,0,.1);display:none";
41
- const SVG = "http://www.w3.org/2000/svg";
42
- function strokeSvg(points) { // a freehand stroke, drawn over the whole window (fixed, like the pins)
43
- const svg = document.createElementNS(SVG, "svg");
44
- svg.style.cssText = "position:fixed;inset:0;width:100vw;height:100vh;pointer-events:none";
45
- const line = document.createElementNS(SVG, "polyline");
46
- line.style.cssText = "fill:none;stroke:#f5b700;stroke-width:3;stroke-linecap:round;stroke-linejoin:round;opacity:.85";
47
- line.setAttribute("points", points.map((p) => p.join(",")).join(" "));
48
- svg.append(line);
49
- return svg;
50
- }
51
- const ink = strokeSvg([]); // the stroke being drawn
52
- ink.style.zIndex = 9999;
53
- ink.style.display = "none";
54
- let current = null; // {target, x, y, stroke} of the open note
55
- let stroke = null; // [[x, y], ...] while the mouse button is held down
56
- document.body.append(layer, ink, input);
57
-
58
- const button = document.createElement("button");
59
- button.innerHTML = `<svg viewBox="0 0 24 24" aria-hidden="true"><path d="M4 20h4L19 9l-4-4L4 16z"/><path d="m13 7 4 4"/></svg>`;
60
- button.title = "Point: click anything, or hold and draw around an area, then write a note (Esc to stop)";
61
- page.slot("status").prepend(button); // the first tool in the palette, whichever script loads first
62
- const setOn = (on) => {
63
- layer.style.display = on ? "block" : "none";
64
- button.classList.toggle("on", on);
65
- if (!on) input.style.display = ink.style.display = "none";
66
- if (!on) current = stroke = null;
67
- if (on) document.dispatchEvent(new CustomEvent("tool-on", { detail: "point" })); // the other tools turn off
68
- };
69
- button.onclick = () => setOn(layer.style.display === "none");
70
- document.addEventListener("tool-on", (e) => { if (e.detail !== "point" && layer.style.display !== "none") setOn(false); }); // one tool at a time
71
- document.addEventListener("keydown", (e) => { if (e.key === "Escape") setOn(false); });
72
-
73
- const describe = (e) => e.localName + (e.id ? `#${e.id}` : "") + (e.dataset?.name ? `[data-name="${e.dataset.name}"]` : "") +
74
- [...e.classList].slice(0, 2).map((c) => `.${c}`).join("");
75
-
76
- function inMain(x, y) { // window px -> main-space px (the main space scrolls; ui_render's place uses these)
77
- const main = page.slot("top"), m = main.getBoundingClientRect();
78
- return { x: Math.round(x - m.left + main.scrollLeft), y: Math.round(y - m.top + main.scrollTop) };
79
- }
80
-
81
- function target(x, y) {
82
- const el = document.elementsFromPoint(x, y).find((e) => e !== layer);
83
- if (!el) return null;
84
- let inner = null, frame = null, box = el.getBoundingClientRect();
85
- if (el.localName === "iframe") { // look inside the frame (an App): same origin, so its document is readable
86
- try {
87
- const r = el.getBoundingClientRect(), dx = r.left + el.clientLeft, dy = r.top + el.clientTop;
88
- inner = el.contentDocument?.elementFromPoint(x - dx, y - dy);
89
- if (inner) { frame = el; const b = inner.getBoundingClientRect(); box = new DOMRect(b.x + dx, b.y + dy, b.width, b.height); }
90
- } catch {}
91
- }
92
- const hitEl = inner || el;
93
- const position = inMain(x, y);
94
- if (el === page.slot("top")) return { owner: null, surface: null, components: [], element: "empty space in the main space",
95
- text: "", file: "", position, box: new DOMRect(x - 12, y - 12, 24, 24) };
96
- const components = []; // outermost first, e.g. ["root (Card)", "chart1 (Table)"]
97
- for (let e = el.closest("[data-cid]"); e; e = e.parentElement?.closest("[data-cid]"))
98
- components.unshift(`${e.dataset.cid} (${names[e.localName] || e.localName})`);
99
- const path = []; // a short selector: up to the nearest element with an id, at most 4 steps
100
- for (let e = hitEl; e && path.length < 4 && e.localName !== "body"; e = e.parentElement) {
101
- path.unshift(describe(e));
102
- if (e.id) break;
103
- }
104
- const innermost = el.closest("[data-cid]");
105
- let part = null; // what the thing itself says is at this spot (partAt), e.g. {"name": "enemy-2"}
106
- try {
107
- if (frame?.contentWindow?.partAt) {
108
- const r = frame.getBoundingClientRect();
109
- part = frame.contentWindow.partAt(x - r.left - frame.clientLeft, y - r.top - frame.clientTop);
110
- } else {
111
- const owner = [hitEl, ...[...document.elementsFromPoint(x, y)]].find((e) => typeof e?.partAt === "function");
112
- if (owner) { const r = owner.getBoundingClientRect(); part = owner.partAt(x - r.left, y - r.top); }
113
- }
114
- } catch (e) { console.warn("partAt:", e); }
115
- const file = frame ? "/app" + new URL(frame.contentWindow.location.href).pathname // /ui/apps/x/... -> /app/ui/apps/x/...
116
- : innermost ? `/app/extensions/ui/components/${names[innermost.localName] || innermost.localName}.js`
117
- : "/app/extensions/page/page.html (the page itself)"; // /app/: the code folder (the server puts in the real one)
118
- return {
119
- owner: el.closest("[data-owner]")?.dataset.owner || null,
120
- surface: el.closest("[data-surface]")?.dataset.surface || null,
121
- components,
122
- element: path.join(" > "),
123
- part,
124
- text: (hitEl.innerText || hitEl.value || hitEl.alt || "").trim().replace(/\s+/g, " ").slice(0, 60),
125
- file,
126
- at: { x: Math.round(((x - box.x) / box.width) * 100) + "%", y: Math.round(((y - box.y) / box.height) * 100) + "%" },
127
- position,
128
- box,
129
- };
130
- }
131
-
132
- // The note box at (x, y). `note`: what Enter sends besides the text (target, combine, files);
133
- // `suggestion`: the faded text, sent as it is if the box is left empty (else the box needs words).
134
- function openBox(x, y, note, suggestion = "") {
135
- input.style.display = "block";
136
- input.style.left = Math.min(x + 12, innerWidth - input.offsetWidth - 8) + "px";
137
- input.style.top = Math.min(y + 12, innerHeight - input.offsetHeight - 8) + "px";
138
- input.placeholder = suggestion || "Write a note, Enter to send";
139
- current = { x, y, suggestion, ...note };
140
- input.value = "";
141
- input.focus();
142
- }
143
-
144
- function openNote(t, x, y, drawn = null) { // a note on target t (and the drawn stroke)
145
- if (!t) return;
146
- const { box, ...rest } = t;
147
- openBox(x, y, { type: "point_note", target: rest, stroke: drawn, to: rest.owner });
148
- }
149
-
150
- // Combine: a surface was dropped onto another (ui.js). The note goes to the target's owner.
151
- document.addEventListener("ui-combine", ({ detail: { a, b, x, y } }) => {
152
- const to = document.querySelector(`[data-surface="${CSS.escape(b)}"]`)?.dataset.owner;
153
- openBox(x, y, { type: "point_combine", a, b, to }, `Add ${a} to ${b}`);
154
- });
155
-
156
- // Files dropped on the page (the files plugin: from the computer, or a row from its panel; already
157
- // stored in /workspace/drops): a note on what they landed on. Esc closes the box; the files stay stored.
158
- document.addEventListener("files-dropped", ({ detail: { x, y, files } }) => {
159
- const t = target(x, y) || { owner: null, surface: null, components: [], element: "", text: "", file: "", position: inMain(x, y) };
160
- const { box, ...rest } = t;
161
- const paths = files.map((f) => `${f.path} (${Math.ceil(f.size / 1024)} KB)`);
162
- openBox(x, y, { type: "point_note", target: rest, files: paths, to: rest.owner }, `Use ${files.map((f) => f.name).join(", ")} here`);
163
- });
164
-
165
- // Press, move, release: a small movement is a click on one thing; a bigger one is a drawing.
166
- layer.onmousedown = (e) => {
167
- stroke = [[e.clientX, e.clientY]];
168
- input.style.display = ink.style.display = "none";
169
- };
170
- layer.onmousemove = (e) => {
171
- if (!stroke) {
172
- const t = target(e.clientX, e.clientY);
173
- layer.style.cursor = t && t.surface ? PEN_ON : PEN;
174
- return;
175
- }
176
- const [lx, ly] = stroke.at(-1);
177
- if (Math.hypot(e.clientX - lx, e.clientY - ly) < 3) return; // keep the stroke light
178
- stroke.push([e.clientX, e.clientY]);
179
- ink.style.display = "block";
180
- ink.firstChild.setAttribute("points", stroke.map((p) => p.join(",")).join(" "));
181
- };
182
- async function partIn(x, y, points) { // what the thing at (x, y) says is inside the drawing, or null
183
- const el = document.elementsFromPoint(x, y).find((e) => e !== layer && e !== ink);
184
- if (el?.localName === "iframe") {
185
- try {
186
- const r = el.getBoundingClientRect(), dx = r.left + el.clientLeft, dy = r.top + el.clientTop;
187
- if (typeof el.contentWindow.partIn === "function") return await el.contentWindow.partIn(points.map(([px, py]) => [px - dx, py - dy]));
188
- } catch (err) { console.warn("partIn:", err); }
189
- }
190
- const owner = document.elementsFromPoint(x, y).find((e) => typeof e.partIn === "function");
191
- try { return owner ? await owner.partIn(points) : null; } catch (err) { console.warn("partIn:", err); return null; }
192
- }
193
-
194
- layer.onmouseup = async (e) => {
195
- const points = stroke;
196
- stroke = null;
197
- if (!points) return;
198
- const xs = points.map((p) => p[0]), ys = points.map((p) => p[1]);
199
- const left = Math.min(...xs), top = Math.min(...ys), width = Math.max(...xs) - left, height = Math.max(...ys) - top;
200
- if (width < 12 && height < 12) return openNote(target(e.clientX, e.clientY), e.clientX, e.clientY);
201
- const t = target(left + width / 2, top + height / 2); // what is under the middle of the drawing
202
- if (t) t.area = { ...inMain(left, top), width: Math.round(width), height: Math.round(height) };
203
- const inside = t && (await partIn(left + width / 2, top + height / 2, points));
204
- if (inside) t.part = inside;
205
- openNote(t, e.clientX, e.clientY, points);
206
- };
207
-
208
- input.onkeydown = (e) => {
209
- const text = input.value.trim() || current?.suggestion;
210
- if (e.key !== "Enter" || !text || !current) return;
211
- const { suggestion, ...note } = current;
212
- page.send({ ...note, text, to: note.to || document.getElementById("to")?.value }); // else the agent picked in the chat
213
- setOn(false);
214
- };
215
-
216
- const pins = {}; // note id -> pin element: its label, and its drawing if it has one
217
- function pin({ id, text, to, x, y, stroke }) { // stays where the note was written (doesn't follow scrolling yet)
218
- if (pins[id]) return;
219
- const el = (pins[id] = document.createElement("div"));
220
- el.style.cssText = "position:fixed;inset:0;z-index:9997;pointer-events:none;transition:opacity 1.5s";
221
- if (stroke) el.append(strokeSvg(stroke));
222
- const label = document.createElement("div");
223
- label.style.cssText = `position:absolute;left:${x}px;top:${y}px;max-width:240px;` +
224
- "background:#fff8c4;border:1px solid #e6c200;border-radius:4px;padding:3px 20px 3px 6px;font:12px sans-serif;" +
225
- "box-shadow:0 1px 3px #0003";
226
- label.textContent = `✎ ${text} · sent to ${to}`;
227
- const close = document.createElement("button");
228
- close.textContent = "✕";
229
- close.title = "Close this note (its agent is told)";
230
- close.style.cssText = "position:absolute;top:1px;right:3px;line-height:1;pointer-events:auto;border:0;background:none;cursor:pointer;padding:0 2px;font:inherit;color:#8a7400";
231
- close.onclick = () => page.send({ type: "point_dismiss", id });
232
- label.append(close);
233
- el.append(label);
234
- document.body.append(el);
235
- }
236
- page.on("point_pin", pin);
237
- page.on("point_done", ({ id }) => {
238
- const el = pins[id];
239
- if (!el) return;
240
- delete pins[id];
241
- el.style.opacity = "0";
242
- setTimeout(() => el.remove(), 1500);
243
- });
244
- try { for (const [id, n] of Object.entries(await (await fetch("/point/notes")).json())) pin({ id, ...n }); } catch {}
245
- }
@@ -1,6 +0,0 @@
1
- # Notes on the UI
2
- The user clicks anything on the page (or draws around it, or drops files there) and writes a note: you get
3
- "[note on the UI] <their words>" with exactly what it is on.
4
- - Change that exact thing. If its text is data (a row, a score), change the surface's data, not the component.
5
- - When you have dealt with it, call note_done(id).
6
- Before you handle your first note, read README.md.
@@ -1,97 +0,0 @@
1
- // Notes on the UI (port of the Python point plugin; the browser half, browser.js, is its point.js): the user points at
2
- // things and writes a note there; the note and its target go to an agent, so it knows exactly what it is about.
3
- // point_note click anything on the page (or draw around an area) and write: the target is the surface,
4
- // component, element, part (partAt), its text and source file, the spot and the area.
5
- // Files dropped on the page come as a note too ("files": paths in <workspace>/drops).
6
- // point_combine the user dragged surface a onto surface b and wrote how to combine them. The agent changes b to
7
- // take a in, reading a's data live ({"ref"} props); a stays as it is.
8
- // Each note goes to the surface's owner (else the agent picked in the chat), gets an id (n1, n2, ...: never reused)
9
- // and stays pinned (state/notes.json: {"last", "open": {id: note}}, redrawn after a reload) until the agent calls
10
- // note_done(id). The user can also close a pin (its ✕): the note is dropped, and taken out of its agent's inbox if
11
- // its turn hasn't started (the agent never saw it); else its agent is told (a note).
12
- // guide.md: the short prompt section; README.md (copied from point.py): the whole of it, read before a first note.
13
- import fs from "node:fs";
14
- import path from "node:path";
15
- import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
16
- import { Type } from "typebox";
17
-
18
- const host = (globalThis as any).chalky;
19
- const FILE = path.join(host.state, "notes.json");
20
- if (!host.shared.has("point"))
21
- host.shared.set("point", fs.existsSync(FILE) ? JSON.parse(fs.readFileSync(FILE, "utf8")) : { last: 0, open: {} });
22
- const notes = host.shared.get("point"); // {last, open: {id: {text, to, x, y, stroke}}}
23
- if (!host.shared.has("point-items")) host.shared.set("point-items", new Map());
24
- const items: Map<string, any> = host.shared.get("point-items"); // note id -> its swarm inbox item (not saved)
25
- const save = () => fs.writeFileSync(FILE, JSON.stringify(notes));
26
-
27
- function pin(text: string, to: string, msg: any) { // a new open note: saved, pinned on every page
28
- const id = `n${++notes.last}`;
29
- notes.open[id] = { text, to, x: msg.x ?? 0, y: msg.y ?? 0, stroke: msg.stroke ?? null }; // with its drawing, if any
30
- save();
31
- host.broadcast("point_pin", { id, ...notes.open[id] });
32
- return id;
33
- }
34
-
35
- function describe(sid: string) { // a source of a combine, for the agent: cut short if long
36
- const r = host.shared.get("ui")?.surfaces[sid] ?? {};
37
- const text = JSON.stringify({ owner: r.owner, about: r.about, components: r.components, data: r.data });
38
- return text.length < 3000 ? text : text.slice(0, 3000) + ` … (more in ${host.app}/ui/surfaces.json and ${host.state}/ui-data.json)`;
39
- }
40
-
41
- export default function (pi: ExtensionAPI) {
42
- const guide = fs.readFileSync(path.join(import.meta.dirname, "guide.md"), "utf8") // short; README.md: the whole of it, read when it says
43
- .replace("README.md", path.join(import.meta.dirname, "README.md"));
44
- pi.on("before_agent_start", (event) => { event.systemPromptOptions.sections.point = guide; });
45
-
46
- pi.registerTool({
47
- name: "note_done", label: "Note done", description: "Mark a note the user pinned on the UI as dealt with; its pin fades.",
48
- parameters: Type.Object({ id: Type.String({ description: "e.g. n3" }) }),
49
- async execute(_id, a) {
50
- if (!notes.open[a.id]) return { content: [{ type: "text", text: `error: no open note '${a.id}'. Open notes: ${Object.keys(notes.open).join(", ")}` }], details: undefined };
51
- delete notes.open[a.id];
52
- items.delete(a.id);
53
- save();
54
- host.broadcast("point_done", { id: a.id });
55
- return { content: [{ type: "text", text: `note ${a.id} is done; its pin fades from the page` }], details: undefined };
56
- },
57
- });
58
-
59
- host.routes.set("/point/notes", (_req: any, res: any) => res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify(notes.open)));
60
-
61
- host.pageHandlers.set("point", (msg: any) => {
62
- const swarm = host.api.swarm;
63
- if (msg.type === "point_dismiss" && notes.open[msg.id]) {
64
- const note = notes.open[msg.id];
65
- delete notes.open[msg.id];
66
- save();
67
- host.broadcast("point_done", { id: msg.id });
68
- const item = items.get(msg.id);
69
- items.delete(msg.id);
70
- if (!(item && swarm?.withdraw(item)))
71
- swarm?.send("user", note.to, `[note on the UI] the user closed note ${msg.id} ("${note.text}"): drop it if you haven't finished it.`, true);
72
- }
73
- if (!["point_note", "point_combine"].includes(msg.type) || !swarm) return;
74
- const to = host.agents.has(msg.to) ? msg.to : "agent1";
75
- const text = msg.text ?? "";
76
- if (msg.type === "point_combine") {
77
- const { a, b } = msg;
78
- const madeFrom = [...(host.shared.get("ui")?.surfaces[b]?.made_from ?? []).filter((m: string) => m !== a), a];
79
- const id = pin(`combine ${a} into ${b}: ${text}`, to, msg);
80
- return items.set(id, swarm.send("user", to, [`[combine on the UI] ${text}`, ` id: ${id} (call note_done("${id}") when dealt with)`,
81
- ` dragged: ${a} (keep it as it is)`, ` onto: ${b} (change this one)`,
82
- ` so: ui_render(surface="${b}", ..., made_from=${JSON.stringify(madeFrom)}, instruction=<their words>), reading ${a}'s data with {"ref": "${a}", "path": ...}`,
83
- ` ${a}: ${describe(a)}`, ` ${b}: ${describe(b)}`].join("\n")));
84
- }
85
- const t = msg.target ?? {};
86
- const at = t.at && `${t.at.x} across, ${t.at.y} down the element`;
87
- const pos = t.position && `x=${t.position.x}, y=${t.position.y} px in the main space (for ui_render's place)`;
88
- const area = t.area && `x=${t.area.x}, y=${t.area.y}, width=${t.area.width}, height=${t.area.height} px in the main space `
89
- + (t.surface ? "(the user drew around this part of the surface: the note is about it)" : "(the user drew around empty space: build inside it)");
90
- const id = pin(text, to, msg);
91
- const lines = [`[note on the UI] ${text}`, ` id: ${id} (call note_done("${id}") when dealt with)`,
92
- ...([["surface", t.surface], ["component", (t.components ?? []).join(" → ")], ["element", t.element],
93
- ["part", t.part && JSON.stringify(t.part)], ["at", at], ["text", t.text], ["file", t.file?.replace(/^\/app\//, host.app + "/")], ["position", pos],
94
- ["area", area], ["files", msg.files?.join(", ")]] as [string, any][]).filter(([, v]) => v).map(([k, v]) => ` ${k}: ${v}`)];
95
- items.set(id, swarm.send("user", to, lines.join("\n")));
96
- });
97
- }
@@ -1,19 +0,0 @@
1
- // Where things are: the paths the prompt needs that depend on this machine, as the <paths> section. The prompt's
2
- // words are app/SYSTEM.md (pi's documented way to replace its default prompt), which is plain text and can't hold
3
- // these. pi adds the skills list and the working folder (<cwd>) itself.
4
- import { getDocsPath, getExamplesPath, getReadmePath, type ExtensionAPI } from "@earendil-works/pi-coding-agent";
5
-
6
- export default function (pi: ExtensionAPI) {
7
- pi.on("before_agent_start", (event) => {
8
- const host = (globalThis as any).chalky;
9
- event.systemPromptOptions.sections.paths = [
10
- `{code} = ${host.app}: your code (versioned: each turn that changes it is saved, and the user can roll back)`,
11
- `{work} = ${host.work}: your working folder (<cwd>), for project work and the user's files`,
12
- `{data} = ${host.state}: what the agents and apps keep (conversations, app data, surfaces' data; not versioned)`,
13
- `{page} = http://127.0.0.1:${host.port}: the user's page, served by the host`,
14
- `The host: ${host.root}/server.ts`,
15
- `pi's docs: ${getDocsPath()}/extensions.md for extensions, examples in ${getExamplesPath()}/extensions/; the rest in `
16
- + `${getReadmePath()} and ${getDocsPath()}/. Read pi's .md files completely and follow their links.`,
17
- ].join("\n");
18
- });
19
- }
@@ -1,69 +0,0 @@
1
- # Showing UI to the user (the ui extension)
2
- Show forms, tables, dashboards or games as UI surfaces: cards in the page's main space, described as JSON.
3
- Prefer a surface over editing the page ({code}/extensions/page/page.html). Aim for one screen; make it scroll only if the user asks.
4
- A surface shows only what you build, with no frame: when asked for a thing (a smiley, a chart), show just
5
- that thing. Add a background, border or Card only if the thing itself needs one. An app's page should keep
6
- a transparent background, so it sits on the workspace.
7
- Name what you build (a clear surface id, ids or data-name on elements): the user points at things by name.
8
- Compose from existing components; write a new component only if none fits.
9
- Before building or restyling what the user sees, load the ui-design skill (how it should look and read).
10
- - ui_catalog: lists the components and their props. Read it first.
11
- - ui_render(surface, components, data): draw or replace a surface. components is a flat list; each has
12
- "id", "component" (a catalog name), its props, and "children": [ids] or "child": id. One has id "root".
13
- A prop can be {"path": "/x"} to bind it to this surface's data (inputs write to it), or
14
- {"ref": "other-surface", "path": "/x"} to read another surface's data live (no copy: it follows changes).
15
- Errors come back to you; fix and retry.
16
- - about="...": one or two sentences on what the surface is for and why it is built this way. The user reads
17
- it on the surface's back side; you read it in {code}/ui/surfaces.json. Keep it true when you change things.
18
- - options=[{"label", "components", "data"}, ...] instead of components: when you are not sure what the user
19
- meant, offer 2-3 versions; they flip through them and keep one.
20
- - Pinned surfaces (the user held their label): your ui_render, ui_update and ui_delete on them fail. Don't
21
- work around it: make a new surface (made_from=[the pinned one]) or ask the user.
22
- - Surfaces stack in the main space and are as wide as their content. place={"x", "y", "width"?, "height"?}
23
- (px from the main space's top-left) pins one at a spot instead: use it when the user points at a spot
24
- ("here") or draws an area (then fill that area: its x, y, width and height).
25
- - Everything you show is draggable by default: the user moves a surface by its label (shown on hover), and
26
- it stays where they put it (ui_render without place keeps it there; where things are now: the "place" of
27
- each surface in {code}/ui/surfaces.json). Pass draggable=false only when the
28
- user wants something fixed. In an app you build, make separate pieces draggable too, unless told otherwise.
29
- - Everything you show is resizable by default too: the user drags its bottom-right corner (shown on hover),
30
- and the surface's root component fills the new size. So build things that fill their space (sizes in %,
31
- flex, or scaled to their box, e.g. an SVG with a viewBox), not fixed px. Pass resizable=false only when the
32
- user wants a fixed size.
33
- - ui_update(surface, path, value): change data only (fast; components keep their state). ui_delete(surface).
34
- - Data (what users typed, rows, scores) is kept in {data}/ui-data.json (not versioned); the rest of each
35
- surface in {code}/ui/surfaces.json. The user can undo any change (theirs or yours), so don't keep backups.
36
- - When the user clicks a Button you get "[ui action] <name> ..." with the surface's data.
37
- - Voice input (dictation, meeting notes): import /ui/voice.js in the app (read its header), never the
38
- browser's speech API or a provider directly: it uses the sandbox's speech-to-text API, or the browser's.
39
- - A new component: {code}/extensions/ui/components/<Name>.js. Start with a comment line "// Name: what it does" (more
40
- comment lines if needed), then its about block, then
41
- `export default class extends HTMLElement { update(props, ui) {...} }`.
42
- The about block of a component (strict JSON after `export const about = `):
43
- "version": 1 the shape of its props; raise it when they change, and export
44
- upgrade(props, fromVersion) -> props, so surfaces saved earlier still draw
45
- "props": {"name": {"type": "text" | "number" | "boolean" | "choice" | "list" | "rows" | "object" | "action",
46
- "options": [...] (for choice), "default": ..., "doc": "what it is"}}
47
- "children": "none" | "one" | "many"
48
- "levers": ["Shown as {variant}."] the back side: plain sentences; each {prop} becomes a control the user
49
- can change there (choice and boolean cycle, text and number are typed)
50
- "accepts": "rows with a number column" optional: what it can take when something is dropped on it
51
- "example": {...props} one sample, for the catalog and for checks
52
- When you change a component's props in a way old surfaces would break on, raise "version" and export
53
- upgrade(props, fromVersion) returning props in the new shape.
54
- update() runs on every change, on the same element: keep internal state there (e.g. a game loop).
55
- ui.children: child elements, placed with ui.mount(container); ui.set(prop, value) writes a bound prop;
56
- ui.action(name, context) sends an action to you. Make it general so it can be reused. New components work
57
- immediately; edited components and App files show when your turn ends (the page refreshes itself). Big multi-file apps: {code}/ui/apps/<name>/, shown with App.
58
- Pointing inside a canvas: give the element (or an App page's window) a partAt(x, y) function returning
59
- {"name", ...} for what is at that spot, so the user can point at parts (an enemy, a cell).
60
- - Agents inside what you build (a simulation's members, a debate, characters): load the agent-apps
61
- skill first. They are real agents that belong to the app, not workers helping you build it.
62
- - Libraries: components import them by name (import { marked } from "marked"), as mapped in the import map
63
- at the top of {code}/extensions/page/page.html; installed ones are in {code}/extensions/ui/package.json. To add
64
- one, download its ES-module build into {code}/extensions/ui/node_modules/<pkg>/ and add a line to the import map.
65
- - Component pitfalls (you can't see the browser, so avoid them up front):
66
- keep your own state in one object, this.state = {...}: fields like this.dir, this.title, this.hidden,
67
- this.id, this.style, this.lang are built-in HTMLElement properties and silently break;
68
- set a canvas's width/height BEFORE drawing (resizing clears it); don't create things twice in update();
69
- remove document-level listeners/timers when the element is no longer connected.
@@ -1,28 +0,0 @@
1
- // App: shows a full mini-site from {code}/ui/apps/<name>/ in a frame. For big things (multi-file games, editors)
2
- // that don't fit one component. Put index.html (and its scripts) in {code}/ui/apps/<name>/.
3
- // No frame of its own and a clear background: only the app's page shows (keep that page's background
4
- // transparent unless the app needs one). As wide as its surface; it fills its surface's placed area,
5
- // else it is `height` px tall. The app can talk to the agents and see their messages: /ui/agents.js.
6
- export const about = {
7
- "version": 1,
8
- "props": {
9
- "src": {"type": "text", "doc": "path inside {code}/ui/apps, e.g. \"snake/index.html\""},
10
- "height": {"type": "number", "default": 400, "doc": "px, when nothing else sizes it"}
11
- },
12
- "children": "none",
13
- "levers": ["Shows {src}.", "{height} px tall when not placed."],
14
- "example": {"src": "snake/index.html", "height": 400}
15
- };
16
- export default class extends HTMLElement {
17
- update({ src = "", height = 400 }) {
18
- if (!this.frame) {
19
- this.style.cssText = "display:flex;flex-direction:column;align-self:stretch";
20
- this.frame = document.createElement("iframe");
21
- this.frame.style.cssText = "width:100%;min-height:0;border:0;background:transparent;color-scheme:normal";
22
- this.append(this.frame);
23
- }
24
- this.frame.style.flex = `1 1 ${height}px`; // height when nothing sizes it; grows or shrinks to a placed area
25
- const url = `/ui/apps/${src}`;
26
- if (this.frame.getAttribute("src") !== url) this.frame.setAttribute("src", url); // don't restart a running app
27
- }
28
- }
@@ -1,28 +0,0 @@
1
- // Button: a button that sends an action to the agent when clicked.
2
- export const about = {
3
- "version": 1,
4
- "props": {
5
- "label": {"type": "text", "default": "OK"},
6
- "action": {"type": "action", "default": "click", "doc": "the action name you will receive"},
7
- "context": {"type": "object", "doc": "sent with the action; values may be {\"path\": ...}"},
8
- "variant": {"type": "choice", "options": ["primary", "secondary"], "default": "primary"}
9
- },
10
- "children": "none",
11
- "levers": ["Says {label}.", "Looks {variant}."],
12
- "example": {"label": "Send order", "action": "send", "variant": "primary"}
13
- };
14
- export default class extends HTMLElement {
15
- update(props, ui) {
16
- if (!this.button) {
17
- this.button = document.createElement("button");
18
- this.button.onclick = () => this.ui.action(this.props.action || "click", this.props.context || {});
19
- this.append(this.button);
20
- }
21
- this.props = props;
22
- this.ui = ui;
23
- this.button.textContent = props.label || "OK";
24
- this.button.style.cssText = "border-radius:980px;padding:7px 16px;font-weight:500;" + (props.variant === "secondary"
25
- ? "background:var(--glass-strong);color:var(--accent);border:1px solid var(--line)"
26
- : "background:var(--accent);color:#fff;border:0");
27
- }
28
- }
@@ -1,37 +0,0 @@
1
- // Markdown: formatted text from a markdown string (headings, lists, tables, code, links, quotes).
2
- // HTML in the text is sanitised; links open in a new tab. The chat draws message bodies with it too.
3
- export const about = {
4
- "version": 1,
5
- "props": {
6
- "text": {"type": "text", "doc": "markdown"}
7
- },
8
- "children": "none",
9
- "levers": [],
10
- "example": {"text": "**Lunch** closes *Friday 11:00*"}
11
- };
12
- import { marked } from "marked"; // bare names: resolved by the import map in chat_ui.html
13
- import DOMPurify from "dompurify";
14
-
15
- const CSS = `
16
- ui-markdown { display: block; white-space: normal; }
17
- ui-markdown p, ui-markdown ul, ui-markdown ol, ui-markdown pre, ui-markdown table, ui-markdown blockquote { margin: 6px 0; }
18
- ui-markdown ul, ui-markdown ol { padding-left: 22px; }
19
- ui-markdown h1, ui-markdown h2, ui-markdown h3, ui-markdown h4 { font-size: 1.05em; margin: 8px 0 4px; }
20
- ui-markdown code { font: 12.5px monospace; background: rgba(0, 0, 0, .06); padding: 1px 4px; border-radius: 3px; }
21
- ui-markdown pre { background: var(--hover); padding: 8px; border-radius: 4px; overflow-x: auto; }
22
- ui-markdown pre code { background: none; padding: 0; }
23
- ui-markdown table { border-collapse: collapse; }
24
- ui-markdown th, ui-markdown td { border: 1px solid var(--line); padding: 3px 8px; }
25
- ui-markdown blockquote { border-left: 3px solid var(--line); padding-left: 8px; color: var(--soft); }`;
26
-
27
- export default class extends HTMLElement {
28
- update({ text = "" }) {
29
- if (!document.getElementById("ui-markdown-css")) {
30
- document.head.insertAdjacentHTML("beforeend", `<style id="ui-markdown-css">${CSS}</style>`);
31
- }
32
- if (this.state?.text === text) return; // unchanged: keep the DOM (and any selection)
33
- this.state = { text };
34
- this.innerHTML = DOMPurify.sanitize(marked.parse(text, { gfm: true, breaks: true }));
35
- for (const a of this.querySelectorAll("a")) { a.target = "_blank"; a.rel = "noopener"; }
36
- }
37
- }
@@ -1,7 +0,0 @@
1
- # Showing UI to the user
2
- Show what you build as UI surfaces: cards in the page's main space, described as JSON (ui_catalog, ui_render,
3
- ui_update, ui_delete), rather than by editing the page. Load the ui-design skill before building or restyling.
4
- - Name what you build (surface ids, element ids): the user points at things by name.
5
- - The user moves and resizes surfaces: build things that fill their box (%, flex, a viewBox), not fixed px.
6
- - A surface the user pinned: don't work around it (make a new one, made_from=[it], or ask).
7
- Before your first ui_render in a conversation, and before writing a component or an App, read README.md.