@chalky/chalky 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -5
- package/agent/SYSTEM.md +36 -0
- package/agent/extensions/app/README.md +72 -0
- package/{app/extensions/ui → agent/extensions/app}/agents.js +22 -16
- package/agent/extensions/app/guide.md +5 -0
- package/agent/extensions/app/index.ts +66 -0
- package/{app/extensions/ui/skills/agent-apps → agent/extensions/app/skills/app-agents}/SKILL.md +17 -18
- package/agent/extensions/app/skills/test-your-app/SKILL.md +23 -0
- package/{app/extensions/boards → agent/extensions/board}/browser.js +30 -32
- package/{app/extensions/boards → agent/extensions/board}/guide.md +1 -1
- package/{app/extensions/boards → agent/extensions/board}/index.ts +32 -30
- package/agent/extensions/card/README.md +88 -0
- package/agent/extensions/card/blocks/Button.js +34 -0
- package/{app/extensions/ui/components → agent/extensions/card/blocks}/Column.js +2 -2
- package/agent/extensions/card/blocks/Markdown.js +37 -0
- package/{app/extensions/ui/components/Card.js → agent/extensions/card/blocks/Panel.js} +3 -3
- package/{app/extensions/ui/components → agent/extensions/card/blocks}/Row.js +2 -2
- package/{app/extensions/ui/components → agent/extensions/card/blocks}/TextField.js +3 -3
- package/{app/extensions/ui → agent/extensions/card}/browser.js +155 -129
- package/agent/extensions/card/guide.md +13 -0
- package/agent/extensions/card/index.ts +370 -0
- package/{app/extensions/commands → agent/extensions/command}/browser.js +3 -3
- package/agent/extensions/command/guide.md +2 -0
- package/{app/extensions/commands → agent/extensions/command}/index.ts +14 -16
- package/agent/extensions/doodle/README.md +57 -0
- package/agent/extensions/doodle/browser.js +1013 -0
- package/agent/extensions/doodle/guide.md +22 -0
- package/agent/extensions/doodle/index.ts +274 -0
- package/agent/extensions/file/browser.js +182 -0
- package/agent/extensions/file/guide.md +5 -0
- package/{app/extensions/files → agent/extensions/file}/index.ts +37 -18
- package/{app/extensions/frames → agent/extensions/frame}/README.md +13 -13
- package/{app/extensions/frames → agent/extensions/frame}/browser.js +20 -18
- package/agent/extensions/frame/guide.md +3 -0
- package/agent/extensions/frame/index.ts +90 -0
- package/{app/extensions/ui_history → agent/extensions/history}/browser.js +18 -18
- package/{app/extensions/ui_history → agent/extensions/history}/index.ts +34 -33
- package/{app → agent}/extensions/mcp/browser.js +1 -1
- package/agent/extensions/mcp/guide.md +3 -0
- package/{app → agent}/extensions/mcp/index.ts +2 -4
- package/{app/extensions/login → agent/extensions/model}/browser.js +67 -67
- package/{app → agent}/extensions/model/index.ts +8 -4
- package/{app/extensions/login/index.ts → agent/extensions/model/login.ts} +47 -46
- package/agent/extensions/note/README.md +37 -0
- package/agent/extensions/note/browser.js +382 -0
- package/agent/extensions/note/guide.md +9 -0
- package/agent/extensions/note/index.ts +215 -0
- package/agent/extensions/page/browser.js +453 -0
- package/{app → agent}/extensions/page/index.ts +72 -3
- package/{app → agent}/extensions/page/page.html +102 -49
- package/agent/extensions/page/skills/interact-with-page/SKILL.md +59 -0
- package/agent/extensions/prompt/index.ts +38 -0
- package/{app → agent}/extensions/reload/index.ts +3 -3
- package/{app → agent}/extensions/screen/browser.js +10 -23
- package/{app → agent}/extensions/screen/index.ts +11 -11
- package/{app/extensions/steps → agent/extensions/step}/index.ts +1 -1
- package/{app/extensions/swarm → agent/extensions/team}/README.md +19 -19
- package/{app/extensions/swarm → agent/extensions/team}/guide.md +4 -4
- package/{app/extensions/swarm → agent/extensions/team}/index.ts +41 -66
- package/{app/extensions/versions → agent/extensions/version}/browser.js +19 -10
- package/agent/extensions/version/index.ts +205 -0
- package/{app → agent}/extensions/viewer/README.md +4 -4
- package/{app → agent}/extensions/viewer/browser.js +8 -8
- package/agent/extensions/viewer/guide.md +3 -0
- package/{app → agent}/extensions/viewer/index.ts +22 -17
- package/{app → agent}/extensions/voice/browser.js +4 -3
- package/{app → agent}/extensions/voice/index.ts +8 -6
- package/{app/extensions/ui → agent/extensions/voice}/voice.js +5 -5
- package/agent/settings.json +8 -0
- package/{app/extensions/ui/skills/ui-check → agent/skills/browser-check}/SKILL.md +4 -4
- package/{app/extensions/ui/skills/ui-check → agent/skills/browser-check}/scripts/check.mjs +1 -1
- package/{app/extensions/ui/skills/ui-design → agent/skills/design}/SKILL.md +15 -15
- package/agent/skills/extension-map/SKILL.md +56 -0
- package/agent/skills/show-me/SKILL.md +66 -0
- package/{app → agent}/skills/write-skill/SKILL.md +1 -1
- package/bin/chalky.mjs +2 -2
- package/package.json +9 -8
- package/server.ts +313 -58
- package/app/SYSTEM.md +0 -31
- package/app/extensions/appdata/README.md +0 -10
- package/app/extensions/appdata/guide.md +0 -4
- package/app/extensions/appdata/index.ts +0 -47
- package/app/extensions/commands/guide.md +0 -3
- package/app/extensions/doodle/README.md +0 -15
- package/app/extensions/doodle/browser.js +0 -398
- package/app/extensions/doodle/guide.md +0 -6
- package/app/extensions/doodle/index.ts +0 -54
- package/app/extensions/files/browser.js +0 -96
- package/app/extensions/files/guide.md +0 -7
- package/app/extensions/frames/guide.md +0 -7
- package/app/extensions/frames/index.ts +0 -85
- package/app/extensions/mcp/guide.md +0 -5
- package/app/extensions/point/README.md +0 -22
- package/app/extensions/point/browser.js +0 -247
- package/app/extensions/point/guide.md +0 -6
- package/app/extensions/point/index.ts +0 -112
- package/app/extensions/prompt/index.ts +0 -19
- package/app/extensions/ui/README.md +0 -69
- package/app/extensions/ui/components/App.js +0 -28
- package/app/extensions/ui/components/Button.js +0 -28
- package/app/extensions/ui/components/Markdown.js +0 -37
- package/app/extensions/ui/guide.md +0 -7
- package/app/extensions/ui/index.ts +0 -291
- package/app/extensions/versions/index.ts +0 -168
- package/app/extensions/viewer/guide.md +0 -5
- package/app/mcp.json +0 -3
- package/app/settings.json +0 -3
- package/app/skills/plugin-map/SKILL.md +0 -49
- package/app/ui/apps/.keep +0 -0
- package/node_modules/xlsx/CHANGELOG.md +0 -361
- package/node_modules/xlsx/LICENSE +0 -201
- package/node_modules/xlsx/README.md +0 -41
- package/node_modules/xlsx/bin/xlsx.njs +0 -310
- package/node_modules/xlsx/bower.json +0 -22
- package/node_modules/xlsx/dist/LICENSE +0 -201
- package/node_modules/xlsx/dist/cpexcel.d.ts +0 -39
- package/node_modules/xlsx/dist/cpexcel.full.mjs +0 -1502
- package/node_modules/xlsx/dist/cpexcel.js +0 -1506
- package/node_modules/xlsx/dist/shim.min.js +0 -2
- package/node_modules/xlsx/dist/xlsx.core.min.js +0 -18
- package/node_modules/xlsx/dist/xlsx.core.min.map +0 -1
- package/node_modules/xlsx/dist/xlsx.extendscript.js +0 -28259
- package/node_modules/xlsx/dist/xlsx.full.min.js +0 -24
- package/node_modules/xlsx/dist/xlsx.full.min.map +0 -1
- package/node_modules/xlsx/dist/xlsx.mini.min.js +0 -10
- package/node_modules/xlsx/dist/xlsx.mini.min.map +0 -1
- package/node_modules/xlsx/dist/xlsx.zahl.js +0 -4
- package/node_modules/xlsx/dist/xlsx.zahl.mjs +0 -2
- package/node_modules/xlsx/dist/zahl.d.ts +0 -4
- package/node_modules/xlsx/package.json +0 -183
- package/node_modules/xlsx/types/index.d.ts +0 -1065
- package/node_modules/xlsx/types/tsconfig.json +0 -15
- package/node_modules/xlsx/xlsx.js +0 -28103
- package/node_modules/xlsx/xlsx.mjs +0 -28228
- package/node_modules/xlsx/xlsxworker.js +0 -14
- /package/{app/extensions/ui/components → agent/extensions/card/blocks}/Table.js +0 -0
- /package/{app/extensions/ui/components → agent/extensions/card/blocks}/Text.js +0 -0
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
<!-- The page the user sees, served fresh on every load by the page extension (extensions/page/; edits show on reload).
|
|
3
3
|
Design (pivot 2026-10-14, see the Chalky repo's NOTES.md "Direction"): what the agents build fills the screen; talk is secondary.
|
|
4
4
|
Layout, every part fixed over the full window:
|
|
5
|
-
#top main space: what the agents built (the
|
|
5
|
+
#top main space: what the agents built (the card extension draws its cards here). Scrolls.
|
|
6
6
|
#rail left edge: icons that open a panel; clicking the open one closes it.
|
|
7
7
|
#panel a glass panel over the main space: #messages (the conversation) or #versions (history,
|
|
8
8
|
a screenshot of each version, rollback; the user's way to undo the agent's own changes: keep it working).
|
|
@@ -14,9 +14,12 @@
|
|
|
14
14
|
A tool with sub-tools (frames' Implement) is a <span class="tool">: its button, then a
|
|
15
15
|
smaller row (.sub) that hangs directly under it, hidden while none of its buttons shows (pageApi.tool).
|
|
16
16
|
#side right edge: empty; a browser plugin fills it (the files plugin's file panel).
|
|
17
|
+
Names (data-name) an agent's pen can point at: tools (the palette), rail, messages, versions, message-line; the
|
|
18
|
+
palette's buttons get theirs from their plugins (pen, text, eraser, show-marks, note, frames, files, dictate).
|
|
17
19
|
Look: glass layers on one soft background, tokens in :root (colours, glass, blur, sizes); icons are inline
|
|
18
20
|
SVG from ICONS (data-icon="name").
|
|
19
|
-
Messages: only words are full text (the user's, and an agent's reply);
|
|
21
|
+
Messages: only words are full text (the user's, and an agent's reply); a message the page wrote for the user
|
|
22
|
+
("[note on the page] ...") shows their words, the whole text in its data-raw (plugins find their ids there); everything else an agent does
|
|
20
23
|
(reasoning, tool calls, results, messages between agents) is a small icon: hover to peek, click to read.
|
|
21
24
|
App agents (status "app": part of something built, e.g. a simulation's members) stay out of the messages,
|
|
22
25
|
the picker and the working rows (one row per app): their app shows their talk.
|
|
@@ -24,7 +27,10 @@
|
|
|
24
27
|
"status", "message", "version", errors. Sending: {to, text} is a message, {type: "stop"} stops all.
|
|
25
28
|
Extending without editing this file: an extension's browser.js (extensions/<name>/browser.js; GET /page/scripts lists them);
|
|
26
29
|
each script's default export gets pageApi: on(type, fn), send(obj) (-> the extensions' page handlers),
|
|
27
|
-
slot("top" | "status" | "side"), tool(button) -> the row under that palette tool (for its sub-tools), note(text), load(name),
|
|
30
|
+
slot("top" | "status" | "side"), tool(button) -> the row under that palette tool (for its sub-tools), note(text) -> the line, load(name), block(name, props) (blocks from extensions/card/blocks),
|
|
31
|
+
notice(content, onClose) -> {el, set(...content), close(), open}: a notice in the column at the top right, newest on
|
|
32
|
+
top (content: a string, a node, or a list of them; a <button class="go"> is the main action); its ✕ calls onClose,
|
|
33
|
+
then closes it; an action closes it with close(). Notices of every extension go here, so they never overlap.
|
|
28
34
|
command(name, fn, description): "/name rest" typed in the input calls fn(rest) instead of sending (other "/..." text
|
|
29
35
|
is sent); typing "/" lists the commands above the input, narrowing as you type.
|
|
30
36
|
provide(name, obj) / use(name) -> promise of obj: what one browser plugin offers others (scripts load in any
|
|
@@ -32,11 +38,11 @@
|
|
|
32
38
|
<meta charset="utf-8">
|
|
33
39
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
34
40
|
<title>Chalky</title>
|
|
35
|
-
<!-- Libraries that
|
|
41
|
+
<!-- Libraries that blocks import by name (the host's dependencies, served by the card extension). Must come before any module. -->
|
|
36
42
|
<script type="importmap">
|
|
37
43
|
{ "imports": {
|
|
38
|
-
"marked": "/
|
|
39
|
-
"dompurify": "/
|
|
44
|
+
"marked": "/card/node_modules/marked/lib/marked.esm.js",
|
|
45
|
+
"dompurify": "/card/node_modules/dompurify/dist/purify.es.mjs"
|
|
40
46
|
} }
|
|
41
47
|
</script>
|
|
42
48
|
<style>
|
|
@@ -79,6 +85,24 @@
|
|
|
79
85
|
#status .sub:not(:has(> :not([hidden]))) { display: none; } /* only hidden buttons (or none): no row */
|
|
80
86
|
#status .tool .badge { top: -4px; right: -4px; } /* on the corner, off the icon */
|
|
81
87
|
#side { position: fixed; top: 0; right: 0; z-index: 20; display: flex; flex-direction: column; align-items: flex-end; }
|
|
88
|
+
/* notices (pageApi.notice): a column at the top right, newest on top; above cards and frames */
|
|
89
|
+
#notices { position: fixed; top: 14px; right: 16px; z-index: 200000; width: min(340px, calc(100vw - 32px));
|
|
90
|
+
display: flex; flex-direction: column; pointer-events: none; }
|
|
91
|
+
#notices .notice { display: grid; grid-template-rows: 1fr; pointer-events: auto;
|
|
92
|
+
transition: grid-template-rows .32s cubic-bezier(.3,.7,.3,1), opacity .25s, transform .32s cubic-bezier(.3,.7,.3,1); }
|
|
93
|
+
#notices .notice.out { grid-template-rows: 0fr; opacity: 0; transform: translateX(28px); } /* arriving or leaving: no height, so the rest slide */
|
|
94
|
+
#notices .notice > div { min-height: 0; overflow: hidden; }
|
|
95
|
+
#notices .notice-box { position: relative; margin-bottom: 8px; padding: 10px 34px 10px 14px; border-radius: 14px; display: flex;
|
|
96
|
+
flex-wrap: wrap; align-items: center; gap: 8px 10px; font-size: 13px; color: var(--text);
|
|
97
|
+
background: var(--glass-strong, #fff); border: 1px solid var(--edge, #eee); box-shadow: 0 8px 26px rgba(0,0,0,.12);
|
|
98
|
+
backdrop-filter: var(--blur); -webkit-backdrop-filter: var(--blur); }
|
|
99
|
+
#notices .notice-box button { border: 0; border-radius: 8px; padding: 5px 11px; font: inherit; font-size: 12px; cursor: pointer;
|
|
100
|
+
background: var(--line, #eee); color: var(--text); }
|
|
101
|
+
#notices .notice-box button.go { background: var(--accent, #0071e3); color: #fff; }
|
|
102
|
+
#notices .notice-box button.notice-x { position: absolute; top: 6px; right: 6px; width: 22px; height: 22px; padding: 0;
|
|
103
|
+
background: none; color: var(--muted); font-size: 13px; line-height: 22px; }
|
|
104
|
+
#notices .notice-box button.notice-x:hover { background: var(--hover, #eee); color: var(--text); }
|
|
105
|
+
@media (prefers-reduced-motion: reduce) { #notices .notice { transition: opacity .15s; } }
|
|
82
106
|
#rail button, #status > button, #status > details > summary, #status > .tool > button, #status .sub > button {
|
|
83
107
|
width: 38px; height: 38px; display: grid; place-items: center; border-radius: 12px; color: var(--muted);
|
|
84
108
|
position: relative; list-style: none; cursor: pointer; transition: background 0.15s, color 0.15s; }
|
|
@@ -113,17 +137,17 @@
|
|
|
113
137
|
mask-image: var(--fade); -webkit-mask-image: var(--fade); }
|
|
114
138
|
#log:hover { --fade: linear-gradient(transparent, #000 56px); }
|
|
115
139
|
#log > :first-child { margin-top: auto; } /* few messages: they sit at the bottom (flex-end would break scrolling) */
|
|
116
|
-
.
|
|
140
|
+
.group { margin: 0 0 22px; animation: rise 0.4s ease-out; }
|
|
117
141
|
@keyframes rise { from { opacity: 0; transform: translateY(14px); } }
|
|
118
142
|
.who { display: flex; align-items: center; gap: 6px; font-size: 11.5px; font-weight: 500; letter-spacing: 0.01em;
|
|
119
143
|
color: var(--c, var(--muted)); }
|
|
120
144
|
.who svg { width: 16px; height: 16px; }
|
|
121
145
|
.who time { margin-left: auto; font-weight: 400; color: var(--muted); opacity: 0; transition: opacity 0.2s; }
|
|
122
|
-
.
|
|
146
|
+
.group:hover .who time { opacity: 1; }
|
|
123
147
|
.text { margin: 3px 0 0 22px; color: var(--soft); white-space: pre-wrap; overflow-wrap: anywhere; letter-spacing: -0.005em; }
|
|
124
|
-
.
|
|
148
|
+
.group:not([data-who="you"]) .text { color: var(--text); }
|
|
125
149
|
.text .to { color: var(--muted); font-size: 12px; margin-right: 4px; }
|
|
126
|
-
.text >
|
|
150
|
+
.text > block-markdown, .text > block-markdown > p:first-child { display: inline; } /* first line next to .to */
|
|
127
151
|
.steps { display: flex; flex-wrap: wrap; gap: 1px; margin: 4px 0 0 19px; }
|
|
128
152
|
.step { color: var(--muted); padding: 3px; border-radius: 6px; opacity: 0.8; }
|
|
129
153
|
.step svg { width: 14px; height: 14px; }
|
|
@@ -187,9 +211,9 @@
|
|
|
187
211
|
</style>
|
|
188
212
|
<body data-panel="messages">
|
|
189
213
|
<div id="top"></div>
|
|
190
|
-
<nav id="rail">
|
|
191
|
-
<button id="show-messages" data-icon="messages" title="Messages"><span class="badge" id="unread"></span></button>
|
|
192
|
-
<button id="show-versions" data-icon="versions" title="Versions"></button>
|
|
214
|
+
<nav id="rail" data-name="rail">
|
|
215
|
+
<button id="show-messages" data-name="messages" data-icon="messages" title="Messages"><span class="badge" id="unread"></span></button>
|
|
216
|
+
<button id="show-versions" data-name="versions" data-icon="versions" title="Versions"></button>
|
|
193
217
|
</nav>
|
|
194
218
|
<aside id="panel">
|
|
195
219
|
<section id="messages"><div id="log"></div></section>
|
|
@@ -201,14 +225,15 @@
|
|
|
201
225
|
</section>
|
|
202
226
|
</aside>
|
|
203
227
|
<div id="working" hidden><div id="busy"></div><button id="stop" data-icon="stop" title="Stop all agents"></button></div>
|
|
204
|
-
<form id="f">
|
|
228
|
+
<form id="f" data-name="message-line">
|
|
205
229
|
<div id="cmds" hidden></div>
|
|
206
230
|
<select id="to" title="Send to" hidden></select>
|
|
207
231
|
<input id="text" placeholder="Tell the agents what to build or change" autocomplete="off">
|
|
208
232
|
<button data-icon="send" title="Send"></button>
|
|
209
233
|
</form>
|
|
210
|
-
<nav id="status"></nav>
|
|
234
|
+
<nav id="status" data-name="tools"></nav>
|
|
211
235
|
<aside id="side"></aside>
|
|
236
|
+
<div id="notices" aria-live="polite"></div>
|
|
212
237
|
<script>
|
|
213
238
|
const log = document.getElementById("log"), to = document.getElementById("to");
|
|
214
239
|
const ws = new WebSocket((location.protocol === "https:" ? "wss://" : "ws://") + location.host + "/ws");
|
|
@@ -247,15 +272,15 @@
|
|
|
247
272
|
to.hidden = to.options.length < 2;
|
|
248
273
|
}
|
|
249
274
|
|
|
250
|
-
// ----
|
|
251
|
-
//
|
|
252
|
-
//
|
|
275
|
+
// ---- Block runtime: the catalog of blocks (/card/blocks/<Name>.js), each a custom element <block-name> with
|
|
276
|
+
// update(props, card). The page draws replies with it; browser parts use page.load / page.block (the card
|
|
277
|
+
// extension draws the agents' cards from the same catalog).
|
|
253
278
|
let catalog = null;
|
|
254
|
-
const loading = {}; // name -> promise: true once <
|
|
279
|
+
const loading = {}; // name -> promise: true once <block-name> is defined, false if there is no such block
|
|
255
280
|
const loaded = {}; // name -> version loaded (custom elements can't be redefined: changes need a reload)
|
|
256
281
|
async function fetchCatalog() { // false: the agent didn't answer (e.g. restarting); the catalog stays as it was
|
|
257
282
|
try {
|
|
258
|
-
const r = await fetch("/
|
|
283
|
+
const r = await fetch("/card/blocks");
|
|
259
284
|
if (r.ok) { catalog = await r.json(); return true; }
|
|
260
285
|
} catch {}
|
|
261
286
|
catalog ??= {};
|
|
@@ -265,32 +290,32 @@
|
|
|
265
290
|
return loading[name] ??= (async () => {
|
|
266
291
|
if (!catalog || !(name in catalog)) await fetchCatalog(); // maybe just written
|
|
267
292
|
if (!(name in catalog)) return false;
|
|
268
|
-
const tag = `
|
|
269
|
-
if (!customElements.get(tag)) customElements.define(tag, (await import(`/
|
|
293
|
+
const tag = `block-${name.toLowerCase()}`;
|
|
294
|
+
if (!customElements.get(tag)) customElements.define(tag, (await import(`/card/blocks/${name}.js?v=${catalog[name]}`)).default);
|
|
270
295
|
loaded[name] = catalog[name];
|
|
271
296
|
return true;
|
|
272
297
|
})().catch((e) => { delete loading[name]; throw e; }); // a broken file: try again next time
|
|
273
298
|
}
|
|
274
|
-
async function
|
|
275
|
-
try { if (!(await load(name))) return null; } catch (e) { console.warn(`
|
|
276
|
-
const el = document.createElement(`
|
|
299
|
+
async function block(name, props = {}) { // a ready element, or null if the block is missing/broken
|
|
300
|
+
try { if (!(await load(name))) return null; } catch (e) { console.warn(`block ${name}:`, e); return null; }
|
|
301
|
+
const el = document.createElement(`block-${name.toLowerCase()}`);
|
|
277
302
|
el.update(props, { children: [], mount() {}, set() {}, action() {} });
|
|
278
303
|
return el;
|
|
279
304
|
}
|
|
280
305
|
// After an agent saves a version (end of its turn), show what it changed without a manual refresh:
|
|
281
|
-
// an
|
|
282
|
-
// (the browser can't redefine a custom element). New
|
|
306
|
+
// an app's files -> reload that app's frame; this page, a browser.js or a block in use -> reload the page
|
|
307
|
+
// (the browser can't redefine a custom element). New blocks need nothing: they load on first use.
|
|
283
308
|
async function refreshAfter(files) {
|
|
284
309
|
if (!(await fetchCatalog())) return reloadWhenUp(); // can't tell what changed: reload once the agent answers
|
|
285
310
|
for (const [n, p] of Object.entries(loading)) p.then((ok) => { if (!ok) delete loading[n]; }); // maybe created now
|
|
286
311
|
const changed = Object.keys(loaded).filter((n) => catalog[n] !== loaded[n]);
|
|
287
|
-
if (changed.length || files.some((f) => f === "extensions/page/page.html" || f.endsWith("/browser.js"))) return location.reload();
|
|
288
|
-
const apps = files.filter((f) => f.startsWith("
|
|
312
|
+
if (changed.length || files.some((f) => f === "agent/extensions/page/page.html" || f.endsWith("/browser.js"))) return location.reload();
|
|
313
|
+
const apps = files.filter((f) => f.startsWith("built/apps/")).map((f) => `/built/apps/${f.split("/")[2]}/`);
|
|
289
314
|
for (const frame of document.querySelectorAll("iframe"))
|
|
290
315
|
if (apps.some((a) => new URL(frame.src, location.href).pathname.startsWith(a))) frame.contentWindow.location.reload();
|
|
291
316
|
}
|
|
292
317
|
|
|
293
|
-
// ---- Messages:
|
|
318
|
+
// ---- Messages: groups of consecutive messages from one speaker (an agent, or "you"). A group has a
|
|
294
319
|
// header (robot in the agent's colour, name, time), then words as text and other work as rows of icons.
|
|
295
320
|
let ts = null; // time of the event being shown (set per event)
|
|
296
321
|
let unread = 0; // replies that arrived while the messages panel was closed
|
|
@@ -300,11 +325,11 @@
|
|
|
300
325
|
if (atBottom) log.scrollTop = log.scrollHeight;
|
|
301
326
|
}
|
|
302
327
|
|
|
303
|
-
function
|
|
328
|
+
function group(who) {
|
|
304
329
|
const last = log.lastElementChild;
|
|
305
330
|
if (last?.dataset.who === who) return last;
|
|
306
331
|
const b = document.createElement("div");
|
|
307
|
-
b.className = "
|
|
332
|
+
b.className = "group";
|
|
308
333
|
b.dataset.who = who;
|
|
309
334
|
if (who !== "you") b.style.setProperty("--c", colour(who));
|
|
310
335
|
b.innerHTML = `<div class="who">${icon(who === "you" ? "person" : "robot")}<span></span><time></time></div>`;
|
|
@@ -318,18 +343,19 @@
|
|
|
318
343
|
const div = document.createElement("div");
|
|
319
344
|
div.className = "text";
|
|
320
345
|
if (toWhom) div.append(Object.assign(document.createElement("span"), { className: "to", textContent: `→ ${toWhom}` }));
|
|
321
|
-
const span = document.createElement("span"); // plain at once; swapped for the Markdown
|
|
346
|
+
const span = document.createElement("span"); // plain at once; swapped for the Markdown block when ready
|
|
322
347
|
span.textContent = words;
|
|
323
348
|
div.append(span);
|
|
324
|
-
follow(() =>
|
|
325
|
-
|
|
349
|
+
follow(() => group(who).append(div));
|
|
350
|
+
block("Markdown", { text: words }).then((el) => el && follow(() => span.replaceWith(el)));
|
|
326
351
|
if (who !== "you" && document.body.dataset.panel !== "messages")
|
|
327
352
|
document.getElementById("unread").textContent = ++unread;
|
|
353
|
+
return div;
|
|
328
354
|
}
|
|
329
355
|
|
|
330
356
|
function step(agent, kind, title, content, error = false) { // one icon in the agent's row; click to read it
|
|
331
357
|
follow(() => {
|
|
332
|
-
const b =
|
|
358
|
+
const b = group(agent);
|
|
333
359
|
let row = b.lastElementChild;
|
|
334
360
|
if (row.classList.contains("detail")) row = row.previousElementSibling; // the row whose step is open
|
|
335
361
|
if (!row.classList.contains("steps")) b.append((row = Object.assign(document.createElement("div"), { className: "steps" })));
|
|
@@ -351,8 +377,10 @@
|
|
|
351
377
|
});
|
|
352
378
|
}
|
|
353
379
|
|
|
354
|
-
function note(words) { // a quiet line: joined, saved a version, errors, ...
|
|
355
|
-
|
|
380
|
+
function note(words) { // a quiet line: joined, saved a version, errors, ... (returns it, for a plugin to act on)
|
|
381
|
+
const line = Object.assign(document.createElement("div"), { className: "note", textContent: words });
|
|
382
|
+
follow(() => log.append(line));
|
|
383
|
+
return line;
|
|
356
384
|
}
|
|
357
385
|
|
|
358
386
|
// ---- Working: animated dots for each agent that isn't idle (from "status" events), Stop beside them
|
|
@@ -391,11 +419,36 @@
|
|
|
391
419
|
send: (obj) => ws.send(JSON.stringify(obj)), // arrives at the extensions' page handlers
|
|
392
420
|
slot: (id) => document.getElementById(id), // "top": the main space; "status": the tool palette (top); "side": right edge
|
|
393
421
|
note, // note(text): a quiet line in the messages
|
|
394
|
-
load, // load(name): define
|
|
395
|
-
|
|
422
|
+
load, // load(name): define block <block-name>; resolves false if there is no such block
|
|
423
|
+
block, // block(name, props): a ready element, or null
|
|
396
424
|
command: (name, fn, description = "") => { commands[name] = { fn, description }; },
|
|
397
425
|
provide: (name, obj) => { offered[name] = obj; for (const ok of wanted[name] || []) ok(obj); delete wanted[name]; },
|
|
398
426
|
use: (name) => name in offered ? Promise.resolve(offered[name]) : new Promise((ok) => (wanted[name] ??= []).push(ok)),
|
|
427
|
+
notice: (content, onClose) => { // a notice at the top right, newest on top; ✕ calls onClose, then it goes
|
|
428
|
+
const item = Object.assign(document.createElement("div"), { className: "notice out" });
|
|
429
|
+
const inner = document.createElement("div"), box = Object.assign(document.createElement("div"), { className: "notice-box" });
|
|
430
|
+
const x = Object.assign(document.createElement("button"), { className: "notice-x", textContent: "✕", title: "Close" });
|
|
431
|
+
let open = true;
|
|
432
|
+
const handle = {
|
|
433
|
+
el: box,
|
|
434
|
+
set: (...nodes) => box.replaceChildren(...nodes, x), // new content, no new animation
|
|
435
|
+
close: () => {
|
|
436
|
+
if (!open) return;
|
|
437
|
+
open = false;
|
|
438
|
+
item.classList.add("out");
|
|
439
|
+
setTimeout(() => item.remove(), 360);
|
|
440
|
+
},
|
|
441
|
+
get open() { return open; },
|
|
442
|
+
};
|
|
443
|
+
x.onclick = () => { onClose?.(); handle.close(); };
|
|
444
|
+
handle.set(...[content].flat());
|
|
445
|
+
inner.append(box);
|
|
446
|
+
item.append(inner);
|
|
447
|
+
document.getElementById("notices").prepend(item);
|
|
448
|
+
item.offsetHeight; // laid out closed first, so opening animates
|
|
449
|
+
item.classList.remove("out");
|
|
450
|
+
return handle;
|
|
451
|
+
},
|
|
399
452
|
tool: (button) => { // puts button in the palette with a row under it; returns the row, for its sub-tools
|
|
400
453
|
const sub = Object.assign(document.createElement("div"), { className: "sub" });
|
|
401
454
|
const wrap = Object.assign(document.createElement("span"), { className: "tool" });
|
|
@@ -409,7 +462,7 @@
|
|
|
409
462
|
});
|
|
410
463
|
|
|
411
464
|
const lastFrom = {}; // agent -> who its current turn's answer goes to
|
|
412
|
-
const asked = {}; //
|
|
465
|
+
const asked = {}; // team_ask / team_tell tool-call ids
|
|
413
466
|
|
|
414
467
|
ws.onmessage = (e) => {
|
|
415
468
|
const ev = JSON.parse(e.data);
|
|
@@ -433,11 +486,11 @@
|
|
|
433
486
|
const m = /^\[(\w+) from ([^\s\]]+)[^\]]*\][^\n]*\n?/.exec(content);
|
|
434
487
|
const from = m ? m[2] : "user", words = m ? content.slice(m[0].length) : content;
|
|
435
488
|
if (ev.reply_to) lastFrom[a] = ev.reply_to; // set on the message that starts a turn
|
|
436
|
-
// a message the page wrote for the user ("[note on the
|
|
489
|
+
// a message the page wrote for the user ("[note on the page] <their words>" + lines for the agent):
|
|
437
490
|
// show their words; the lines meant for the agent go behind an icon
|
|
438
491
|
const tagged = from === "user" && /^\[([^\]]+)\] ?(.*)/.exec(words);
|
|
439
492
|
if (tagged) {
|
|
440
|
-
text("you", tagged[2] || tagged[1], agents.size > 1 ? a : null);
|
|
493
|
+
text("you", tagged[2] || tagged[1], agents.size > 1 ? a : null).dataset.raw = words; // all of it, for plugins (doodle: its group)
|
|
441
494
|
if (words.includes("\n")) step("you", "inbox", `${tagged[1]}: what ${a} got`, words);
|
|
442
495
|
} else if (from === "user") text("you", words, agents.size > 1 ? a : null);
|
|
443
496
|
// from another agent: already shown as its "asks"/"tells" step, or as its answer
|
|
@@ -454,12 +507,12 @@
|
|
|
454
507
|
for (const c of ev.tool_calls || []) {
|
|
455
508
|
let args = {};
|
|
456
509
|
try { args = JSON.parse(c.function.arguments); } catch {}
|
|
457
|
-
if (c.function.name === "
|
|
458
|
-
else if (c.function.name === "
|
|
510
|
+
if (c.function.name === "team_ask") { asked[c.id] = true; step(a, "ask", `${a} asks ${args.to}`, args.text || ""); }
|
|
511
|
+
else if (c.function.name === "team_tell") { asked[c.id] = true; step(a, "ask", `${a} tells ${args.to}`, args.text || ""); }
|
|
459
512
|
else step(a, "tool", `${a} → ${c.function.name}`, c.function.arguments);
|
|
460
513
|
}
|
|
461
514
|
}
|
|
462
|
-
//
|
|
515
|
+
// a team_ask result is the other agent's answer, already shown as its reply; a team_tell result is
|
|
463
516
|
// just "sent" (errors still shown)
|
|
464
517
|
// a result may be a list of parts (view_image): text, and [image <path>] for an image
|
|
465
518
|
const res = typeof ev.content === "string" ? ev.content
|
|
@@ -562,7 +615,7 @@
|
|
|
562
615
|
};
|
|
563
616
|
|
|
564
617
|
async function loadVersions() {
|
|
565
|
-
try { versions = await (await fetch("/
|
|
618
|
+
try { versions = await (await fetch("/version")).json(); } catch { return; } // no versions plugin
|
|
566
619
|
sel = Math.max(0, versions.findIndex((v) => v.current));
|
|
567
620
|
renderWheel();
|
|
568
621
|
}
|
|
@@ -587,7 +640,7 @@
|
|
|
587
640
|
detail.textContent = `${v.short} · ${v.author} · ${new Date(v.date).toLocaleString()}` + (v.current ? " · current" : "");
|
|
588
641
|
rollbackBtn.disabled = v.current;
|
|
589
642
|
preview.hidden = !v.shot; // a version made while no page was open has no picture
|
|
590
|
-
if (v.shot) preview.src = `/
|
|
643
|
+
if (v.shot) preview.src = `/version/${v.sha}/shot`;
|
|
591
644
|
}
|
|
592
645
|
|
|
593
646
|
function select(i) {
|
|
@@ -602,7 +655,7 @@
|
|
|
602
655
|
if (!confirm(`Roll back to "${v.message}" (${v.short})?\n\nThe agent restarts and busy agents are interrupted. ` +
|
|
603
656
|
`Later changes stay in the history, so you can roll forward again.`)) return;
|
|
604
657
|
note(`rolling back to ${v.short}…`);
|
|
605
|
-
await fetch(`/
|
|
658
|
+
await fetch(`/version/${v.sha}/rollback`, { method: "POST" });
|
|
606
659
|
};
|
|
607
660
|
|
|
608
661
|
openPanel(SHOT ? "" : document.body.dataset.panel);
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: interact-with-page
|
|
3
|
+
description: Use the page the user has open the way they would, while they watch - read what's on screen, click, type, press keys, drag - in any card, app or part of the page. Use to check what the user sees, show them something, fill something in for them, reproduce what they pointed at, or try something you built.
|
|
4
|
+
---
|
|
5
|
+
# Using the user's page
|
|
6
|
+
|
|
7
|
+
The page_run tool (load it: tool_search "page_run") runs JavaScript in the page the user has open, as it is right
|
|
8
|
+
now. With card=<id> it runs inside that card's app: `document` is the app's page. The code is the body of an async
|
|
9
|
+
function: `await` works, and you `return` what you want back.
|
|
10
|
+
|
|
11
|
+
## Act with the helpers
|
|
12
|
+
They act as a person would: your pointer moves to the thing first, what you touch gets an outline with your name,
|
|
13
|
+
and typing goes letter by letter, so the user can follow.
|
|
14
|
+
- `await click(t)`
|
|
15
|
+
- `await type(t, text)`: replaces what the field holds (`{add: true}` adds to it). A dropdown picks the option with
|
|
16
|
+
that text; a slider, date or colour field is set to that value.
|
|
17
|
+
- `await press(key, t?)`: Enter, Escape, Tab, ArrowDown, "Ctrl+z"... to t, else to what has focus. Enter in a form
|
|
18
|
+
field submits the form.
|
|
19
|
+
- `await drag(t, dx, dy)` by pixels, or `await drag(t, other)` onto another thing. A draggable thing (a file row)
|
|
20
|
+
is dragged and dropped as the browser does it, so drop targets get it.
|
|
21
|
+
- `read(t?)`: what's on screen as short lines: headings, controls with their value and state ([selected],
|
|
22
|
+
[checked], [disabled]), short texts. Only what's in view; `read(null, {all: true})` for the whole page. The
|
|
23
|
+
conversation (the chat log) is left out: you know it; `read("#log")` if you need it.
|
|
24
|
+
- `await wait(ms)`, `await wait(".row.new")` (until it appears), `await wait(() => condition)`.
|
|
25
|
+
|
|
26
|
+
t is an element, a CSS selector, or the visible text: a button's words, a field's placeholder or label. Or a spot
|
|
27
|
+
`{x: 600, y: 300}` in window px (screen_see's picture, or an element's getBoundingClientRect()): whatever is on top
|
|
28
|
+
there gets it, as with a real mouse. Use a spot for empty space (`await drag({x: 400, y: 300}, 300, 200)` draws a
|
|
29
|
+
frame there) and for tools that work through a layer over the page (Point: click the tool, then the spot).
|
|
30
|
+
```
|
|
31
|
+
page_run(card="habit-tracker", code=`await type("New habit", "Drink water");
|
|
32
|
+
await click("Add habit");
|
|
33
|
+
return read()`)
|
|
34
|
+
```
|
|
35
|
+
Plain DOM code is fine for reading and checking precisely: `getComputedStyle(el)`, `el.scrollWidth > el.clientWidth`
|
|
36
|
+
(text cut off), state the app exposes. For acting, use the helpers: a bare `el.click()` happens at once, and the user
|
|
37
|
+
only sees an outline afterwards.
|
|
38
|
+
|
|
39
|
+
## Look after each step, cheaply
|
|
40
|
+
One step, or a few, per page_run, ending with a small read: `return read()` or the one value you need. You see what
|
|
41
|
+
happened before you decide the next step. A read is about a hundred tokens; a picture (screen_see) is over a thousand
|
|
42
|
+
and stays in your conversation, so look at a picture when the look matters (layout, colours, clipped or overlapping
|
|
43
|
+
things), not after every step. From bash the same is `curl -X POST --data '{"code": "...", "card": "<id>"}' {page}/page/run`,
|
|
44
|
+
for a loop or a script.
|
|
45
|
+
|
|
46
|
+
## The user is watching, and can take over
|
|
47
|
+
While you use the page a bar says so, with a Stop button. If the user clicks or types on the page, page_run answers
|
|
48
|
+
"the user took over": stop there, don't retry, and say in a sentence what you were doing. When they press
|
|
49
|
+
"Let <you> continue" you get a message: read the page again first, it may have changed.
|
|
50
|
+
|
|
51
|
+
## Care
|
|
52
|
+
- It's the live page: clicks really happen and data really changes. Touch only what your task needs, and don't send
|
|
53
|
+
messages through the chat box.
|
|
54
|
+
- The page's Stop button and rollbacks refuse code: they are the user's.
|
|
55
|
+
- Events from code are not quite real input: hover styles, native dropdown lists, file pickers, and copy and paste
|
|
56
|
+
don't respond. For those, and for other window sizes, use the browser-check skill
|
|
57
|
+
(a separate headless browser).
|
|
58
|
+
- Errors the page logs while your code runs come back with the answer: read them.
|
|
59
|
+
- An answer marked "not on the board the user is looking at" ran on a card they can't see right now.
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
// The prompt after SYSTEM.md (pi's documented way to replace its default prompt, plain text): one section per
|
|
2
|
+
// extension, its guide.md, then <paths>. A guide.md is found by looking in each extension folder, as the page finds
|
|
3
|
+
// browser.js, so an extension has nothing to register. Read again on every input (an edited guide shows at once).
|
|
4
|
+
// In a guide: {agent} {built} {data} {workspace} {page} become the full paths and address, {name} and {role} this
|
|
5
|
+
// agent's, and README.md the full path of that extension's README.md. pi adds the skills list and <cwd> itself.
|
|
6
|
+
// READMEs and skills keep the placeholders as written (agents edit them; <paths> says what each one is).
|
|
7
|
+
import fs from "node:fs";
|
|
8
|
+
import path from "node:path";
|
|
9
|
+
import { getDocsPath, getExamplesPath, getReadmePath, type ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
10
|
+
|
|
11
|
+
const host = (globalThis as any).chalky;
|
|
12
|
+
const EXTENSIONS = path.dirname(import.meta.dirname);
|
|
13
|
+
|
|
14
|
+
export default function (pi: ExtensionAPI) {
|
|
15
|
+
pi.on("before_agent_start", (event) => {
|
|
16
|
+
const name = pi.getSessionName() ?? "", page = `http://127.0.0.1:${host.port}`;
|
|
17
|
+
const values: Record<string, string> = { agent: host.agent, built: host.built, data: host.data, workspace: host.workspace,
|
|
18
|
+
page, name, role: host.agents.get(name)?.role ?? "" };
|
|
19
|
+
const sections = event.systemPromptOptions.sections;
|
|
20
|
+
for (const ext of fs.readdirSync(EXTENSIONS).sort()) {
|
|
21
|
+
const guide = path.join(EXTENSIONS, ext, "guide.md");
|
|
22
|
+
if (!fs.existsSync(guide)) continue;
|
|
23
|
+
sections[ext] = fs.readFileSync(guide, "utf8").replace(/\{(\w+)\}/g, (m, k) => values[k] ?? m)
|
|
24
|
+
.replaceAll("README.md", path.join(EXTENSIONS, ext, "README.md"));
|
|
25
|
+
}
|
|
26
|
+
sections.paths = [
|
|
27
|
+
`{agent} = ${host.agent}: you: your prompt, settings, extensions/ and skills/`,
|
|
28
|
+
`{built} = ${host.built}: what you make for the user: card.json (the cards), board.json (the boards), apps/<name>/ (the apps)`,
|
|
29
|
+
` {agent} and {built} are versioned together: each turn that changes them is saved, and the user can roll both back.`,
|
|
30
|
+
`{data} = ${host.data}: what is kept while running (conversations, what users typed into cards and apps, notes; not versioned)`,
|
|
31
|
+
`{workspace} = ${host.workspace}: your working folder (<cwd>), for project work; drops/ in it holds the user's files`,
|
|
32
|
+
`{page} = ${page}: the user's page; a file in {built} is at {page}/built/<its path in {built}>`,
|
|
33
|
+
`The host: ${host.root}/server.ts`,
|
|
34
|
+
`pi's docs: ${getDocsPath()}/extensions.md for extensions, examples in ${getExamplesPath()}/extensions/; the rest in `
|
|
35
|
+
+ `${getReadmePath()} and ${getDocsPath()}/. Read pi's .md files completely and follow their links.`,
|
|
36
|
+
].join("\n");
|
|
37
|
+
});
|
|
38
|
+
}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
//
|
|
1
|
+
// Reload: an agent's edits to itself take effect. The tool only asks; when this agent's turn
|
|
2
2
|
// has ended, the host calls pi's session.reload() (extensions, skills and prompt load again; the conversation stays).
|
|
3
3
|
// Not pi's reload-runtime.ts example: its pi.sendUserMessage("/reload-runtime") reaches the model as text in 1.4.0
|
|
4
4
|
// (expandPromptTemplates defaults to false), and the model kept calling the tool again.
|
|
5
|
-
// No process restart, so no start.sh (a rollback: the
|
|
5
|
+
// No process restart, so no start.sh (a rollback: the version extension, which reloads every agent).
|
|
6
6
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
7
7
|
import { Type } from "typebox";
|
|
8
8
|
|
|
@@ -13,7 +13,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
13
13
|
async execute(_id, a) {
|
|
14
14
|
const host = (globalThis as any).chalky, name = pi.getSessionName();
|
|
15
15
|
host.reloadAfterTurn.set(name, false); // not quiet: the page notes the reload
|
|
16
|
-
host.shared.set(`reload-reason ${name}`, a.reason); // the
|
|
16
|
+
host.shared.set(`reload-reason ${name}`, a.reason); // the version extension's commit message for this turn
|
|
17
17
|
return { content: [{ type: "text", text: "Reload scheduled for when your turn ends. End your turn now; don't call reload again." }],
|
|
18
18
|
details: undefined };
|
|
19
19
|
},
|
|
@@ -1,23 +1,15 @@
|
|
|
1
|
-
//
|
|
2
|
-
// of what the user sees. Offered as page.provide("screen", {...});
|
|
1
|
+
// The page side of the screen extension: this tab, shared once (getDisplayMedia), for every extension that needs a picture
|
|
2
|
+
// of what the user sees. Offered as page.provide("screen", {...}); an extension gets it with await page.use("screen"):
|
|
3
3
|
// share() ask the browser to share this tab (only works right after a click: transient activation);
|
|
4
4
|
// resolves true if shared. The user can stop sharing any time (the browser's bar).
|
|
5
5
|
// shared() whether the tab is shared now
|
|
6
6
|
// frame() the screen right now, as a canvas (null when not shared)
|
|
7
7
|
// picture(full, box) crop a frame to box (window px {x, y, w, h}), upload it (POST /screen/shot): {path, url}
|
|
8
|
-
// An agent can ask to see the screen (its tool
|
|
9
|
-
// of the whole window goes back at once; else a
|
|
8
|
+
// An agent can ask to see the screen (its tool screen_see, event screen_ask): if the tab is shared, a picture
|
|
9
|
+
// of the whole window goes back at once; else a notice asks "Share" (✕: no) first, since the agent's ask is no click
|
|
10
10
|
// (screen_reply {id, shot, window, main}).
|
|
11
11
|
export default async function (page) {
|
|
12
12
|
const area = page.slot("top");
|
|
13
|
-
const style = document.createElement("style");
|
|
14
|
-
style.textContent = `
|
|
15
|
-
.screen-ask { position: fixed; top: 76px; left: 50%; transform: translateX(-50%); z-index: 100; display: flex; align-items: center; gap: 10px;
|
|
16
|
-
padding: 8px 8px 8px 14px; border-radius: 14px; background: var(--glass-strong, #fff); border: 1px solid var(--edge, #eee);
|
|
17
|
-
box-shadow: 0 8px 30px rgba(0, 0, 0, .12); font-size: 13px; color: var(--text); }
|
|
18
|
-
.screen-ask button { border: 0; border-radius: 9px; padding: 5px 12px; font: inherit; cursor: pointer; background: var(--line, #eee); color: var(--text); }
|
|
19
|
-
.screen-ask button.yes { background: var(--accent, #0071e3); color: #fff; }`;
|
|
20
|
-
document.head.append(style);
|
|
21
13
|
|
|
22
14
|
let stream = null, video = null;
|
|
23
15
|
async function share() {
|
|
@@ -53,24 +45,19 @@ export default async function (page) {
|
|
|
53
45
|
// ---- an agent asks to see the screen ----
|
|
54
46
|
const asks = {}; // id -> its card, while the agent waits
|
|
55
47
|
async function answer(id) {
|
|
56
|
-
for (const card of Object.values(asks)) card.
|
|
57
|
-
await new Promise((ok) => setTimeout(ok,
|
|
48
|
+
for (const card of Object.values(asks)) card.close(); // not in the picture
|
|
49
|
+
await new Promise((ok) => setTimeout(ok, 400)); // it has slid away; a fresh frame from the shared tab
|
|
58
50
|
const full = frame();
|
|
59
51
|
const s = full && await picture(full, { x: 0, y: 0, w: innerWidth, h: innerHeight }).catch(() => null);
|
|
60
|
-
const m = area.getBoundingClientRect(); // window (0, 0) in main-space px (as
|
|
52
|
+
const m = area.getBoundingClientRect(); // window (0, 0) in main-space px (as note/browser.js)
|
|
61
53
|
page.send({ type: "screen_reply", id, shot: s?.path || null, window: { width: innerWidth, height: innerHeight },
|
|
62
54
|
main: { x: area.scrollLeft - m.left, y: area.scrollTop - m.top } });
|
|
63
55
|
}
|
|
64
56
|
page.on("screen_ask", ({ id, agent }) => {
|
|
65
57
|
if (shared()) return answer(id);
|
|
66
|
-
const
|
|
67
|
-
const yes = Object.assign(document.createElement("button"), { className: "yes", textContent: "Share" });
|
|
68
|
-
const no = Object.assign(document.createElement("button"), { textContent: "No" });
|
|
69
|
-
card.append(`${agent} wants to see your screen`, yes, no);
|
|
58
|
+
const yes = Object.assign(document.createElement("button"), { className: "go", textContent: "Share" });
|
|
70
59
|
yes.onclick = async () => { await share(); answer(id); };
|
|
71
|
-
|
|
72
|
-
asks[id] = card;
|
|
73
|
-
document.body.append(card);
|
|
60
|
+
asks[id] = page.notice([`${agent} wants to see your screen`, yes], () => page.send({ type: "screen_reply", id, shot: null })); // ✕: no
|
|
74
61
|
});
|
|
75
|
-
page.on("screen_ask_done", ({ id }) => { asks[id]?.
|
|
62
|
+
page.on("screen_ask_done", ({ id }) => { asks[id]?.close(); delete asks[id]; }); // answered (here or in another tab), or timed out
|
|
76
63
|
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
// Screen: what the user sees, as pictures
|
|
1
|
+
// Screen: what the user sees, as pictures. The
|
|
2
2
|
// browser half shares the tab once (for every extension: doodle's steps, an agent's look) and uploads pictures
|
|
3
|
-
// cropped from it. This half keeps them and gives agents the tool
|
|
4
|
-
// (their size, scroll, panels, running apps;
|
|
5
|
-
// POST /screen/shot a png (at most MAX_BYTES): saved in
|
|
6
|
-
// event "screen_ask" {id, agent} to every open page, when an agent calls
|
|
3
|
+
// cropped from it. This half keeps them and gives agents the tool screen_see: the user's window as it is now
|
|
4
|
+
// (their size, scroll, panels, running apps; browser-check's headless Chrome is a fresh window of its own).
|
|
5
|
+
// POST /screen/shot a png (at most MAX_BYTES): saved in data/screen, {path} back
|
|
6
|
+
// event "screen_ask" {id, agent} to every open page, when an agent calls screen_see
|
|
7
7
|
// page message screen_reply {id, shot, window: {width, height}, main: {x, y}}: the first answers
|
|
8
8
|
// (shot null: the user said no; main: window (0, 0) in main-space px)
|
|
9
9
|
// event "screen_ask_done" {id} to every page when answered or after ASK_WAIT s, so the asks go away
|
|
@@ -14,14 +14,14 @@ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
|
14
14
|
import { Type } from "typebox";
|
|
15
15
|
|
|
16
16
|
const host = (globalThis as any).chalky;
|
|
17
|
-
const SHOTS = path.join(host.
|
|
17
|
+
const SHOTS = path.join(host.data, "screen");
|
|
18
18
|
const MAX_BYTES = 8 * 1024 * 1024;
|
|
19
19
|
const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
|
20
|
-
const ASK_WAIT = 90; // s for the user to answer
|
|
20
|
+
const ASK_WAIT = 90; // s for the user to answer screen_see
|
|
21
21
|
|
|
22
22
|
const isShot = (p: unknown) => typeof p === "string" && path.dirname(p) === SHOTS && fs.existsSync(p);
|
|
23
23
|
host.api.screen = { isShot };
|
|
24
|
-
// id -> resolve, while
|
|
24
|
+
// id -> resolve, while screen_see waits for a page (on the host: the tool is per agent, the page handler one)
|
|
25
25
|
const asks: Map<string, (msg: any) => void> = host.shared.get("screen-asks") ?? new Map();
|
|
26
26
|
host.shared.set("screen-asks", asks);
|
|
27
27
|
|
|
@@ -50,9 +50,9 @@ const result = (text: string, png?: string) => ({ details: undefined, content: [
|
|
|
50
50
|
|
|
51
51
|
export default function (pi: ExtensionAPI) {
|
|
52
52
|
pi.registerTool({
|
|
53
|
-
name: "
|
|
53
|
+
name: "screen_see", label: "Screen see", exposure: "deferred",
|
|
54
54
|
description: "See the user's screen as they see it right now: a picture of their whole window (their size, scroll, "
|
|
55
|
-
+ "open panels, apps as they are running).
|
|
55
|
+
+ "open panels, apps as they are running). browser-check's headless Chrome is a fresh window of its own, not theirs: when "
|
|
56
56
|
+ "the user talks about what they see, or before you say something looks right to them, look here. The first "
|
|
57
57
|
+ "time, the user is asked to share the tab (they may say no); then it's instant.",
|
|
58
58
|
parameters: Type.Object({}),
|
|
@@ -74,7 +74,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
74
74
|
const mx = Math.round(m.x ?? 0), my = Math.round(m.y ?? 0);
|
|
75
75
|
return result(`the user's screen right now: their window is ${w.width}x${w.height} px; the picture is the whole window, `
|
|
76
76
|
+ `${pw}x${ph} px (${+k.toFixed(3)} picture px per window px). Window (0, 0) is main-space (${mx}, ${my}): a spot in `
|
|
77
|
-
+ `the picture is main-space x = ${mx} + picture x / ${+k.toFixed(3)}, the same for y (what
|
|
77
|
+
+ `the picture is main-space x = ${mx} + picture x / ${+k.toFixed(3)}, the same for y (what card_show's place uses).`, reply.shot);
|
|
78
78
|
},
|
|
79
79
|
});
|
|
80
80
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
//
|
|
1
|
+
// Step: a limit on steps (model calls) per input: pi's loop has none, and a model that
|
|
2
2
|
// keeps calling tools would run on (it did once: 55 rounds). At the limit the run is stopped and the agent says so.
|
|
3
3
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
4
4
|
|