agello 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -7
- package/bin/agello.ts +113 -23
- package/package.json +1 -1
- package/src/screencast.ts +103 -10
- package/src/server.ts +88 -9
- package/web/agent-bridge.js +224 -6
package/README.md
CHANGED
|
@@ -38,16 +38,19 @@ Run inside the herdr pane where the agent is running (`$HERDR_PANE_ID` is picked
|
|
|
38
38
|
|
|
39
39
|
```sh
|
|
40
40
|
agello start --open # start in the background and open the page
|
|
41
|
-
agello status # list running servers
|
|
42
|
-
agello stop # stop
|
|
41
|
+
agello status # list running servers (pane, agent state, screen browser)
|
|
42
|
+
agello stop # stop this pane's server
|
|
43
43
|
```
|
|
44
44
|
|
|
45
45
|
Or target another pane:
|
|
46
46
|
|
|
47
47
|
```sh
|
|
48
|
-
agello start --pane w1:p2
|
|
48
|
+
agello start --pane w1:p2
|
|
49
|
+
agello stop --pane w1:p2
|
|
49
50
|
```
|
|
50
51
|
|
|
52
|
+
One server per pane. `start` reuses the pane's server if it is already running, and otherwise picks the first free port from 8765 (so servers for different panes never collide). The screen panel only uses the terminal-browser given with `--browser` or the one in the agent's own herdr tab, never another tab's.
|
|
53
|
+
|
|
51
54
|
Messages sent from the page arrive in the agent as:
|
|
52
55
|
|
|
53
56
|
```
|
|
@@ -61,14 +64,29 @@ The approve / reject buttons send `action=approve` / `action=reject`.
|
|
|
61
64
|
| Command | Description |
|
|
62
65
|
|---|---|
|
|
63
66
|
| `agello start [options]` | Start a server in the background |
|
|
64
|
-
| `agello stop [--port N \| --all]` | Stop a server |
|
|
65
|
-
| `agello status [--json]` | List running servers
|
|
66
|
-
| `agello open [--port N]` | Open the page
|
|
67
|
+
| `agello stop [--port N \| --pane ID \| --all]` | Stop a server (default: this pane's) |
|
|
68
|
+
| `agello status [--json]` | List running servers, agent state and screen browser |
|
|
69
|
+
| `agello open [--port N \| --pane ID]` | Open the page (default: this pane's server) |
|
|
70
|
+
| `agello present [x,y,w,h] [--port N \| --pane ID]` | Presentation mode (see below) |
|
|
71
|
+
| `agello present stop` | End presentation mode |
|
|
67
72
|
|
|
68
|
-
`start` options: `--pane <id>`, `--port <n>` (default 8765), `--session <id>`, `--browser <terminal-browser key>`, `--allow-origin <origin>` (repeatable), `--open`, `--foreground`.
|
|
73
|
+
`start` options: `--pane <id>`, `--port <n>` (default: first free from 8765), `--session <id>`, `--browser <terminal-browser key>`, `--allow-origin <origin>` (repeatable), `--open`, `--foreground`.
|
|
69
74
|
|
|
70
75
|
State and logs: `~/.local/state/agello/<port>.json`, `<port>.log`.
|
|
71
76
|
|
|
77
|
+
### Presentation mode
|
|
78
|
+
|
|
79
|
+
`agello present x,y,w,h` (CSS px of the browser's visible viewport; omit for the whole screen):
|
|
80
|
+
|
|
81
|
+
- only that rectangle is sent to the page (`Page.captureScreenshot` with a clip, at device resolution)
|
|
82
|
+
- the screen panel expands to fill the window
|
|
83
|
+
- agent replies appear over it as bubbles that fade after 10s
|
|
84
|
+
- `agello present stop` returns to the normal layout; the replies are in the chat as usual
|
|
85
|
+
- the viewer can ask a question with the raise-hand button (its bubble fades after 10s too) or end the presentation with the end button; the agent then receives `[browser] action=present-stop`
|
|
86
|
+
- raising the hand tells the agent at once (`action=hand-raise`) so it can pause; closing the input without asking sends `action=hand-lower`
|
|
87
|
+
|
|
88
|
+
Run `present` again to move the crop. User control is off while presenting.
|
|
89
|
+
|
|
72
90
|
## Embed
|
|
73
91
|
|
|
74
92
|
```html
|
package/bin/agello.ts
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
import { spawn } from "node:child_process";
|
|
5
5
|
import { existsSync, mkdirSync, openSync, readdirSync, rmSync } from "node:fs";
|
|
6
|
+
import { createServer } from "node:net";
|
|
6
7
|
import { homedir } from "node:os";
|
|
7
8
|
import { join } from "node:path";
|
|
8
9
|
import { parseArgs } from "node:util";
|
|
@@ -11,22 +12,28 @@ import { startServer } from "../src/server.ts";
|
|
|
11
12
|
|
|
12
13
|
const NAME = "agello";
|
|
13
14
|
const DEFAULT_PORT = 8765;
|
|
15
|
+
const PORT_RANGE_END = 8799; // auto-picked ports: DEFAULT_PORT..PORT_RANGE_END
|
|
14
16
|
const STATE_DIR = join(process.env.XDG_STATE_HOME ?? join(homedir(), ".local", "state"), NAME);
|
|
15
17
|
|
|
16
18
|
const HELP = `${NAME} ${pkg.version}: talk to a coding agent in a herdr pane from the browser
|
|
17
19
|
|
|
18
20
|
Usage:
|
|
19
|
-
${NAME} start [options] Start a bridge server in the background
|
|
20
|
-
${NAME} stop [--port N|--all]
|
|
21
|
+
${NAME} start [options] Start a bridge server for a pane in the background
|
|
22
|
+
${NAME} stop [--port N|--pane ID|--all] Stop a server (default: this pane's)
|
|
21
23
|
${NAME} status [--json] List running servers
|
|
22
|
-
${NAME} open [--port N]
|
|
24
|
+
${NAME} open [--port N|--pane ID] Open the page (default: this pane's server)
|
|
25
|
+
${NAME} present [x,y,w,h] [--port N|--pane ID] Presentation mode: the page shows only the screen,
|
|
26
|
+
full size, cropped to the rect (CSS px of the visible viewport; omit for the
|
|
27
|
+
whole screen), with agent replies as bubbles that fade after 10s.
|
|
28
|
+
Run again to move the crop.
|
|
29
|
+
${NAME} present stop [--port N|--pane ID] End presentation mode (bubbles stay in the chat)
|
|
23
30
|
${NAME} help | --version
|
|
24
31
|
|
|
25
32
|
start options:
|
|
26
33
|
--pane <id> herdr pane of the agent (default: $HERDR_PANE_ID)
|
|
27
|
-
--port <n> port (default: ${DEFAULT_PORT})
|
|
34
|
+
--port <n> port (default: first free port from ${DEFAULT_PORT})
|
|
28
35
|
--session <id> expected agent session id (default: session seen at start)
|
|
29
|
-
--browser <key> terminal-browser key for the screen panel (default: same herdr tab)
|
|
36
|
+
--browser <key> terminal-browser key for the screen panel (default: the one in the same herdr tab)
|
|
30
37
|
--allow-origin <o> extra allowed origin for embedding, repeatable ("null" for file://)
|
|
31
38
|
--open open the page after starting
|
|
32
39
|
--foreground run in this process instead of the background
|
|
@@ -40,6 +47,7 @@ type Instance = {
|
|
|
40
47
|
port: number;
|
|
41
48
|
pane: string;
|
|
42
49
|
session?: string;
|
|
50
|
+
browser?: string; // --browser given at start (unset: same herdr tab)
|
|
43
51
|
url: string;
|
|
44
52
|
log: string;
|
|
45
53
|
startedAt: string;
|
|
@@ -89,6 +97,33 @@ async function fetchStatus(url: string): Promise<any | null> {
|
|
|
89
97
|
}
|
|
90
98
|
}
|
|
91
99
|
|
|
100
|
+
function portFree(port: number): Promise<boolean> {
|
|
101
|
+
return new Promise((resolve) => {
|
|
102
|
+
const s = createServer();
|
|
103
|
+
s.once("error", () => resolve(false));
|
|
104
|
+
s.listen(port, "127.0.0.1", () => s.close(() => resolve(true)));
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Which server a stop/open without --all targets: --port, else --pane,
|
|
109
|
+
// else this pane's ($HERDR_PANE_ID), else the only running one.
|
|
110
|
+
async function resolveTarget(o: { port?: string; pane?: string }): Promise<Instance> {
|
|
111
|
+
const all = await listInstances();
|
|
112
|
+
if (o.port) {
|
|
113
|
+
const i = all.find((x) => x.port === Number(o.port));
|
|
114
|
+
return i ?? fail(`nothing running on port ${o.port}`);
|
|
115
|
+
}
|
|
116
|
+
const pane = o.pane ?? process.env.HERDR_PANE_ID;
|
|
117
|
+
if (pane) {
|
|
118
|
+
const i = all.find((x) => x.pane === pane);
|
|
119
|
+
if (i) return i;
|
|
120
|
+
if (o.pane) fail(`nothing running for pane ${pane}`);
|
|
121
|
+
}
|
|
122
|
+
if (all.length === 1) return all[0];
|
|
123
|
+
if (!all.length) fail(`not running; run \`${NAME} start\``);
|
|
124
|
+
return fail(`several servers running; pass --port or --pane:\n${all.map((i) => ` ${i.url} pane=${i.pane}`).join("\n")}`);
|
|
125
|
+
}
|
|
126
|
+
|
|
92
127
|
function openUrl(url: string) {
|
|
93
128
|
const cmd = process.platform === "darwin" ? "open" : process.platform === "win32" ? "start" : "xdg-open";
|
|
94
129
|
Bun.spawn([cmd, url], { stdout: "ignore", stderr: "ignore" });
|
|
@@ -106,7 +141,7 @@ async function cmdStart(argv: string[]) {
|
|
|
106
141
|
args: argv,
|
|
107
142
|
options: {
|
|
108
143
|
pane: { type: "string", default: process.env.HERDR_PANE_ID },
|
|
109
|
-
port: { type: "string"
|
|
144
|
+
port: { type: "string" },
|
|
110
145
|
session: { type: "string" },
|
|
111
146
|
browser: { type: "string" },
|
|
112
147
|
"allow-origin": { type: "string", multiple: true, default: [] },
|
|
@@ -114,18 +149,33 @@ async function cmdStart(argv: string[]) {
|
|
|
114
149
|
foreground: { type: "boolean", default: false },
|
|
115
150
|
},
|
|
116
151
|
});
|
|
117
|
-
const port = Number(o.port);
|
|
118
152
|
if (!o.pane) fail("--pane is required (HERDR_PANE_ID is not set; run inside herdr or pass --pane)");
|
|
119
153
|
if (!Bun.which("herdr")) fail("herdr not found on PATH");
|
|
120
154
|
mkdirSync(STATE_DIR, { recursive: true });
|
|
155
|
+
const pane = o.pane!;
|
|
121
156
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
157
|
+
// One server per pane: reuse this pane's server, never another pane's.
|
|
158
|
+
const all = await listInstances();
|
|
159
|
+
const mine = all.find((i) => i.pane === pane);
|
|
160
|
+
if (mine) {
|
|
161
|
+
console.log(`already running: ${mine.url} (pane ${mine.pane}, pid ${mine.pid})`);
|
|
162
|
+
if (o.open) openUrl(mine.url);
|
|
126
163
|
return;
|
|
127
164
|
}
|
|
128
165
|
|
|
166
|
+
let port: number;
|
|
167
|
+
if (o.port) {
|
|
168
|
+
port = Number(o.port);
|
|
169
|
+
const other = all.find((i) => i.port === port);
|
|
170
|
+
if (other) fail(`port ${port} is used by the server for pane ${other.pane} (${other.url}); omit --port to pick a free one`);
|
|
171
|
+
if (!(await portFree(port))) fail(`port ${port} is in use by another program`);
|
|
172
|
+
} else {
|
|
173
|
+
let found = 0;
|
|
174
|
+
for (let p = DEFAULT_PORT; p <= PORT_RANGE_END && !found; p++)
|
|
175
|
+
if (!all.some((i) => i.port === p) && (await portFree(p))) found = p;
|
|
176
|
+
port = found || fail(`no free port in ${DEFAULT_PORT}-${PORT_RANGE_END}; pass --port`);
|
|
177
|
+
}
|
|
178
|
+
|
|
129
179
|
if (o.foreground) {
|
|
130
180
|
const srv = await startServer({
|
|
131
181
|
port,
|
|
@@ -139,6 +189,7 @@ async function cmdStart(argv: string[]) {
|
|
|
139
189
|
port,
|
|
140
190
|
pane: srv.pane,
|
|
141
191
|
session: srv.session,
|
|
192
|
+
browser: o.browser,
|
|
142
193
|
url: srv.url,
|
|
143
194
|
log: logPath(port),
|
|
144
195
|
startedAt: new Date().toISOString(),
|
|
@@ -157,7 +208,10 @@ async function cmdStart(argv: string[]) {
|
|
|
157
208
|
}
|
|
158
209
|
|
|
159
210
|
// Background: re-run this CLI with --foreground, detached, logging to a file.
|
|
160
|
-
const args =
|
|
211
|
+
const args = ["--pane", pane, "--port", String(port)];
|
|
212
|
+
if (o.session) args.push("--session", o.session);
|
|
213
|
+
if (o.browser) args.push("--browser", o.browser);
|
|
214
|
+
for (const origin of o["allow-origin"] as string[]) args.push("--allow-origin", origin);
|
|
161
215
|
const log = openSync(logPath(port), "a");
|
|
162
216
|
const child = spawn(process.execPath, [import.meta.path, "start", "--foreground", ...args], {
|
|
163
217
|
detached: true,
|
|
@@ -171,7 +225,7 @@ async function cmdStart(argv: string[]) {
|
|
|
171
225
|
while (Date.now() < deadline) {
|
|
172
226
|
if (child.exitCode !== null || !isAlive(child.pid!)) fail(`server exited; see ${logPath(port)}`);
|
|
173
227
|
if ((await readInstance(port)) && (await fetchStatus(url))) {
|
|
174
|
-
console.log(`started: ${url} (pane ${
|
|
228
|
+
console.log(`started: ${url} (pane ${pane}, pid ${child.pid})`);
|
|
175
229
|
console.log(`log: ${logPath(port)}`);
|
|
176
230
|
if (o.open) openUrl(url);
|
|
177
231
|
return;
|
|
@@ -184,11 +238,9 @@ async function cmdStart(argv: string[]) {
|
|
|
184
238
|
async function cmdStop(argv: string[]) {
|
|
185
239
|
const { values: o } = parseArgs({
|
|
186
240
|
args: argv,
|
|
187
|
-
options: { port: { type: "string" }, all: { type: "boolean", default: false } },
|
|
241
|
+
options: { port: { type: "string" }, pane: { type: "string" }, all: { type: "boolean", default: false } },
|
|
188
242
|
});
|
|
189
|
-
const targets = o.all
|
|
190
|
-
? await listInstances()
|
|
191
|
-
: [await readInstance(Number(o.port ?? DEFAULT_PORT))].filter((i): i is Instance => !!i);
|
|
243
|
+
const targets = o.all ? await listInstances() : [await resolveTarget(o)];
|
|
192
244
|
if (!targets.length) {
|
|
193
245
|
console.log("not running");
|
|
194
246
|
return;
|
|
@@ -219,16 +271,51 @@ async function cmdStatus(argv: string[]) {
|
|
|
219
271
|
const a = r.agent;
|
|
220
272
|
const state = !a ? "unreachable" : a.alive ? a.status : `dead (${a.reason})`;
|
|
221
273
|
const name = a?.label ? `${a.label} (${r.pane})` : r.pane;
|
|
222
|
-
|
|
274
|
+
const scr = a?.screen;
|
|
275
|
+
const browser = scr?.browser ?? (scr?.reason ? `none (${scr.reason})` : "unknown");
|
|
276
|
+
console.log(`${r.url} pane=${name} agent=${state} browser=${browser} pid=${r.pid}`);
|
|
223
277
|
}
|
|
224
278
|
}
|
|
225
279
|
|
|
226
280
|
async function cmdOpen(argv: string[]) {
|
|
227
|
-
const { values: o } = parseArgs({ args: argv, options: { port: { type: "string" } } });
|
|
228
|
-
const inst = await
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
281
|
+
const { values: o } = parseArgs({ args: argv, options: { port: { type: "string" }, pane: { type: "string" } } });
|
|
282
|
+
const inst = await resolveTarget(o);
|
|
283
|
+
openUrl(inst.url);
|
|
284
|
+
console.log(inst.url);
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
async function cmdPresent(argv: string[]) {
|
|
288
|
+
const { values: o, positionals } = parseArgs({
|
|
289
|
+
args: argv,
|
|
290
|
+
options: { port: { type: "string" }, pane: { type: "string" } },
|
|
291
|
+
allowPositionals: true,
|
|
292
|
+
});
|
|
293
|
+
const inst = await resolveTarget(o);
|
|
294
|
+
const arg = positionals.join(",");
|
|
295
|
+
let body: object;
|
|
296
|
+
if (arg === "stop") body = { stop: true };
|
|
297
|
+
else if (!arg) body = {};
|
|
298
|
+
else {
|
|
299
|
+
const v = arg.split(/[\s,]+/).filter(Boolean).map(Number);
|
|
300
|
+
if (v.length !== 4 || !v.every(Number.isFinite)) fail(`rect must be x,y,w,h (got "${arg}")`);
|
|
301
|
+
body = { rect: { x: v[0], y: v[1], w: v[2], h: v[3] } };
|
|
302
|
+
}
|
|
303
|
+
let res: any;
|
|
304
|
+
try {
|
|
305
|
+
const r = await fetch(`${inst.url}/present`, {
|
|
306
|
+
method: "POST",
|
|
307
|
+
headers: { "Content-Type": "application/json" },
|
|
308
|
+
body: JSON.stringify(body),
|
|
309
|
+
signal: AbortSignal.timeout(4000),
|
|
310
|
+
});
|
|
311
|
+
res = await r.json();
|
|
312
|
+
} catch {
|
|
313
|
+
fail(`server unreachable: ${inst.url}`);
|
|
314
|
+
}
|
|
315
|
+
if (!res?.ok) fail(res?.error ?? "failed");
|
|
316
|
+
const p = res.present;
|
|
317
|
+
if (!p.on) console.log("presentation: off");
|
|
318
|
+
else console.log(`presentation: on ${p.rect ? `rect=${p.rect.x},${p.rect.y},${p.rect.w},${p.rect.h}` : "(whole screen)"}`);
|
|
232
319
|
}
|
|
233
320
|
|
|
234
321
|
// ---------- main ----------
|
|
@@ -247,6 +334,9 @@ switch (cmd) {
|
|
|
247
334
|
case "open":
|
|
248
335
|
await cmdOpen(rest);
|
|
249
336
|
break;
|
|
337
|
+
case "present":
|
|
338
|
+
await cmdPresent(rest);
|
|
339
|
+
break;
|
|
250
340
|
case "--version":
|
|
251
341
|
case "-v":
|
|
252
342
|
console.log(pkg.version);
|
package/package.json
CHANGED
package/src/screencast.ts
CHANGED
|
@@ -14,8 +14,12 @@ type Browser = {
|
|
|
14
14
|
tabs: { id: number; url: string; title: string; active: boolean; targetId: string; agentControlled: boolean }[];
|
|
15
15
|
};
|
|
16
16
|
|
|
17
|
+
// Presentation crop: a rectangle of the visible viewport, in CSS pixels.
|
|
18
|
+
export type Rect = { x: number; y: number; w: number; h: number };
|
|
19
|
+
|
|
17
20
|
export type ScreenMeta = {
|
|
18
21
|
connected: boolean;
|
|
22
|
+
reason?: "browser_not_found" | "no_browser_in_tab" | "tab_unknown" | "no_active_tab";
|
|
19
23
|
browser?: string;
|
|
20
24
|
url?: string;
|
|
21
25
|
title?: string;
|
|
@@ -27,11 +31,15 @@ export function createScreencast(opts: { browserKey?: string; herdrTab?: () => P
|
|
|
27
31
|
const enc = new TextEncoder();
|
|
28
32
|
|
|
29
33
|
let meta: ScreenMeta = { connected: false };
|
|
30
|
-
|
|
34
|
+
type Frame = { data: string; w: number; h: number; clip?: Rect };
|
|
35
|
+
let lastFrame: Frame | null = null; // full screencast frame
|
|
36
|
+
let lastCrop: Frame | null = null; // cropped frame while a clip is set
|
|
37
|
+
let clip: Rect | null = null;
|
|
31
38
|
let ws: WebSocket | null = null;
|
|
32
39
|
let wsTarget = "";
|
|
33
40
|
let msgId = 0;
|
|
34
41
|
let timer: Timer | null = null;
|
|
42
|
+
const pending = new Map<number, (result: any) => void>();
|
|
35
43
|
|
|
36
44
|
const emit = (c: ReadableStreamDefaultController, event: string, data: unknown) => {
|
|
37
45
|
try {
|
|
@@ -53,17 +61,83 @@ export function createScreencast(opts: { browserKey?: string; herdrTab?: () => P
|
|
|
53
61
|
return (await runJson(["terminal-browser", "ls", "--all", "--json"], 3000))?.browsers ?? [];
|
|
54
62
|
}
|
|
55
63
|
|
|
56
|
-
|
|
64
|
+
// Only ever relay (and forward input to) the intended browser: the one given
|
|
65
|
+
// with --browser, or one in the agent's own herdr tab. Never fall back to
|
|
66
|
+
// another tab's browser: that would expose another agent's screen and let
|
|
67
|
+
// "my control" type into it.
|
|
68
|
+
async function pickBrowser(): Promise<{ browser?: Browser; reason?: ScreenMeta["reason"] }> {
|
|
57
69
|
const all = await listBrowsers();
|
|
58
|
-
if (opts.browserKey)
|
|
70
|
+
if (opts.browserKey) {
|
|
71
|
+
const browser = all.find((b) => b.key === opts.browserKey);
|
|
72
|
+
return browser ? { browser } : { reason: "browser_not_found" };
|
|
73
|
+
}
|
|
59
74
|
const tab = await opts.herdrTab?.();
|
|
60
|
-
|
|
75
|
+
if (!tab) return { reason: "tab_unknown" };
|
|
76
|
+
const browser = all.find((b) => b.pane?.tab === tab);
|
|
77
|
+
return browser ? { browser } : { reason: "no_browser_in_tab" };
|
|
61
78
|
}
|
|
62
79
|
|
|
63
80
|
function disconnect() {
|
|
64
81
|
ws?.close();
|
|
65
82
|
ws = null;
|
|
66
83
|
wsTarget = "";
|
|
84
|
+
pending.forEach((r) => r(null));
|
|
85
|
+
pending.clear();
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// CDP call that resolves with the result (null on error or closed socket).
|
|
89
|
+
function request(method: string, params: object = {}): Promise<any> {
|
|
90
|
+
const sock = ws;
|
|
91
|
+
if (!sock || sock.readyState !== WebSocket.OPEN) return Promise.resolve(null);
|
|
92
|
+
return new Promise((resolve) => {
|
|
93
|
+
const id = ++msgId;
|
|
94
|
+
const t = setTimeout(() => pending.delete(id) && resolve(null), 5000);
|
|
95
|
+
pending.set(id, (r) => (clearTimeout(t), resolve(r)));
|
|
96
|
+
sock.send(JSON.stringify({ id, method, params }));
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// While a clip is set, only the clipped region is sent to viewers: every
|
|
101
|
+
// screencast frame (= the page changed) triggers one Page.captureScreenshot
|
|
102
|
+
// of the rectangle at full device resolution. At most one capture runs at a
|
|
103
|
+
// time; changes during a capture are coalesced into one more capture.
|
|
104
|
+
let capturing = false;
|
|
105
|
+
let dirty = false;
|
|
106
|
+
async function capture() {
|
|
107
|
+
if (!clip) return;
|
|
108
|
+
if (capturing) {
|
|
109
|
+
dirty = true;
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
capturing = true;
|
|
113
|
+
try {
|
|
114
|
+
do {
|
|
115
|
+
dirty = false;
|
|
116
|
+
const c = clip;
|
|
117
|
+
if (!c) break;
|
|
118
|
+
// clip coordinates are document-relative; the rect is viewport-relative
|
|
119
|
+
const vv = (await request("Page.getLayoutMetrics"))?.cssVisualViewport;
|
|
120
|
+
if (!vv) break;
|
|
121
|
+
const shot = await request("Page.captureScreenshot", {
|
|
122
|
+
format: "jpeg",
|
|
123
|
+
quality: 85,
|
|
124
|
+
clip: { x: c.x + vv.pageX, y: c.y + vv.pageY, width: c.w, height: c.h, scale: 1 },
|
|
125
|
+
});
|
|
126
|
+
if (!shot?.data || clip !== c) continue;
|
|
127
|
+
lastCrop = { data: shot.data, w: c.w, h: c.h, clip: c };
|
|
128
|
+
broadcast("frame", lastCrop);
|
|
129
|
+
await Bun.sleep(100); // cap at ~10 captures/s
|
|
130
|
+
} while (dirty && clip);
|
|
131
|
+
} finally {
|
|
132
|
+
capturing = false;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function setClip(next: Rect | null) {
|
|
137
|
+
clip = next;
|
|
138
|
+
lastCrop = null;
|
|
139
|
+
if (clip) capture();
|
|
140
|
+
else if (lastFrame) broadcast("frame", lastFrame);
|
|
67
141
|
}
|
|
68
142
|
|
|
69
143
|
function connect(port: number, targetId: string) {
|
|
@@ -78,14 +152,22 @@ export function createScreencast(opts: { browserKey?: string; herdrTab?: () => P
|
|
|
78
152
|
sock.onopen = () => {
|
|
79
153
|
call("Page.enable");
|
|
80
154
|
call("Page.startScreencast", { format: "jpeg", quality: 70, maxWidth: 1600, maxHeight: 1600 });
|
|
155
|
+
if (clip) capture();
|
|
81
156
|
};
|
|
82
157
|
sock.onmessage = (ev) => {
|
|
83
158
|
const msg = JSON.parse(String(ev.data));
|
|
159
|
+
if (msg.id && pending.has(msg.id)) {
|
|
160
|
+
const resolve = pending.get(msg.id)!;
|
|
161
|
+
pending.delete(msg.id);
|
|
162
|
+
resolve(msg.result ?? null);
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
84
165
|
if (msg.method !== "Page.screencastFrame") return;
|
|
85
166
|
const { data, metadata, sessionId } = msg.params;
|
|
86
167
|
call("Page.screencastFrameAck", { sessionId });
|
|
87
168
|
lastFrame = { data, w: metadata.deviceWidth, h: metadata.deviceHeight };
|
|
88
|
-
|
|
169
|
+
if (clip) capture();
|
|
170
|
+
else broadcast("frame", lastFrame);
|
|
89
171
|
};
|
|
90
172
|
sock.onclose = () => {
|
|
91
173
|
if (ws === sock) {
|
|
@@ -107,12 +189,13 @@ export function createScreencast(opts: { browserKey?: string; herdrTab?: () => P
|
|
|
107
189
|
}
|
|
108
190
|
|
|
109
191
|
async function syncOnce() {
|
|
110
|
-
const b = await pickBrowser();
|
|
192
|
+
const { browser: b, reason } = await pickBrowser();
|
|
111
193
|
const tab = b?.tabs.find((t) => t.active);
|
|
112
194
|
if (!b || !tab) {
|
|
113
195
|
disconnect();
|
|
114
196
|
lastFrame = null;
|
|
115
|
-
|
|
197
|
+
lastCrop = null;
|
|
198
|
+
setMeta({ connected: false, reason: reason ?? "no_active_tab" });
|
|
116
199
|
return;
|
|
117
200
|
}
|
|
118
201
|
const url = `ws://127.0.0.1:${b.cdpPort}/devtools/page/${tab.targetId}`;
|
|
@@ -136,6 +219,7 @@ export function createScreencast(opts: { browserKey?: string; herdrTab?: () => P
|
|
|
136
219
|
timer = null;
|
|
137
220
|
disconnect();
|
|
138
221
|
lastFrame = null;
|
|
222
|
+
lastCrop = null;
|
|
139
223
|
meta = { connected: false };
|
|
140
224
|
}
|
|
141
225
|
|
|
@@ -148,7 +232,8 @@ export function createScreencast(opts: { browserKey?: string; herdrTab?: () => P
|
|
|
148
232
|
ctrl = c;
|
|
149
233
|
viewers.add(c);
|
|
150
234
|
emit(c, "meta", meta);
|
|
151
|
-
|
|
235
|
+
const f = clip ? lastCrop : lastFrame;
|
|
236
|
+
if (f) emit(c, "frame", f);
|
|
152
237
|
ping = setInterval(() => emit(c, "ping", {}), 15000);
|
|
153
238
|
start();
|
|
154
239
|
},
|
|
@@ -171,9 +256,17 @@ export function createScreencast(opts: { browserKey?: string; herdrTab?: () => P
|
|
|
171
256
|
} catch {
|
|
172
257
|
return;
|
|
173
258
|
}
|
|
174
|
-
|
|
259
|
+
// input coordinates assume the full frame: ignore input while cropped
|
|
260
|
+
if (clip || !INPUT_METHODS.has(m?.method) || !ws || ws.readyState !== WebSocket.OPEN) return;
|
|
175
261
|
ws.send(JSON.stringify({ id: ++msgId, method: m.method, params: m.params ?? {} }));
|
|
176
262
|
}
|
|
177
263
|
|
|
178
|
-
|
|
264
|
+
// Which browser the relay uses (or would use), for status reporting.
|
|
265
|
+
async function describe(): Promise<ScreenMeta> {
|
|
266
|
+
if (meta.connected) return meta;
|
|
267
|
+
const { browser, reason } = await pickBrowser();
|
|
268
|
+
return browser ? { connected: false, browser: browser.key } : { connected: false, reason };
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
return { handle, input, describe, setClip };
|
|
179
272
|
}
|
package/src/server.ts
CHANGED
|
@@ -7,11 +7,15 @@
|
|
|
7
7
|
// POST /send {action, text} -> herdr agent prompt
|
|
8
8
|
// GET /screen SSE: terminal-browser screen frames (CDP screencast)
|
|
9
9
|
// WS /input user control: CDP Input.* commands forwarded to the page
|
|
10
|
+
// GET /present presentation state
|
|
11
|
+
// POST /present {rect?: {x,y,w,h}} start (or move the crop), {stop: true} end
|
|
12
|
+
// ({stop: true, from: "viewer"}: ended from the page, the agent is told
|
|
13
|
+
// with "[browser] action=present-stop")
|
|
10
14
|
|
|
11
15
|
import { homedir } from "node:os";
|
|
12
16
|
import { join } from "node:path";
|
|
13
17
|
import { runJson } from "./run.ts";
|
|
14
|
-
import { createScreencast } from "./screencast.ts";
|
|
18
|
+
import { createScreencast, type Rect } from "./screencast.ts";
|
|
15
19
|
|
|
16
20
|
export type ServerOptions = {
|
|
17
21
|
port: number;
|
|
@@ -58,11 +62,36 @@ function toolSummary(input: any): string {
|
|
|
58
62
|
return text.length > 80 ? text.slice(0, 80) + "…" : text;
|
|
59
63
|
}
|
|
60
64
|
|
|
65
|
+
// Some models emit user-facing narration as a `thinking` block whose signature
|
|
66
|
+
// carries a "narration" marker. Claude Code renders these like normal replies,
|
|
67
|
+
// so treat them as assistant text. Plain (unmarked) thinking stays hidden.
|
|
68
|
+
// The marker is an undocumented detail of the signature; if it changes, these
|
|
69
|
+
// blocks are simply not shown (same as before).
|
|
70
|
+
export function isNarration(b: any): boolean {
|
|
71
|
+
if (b?.type !== "thinking" || typeof b.thinking !== "string" || !b.thinking.trim()) return false;
|
|
72
|
+
try {
|
|
73
|
+
return Buffer.from(String(b.signature ?? "").slice(0, 200), "base64").includes("narration");
|
|
74
|
+
} catch {
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
61
79
|
// Every herdr call is killed after `timeout` ms so a hung herdr cannot pile up processes.
|
|
62
80
|
async function herdr(timeout: number, ...cmd: string[]): Promise<any> {
|
|
63
81
|
return (await runJson(["herdr", ...cmd], timeout)) ?? { error: { code: "herdr_failed" } };
|
|
64
82
|
}
|
|
65
83
|
|
|
84
|
+
// Presentation rect from the API: finite, non-negative origin, positive size (CSS px).
|
|
85
|
+
export function parseRect(r: any): Rect | null {
|
|
86
|
+
if (!r || typeof r !== "object") return null;
|
|
87
|
+
const v = [r.x, r.y, r.w, r.h].map(Number);
|
|
88
|
+
if (!v.every(Number.isFinite) || v[0] < 0 || v[1] < 0 || v[2] < 1 || v[3] < 1) return null;
|
|
89
|
+
const [x, y, w, h] = v.map(Math.round);
|
|
90
|
+
return { x, y, w, h };
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export type Present = { on: boolean; rect?: Rect; since?: string };
|
|
94
|
+
|
|
66
95
|
type Status =
|
|
67
96
|
| { alive: false; reason: string; pane: string }
|
|
68
97
|
| { alive: true; status: string; pane: string; label?: string; session: string; title?: string };
|
|
@@ -229,6 +258,8 @@ export async function startServer(opts: ServerOptions) {
|
|
|
229
258
|
for (const b of content) {
|
|
230
259
|
if (b.type === "text" && b.text.trim()) {
|
|
231
260
|
broadcast("message", { role: "assistant", text: b.text, ts: d.timestamp });
|
|
261
|
+
} else if (isNarration(b)) {
|
|
262
|
+
broadcast("message", { role: "assistant", text: b.thinking, ts: d.timestamp, narration: true });
|
|
232
263
|
} else if (b.type === "tool_use") {
|
|
233
264
|
const t = { id: b.id, name: b.name, summary: toolSummary(b.input), ts: d.timestamp };
|
|
234
265
|
pendingTools.set(b.id, t);
|
|
@@ -275,6 +306,40 @@ export async function startServer(opts: ServerOptions) {
|
|
|
275
306
|
herdrTab: async () => (await agentInfo())?.tab_id,
|
|
276
307
|
});
|
|
277
308
|
|
|
309
|
+
// ----- presentation mode -----
|
|
310
|
+
// on: viewers show only the screen, full size, with agent replies as
|
|
311
|
+
// short-lived bubbles. rect: crop of the visible viewport (unset = whole screen).
|
|
312
|
+
|
|
313
|
+
let present: Present = { on: false };
|
|
314
|
+
function setPresent(next: Present) {
|
|
315
|
+
present = next;
|
|
316
|
+
screen.setClip(next.on ? (next.rect ?? null) : null);
|
|
317
|
+
broadcast("present", present);
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
async function presentRequest(req: Request): Promise<Response> {
|
|
321
|
+
const data = (await req.json().catch(() => ({}))) as { rect?: unknown; stop?: boolean; from?: string };
|
|
322
|
+
if (data.stop) {
|
|
323
|
+
const was = present.on;
|
|
324
|
+
setPresent({ on: false });
|
|
325
|
+
// Ended from the page: the agent may still be presenting, so tell it.
|
|
326
|
+
// (The CLI stop comes from the agent itself and needs no notice.)
|
|
327
|
+
let notified: string | undefined;
|
|
328
|
+
if (was && data.from === "viewer") {
|
|
329
|
+
const r = await promptAgent("present-stop", "사용자가 페이지에서 발표를 종료함");
|
|
330
|
+
notified = r.ok ? "ok" : r.error;
|
|
331
|
+
}
|
|
332
|
+
return Response.json({ ok: true, present, notified });
|
|
333
|
+
}
|
|
334
|
+
let rect: Rect | undefined;
|
|
335
|
+
if (data.rect != null) {
|
|
336
|
+
rect = parseRect(data.rect) ?? undefined;
|
|
337
|
+
if (!rect) return Response.json({ ok: false, error: "invalid_rect" }, { status: 400 });
|
|
338
|
+
}
|
|
339
|
+
setPresent({ on: true, rect, since: present.on ? present.since : new Date().toISOString() });
|
|
340
|
+
return Response.json({ ok: true, present });
|
|
341
|
+
}
|
|
342
|
+
|
|
278
343
|
// ----- HTTP -----
|
|
279
344
|
|
|
280
345
|
// Same-origin and localhost pages are allowed by default; add more with
|
|
@@ -313,6 +378,7 @@ export async function startServer(opts: ServerOptions) {
|
|
|
313
378
|
tools: [...pendingTools.keys()],
|
|
314
379
|
});
|
|
315
380
|
if (lastStatus) send(c, "status", lastStatus);
|
|
381
|
+
send(c, "present", present);
|
|
316
382
|
for (const t of pendingTools.values()) send(c, "tool_start", t);
|
|
317
383
|
ping = setInterval(() => send(c, "ping", {}), 15000);
|
|
318
384
|
},
|
|
@@ -326,22 +392,32 @@ export async function startServer(opts: ServerOptions) {
|
|
|
326
392
|
});
|
|
327
393
|
}
|
|
328
394
|
|
|
329
|
-
|
|
330
|
-
|
|
395
|
+
// Type a [browser] prompt into the agent session.
|
|
396
|
+
async function promptAgent(action: string, text: string): Promise<{ ok: boolean; error?: string; status?: number }> {
|
|
331
397
|
const st = await checkStatus();
|
|
332
|
-
if (!st.alive) return
|
|
333
|
-
if (st.status === "blocked") return
|
|
398
|
+
if (!st.alive) return { ok: false, error: st.reason, status: 409 };
|
|
399
|
+
if (st.status === "blocked") return { ok: false, error: "agent_blocked", status: 409 };
|
|
400
|
+
const res = await herdr(10000, "agent", "prompt", PANE, formatPrompt(action, text.trim()));
|
|
401
|
+
if (res?.error) return { ok: false, error: res.error.code, status: 502 };
|
|
402
|
+
return { ok: true };
|
|
403
|
+
}
|
|
334
404
|
|
|
405
|
+
async function sendPrompt(req: Request): Promise<Response> {
|
|
406
|
+
const data = (await req.json().catch(() => ({}))) as { action?: string; text?: string };
|
|
335
407
|
const action = /^[\w-]{1,32}$/.test(data.action ?? "") ? data.action! : "message";
|
|
336
|
-
const
|
|
337
|
-
|
|
338
|
-
return Response.json({ ok: true });
|
|
408
|
+
const r = await promptAgent(action, data.text ?? "");
|
|
409
|
+
return r.ok ? Response.json({ ok: true }) : Response.json({ ok: false, error: r.error }, { status: r.status });
|
|
339
410
|
}
|
|
340
411
|
|
|
341
412
|
async function route(req: Request, url: URL): Promise<Response> {
|
|
342
413
|
switch (url.pathname) {
|
|
343
414
|
case "/status":
|
|
344
|
-
return Response.json(
|
|
415
|
+
return Response.json({
|
|
416
|
+
...(await checkStatus()),
|
|
417
|
+
browser: opts.browser,
|
|
418
|
+
screen: await screen.describe(),
|
|
419
|
+
present,
|
|
420
|
+
});
|
|
345
421
|
case "/events":
|
|
346
422
|
return events();
|
|
347
423
|
case "/screen":
|
|
@@ -352,6 +428,9 @@ export async function startServer(opts: ServerOptions) {
|
|
|
352
428
|
});
|
|
353
429
|
case "/send":
|
|
354
430
|
if (req.method === "POST") return sendPrompt(req);
|
|
431
|
+
break;
|
|
432
|
+
case "/present":
|
|
433
|
+
return req.method === "POST" ? presentRequest(req) : Response.json(present);
|
|
355
434
|
}
|
|
356
435
|
return new Response(HTML, { headers: { "Content-Type": "text/html; charset=utf-8", "Cache-Control": "no-store" } });
|
|
357
436
|
}
|
package/web/agent-bridge.js
CHANGED
|
@@ -23,10 +23,24 @@
|
|
|
23
23
|
// agent-tool detail: { phase: 'start'|'end', id, name?, summary?, isError? }
|
|
24
24
|
// (tool calls are also kept in the conversation as one-line rows, like the terminal)
|
|
25
25
|
// agent-screen detail: { connected, browser?, url?, title?, agentControlled? }
|
|
26
|
+
// agent-present detail: { on, rect?: {x,y,w,h}, since? }
|
|
27
|
+
//
|
|
28
|
+
// Presentation mode (started by the agent: `agello present [x,y,w,h]`)
|
|
29
|
+
// The screen panel expands to fill the window and shows only the given
|
|
30
|
+
// rectangle of the browser. Agent replies appear over it as bubbles that fade
|
|
31
|
+
// after 10s; they are also added to the chat, so it is all there when the
|
|
32
|
+
// presentation ends. User control is off while presenting.
|
|
33
|
+
// A raise-hand button next to the bubbles opens a small input so the viewer
|
|
34
|
+
// can ask the agent something mid-presentation (sent as action=message).
|
|
35
|
+
// Opening it tells the agent right away (action=hand-raise) so it can pause;
|
|
36
|
+
// closing it without asking sends action=hand-lower so it can go on.
|
|
37
|
+
// The viewer's questions show as bubbles too, with the same 10s lifetime.
|
|
38
|
+
// An end button above it stops the presentation for every viewer.
|
|
26
39
|
//
|
|
27
40
|
// Methods
|
|
28
41
|
// el.send(text, action = 'message') -> Promise<{ok, error?}>
|
|
29
42
|
// el.clearHistory() clear saved chat for the current pane
|
|
43
|
+
// el.stopPresent() end presentation mode, tell the agent -> Promise<{ok, error?, notified?}>
|
|
30
44
|
|
|
31
45
|
const SCRIPT_ORIGIN = new URL(import.meta.url).origin;
|
|
32
46
|
|
|
@@ -47,7 +61,16 @@ const REASONS = {
|
|
|
47
61
|
session_changed: "다른 세션으로 교체됨",
|
|
48
62
|
};
|
|
49
63
|
const STATES = { idle: "대기 중", working: "작업 중", blocked: "입력 대기", done: "완료", unknown: "상태 불명" };
|
|
50
|
-
const
|
|
64
|
+
const SCREEN_REASONS = {
|
|
65
|
+
browser_not_found: "지정한 terminal-browser(--browser)를 찾을 수 없음",
|
|
66
|
+
no_browser_in_tab: "이 herdr 탭에 열린 terminal-browser 없음",
|
|
67
|
+
tab_unknown: "에이전트의 herdr 탭을 확인할 수 없음",
|
|
68
|
+
no_active_tab: "브라우저에 활성 탭 없음",
|
|
69
|
+
};
|
|
70
|
+
const ACTIONS = {
|
|
71
|
+
message: "메시지", approve: "승인", reject: "거절",
|
|
72
|
+
"present-stop": "발표 종료", "hand-raise": "손들기", "hand-lower": "손 내림",
|
|
73
|
+
};
|
|
51
74
|
|
|
52
75
|
const CSS = `
|
|
53
76
|
:host {
|
|
@@ -148,6 +171,60 @@ h1 { font-size: 14px; margin: 0; font-weight: 600; }
|
|
|
148
171
|
.control-btn:focus-visible { outline: 2px solid var(--_accent); outline-offset: 2px; border-radius: 6px; }
|
|
149
172
|
@media (prefers-reduced-motion: reduce) { .control-btn .track, .control-btn .knob { transition: none; } }
|
|
150
173
|
.placeholder { font-size: 13px; color: var(--_muted); text-align: center; }
|
|
174
|
+
|
|
175
|
+
/* presentation mode: the screen panel covers the window, chat stays hidden underneath.
|
|
176
|
+
container-type would make the host the containing block of the fixed layer, so drop it. */
|
|
177
|
+
:host([presenting]) { container-type: normal; }
|
|
178
|
+
.body.present .screen { display: flex; position: fixed; inset: 0; z-index: 2147483000; border: 0;
|
|
179
|
+
width: auto; height: auto; background: #0b0b0c; }
|
|
180
|
+
.body.present .screen-bar, .body.present .ring, .body.present .glow, .body.present .agent-badge { display: none; }
|
|
181
|
+
.body.present .viewport { padding: 0; background: #0b0b0c; }
|
|
182
|
+
.body.present .viewport img { width: 100%; height: 100%; max-width: none; max-height: none; border-radius: 0;
|
|
183
|
+
box-shadow: none; }
|
|
184
|
+
.captions { display: none; }
|
|
185
|
+
.body.present .captions { position: absolute; left: 0; right: 0; bottom: 0; z-index: 4; display: flex;
|
|
186
|
+
flex-direction: column; align-items: center; gap: 10px; padding: 0 88px 28px; pointer-events: none; }
|
|
187
|
+
.caption { max-width: min(900px, 86%); padding: 12px 18px; border-radius: 16px; overflow-wrap: anywhere;
|
|
188
|
+
font-size: 18px; line-height: 1.5; color: #fff; background: rgba(20, 20, 22, .72);
|
|
189
|
+
-webkit-backdrop-filter: blur(16px) saturate(1.3); backdrop-filter: blur(16px) saturate(1.3);
|
|
190
|
+
box-shadow: 0 8px 30px rgba(0, 0, 0, .35); animation: capIn .3s ease; }
|
|
191
|
+
.caption.plain { white-space: pre-wrap; }
|
|
192
|
+
.caption.user { background: color-mix(in srgb, var(--_accent) 88%, transparent); }
|
|
193
|
+
.caption.out { opacity: 0; translate: 0 6px; transition: opacity .5s ease, translate .5s ease; }
|
|
194
|
+
.caption > :first-child { margin-top: 0; } .caption > :last-child { margin-bottom: 0; }
|
|
195
|
+
.caption p, .caption ul, .caption ol { margin: .35em 0; }
|
|
196
|
+
.caption code { font-family: ui-monospace, Menlo, monospace; font-size: .85em; background: rgba(255,255,255,.14);
|
|
197
|
+
padding: 1px 5px; border-radius: 4px; }
|
|
198
|
+
.caption pre { background: rgba(255,255,255,.1); padding: 8px 10px; border-radius: 8px; overflow-x: auto; }
|
|
199
|
+
.caption a { color: inherit; }
|
|
200
|
+
@keyframes capIn { from { opacity: 0; translate: 0 10px; } }
|
|
201
|
+
|
|
202
|
+
/* raise hand: ask the agent mid-presentation */
|
|
203
|
+
.hand-wrap { display: none; }
|
|
204
|
+
.body.present .hand-wrap { position: absolute; right: 22px; bottom: 28px; z-index: 5; display: flex;
|
|
205
|
+
flex-direction: column; align-items: flex-end; gap: 10px; }
|
|
206
|
+
.hand, .end { width: 48px; height: 48px; padding: 0; border-radius: 50%; display: grid; place-items: center;
|
|
207
|
+
color: #fff; border: 1px solid rgba(255,255,255,.18); background: rgba(20, 20, 22, .72);
|
|
208
|
+
-webkit-backdrop-filter: blur(16px); backdrop-filter: blur(16px); box-shadow: 0 8px 30px rgba(0,0,0,.35);
|
|
209
|
+
transition: background .2s ease, transform .2s ease; }
|
|
210
|
+
.hand:hover, .end:hover { background: rgba(40, 40, 44, .85); transform: translateY(-2px); }
|
|
211
|
+
.hand[aria-expanded="true"] { background: var(--_accent); border-color: var(--_accent); }
|
|
212
|
+
.hand.sent { background: var(--_ok); border-color: var(--_ok); }
|
|
213
|
+
.end:hover { background: var(--_dead); border-color: var(--_dead); }
|
|
214
|
+
.end:disabled { opacity: .5; cursor: progress; }
|
|
215
|
+
.hand:focus-visible, .end:focus-visible { outline: 2px solid #fff; outline-offset: 3px; }
|
|
216
|
+
.hand svg, .end svg { width: 22px; height: 22px; }
|
|
217
|
+
.ask { width: min(420px, calc(100vw - 44px)); padding: 12px; border-radius: 16px; color: #fff;
|
|
218
|
+
background: rgba(20, 20, 22, .82); -webkit-backdrop-filter: blur(16px); backdrop-filter: blur(16px);
|
|
219
|
+
box-shadow: 0 8px 30px rgba(0,0,0,.4); animation: capIn .2s ease; }
|
|
220
|
+
.ask[hidden] { display: none; }
|
|
221
|
+
.ask textarea { min-height: 64px; background: rgba(255,255,255,.08); color: #fff;
|
|
222
|
+
border-color: rgba(255,255,255,.16); font-size: 15px; }
|
|
223
|
+
.ask textarea::placeholder { color: rgba(255,255,255,.5); }
|
|
224
|
+
.ask .ask-row { display: flex; align-items: center; gap: 8px; margin-top: 8px; }
|
|
225
|
+
.ask .ask-hint { font-size: 12px; color: rgba(255,255,255,.6); flex: 1; }
|
|
226
|
+
@media (prefers-reduced-motion: reduce) { .hand, .ask { transition: none; animation: none; } }
|
|
227
|
+
@media (prefers-reduced-motion: reduce) { .caption { animation: none; } .caption.out { transition: none; } }
|
|
151
228
|
@container (max-width: 820px) {
|
|
152
229
|
.body { flex-direction: column; }
|
|
153
230
|
:host([screen]) .screen { order: -1; flex: none; height: 45%; border-left: 0; border-bottom: 1px solid var(--_line); }
|
|
@@ -250,7 +327,17 @@ const TEMPLATE = `
|
|
|
250
327
|
<div class="screen-bar"><span class="live" hidden>에이전트 조작 중</span><span class="url"></span>
|
|
251
328
|
<button class="control-btn" role="switch" aria-checked="false" title="켜면 마우스와 키보드 입력이 브라우저로 전달돼요 (끄기: Shift+Esc)">
|
|
252
329
|
<span>내 조작</span><span class="track"><span class="knob"></span></span><span class="state-label">OFF</span></button></div>
|
|
253
|
-
<div class="viewport"><div class="ring"></div><div class="glow"></div><div class="agent-badge">에이전트 조작 중</div><img alt="에이전트 브라우저 화면" draggable="false"><textarea class="kbd" aria-label="브라우저 키보드 입력"></textarea><div class="placeholder">열린 terminal-browser 없음</div></div>
|
|
330
|
+
<div class="viewport"><div class="ring"></div><div class="glow"></div><div class="agent-badge">에이전트 조작 중</div><img alt="에이전트 브라우저 화면" draggable="false"><textarea class="kbd" aria-label="브라우저 키보드 입력"></textarea><div class="placeholder">열린 terminal-browser 없음</div><div class="captions" part="captions" aria-live="polite"></div>
|
|
331
|
+
<div class="hand-wrap">
|
|
332
|
+
<form class="ask" part="ask" hidden><textarea aria-label="에이전트에게 질문" placeholder="에이전트에게 질문 (Enter 전송, Esc 닫기)"></textarea>
|
|
333
|
+
<div class="ask-row"><span class="ask-hint"></span><button type="submit" class="primary">보내기</button></div></form>
|
|
334
|
+
<button class="end" part="end" type="button" aria-label="발표 종료" title="발표 종료">
|
|
335
|
+
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M18 6 6 18"/><path d="m6 6 12 12"/></svg>
|
|
336
|
+
</button>
|
|
337
|
+
<button class="hand" part="hand" type="button" aria-expanded="false" aria-label="손들기: 에이전트에게 질문" title="손들기: 에이전트에게 질문">
|
|
338
|
+
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M18 11V6a2 2 0 0 0-2-2a2 2 0 0 0-2 2"/><path d="M14 10V4a2 2 0 0 0-2-2a2 2 0 0 0-2 2v2"/><path d="M10 10.5V6a2 2 0 0 0-2-2a2 2 0 0 0-2 2v8"/><path d="M18 8a2 2 0 1 1 4 0v6a8 8 0 0 1-8 8h-2c-2.8 0-4.5-.86-5.99-2.34l-3.6-3.6a2 2 0 0 1 2.83-2.82L7 15"/></svg>
|
|
339
|
+
</button>
|
|
340
|
+
</div></div>
|
|
254
341
|
</aside></div>`;
|
|
255
342
|
|
|
256
343
|
class AgentBridge extends HTMLElement {
|
|
@@ -267,7 +354,10 @@ class AgentBridge extends HTMLElement {
|
|
|
267
354
|
#md = null;
|
|
268
355
|
#historyKey = null;
|
|
269
356
|
#history = [];
|
|
357
|
+
#present = { on: false };
|
|
270
358
|
static HISTORY_LIMIT = 500;
|
|
359
|
+
static CAPTION_MS = 10000; // presentation bubble lifetime
|
|
360
|
+
static CAPTION_MAX = 3; // bubbles on screen at once (oldest leaves early)
|
|
271
361
|
|
|
272
362
|
constructor() {
|
|
273
363
|
super();
|
|
@@ -279,6 +369,8 @@ class AgentBridge extends HTMLElement {
|
|
|
279
369
|
buttons: this.#root.querySelectorAll("button[data-action]"),
|
|
280
370
|
live: q(".live"), url: q(".screen-bar .url"), viewport: q(".viewport"), img: q(".viewport img"),
|
|
281
371
|
controlBtn: q(".control-btn"), kbd: q(".kbd"), clearBtn: q(".clear-btn"),
|
|
372
|
+
body: q(".body"), screen: q(".screen"), captions: q(".captions"),
|
|
373
|
+
hand: q(".hand"), end: q(".end"), ask: q(".ask"), askBox: q(".ask textarea"), askHint: q(".ask-hint"),
|
|
282
374
|
};
|
|
283
375
|
this.$.buttons.forEach((b) => b.addEventListener("click", () => this.#submit(b.dataset.action)));
|
|
284
376
|
this.#bindGrip();
|
|
@@ -290,6 +382,7 @@ class AgentBridge extends HTMLElement {
|
|
|
290
382
|
});
|
|
291
383
|
loadMarkdown().then((md) => (this.#md = md));
|
|
292
384
|
this.#bindControl();
|
|
385
|
+
this.#bindHand();
|
|
293
386
|
this.$.clearBtn.addEventListener("click", () => this.clearHistory());
|
|
294
387
|
}
|
|
295
388
|
|
|
@@ -328,6 +421,19 @@ class AgentBridge extends HTMLElement {
|
|
|
328
421
|
}
|
|
329
422
|
}
|
|
330
423
|
|
|
424
|
+
async stopPresent() {
|
|
425
|
+
try {
|
|
426
|
+
const res = await fetch(`${this.server}/present`, {
|
|
427
|
+
method: "POST",
|
|
428
|
+
headers: { "Content-Type": "application/json" },
|
|
429
|
+
body: JSON.stringify({ stop: true, from: "viewer" }),
|
|
430
|
+
});
|
|
431
|
+
return await res.json();
|
|
432
|
+
} catch {
|
|
433
|
+
return { ok: false, error: "server_unreachable" };
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
|
|
331
437
|
#emit(type, detail) {
|
|
332
438
|
this.dispatchEvent(new CustomEvent(type, { detail, bubbles: true, composed: true }));
|
|
333
439
|
}
|
|
@@ -343,7 +449,14 @@ class AgentBridge extends HTMLElement {
|
|
|
343
449
|
on("delivered", ({ id }) => { this.#deliver(id); this.#emit("agent-queue", { phase: "delivered", id }); });
|
|
344
450
|
on("unqueued", ({ id }) => { this.#unqueue(id); this.#emit("agent-queue", { phase: "unqueued", id }); });
|
|
345
451
|
on("status", (s) => this.#renderStatus(s));
|
|
346
|
-
on("message", (m) => {
|
|
452
|
+
on("message", (m) => {
|
|
453
|
+
this.#addMessage(m);
|
|
454
|
+
this.#saveMessage(m);
|
|
455
|
+
if (this.#present.on && m.role === "assistant") this.#caption(m.text);
|
|
456
|
+
if (this.#present.on && m.role === "browser" && m.action === "message") this.#caption(m.text, "user");
|
|
457
|
+
this.#emit("agent-message", m);
|
|
458
|
+
});
|
|
459
|
+
on("present", (p) => this.#setPresent(p));
|
|
347
460
|
on("tool_start", (t) => {
|
|
348
461
|
this.#tools.set(t.id, t);
|
|
349
462
|
this.#renderActivity();
|
|
@@ -360,11 +473,11 @@ class AgentBridge extends HTMLElement {
|
|
|
360
473
|
this.#setToolState(t.id, t.isError ? "error" : "done");
|
|
361
474
|
this.#emit("agent-tool", { phase: "end", ...t });
|
|
362
475
|
});
|
|
363
|
-
es.onerror = () => this.#renderStatus(null);
|
|
476
|
+
es.onerror = () => { this.#renderStatus(null); this.#setPresent({ on: false }); };
|
|
364
477
|
}
|
|
365
478
|
|
|
366
479
|
#syncScreen() {
|
|
367
|
-
const want = this.hasAttribute("screen");
|
|
480
|
+
const want = this.hasAttribute("screen") || this.#present.on;
|
|
368
481
|
if (!want) { this.#screenEs?.close(); this.#screenEs = null; this.#setControl(false); return; }
|
|
369
482
|
if (this.#screenEs) return;
|
|
370
483
|
const es = (this.#screenEs = new EventSource(`${this.server}/screen`));
|
|
@@ -378,6 +491,110 @@ class AgentBridge extends HTMLElement {
|
|
|
378
491
|
es.onerror = () => this.#renderScreenMeta({ connected: false });
|
|
379
492
|
}
|
|
380
493
|
|
|
494
|
+
// ---------- presentation mode ----------
|
|
495
|
+
|
|
496
|
+
#setPresent(p) {
|
|
497
|
+
const on = !!p?.on;
|
|
498
|
+
const was = this.#present.on;
|
|
499
|
+
this.#present = p ?? { on: false };
|
|
500
|
+
this.#emit("agent-present", this.#present);
|
|
501
|
+
if (on === was) return;
|
|
502
|
+
if (on) this.#setControl(false);
|
|
503
|
+
else { this.$.captions.replaceChildren(); this.#openAsk(false); this.#handUp = false; }
|
|
504
|
+
this.#syncScreen();
|
|
505
|
+
// FLIP: grow from (or shrink back to) the side panel
|
|
506
|
+
const { screen, body } = this.$;
|
|
507
|
+
const before = screen.getBoundingClientRect();
|
|
508
|
+
body.classList.toggle("present", on);
|
|
509
|
+
this.toggleAttribute("presenting", on);
|
|
510
|
+
const after = screen.getBoundingClientRect();
|
|
511
|
+
if (matchMedia("(prefers-reduced-motion: reduce)").matches || !before.width || !after.width) return;
|
|
512
|
+
const t = `translate(${before.left - after.left}px, ${before.top - after.top}px) ` +
|
|
513
|
+
`scale(${before.width / after.width}, ${before.height / after.height})`;
|
|
514
|
+
screen.animate([{ transformOrigin: "0 0", transform: t }, { transformOrigin: "0 0", transform: "none" }],
|
|
515
|
+
{ duration: on ? 420 : 360, easing: "cubic-bezier(.2, .7, .2, 1)" });
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
// Raise hand: open a small input over the presentation and send it as a message.
|
|
519
|
+
#bindHand() {
|
|
520
|
+
const { hand, ask, askBox, askHint } = this.$;
|
|
521
|
+
hand.addEventListener("click", () => {
|
|
522
|
+
if (ask.hidden) this.#raiseHand();
|
|
523
|
+
else this.#lowerHand();
|
|
524
|
+
});
|
|
525
|
+
this.$.end.addEventListener("click", async () => {
|
|
526
|
+
this.$.end.disabled = true;
|
|
527
|
+
const res = await this.stopPresent();
|
|
528
|
+
this.$.end.disabled = false;
|
|
529
|
+
if (!res.ok) { this.#openAsk(true); askHint.textContent = `발표 종료 실패: ${res.error}`; }
|
|
530
|
+
});
|
|
531
|
+
ask.addEventListener("submit", async (e) => {
|
|
532
|
+
e.preventDefault();
|
|
533
|
+
const text = askBox.value.trim();
|
|
534
|
+
if (!text) return;
|
|
535
|
+
askHint.textContent = "보내는 중…";
|
|
536
|
+
const res = await this.send(text, "message");
|
|
537
|
+
if (!res.ok) { askHint.textContent = `전송 실패: ${res.error}`; return; }
|
|
538
|
+
askBox.value = "";
|
|
539
|
+
this.#handUp = false; // the question itself ends the raised hand
|
|
540
|
+
this.#openAsk(false);
|
|
541
|
+
hand.classList.add("sent");
|
|
542
|
+
setTimeout(() => hand.classList.remove("sent"), 1500);
|
|
543
|
+
});
|
|
544
|
+
askBox.addEventListener("keydown", (e) => {
|
|
545
|
+
if (e.key === "Enter" && !e.shiftKey && !e.isComposing) { e.preventDefault(); ask.requestSubmit(); }
|
|
546
|
+
if (e.key === "Escape") { e.preventDefault(); this.#lowerHand(); hand.focus(); }
|
|
547
|
+
});
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
// Raise: open the input and tell the agent to pause. Lower (closed without
|
|
551
|
+
// asking): tell it to go on. Failures only show a hint; asking still works.
|
|
552
|
+
#handUp = false;
|
|
553
|
+
async #raiseHand() {
|
|
554
|
+
this.#openAsk(true);
|
|
555
|
+
if (this.#handUp) return;
|
|
556
|
+
this.#handUp = true;
|
|
557
|
+
const res = await this.send("사용자가 손을 듦: 질문 입력 중", "hand-raise");
|
|
558
|
+
if (!res.ok && !this.$.ask.hidden) this.$.askHint.textContent = `손들기 알림 실패: ${res.error}`;
|
|
559
|
+
}
|
|
560
|
+
async #lowerHand() {
|
|
561
|
+
this.#openAsk(false);
|
|
562
|
+
if (!this.#handUp) return;
|
|
563
|
+
this.#handUp = false;
|
|
564
|
+
await this.send("사용자가 질문 없이 손을 내림", "hand-lower");
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
#openAsk(open) {
|
|
568
|
+
const { hand, ask, askBox, askHint } = this.$;
|
|
569
|
+
ask.hidden = !open;
|
|
570
|
+
hand.setAttribute("aria-expanded", String(open));
|
|
571
|
+
askHint.textContent = "";
|
|
572
|
+
if (open) askBox.focus();
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
// Agent reply as a short-lived bubble over the presented screen.
|
|
576
|
+
// kind "user": the viewer's own question (plain text, accent colour).
|
|
577
|
+
#caption(text, kind = "agent") {
|
|
578
|
+
if (!text?.trim()) return;
|
|
579
|
+
const el = document.createElement("div");
|
|
580
|
+
el.className = `caption ${kind}`;
|
|
581
|
+
el.part = "caption";
|
|
582
|
+
if (kind === "user") { el.textContent = text; el.classList.add("plain"); }
|
|
583
|
+
else if (this.#md) el.innerHTML = this.#md(text);
|
|
584
|
+
else { el.textContent = text; el.classList.add("plain"); }
|
|
585
|
+
const box = this.$.captions;
|
|
586
|
+
box.append(el);
|
|
587
|
+
const leave = () => {
|
|
588
|
+
if (el.classList.contains("out")) return;
|
|
589
|
+
el.classList.add("out");
|
|
590
|
+
setTimeout(() => el.remove(), 500);
|
|
591
|
+
};
|
|
592
|
+
setTimeout(leave, AgentBridge.CAPTION_MS);
|
|
593
|
+
const live = [...box.children].filter((c) => !c.classList.contains("out"));
|
|
594
|
+
live.slice(0, Math.max(0, live.length - AgentBridge.CAPTION_MAX)).forEach((c) => c.__leave?.());
|
|
595
|
+
el.__leave = leave;
|
|
596
|
+
}
|
|
597
|
+
|
|
381
598
|
// ---------- composer resize grip (top edge) ----------
|
|
382
599
|
|
|
383
600
|
#bindGrip() {
|
|
@@ -450,7 +667,7 @@ class AgentBridge extends HTMLElement {
|
|
|
450
667
|
const mods = (e) => (e.altKey ? 1 : 0) | (e.ctrlKey ? 2 : 0) | (e.metaKey ? 4 : 0) | (e.shiftKey ? 8 : 0);
|
|
451
668
|
const BUTTONS = ["left", "middle", "right"];
|
|
452
669
|
|
|
453
|
-
controlBtn.addEventListener("click", () => this.#setControl(!this.#controlWs));
|
|
670
|
+
controlBtn.addEventListener("click", () => { if (!this.#present.on) this.#setControl(!this.#controlWs); });
|
|
454
671
|
|
|
455
672
|
img.addEventListener("mousedown", (e) => {
|
|
456
673
|
if (!on()) return;
|
|
@@ -519,6 +736,7 @@ class AgentBridge extends HTMLElement {
|
|
|
519
736
|
#renderScreenMeta(m) {
|
|
520
737
|
const { live, url, viewport } = this.$;
|
|
521
738
|
if (!m.connected) {
|
|
739
|
+
this.$.viewport.querySelector(".placeholder").textContent = SCREEN_REASONS[m.reason] || "열린 terminal-browser 없음";
|
|
522
740
|
live.hidden = true;
|
|
523
741
|
url.textContent = "";
|
|
524
742
|
viewport.classList.remove("has", "agent");
|