@neta-art/cohub-cli 10.17.1 → 10.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/README.md +6 -0
  2. package/dist/board-export-resources.d.ts +2 -0
  3. package/dist/board-export-resources.js +1 -0
  4. package/dist/board-export.d.ts +11 -2
  5. package/dist/board-export.js +181 -62
  6. package/dist/board-gpu/index.d.ts +32 -0
  7. package/dist/board-gpu/index.js +188 -0
  8. package/dist/board-gpu/page.js +70384 -0
  9. package/dist/board-gpu/protocol.d.ts +38 -0
  10. package/dist/board-gpu/protocol.js +1 -0
  11. package/dist/board-kernel/{getPo2TextureFromSource.js → FilterPipe.js} +251 -1
  12. package/dist/board-kernel/browserAll.js +1 -1
  13. package/dist/board-kernel/init.js +1 -251
  14. package/dist/board-kernel/webworkerAll.js +1 -1
  15. package/dist/board-kernel.d.ts +3 -3
  16. package/dist/board-kernel.js +6501 -414
  17. package/dist/board-media-export.d.ts +16 -0
  18. package/dist/board-media-export.js +147 -0
  19. package/dist/board-media.d.ts +2 -1
  20. package/dist/board-media.js +23 -0
  21. package/dist/board-simulate.d.ts +4 -0
  22. package/dist/board-simulate.js +34 -4
  23. package/dist/client.d.ts +1 -0
  24. package/dist/client.js +4 -1
  25. package/dist/commands/apps.js +25 -3
  26. package/dist/commands/boards/authoring.js +45 -3
  27. package/dist/commands/boards/examples.js +21 -5
  28. package/dist/commands/boards.js +10 -2
  29. package/dist/commands/space-displays.d.ts +1 -1
  30. package/dist/commands/space-displays.js +356 -85
  31. package/dist/commands/spaces.js +11 -2
  32. package/dist/host/runtime.d.ts +12 -2
  33. package/dist/host/runtime.js +155 -10
  34. package/dist/host/serve.js +49 -11
  35. package/dist/host/workspaces.d.ts +2 -0
  36. package/dist/host/workspaces.js +18 -0
  37. package/dist/index.js +1 -0
  38. package/dist/output.d.ts +1 -0
  39. package/dist/output.js +3 -0
  40. package/dist/runtime/cli-path.d.ts +5 -0
  41. package/dist/runtime/cli-path.js +27 -0
  42. package/dist/runtime/connection.d.ts +22 -4
  43. package/dist/runtime/connection.js +13 -3
  44. package/dist/runtime/harnesses/claude-approval.d.ts +31 -0
  45. package/dist/runtime/harnesses/claude-approval.js +119 -0
  46. package/dist/runtime/harnesses/claude.js +42 -3
  47. package/dist/runtime/harnesses/codex-approval.d.ts +25 -0
  48. package/dist/runtime/harnesses/codex-approval.js +69 -0
  49. package/dist/runtime/harnesses/codex.js +36 -5
  50. package/dist/runtime/harnesses/index.js +12 -3
  51. package/dist/runtime/harnesses/stream-json.d.ts +16 -2
  52. package/dist/runtime/harnesses/stream-json.js +26 -6
  53. package/dist/runtime/harnesses/types.d.ts +3 -1
  54. package/dist/runtime/instance.d.ts +1 -0
  55. package/dist/runtime/instance.js +1 -0
  56. package/dist/runtime/launch.js +2 -4
  57. package/dist/runtime/local-workspace.d.ts +15 -0
  58. package/dist/runtime/local-workspace.js +52 -0
  59. package/dist/runtime/model-catalog.js +7 -4
  60. package/dist/runtime/native/approvals.d.ts +18 -0
  61. package/dist/runtime/native/approvals.js +54 -0
  62. package/dist/runtime/native/codex-link.d.ts +7 -0
  63. package/dist/runtime/native/codex-link.js +40 -5
  64. package/dist/runtime/native/daemon.d.ts +5 -1
  65. package/dist/runtime/native/daemon.js +6 -1
  66. package/dist/runtime/native/execution.d.ts +2 -0
  67. package/dist/runtime/preferences.d.ts +7 -0
  68. package/dist/runtime/preferences.js +22 -0
  69. package/dist/runtime/presentation.d.ts +3 -0
  70. package/dist/runtime/presentation.js +2 -1
  71. package/dist/runtime/supervisor.js +92 -9
  72. package/dist/safe-remote-image.d.ts +4 -0
  73. package/dist/safe-remote-image.js +8 -0
  74. package/dist/space-files/transfer.js +3 -2
  75. package/dist/space-files/upload.d.ts +5 -2
  76. package/dist/space-files/upload.js +41 -1
  77. package/package.json +2 -2
@@ -1,17 +1,25 @@
1
- import { mkdir, readFile, writeFile } from "node:fs/promises";
1
+ import { randomUUID } from "node:crypto";
2
+ import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
2
3
  import { homedir } from "node:os";
3
4
  import { dirname, join, resolve } from "node:path";
4
- import { compileDisplayActions, DISPLAY_ELEMENT_REF_PATTERN, DISPLAY_SYSTEM_ACTIONS, DisplayActionError, } from "@neta-art/cohub";
5
+ import { compileDisplayPlan, DISPLAY_ELEMENT_REF_PATTERN, DISPLAY_ELEMENT_ROLES, DISPLAY_SYSTEM_ACTIONS, DISPLAY_HANDOVER_DEFAULT_MS, DISPLAY_HANDOVER_MAX_MS, DISPLAY_HANDOVER_WAIT_MS, DisplayActionError, } from "@neta-art/cohub";
6
+ import { Option } from "commander";
5
7
  import { createClient } from "../client.js";
6
- import { error, handleHttp, json as outJson, jsonRequested, ok, table } from "../output.js";
8
+ import { error, handleHttp, json as outJson, jsonRequested, ok, table, warn } from "../output.js";
7
9
  import { resolveSpace } from "../space.js";
8
10
  import { webUrl } from "../web.js";
9
11
  const DIRECTIONS = ["up", "down", "left", "right"];
10
12
  const BUTTONS = ["primary", "secondary", "middle"];
13
+ const TREES = ["compact", "full", "none"];
11
14
  const DEFAULT_MAX_SIZE = 1280;
12
- const SETTLE_MS = 600;
13
- const FRAMES_PATH = join(homedir(), ".cache", "cohub-cli", "display-frames.json");
14
- const NO_DISPLAY_HINT = "Share one first: the Android app's Runtime menu, `cohub runtime up --display` on a computer, or `cohub spaces displays start` for a virtual screen.";
15
+ const HANDOVER_RETRY_MS = 60_000;
16
+ const HANDOVER_RETRY_GAP_MS = 2_000;
17
+ const CACHE_DIR = join(homedir(), ".cache", "cohub-cli");
18
+ const FRAMES_PATH = join(CACHE_DIR, "display-frames.json");
19
+ // In the workspace, where a Runtime's fenced file tools reach them.
20
+ const IMAGE_DIR = join(".cache", "cohub", "displays");
21
+ const KEPT_IMAGES = 20;
22
+ const NO_DISPLAY_HINT = "Share one first: set up phone control in the Cohub Android app, run `cohub runtime up --display` on a computer, or `cohub spaces displays start` for a virtual screen.";
15
23
  function parseSize(value) {
16
24
  const match = /^(\d{1,5})x(\d{1,5})$/i.exec(value.trim());
17
25
  const width = Number(match?.[1]);
@@ -58,17 +66,23 @@ function pickDisplay(displays, requested) {
58
66
  return error(`Display "${requested}" is not shared`, `Shared displays: ${displays.map((display) => display.id).join(", ") || "none"}`);
59
67
  return found;
60
68
  }
69
+ const agents = displays.filter((display) => display.kind === "agent");
61
70
  if (displays.length === 1 && displays[0])
62
71
  return displays[0];
72
+ if (agents.length === 1 && agents[0])
73
+ return agents[0];
63
74
  if (displays.length === 0)
64
75
  return error("No display is shared with this Space", NO_DISPLAY_HINT);
65
76
  return error("Several displays are shared", `Pass --display with one of: ${displays.map((display) => display.id).join(", ")}`);
66
77
  }
67
- async function target(spacesCmd, opts) {
78
+ async function target(spacesCmd, opts, launching = false) {
68
79
  const spaceId = await resolveSpace(spacesCmd);
69
80
  const api = createClient().space(spaceId).displays;
70
- const { displays } = await api.list();
71
- return { spaceId, api, display: pickDisplay(displays, opts.display) };
81
+ let { displays, virtual } = await api.list();
82
+ if (launching && !opts.display && !displays.some((display) => display.apps) && virtual === "available")
83
+ ({ displays } = await api.startVirtual());
84
+ const launchers = displays.filter((display) => display.apps);
85
+ return { spaceId, api, display: pickDisplay(launching && !opts.display && launchers.length > 0 ? launchers : displays, opts.display) };
72
86
  }
73
87
  function displayUrl(spaceId, displayId) {
74
88
  return webUrl(`/spaces/${spaceId}?window=${encodeURIComponent(`display:${displayId}`)}`);
@@ -77,83 +91,279 @@ function needsHint(display) {
77
91
  const needs = display.needs ?? [];
78
92
  if (needs.length === 0)
79
93
  return null;
94
+ if (needs.includes("phoneControl"))
95
+ return `${display.name || display.id} needs phone control: ask the user to set it up in the Cohub app on the phone.`;
80
96
  if (!display.desktop)
81
- return `${display.name || display.id} needs control: on the phone, choose Allow control in the Space's Runtime menu and enable Cohub in Accessibility settings.`;
97
+ return `${display.name || display.id} needs control: on the phone, enable Cohub in Accessibility settings.`;
82
98
  const names = needs.map((need) => (need === "screenRecording" ? "Screen Recording" : "Accessibility")).join(" and ");
83
99
  return `${display.name || display.id} needs ${names}: allow it for the Runtime's terminal in System Settings → Privacy & Security, then restart the Runtime.`;
84
100
  }
101
+ async function writeOnce(path, content) {
102
+ await writeFile(path, content, { flag: "wx" }).catch((e) => {
103
+ if (e.code !== "EEXIST")
104
+ throw e;
105
+ });
106
+ }
107
+ async function imageDir() {
108
+ const dir = resolve(IMAGE_DIR);
109
+ await mkdir(dir, { recursive: true });
110
+ await Promise.all([writeOnce(join(dir, "..", ".gitignore"), "*\n"), writeOnce(join(dir, ".nomedia"), "")]);
111
+ const images = (await readdir(dir)).filter((name) => name.endsWith(".jpg")).sort();
112
+ await Promise.all(images.slice(0, -(KEPT_IMAGES - 1)).map((name) => rm(join(dir, name), { force: true })));
113
+ return dir;
114
+ }
115
+ async function saveImage(spaceId, display, image, output) {
116
+ const name = `${new Date().toISOString().replace(/[:.]/g, "-")}-${display.id}.jpg`;
117
+ const path = output ? resolve(output) : join(await imageDir().catch(() => join(CACHE_DIR, "displays")), name);
118
+ await mkdir(dirname(path), { recursive: true });
119
+ await writeFile(path, Buffer.from(image.data, "base64"));
120
+ await rememberFrame(spaceId, display, image);
121
+ return { path, width: image.width, height: image.height };
122
+ }
85
123
  async function screenshot(api, spaceId, display, maxSize, output) {
86
- const capture = await api.capture(display.id, { maxSize });
87
- const path = resolve(output ?? `${display.id}-${new Date().toISOString().replace(/[:.]/g, "-")}.jpg`);
88
- await writeFile(path, Buffer.from(capture.data, "base64"));
89
- await rememberFrame(spaceId, display, capture);
90
- return { path, width: capture.width, height: capture.height };
124
+ return saveImage(spaceId, display, await api.capture(display.id, { maxSize }), output);
125
+ }
126
+ function formatElement(element, frame, indent = true, box = true) {
127
+ const scaleX = frame.width - 1 || 1;
128
+ const scaleY = frame.height - 1 || 1;
129
+ const [x, y, width, height] = element.bounds;
130
+ const parts = [`${indent ? " ".repeat(element.depth) : ""}${element.ref}`, element.role];
131
+ if (element.name)
132
+ parts.push(JSON.stringify(element.name));
133
+ if (element.value)
134
+ parts.push(`value=${JSON.stringify(element.value)}`);
135
+ if (element.states?.length)
136
+ parts.push(`[${element.states.join(", ")}]`);
137
+ if (box)
138
+ parts.push(`(${Math.round(x * scaleX)},${Math.round(y * scaleY)} ${Math.round(width * scaleX)}x${Math.round(height * scaleY)})`);
139
+ return parts.join(" ");
140
+ }
141
+ function formatTree(tree, frame, compact = false) {
142
+ return tree.elements.map((element) => formatElement(element, frame, true, !compact || Boolean(element.actions?.length))).join("\n");
143
+ }
144
+ const ACTOR = process.env.COHUB_SESSION_ID
145
+ ? { sessionId: process.env.COHUB_SESSION_ID, ...(process.env.COHUB_TURN_ID ? { turnId: process.env.COHUB_TURN_ID } : {}) }
146
+ : undefined;
147
+ function untilOf(opts) {
148
+ const until = {
149
+ ...(opts.untilText ? { text: opts.untilText } : {}),
150
+ ...(opts.untilGone ? { gone: opts.untilGone } : {}),
151
+ ...(opts.untilApp ? { app: opts.untilApp } : {}),
152
+ ...(opts.untilStable ? { stable: true } : {}),
153
+ };
154
+ if (Object.keys(until).length === 0)
155
+ return undefined;
156
+ return { ...until, ...(opts.timeoutMs ? { timeoutMs: parseNumber("timeout-ms", opts.timeoutMs) } : {}) };
91
157
  }
92
158
  async function act(spacesCmd, opts, actions) {
159
+ let sent = false;
93
160
  try {
94
- const { spaceId, api, display } = await target(spacesCmd, opts);
95
- if (!display.input)
161
+ const launching = actions.some((action) => action.type === "launch" || action.type === "open");
162
+ const { spaceId, api, display } = await target(spacesCmd, opts, launching);
163
+ if (!display.input && actions.some((action) => action.type !== "wait"))
96
164
  return error("This display is view only", needsHint(display) ?? "Allow control on the device first.");
97
165
  const frame = await frameOf(spaceId, display, opts.size);
98
- const events = compileDisplayActions(actions, frame.width, frame.height);
99
- const result = await api.input(display.id, events);
100
- let shot;
101
- if (opts.screenshot) {
102
- await new Promise((done) => setTimeout(done, SETTLE_MS));
103
- shot = await screenshot(api, spaceId, display, Math.max(frame.width, frame.height), typeof opts.screenshot === "string" ? opts.screenshot : undefined);
166
+ const tree = opts.tree ?? "compact";
167
+ if (!TREES.includes(tree))
168
+ return error("Invalid tree", `Use one of: ${TREES.join(", ")}`);
169
+ const last = actions.at(-1);
170
+ if (last) {
171
+ const until = untilOf(opts);
172
+ if (until)
173
+ last.until = until;
174
+ if (opts.label)
175
+ last.label = opts.label;
104
176
  }
105
- if (jsonRequested(opts))
106
- return outJson({ display: display.id, ...result, frame: { width: frame.width, height: frame.height }, ...(shot ? { screenshot: shot } : {}) });
107
- ok(`Performed ${actions.length} action${actions.length === 1 ? "" : "s"} on ${display.name || display.id}, in ${frame.width}x${frame.height} pixels`);
108
- if (shot)
109
- console.log(` Screenshot ${shot.width}x${shot.height}: ${shot.path}`);
177
+ const steps = compileDisplayPlan(actions, frame.width, frame.height);
178
+ sent = steps.length > 0;
179
+ const result = await api.act(display.id, {
180
+ steps,
181
+ observe: { tree: tree, image: opts.screenshot ? "always" : "auto", maxSize: DEFAULT_MAX_SIZE },
182
+ ...(ACTOR ? { actor: ACTOR } : {}),
183
+ });
184
+ const image = result.screen.image
185
+ ? await saveImage(spaceId, display, result.screen.image, typeof opts.screenshot === "string" ? opts.screenshot : undefined)
186
+ : undefined;
187
+ const shown = image ?? frame;
188
+ if (result.stopped)
189
+ process.exitCode = 1;
190
+ if (jsonRequested(opts)) {
191
+ const { image: _, ...screen } = result.screen;
192
+ return outJson({ display: display.id, ...result, screen: { ...screen, ...(image ? { image } : {}) }, frame: { width: shown.width, height: shown.height } });
193
+ }
194
+ printResult(display, actions, result, shown, tree === "compact", image?.path);
110
195
  }
111
196
  catch (e) {
112
197
  if (e instanceof DisplayActionError)
113
198
  return error("Invalid action", e.message);
199
+ if (sent && isLostAnswer(e))
200
+ return error("The answer was lost", "The plan may have run, in part or whole: `look` before repeating any step.");
114
201
  handleHttp(e);
115
202
  }
116
203
  }
117
- function withTarget(command) {
204
+ function isLostAnswer(e) {
205
+ const status = e.status;
206
+ return status === undefined ? e instanceof TypeError : status >= 500 && status !== 501;
207
+ }
208
+ function targetText(target) {
209
+ return target.name ? `${target.role} ${JSON.stringify(target.name)}` : target.role;
210
+ }
211
+ function atPoint(action) {
212
+ return action.x !== undefined && action.ref === undefined && action.name === undefined && action.role === undefined;
213
+ }
214
+ function actionText(action) {
215
+ const point = atPoint(action) ? ` (${action.x},${action.y})` : "";
216
+ switch (action.type) {
217
+ case "tap": return `${action.count === 2 ? "double-tap" : action.count === 3 ? "triple-tap" : "tap"}${point}`;
218
+ case "long_press": return `long press${point}`;
219
+ case "swipe": return `swipe${point} to (${action.to_x},${action.to_y})`;
220
+ case "scroll": return `scroll ${action.direction ?? "down"}`;
221
+ case "type": return `type ${[...(action.text ?? "")].length} characters`; // never echo it: it may be a secret
222
+ case "key": return `key ${action.key}`;
223
+ case "system": return `press ${action.action}`;
224
+ case "launch": return `launch ${action.app}`;
225
+ case "open": return `open ${action.url}`;
226
+ case "wait": return "wait";
227
+ }
228
+ }
229
+ function stepText(action, step) {
230
+ const parts = [action ? actionText(action) : "step"];
231
+ if (step.target)
232
+ parts.push(`${action && atPoint(action) ? "on " : action?.type === "type" ? "into " : ""}${targetText(step.target)}`);
233
+ if (action?.type !== "wait")
234
+ parts.push(`· ${step.effect}`);
235
+ parts.push(`· ${step.findMs + step.actMs + step.waitedMs} ms`);
236
+ return parts.join(" ");
237
+ }
238
+ function printResult(display, actions, result, frame, compact, imageFile) {
239
+ const name = display.name || display.id;
240
+ if (result.steps.length > 0 || result.stopped) {
241
+ const done = result.steps.length;
242
+ const still = result.steps.findIndex((step, index) => step.effect === "none" && actions[index]?.type !== "wait");
243
+ if (!result.stopped) {
244
+ if (still >= 0)
245
+ warn(`Step ${still + 1} changed nothing on ${name}: it may not have landed. Look before repeating it.`);
246
+ else
247
+ ok(`${done} step${done === 1 ? "" : "s"} on ${name}`);
248
+ }
249
+ for (const [index, step] of result.steps.entries())
250
+ console.log(` ${index + 1}. ${stepText(actions[index], step)}`);
251
+ if (result.stopped) {
252
+ console.log(`✗ Stopped (${result.stopped.reason}): ${result.stopped.message}`);
253
+ for (const candidate of result.stopped.candidates ?? [])
254
+ console.log(` ${formatElement(candidate, frame, false)}`);
255
+ }
256
+ }
257
+ const { screen } = result;
258
+ console.log(`\n${[screen.app, screen.quality === "poor" ? "the tree misses much of this screen: read the image" : null].filter(Boolean).join(" · ") || name}`);
259
+ if (screen.unchanged)
260
+ console.log(" The screen is as you last saw it; its refs are gone, so target by name or `look` again.");
261
+ if (screen.tree) {
262
+ console.log(formatTree(screen.tree, frame, compact));
263
+ if (screen.tree.truncated)
264
+ console.log(" More elements exist; pass --tree full or read `tree --max`.");
265
+ }
266
+ if (imageFile)
267
+ console.log(`\n Image ${frame.width}x${frame.height}: ${imageFile} (coordinates are its pixels)`);
268
+ if (screen.tree)
269
+ console.log(`\n Boxes are x,y and size in ${frame.width}x${frame.height} pixels.`);
270
+ if (display.held)
271
+ console.log(" A person at the device has taken it over; wait until they hand it back.");
272
+ else if (display.viewers?.some((viewer) => viewer.control))
273
+ console.log(" A person is watching with control; what you do here pauses while they act.");
274
+ }
275
+ const HANDOVER_ENDS = {
276
+ stopped: () => ["The person stopped this", "Do not go on with this Turn; ask them."],
277
+ gone: () => ["The display went away", "Ask the person what happened before going on."],
278
+ expired: (held) => held
279
+ ? ["Timed out while the person is still working", "Tell them to say so in the chat when done; they will not wake you by handing back."]
280
+ : ["Timed out; the person did not take over", "Tell them what is needed, and to say so in the chat when done."],
281
+ };
282
+ async function handover(spacesCmd, opts) {
283
+ try {
284
+ const timeoutMs = opts.timeoutMs ? parseNumber("timeout-ms", opts.timeoutMs) : DISPLAY_HANDOVER_DEFAULT_MS;
285
+ if (!Number.isInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > DISPLAY_HANDOVER_MAX_MS)
286
+ return error("Invalid timeout", `--timeout-ms must be between 1 and ${DISPLAY_HANDOVER_MAX_MS} milliseconds.`);
287
+ const { api, display } = await target(spacesCmd, opts);
288
+ const params = { id: randomUUID(), reason: opts.reason, timeoutMs, ...(ACTOR ? { actor: ACTOR } : {}) };
289
+ const giveUpAt = Date.now() + timeoutMs + HANDOVER_RETRY_MS;
290
+ let result = await api.handover(display.id, { ...params, waitMs: opts.wait ? DISPLAY_HANDOVER_WAIT_MS : 0 });
291
+ while (opts.wait && result.state === "waiting") {
292
+ try {
293
+ result = await api.handover(display.id, { ...params, waitMs: DISPLAY_HANDOVER_WAIT_MS });
294
+ }
295
+ catch (e) {
296
+ if (!isLostAnswer(e) || Date.now() > giveUpAt)
297
+ throw e;
298
+ await new Promise((done) => setTimeout(done, HANDOVER_RETRY_GAP_MS));
299
+ }
300
+ }
301
+ if (result.state === "waiting") {
302
+ if (jsonRequested(opts))
303
+ return outJson({ display: display.id, ...result });
304
+ return ok(`Asked the person: ${opts.reason}`);
305
+ }
306
+ if (result.state === "returned")
307
+ return act(spacesCmd, { display: display.id, json: opts.json }, []);
308
+ const [title, hint] = HANDOVER_ENDS[result.state](result.held);
309
+ if (jsonRequested(opts)) {
310
+ process.exitCode = 1;
311
+ return outJson({ display: display.id, ...result });
312
+ }
313
+ return error(title, hint);
314
+ }
315
+ catch (e) {
316
+ handleHttp(e);
317
+ }
318
+ }
319
+ function withAct(command) {
118
320
  return command
119
- .option("-d, --display <id>", "Display id; defaults to the only shared display")
120
- .option("--size <WxH>", "Coordinates are pixels of an image this size (default: the latest screenshot)")
121
- .option("--screenshot [path]", "Save a screenshot once the screen settles")
321
+ .option("-d, --display <id>", "Display id; defaults to the only one, or the agent screen")
322
+ .option("--size <WxH>", "Coordinates are pixels of an image this size (default: the latest image)")
323
+ .option("--until-text <text>", "Wait until this text shows")
324
+ .option("--until-gone <text>", "Wait until this text is gone, e.g. a loading label")
325
+ .option("--until-app <id>", "Wait until this app is in front")
326
+ .option("--until-stable", "Wait until the screen stops changing")
327
+ .option("--timeout-ms <ms>", "How long --until-* waits (default 8000)")
328
+ .option("--label <text>", "What this does, shown to people watching")
329
+ .option("--screenshot [path]", "Attach an image of the screen after (by default only when its tree misses much)")
330
+ .option("--tree <kind>", `The tree after: ${TREES.join(", ")}`)
122
331
  .option("--json", "Output as JSON");
123
332
  }
124
- function pointOrRef(x, y, command) {
333
+ function withTarget(command) {
334
+ return command
335
+ .option("--name <text>", "The element that says this: whole, then start, then part")
336
+ .option("--role <role>", `Its role: ${DISPLAY_ELEMENT_ROLES.join(", ")}`)
337
+ .option("--nth <n>", "The nth match, top to bottom")
338
+ .option("--scroll", "Scroll the list on until it shows");
339
+ }
340
+ function targetOf(x, y, opts, command) {
341
+ if (opts.name !== undefined || opts.role !== undefined) {
342
+ if (opts.role !== undefined && !DISPLAY_ELEMENT_ROLES.includes(opts.role)) {
343
+ return error("Invalid role", `Use one of: ${DISPLAY_ELEMENT_ROLES.join(", ")}`);
344
+ }
345
+ return {
346
+ ...(opts.name !== undefined ? { name: opts.name } : {}),
347
+ ...(opts.role !== undefined ? { role: opts.role } : {}),
348
+ ...(opts.nth !== undefined ? { nth: parseNumber("nth", opts.nth) } : {}),
349
+ ...(opts.scroll ? { scroll: true } : {}),
350
+ };
351
+ }
352
+ if (x === undefined)
353
+ return error("Missing target", `Pass --name, an element ref, or x and y, e.g. ${command} --name Send.`);
125
354
  if (y === undefined)
126
355
  return isRef(x) ? { ref: x } : error("Invalid target", `Pass x and y, or an element ref from \`cohub spaces displays tree\`, e.g. ${command} e3.12.`);
127
356
  return { x: parseNumber("x", x), y: parseNumber("y", y) };
128
357
  }
129
- function formatTree(tree, frame) {
130
- const scaleX = (frame.width - 1) || 1;
131
- const scaleY = (frame.height - 1) || 1;
132
- return tree.elements
133
- .map((element) => {
134
- const [x, y, width, height] = element.bounds;
135
- const box = `(${Math.round(x * scaleX)},${Math.round(y * scaleY)} ${Math.round(width * scaleX)}x${Math.round(height * scaleY)})`;
136
- const parts = [`${" ".repeat(element.depth)}${element.ref}`, element.role];
137
- if (element.name)
138
- parts.push(JSON.stringify(element.name));
139
- if (element.value)
140
- parts.push(`value=${JSON.stringify(element.value)}`);
141
- if (element.states?.length)
142
- parts.push(`[${element.states.join(", ")}]`);
143
- parts.push(box);
144
- return parts.join(" ");
145
- })
146
- .join("\n");
147
- }
148
358
  export function registerSpaceDisplays(spacesCmd) {
149
359
  const displays = spacesCmd
150
360
  .command("displays")
151
- .description("Screens shared with a Space: view, read and control")
361
+ .description("Screens shared with a Space: view, read and act on them")
152
362
  .hook("preAction", async () => { await resolveSpace(spacesCmd); });
153
363
  displays
154
364
  .command("ls")
155
365
  .alias("list")
156
- .description("List shared displays and who watches them")
366
+ .description("List shared displays, what they can do, and who uses them")
157
367
  .option("--json", "Output as JSON")
158
368
  .action(async (opts) => {
159
369
  try {
@@ -167,16 +377,20 @@ export function registerSpaceDisplays(spacesCmd) {
167
377
  table(result.displays.map((display) => ({
168
378
  ...display,
169
379
  size: `${display.width}x${display.height}`,
170
- platform: display.platform ?? "—",
171
- abilities: [display.stream && "view", display.capture && "capture", display.input && "control", display.tree && "tree", display.desktop && "desktop"].filter(Boolean).join(", "),
172
- watching: display.viewers?.length ? `${display.viewers.length}${display.viewers.some((viewer) => viewer.control) ? " (controlling)" : ""}` : "—",
380
+ kind: display.kind ?? "screen",
381
+ abilities: [display.stream && "view", display.capture && "capture", display.input && "control", display.tree && "tree", display.apps && "apps", display.desktop && "desktop"].filter(Boolean).join(", "),
382
+ using: [
383
+ display.held && "a person at the device",
384
+ display.activity && `an agent${display.activity.label ? `: ${display.activity.label}` : ""}`,
385
+ display.viewers?.length && `${display.viewers.length} watching${display.viewers.some((viewer) => viewer.control) ? " (controlling)" : ""}`,
386
+ ].filter(Boolean).join("; ") || "—",
173
387
  })), [
174
388
  { key: "id", label: "ID" },
175
389
  { key: "name", label: "Name" },
390
+ { key: "kind", label: "Kind" },
176
391
  { key: "size", label: "Size" },
177
- { key: "platform", label: "Platform" },
178
392
  { key: "abilities", label: "Abilities" },
179
- { key: "watching", label: "Watching" },
393
+ { key: "using", label: "In use by" },
180
394
  ]);
181
395
  for (const display of result.displays) {
182
396
  const hint = needsHint(display);
@@ -193,8 +407,8 @@ export function registerSpaceDisplays(spacesCmd) {
193
407
  });
194
408
  displays
195
409
  .command("start")
196
- .description("Start the machine's virtual screen, e.g. in a cloud sandbox")
197
- .option("--size <WxH>", "Screen size (default 1280x800)")
410
+ .description("Start the machine's virtual screen: a phone's agent screen, or a sandbox's")
411
+ .option("--size <WxH>", "Screen size where the machine lets you choose (default 1280x800)")
198
412
  .option("--json", "Output as JSON")
199
413
  .action(async (opts) => {
200
414
  try {
@@ -202,10 +416,11 @@ export function registerSpaceDisplays(spacesCmd) {
202
416
  if (opts.size)
203
417
  parseSize(opts.size);
204
418
  const result = await createClient().space(spaceId).displays.startVirtual(opts.size ? { size: opts.size } : {});
205
- const url = result.displays[0] ? displayUrl(spaceId, result.displays[0].id) : null;
419
+ const started = result.displays.find((display) => display.kind === "agent") ?? result.displays[0];
420
+ const url = started ? displayUrl(spaceId, started.id) : null;
206
421
  if (jsonRequested(opts))
207
422
  return outJson({ ...result, url });
208
- ok("Virtual screen running; programs started from now on draw on it");
423
+ ok("Virtual screen running; apps and programs you start from now on run on it");
209
424
  if (url)
210
425
  console.log(` Watch and take over: ${url}`);
211
426
  }
@@ -232,8 +447,8 @@ export function registerSpaceDisplays(spacesCmd) {
232
447
  displays
233
448
  .command("capture")
234
449
  .description("Save a screenshot; later coordinates are its pixels")
235
- .option("-d, --display <id>", "Display id; defaults to the only shared display")
236
- .option("-o, --output <path>", "Where to write the image (default: <display>-<time>.jpg)")
450
+ .option("-d, --display <id>", "Display id; defaults to the only one, or the agent screen")
451
+ .option("-o, --output <path>", `Where to write the image (default: ${IMAGE_DIR}/, the latest ${KEPT_IMAGES} kept)`)
237
452
  .option("--max-size <px>", `Longest edge in pixels (default ${DEFAULT_MAX_SIZE})`)
238
453
  .option("--json", "Output as JSON")
239
454
  .action(async (opts) => {
@@ -252,9 +467,10 @@ export function registerSpaceDisplays(spacesCmd) {
252
467
  displays
253
468
  .command("tree")
254
469
  .description("Read the interface as elements: refs to act on, roles, names and boxes")
255
- .option("-d, --display <id>", "Display id; defaults to the only shared display")
470
+ .option("-d, --display <id>", "Display id; defaults to the only one, or the agent screen")
256
471
  .option("--max <n>", "Most elements to return (default 300)")
257
- .option("--actionable", "Only elements that take an action, flat: fewer tokens when looking for a target")
472
+ .option("--full", "Every element, nested, instead of what can be acted on or read")
473
+ .addOption(new Option("--compact").hideHelp())
258
474
  .option("--json", "Output as JSON")
259
475
  .action(async (opts) => {
260
476
  try {
@@ -263,38 +479,63 @@ export function registerSpaceDisplays(spacesCmd) {
263
479
  return error("This display has no element tree", "Use capture and coordinates instead.");
264
480
  const tree = await api.tree(display.id, {
265
481
  ...(opts.max ? { maxElements: parseNumber("max", opts.max) } : {}),
266
- ...(opts.actionable ? { actionable: true } : {}),
482
+ ...(opts.full ? {} : { compact: true }),
267
483
  });
268
484
  if (jsonRequested(opts))
269
485
  return outJson(tree);
270
486
  const frame = await frameOf(spaceId, display);
271
- console.log(formatTree(tree, frame));
272
- console.log(`\n Boxes are x,y and size in ${frame.width}x${frame.height} pixels. Act with: tap <ref> · type <text> --into <ref> · scroll <direction> --in <ref>`);
487
+ if (tree.app)
488
+ console.log(tree.app);
489
+ console.log(formatTree(tree, frame, !opts.full));
490
+ console.log(`\n Boxes are x,y and size in ${frame.width}x${frame.height} pixels. Act with: tap --name <text> · type <text> --into <ref> · act < plan.json`);
273
491
  if (tree.truncated)
274
492
  console.log(" More elements exist; pass --max to read more.");
275
- if (display.viewers?.some((viewer) => viewer.control))
276
- console.log(" A person is watching with control; what you do here pauses while they act.");
277
493
  }
278
494
  catch (e) {
279
495
  handleHttp(e);
280
496
  }
281
497
  });
282
- withTarget(displays.command("tap <x> [y]").description("Tap or click a point, or an element by ref"))
283
- .option("--count <n>", "2 to double-click, 3 to triple-click")
498
+ displays
499
+ .command("apps")
500
+ .description("List the apps a display can launch")
501
+ .option("-d, --display <id>", "Display id; defaults to the only one, or the agent screen")
502
+ .option("--json", "Output as JSON")
503
+ .action(async (opts) => {
504
+ try {
505
+ const { api, display } = await target(spacesCmd, opts);
506
+ if (!display.apps)
507
+ return error("This display does not launch apps");
508
+ const { apps } = await api.apps(display.id);
509
+ if (jsonRequested(opts))
510
+ return outJson({ apps });
511
+ table(apps, [{ key: "id", label: "ID" }, { key: "name", label: "Name" }]);
512
+ }
513
+ catch (e) {
514
+ handleHttp(e);
515
+ }
516
+ });
517
+ withAct(displays.command("look").description("Read the screen as `act` reports it, without acting; --until-* waits first"))
518
+ .action((opts) => act(spacesCmd, opts, untilOf(opts) ? [{ type: "wait" }] : []));
519
+ withAct(displays.command("launch <app>").description("Launch an app by id (see `apps`) and wait until it is in front"))
520
+ .action((app, opts) => act(spacesCmd, { untilApp: app, ...opts }, [{ type: "launch", app }]));
521
+ withAct(displays.command("open <url>").description("Open a link, or a page of the system's settings"))
522
+ .action((url, opts) => act(spacesCmd, opts, [{ type: "open", url }]));
523
+ withAct(withTarget(displays.command("tap [x] [y]").description("Tap an element (--name, or a ref) or a point")))
524
+ .option("--count <n>", "2 to double-tap, 3 to triple-tap")
284
525
  .option("--button <button>", `Mouse button on a desktop: ${BUTTONS.join(", ")}`)
285
526
  .action((x, y, opts) => {
286
527
  const button = BUTTONS.find((candidate) => candidate === (opts.button ?? "primary"));
287
528
  if (!button)
288
529
  return error("Invalid button", `Use one of: ${BUTTONS.join(", ")}`);
289
530
  return act(spacesCmd, opts, [{
290
- type: "tap", ...pointOrRef(x, y, "tap"), button,
531
+ type: "tap", ...targetOf(x, y, opts, "tap"), button,
291
532
  ...(opts.count ? { count: parseNumber("count", opts.count) } : {}),
292
533
  }]);
293
534
  });
294
- withTarget(displays.command("long-press <x> [y]").description("Touch and hold a point, or an element by ref"))
535
+ withAct(withTarget(displays.command("long-press [x] [y]").description("Touch and hold an element or a point")))
295
536
  .option("--duration <ms>", "Hold time in milliseconds", "800")
296
- .action((x, y, opts) => act(spacesCmd, opts, [{ type: "long_press", ...pointOrRef(x, y, "long-press"), duration_ms: parseNumber("duration", opts.duration) }]));
297
- withTarget(displays.command("swipe <x1> <y1> <x2> <y2>").description("Swipe from one point to another"))
537
+ .action((x, y, opts) => act(spacesCmd, opts, [{ type: "long_press", ...targetOf(x, y, opts, "long-press"), duration_ms: parseNumber("duration", opts.duration) }]));
538
+ withAct(displays.command("swipe <x1> <y1> <x2> <y2>").description("Swipe from one point to another"))
298
539
  .option("--duration <ms>", "Swipe time in milliseconds", "300")
299
540
  .action((x1, y1, x2, y2, opts) => act(spacesCmd, opts, [{
300
541
  type: "swipe",
@@ -302,7 +543,7 @@ export function registerSpaceDisplays(spacesCmd) {
302
543
  to_x: parseNumber("x2", x2), to_y: parseNumber("y2", y2),
303
544
  duration_ms: parseNumber("duration", opts.duration),
304
545
  }]));
305
- withTarget(displays.command("scroll <direction>").description("Scroll content up, down, left or right"))
546
+ withAct(displays.command("scroll <direction>").description("Scroll content up, down, left or right"))
306
547
  .option("--amount <fraction>", "Fraction of the display, 0.1–1", "0.5")
307
548
  .option("--in <ref>", "Scroll this element of the tree")
308
549
  .action((direction, opts) => {
@@ -313,20 +554,50 @@ export function registerSpaceDisplays(spacesCmd) {
313
554
  ...(opts.in ? { ref: opts.in } : {}),
314
555
  }]);
315
556
  });
316
- withTarget(displays.command("type <text>").description("Type into the focused field, or replace an element's text"))
557
+ withAct(withTarget(displays.command("type <text>").description("Type into the focused field, or replace a field's text")))
317
558
  .option("--into <ref>", "Replace the text of this element of the tree")
318
- .action((text, opts) => act(spacesCmd, opts, [{ type: "type", text, ...(opts.into ? { ref: opts.into } : {}) }]));
319
- withTarget(displays.command("key <key>").description("Press a key or shortcut, e.g. Enter or Control+a"))
559
+ .action((text, opts) => {
560
+ const field = opts.into ? { ref: opts.into } : opts.name !== undefined || opts.role !== undefined ? targetOf(undefined, undefined, opts, "type") : {};
561
+ return act(spacesCmd, opts, [{ type: "type", text, ...field }]);
562
+ });
563
+ withAct(displays.command("key <key>").description("Press a key or shortcut, e.g. Enter or Control+a"))
320
564
  .action((key, opts) => act(spacesCmd, opts, [{ type: "key", key }]));
321
- withTarget(displays.command("press <button>").description(`Press a system button: ${DISPLAY_SYSTEM_ACTIONS.join(", ")}`))
565
+ withAct(displays.command("press <button>").description(`Press a system button: ${DISPLAY_SYSTEM_ACTIONS.join(", ")}`))
322
566
  .action((button, opts) => {
323
567
  const action = DISPLAY_SYSTEM_ACTIONS.find((candidate) => candidate === button);
324
568
  if (!action)
325
569
  return error("Invalid button", `Use one of: ${DISPLAY_SYSTEM_ACTIONS.join(", ")}`);
326
570
  return act(spacesCmd, opts, [{ type: "system", action }]);
327
571
  });
328
- withTarget(displays.command("act").description("Run a JSON array of actions from --actions or stdin"))
329
- .option("--actions <json>", "Actions, e.g. '[{\"type\":\"tap\",\"x\":540,\"y\":1200},{\"type\":\"tap\",\"ref\":\"e3.12\"}]'")
572
+ withAct(displays.command("wait").description("Wait for --until-* (or a pause) without acting"))
573
+ .option("--ms <ms>", "Pause this long first")
574
+ .action((opts) => act(spacesCmd, opts, [{ type: "wait", ...(opts.ms ? { duration_ms: parseNumber("ms", opts.ms) } : {}) }]));
575
+ displays
576
+ .command("handover")
577
+ .description("Ask the person to do what you must not (sign in, pay), and wait until they hand back")
578
+ .requiredOption("--reason <text>", "What they need to do, in their language; shown on the device")
579
+ .option("-d, --display <id>", "Display id; defaults to the only one, or the agent screen")
580
+ .option("--timeout-ms <ms>", `How long to wait (default: ${DISPLAY_HANDOVER_DEFAULT_MS}; max: ${DISPLAY_HANDOVER_MAX_MS})`)
581
+ .option("--no-wait", "Ask and exit without waiting")
582
+ .option("--json", "Output as JSON")
583
+ .action(async (opts) => {
584
+ await handover(spacesCmd, opts);
585
+ });
586
+ withAct(displays.command("act").description("Run a plan: a JSON array of actions from --actions or stdin"))
587
+ .option("--actions <json>", 'Actions, e.g. \'[{"type":"launch","app":"com.android.settings"},{"type":"tap","name":"Wi-Fi"}]\'')
588
+ .addHelpText("after", `
589
+ Each action is one step; the plan runs on the device and stops at the first
590
+ surprise (a target missing or ambiguous, an until that never came, a dialog,
591
+ another app, a person taking over), then reports the screen.
592
+
593
+ {"type":"launch","app":"<id>"} {"type":"open","url":"<link>"}
594
+ {"type":"tap","name":"Send"} {"type":"tap","name":"Ada","scroll":true} {"type":"tap","ref":"e3.12"} {"type":"tap","x":540,"y":1200}
595
+ {"type":"type","text":"hi","name":"Message","role":"textField"} {"type":"type","text":"hi"}
596
+ {"type":"scroll","direction":"down"} {"type":"swipe",...} {"type":"key","key":"Enter"}
597
+ {"type":"system","action":"back"} {"type":"long_press",...} {"type":"wait","duration_ms":500}
598
+
599
+ Any action takes "until": {"text"|"gone"|"app": …, "stable": true, "timeoutMs": 8000}
600
+ and "label": what it does, shown to people watching; name matches take "role" and "nth".`)
330
601
  .action(async (opts) => {
331
602
  let raw = opts.actions ?? "";
332
603
  if (!raw && !process.stdin.isTTY) {