@neta-art/cohub-cli 10.17.1 → 10.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +6 -0
  2. package/dist/board-export-resources.d.ts +2 -0
  3. package/dist/board-export-resources.js +1 -0
  4. package/dist/board-export.d.ts +4 -1
  5. package/dist/board-export.js +109 -23
  6. package/dist/board-gpu/index.d.ts +32 -0
  7. package/dist/board-gpu/index.js +188 -0
  8. package/dist/board-gpu/page.js +70178 -0
  9. package/dist/board-gpu/protocol.d.ts +37 -0
  10. package/dist/board-gpu/protocol.js +1 -0
  11. package/dist/board-kernel/{getPo2TextureFromSource.js → FilterPipe.js} +251 -1
  12. package/dist/board-kernel/browserAll.js +1 -1
  13. package/dist/board-kernel/init.js +1 -251
  14. package/dist/board-kernel/webworkerAll.js +1 -1
  15. package/dist/board-kernel.d.ts +3 -3
  16. package/dist/board-kernel.js +6624 -1462
  17. package/dist/board-simulate.d.ts +4 -0
  18. package/dist/board-simulate.js +34 -4
  19. package/dist/client.d.ts +1 -0
  20. package/dist/client.js +4 -1
  21. package/dist/commands/apps.js +25 -3
  22. package/dist/commands/boards/examples.js +21 -5
  23. package/dist/commands/boards.js +10 -2
  24. package/dist/commands/space-displays.js +301 -82
  25. package/dist/commands/spaces.js +11 -2
  26. package/dist/host/runtime.d.ts +12 -2
  27. package/dist/host/runtime.js +155 -10
  28. package/dist/host/serve.js +49 -11
  29. package/dist/host/workspaces.d.ts +2 -0
  30. package/dist/host/workspaces.js +18 -0
  31. package/dist/index.js +1 -0
  32. package/dist/runtime/cli-path.d.ts +5 -0
  33. package/dist/runtime/cli-path.js +27 -0
  34. package/dist/runtime/connection.d.ts +22 -4
  35. package/dist/runtime/connection.js +13 -3
  36. package/dist/runtime/harnesses/claude-approval.d.ts +31 -0
  37. package/dist/runtime/harnesses/claude-approval.js +119 -0
  38. package/dist/runtime/harnesses/claude.js +42 -3
  39. package/dist/runtime/harnesses/codex-approval.d.ts +25 -0
  40. package/dist/runtime/harnesses/codex-approval.js +69 -0
  41. package/dist/runtime/harnesses/codex.js +36 -5
  42. package/dist/runtime/harnesses/index.js +12 -3
  43. package/dist/runtime/harnesses/stream-json.d.ts +16 -2
  44. package/dist/runtime/harnesses/stream-json.js +26 -6
  45. package/dist/runtime/harnesses/types.d.ts +3 -1
  46. package/dist/runtime/instance.d.ts +1 -0
  47. package/dist/runtime/instance.js +1 -0
  48. package/dist/runtime/launch.js +2 -4
  49. package/dist/runtime/local-workspace.d.ts +15 -0
  50. package/dist/runtime/local-workspace.js +52 -0
  51. package/dist/runtime/model-catalog.js +7 -4
  52. package/dist/runtime/native/approvals.d.ts +18 -0
  53. package/dist/runtime/native/approvals.js +54 -0
  54. package/dist/runtime/native/codex-link.d.ts +7 -0
  55. package/dist/runtime/native/codex-link.js +40 -5
  56. package/dist/runtime/native/daemon.d.ts +5 -1
  57. package/dist/runtime/native/daemon.js +6 -1
  58. package/dist/runtime/native/execution.d.ts +2 -0
  59. package/dist/runtime/preferences.d.ts +7 -0
  60. package/dist/runtime/preferences.js +22 -0
  61. package/dist/runtime/presentation.d.ts +3 -0
  62. package/dist/runtime/presentation.js +2 -1
  63. package/dist/runtime/supervisor.js +92 -9
  64. package/dist/space-files/transfer.js +3 -2
  65. package/dist/space-files/upload.d.ts +5 -2
  66. package/dist/space-files/upload.js +41 -1
  67. package/package.json +2 -2
@@ -1,17 +1,21 @@
1
+ import { randomUUID } from "node:crypto";
1
2
  import { mkdir, readFile, writeFile } from "node:fs/promises";
2
3
  import { homedir } from "node:os";
3
4
  import { dirname, join, resolve } from "node:path";
4
- import { compileDisplayActions, DISPLAY_ELEMENT_REF_PATTERN, DISPLAY_SYSTEM_ACTIONS, DisplayActionError, } from "@neta-art/cohub";
5
+ import { compileDisplayPlan, DISPLAY_ELEMENT_REF_PATTERN, DISPLAY_ELEMENT_ROLES, DISPLAY_SYSTEM_ACTIONS, DISPLAY_HANDOVER_DEFAULT_MS, DISPLAY_HANDOVER_MAX_MS, DISPLAY_HANDOVER_WAIT_MS, DisplayActionError, } from "@neta-art/cohub";
5
6
  import { createClient } from "../client.js";
6
7
  import { error, handleHttp, json as outJson, jsonRequested, ok, table } from "../output.js";
7
8
  import { resolveSpace } from "../space.js";
8
9
  import { webUrl } from "../web.js";
9
10
  const DIRECTIONS = ["up", "down", "left", "right"];
10
11
  const BUTTONS = ["primary", "secondary", "middle"];
12
+ const TREES = ["compact", "full", "none"];
11
13
  const DEFAULT_MAX_SIZE = 1280;
12
- const SETTLE_MS = 600;
13
- const FRAMES_PATH = join(homedir(), ".cache", "cohub-cli", "display-frames.json");
14
- const NO_DISPLAY_HINT = "Share one first: the Android app's Runtime menu, `cohub runtime up --display` on a computer, or `cohub spaces displays start` for a virtual screen.";
14
+ const HANDOVER_RETRY_MS = 60_000;
15
+ const HANDOVER_RETRY_GAP_MS = 2_000;
16
+ const CACHE_DIR = join(homedir(), ".cache", "cohub-cli");
17
+ const FRAMES_PATH = join(CACHE_DIR, "display-frames.json");
18
+ const NO_DISPLAY_HINT = "Share one first: set up phone control in the Cohub Android app, run `cohub runtime up --display` on a computer, or `cohub spaces displays start` for a virtual screen.";
15
19
  function parseSize(value) {
16
20
  const match = /^(\d{1,5})x(\d{1,5})$/i.exec(value.trim());
17
21
  const width = Number(match?.[1]);
@@ -58,17 +62,23 @@ function pickDisplay(displays, requested) {
58
62
  return error(`Display "${requested}" is not shared`, `Shared displays: ${displays.map((display) => display.id).join(", ") || "none"}`);
59
63
  return found;
60
64
  }
65
+ const agents = displays.filter((display) => display.kind === "agent");
61
66
  if (displays.length === 1 && displays[0])
62
67
  return displays[0];
68
+ if (agents.length === 1 && agents[0])
69
+ return agents[0];
63
70
  if (displays.length === 0)
64
71
  return error("No display is shared with this Space", NO_DISPLAY_HINT);
65
72
  return error("Several displays are shared", `Pass --display with one of: ${displays.map((display) => display.id).join(", ")}`);
66
73
  }
67
- async function target(spacesCmd, opts) {
74
+ async function target(spacesCmd, opts, launching = false) {
68
75
  const spaceId = await resolveSpace(spacesCmd);
69
76
  const api = createClient().space(spaceId).displays;
70
- const { displays } = await api.list();
71
- return { spaceId, api, display: pickDisplay(displays, opts.display) };
77
+ let { displays, virtual } = await api.list();
78
+ if (launching && !opts.display && !displays.some((display) => display.apps) && virtual === "available")
79
+ ({ displays } = await api.startVirtual());
80
+ const launchers = displays.filter((display) => display.apps);
81
+ return { spaceId, api, display: pickDisplay(launching && !opts.display && launchers.length > 0 ? launchers : displays, opts.display) };
72
82
  }
73
83
  function displayUrl(spaceId, displayId) {
74
84
  return webUrl(`/spaces/${spaceId}?window=${encodeURIComponent(`display:${displayId}`)}`);
@@ -77,83 +87,232 @@ function needsHint(display) {
77
87
  const needs = display.needs ?? [];
78
88
  if (needs.length === 0)
79
89
  return null;
90
+ if (needs.includes("phoneControl"))
91
+ return `${display.name || display.id} needs phone control: ask the user to set it up in the Cohub app on the phone.`;
80
92
  if (!display.desktop)
81
- return `${display.name || display.id} needs control: on the phone, choose Allow control in the Space's Runtime menu and enable Cohub in Accessibility settings.`;
93
+ return `${display.name || display.id} needs control: on the phone, enable Cohub in Accessibility settings.`;
82
94
  const names = needs.map((need) => (need === "screenRecording" ? "Screen Recording" : "Accessibility")).join(" and ");
83
95
  return `${display.name || display.id} needs ${names}: allow it for the Runtime's terminal in System Settings → Privacy & Security, then restart the Runtime.`;
84
96
  }
97
+ function imagePath(display, output) {
98
+ return resolve(output ?? join(CACHE_DIR, "displays", `${display.id}-${new Date().toISOString().replace(/[:.]/g, "-")}.jpg`));
99
+ }
100
+ async function saveImage(spaceId, display, image, output) {
101
+ const path = imagePath(display, output);
102
+ await mkdir(dirname(path), { recursive: true });
103
+ await writeFile(path, Buffer.from(image.data, "base64"));
104
+ await rememberFrame(spaceId, display, image);
105
+ return { path, width: image.width, height: image.height };
106
+ }
85
107
  async function screenshot(api, spaceId, display, maxSize, output) {
86
- const capture = await api.capture(display.id, { maxSize });
87
- const path = resolve(output ?? `${display.id}-${new Date().toISOString().replace(/[:.]/g, "-")}.jpg`);
88
- await writeFile(path, Buffer.from(capture.data, "base64"));
89
- await rememberFrame(spaceId, display, capture);
90
- return { path, width: capture.width, height: capture.height };
108
+ return saveImage(spaceId, display, await api.capture(display.id, { maxSize }), output ?? resolve(`${display.id}-${new Date().toISOString().replace(/[:.]/g, "-")}.jpg`));
109
+ }
110
+ function formatElement(element, frame, indent = true, box = true) {
111
+ const scaleX = frame.width - 1 || 1;
112
+ const scaleY = frame.height - 1 || 1;
113
+ const [x, y, width, height] = element.bounds;
114
+ const parts = [`${indent ? " ".repeat(element.depth) : ""}${element.ref}`, element.role];
115
+ if (element.name)
116
+ parts.push(JSON.stringify(element.name));
117
+ if (element.value)
118
+ parts.push(`value=${JSON.stringify(element.value)}`);
119
+ if (element.states?.length)
120
+ parts.push(`[${element.states.join(", ")}]`);
121
+ if (box)
122
+ parts.push(`(${Math.round(x * scaleX)},${Math.round(y * scaleY)} ${Math.round(width * scaleX)}x${Math.round(height * scaleY)})`);
123
+ return parts.join(" ");
124
+ }
125
+ function formatTree(tree, frame, compact = false) {
126
+ return tree.elements.map((element) => formatElement(element, frame, true, !compact || Boolean(element.actions?.length))).join("\n");
127
+ }
128
+ const ACTOR = process.env.COHUB_SESSION_ID
129
+ ? { sessionId: process.env.COHUB_SESSION_ID, ...(process.env.COHUB_TURN_ID ? { turnId: process.env.COHUB_TURN_ID } : {}) }
130
+ : undefined;
131
+ function untilOf(opts) {
132
+ const until = {
133
+ ...(opts.untilText ? { text: opts.untilText } : {}),
134
+ ...(opts.untilGone ? { gone: opts.untilGone } : {}),
135
+ ...(opts.untilApp ? { app: opts.untilApp } : {}),
136
+ ...(opts.untilStable ? { stable: true } : {}),
137
+ };
138
+ if (Object.keys(until).length === 0)
139
+ return undefined;
140
+ return { ...until, ...(opts.timeoutMs ? { timeoutMs: parseNumber("timeout-ms", opts.timeoutMs) } : {}) };
91
141
  }
92
142
  async function act(spacesCmd, opts, actions) {
143
+ let sent = false;
93
144
  try {
94
- const { spaceId, api, display } = await target(spacesCmd, opts);
95
- if (!display.input)
145
+ const launching = actions.some((action) => action.type === "launch" || action.type === "open");
146
+ const { spaceId, api, display } = await target(spacesCmd, opts, launching);
147
+ if (!display.input && actions.length > 0)
96
148
  return error("This display is view only", needsHint(display) ?? "Allow control on the device first.");
97
149
  const frame = await frameOf(spaceId, display, opts.size);
98
- const events = compileDisplayActions(actions, frame.width, frame.height);
99
- const result = await api.input(display.id, events);
100
- let shot;
101
- if (opts.screenshot) {
102
- await new Promise((done) => setTimeout(done, SETTLE_MS));
103
- shot = await screenshot(api, spaceId, display, Math.max(frame.width, frame.height), typeof opts.screenshot === "string" ? opts.screenshot : undefined);
150
+ const tree = opts.tree ?? "compact";
151
+ if (!TREES.includes(tree))
152
+ return error("Invalid tree", `Use one of: ${TREES.join(", ")}`);
153
+ const last = actions.at(-1);
154
+ if (last) {
155
+ const until = untilOf(opts);
156
+ if (until)
157
+ last.until = until;
158
+ if (opts.label)
159
+ last.label = opts.label;
160
+ }
161
+ const steps = compileDisplayPlan(actions, frame.width, frame.height);
162
+ sent = steps.length > 0;
163
+ const result = await api.act(display.id, {
164
+ steps,
165
+ observe: { tree: tree, image: opts.screenshot ? "always" : "auto", maxSize: DEFAULT_MAX_SIZE },
166
+ ...(ACTOR ? { actor: ACTOR } : {}),
167
+ });
168
+ const image = result.screen.image
169
+ ? await saveImage(spaceId, display, result.screen.image, typeof opts.screenshot === "string" ? opts.screenshot : undefined)
170
+ : undefined;
171
+ const shown = image ?? frame;
172
+ if (result.stopped)
173
+ process.exitCode = 1;
174
+ if (jsonRequested(opts)) {
175
+ const { image: _, ...screen } = result.screen;
176
+ return outJson({ display: display.id, ...result, screen: { ...screen, ...(image ? { image } : {}) }, frame: { width: shown.width, height: shown.height } });
104
177
  }
105
- if (jsonRequested(opts))
106
- return outJson({ display: display.id, ...result, frame: { width: frame.width, height: frame.height }, ...(shot ? { screenshot: shot } : {}) });
107
- ok(`Performed ${actions.length} action${actions.length === 1 ? "" : "s"} on ${display.name || display.id}, in ${frame.width}x${frame.height} pixels`);
108
- if (shot)
109
- console.log(` Screenshot ${shot.width}x${shot.height}: ${shot.path}`);
178
+ printResult(display, result, shown, tree === "compact", image?.path);
110
179
  }
111
180
  catch (e) {
112
181
  if (e instanceof DisplayActionError)
113
182
  return error("Invalid action", e.message);
183
+ if (sent && isLostAnswer(e))
184
+ return error("The answer was lost", "The plan may have run, in part or whole: `look` before repeating any step.");
114
185
  handleHttp(e);
115
186
  }
116
187
  }
117
- function withTarget(command) {
188
+ function isLostAnswer(e) {
189
+ const status = e.status;
190
+ return status === undefined ? e instanceof TypeError : status >= 500 && status !== 501;
191
+ }
192
+ function printResult(display, result, frame, compact, imageFile) {
193
+ const name = display.name || display.id;
194
+ if (result.steps.length > 0 || result.stopped) {
195
+ const done = result.steps.length;
196
+ if (!result.stopped)
197
+ ok(`${done} step${done === 1 ? "" : "s"} on ${name}`);
198
+ for (const [index, step] of result.steps.entries())
199
+ console.log(` ${index + 1}. ${step.effect} · ${step.findMs + step.actMs + step.waitedMs} ms`);
200
+ if (result.stopped) {
201
+ console.log(`✗ Stopped (${result.stopped.reason}): ${result.stopped.message}`);
202
+ for (const candidate of result.stopped.candidates ?? [])
203
+ console.log(` ${formatElement(candidate, frame, false)}`);
204
+ }
205
+ }
206
+ const { screen } = result;
207
+ console.log(`\n${[screen.app, screen.quality === "poor" ? "the tree misses much of this screen: read the image" : null].filter(Boolean).join(" · ") || name}`);
208
+ if (screen.unchanged)
209
+ console.log(" The screen is as you last saw it; its refs are gone, so target by name or `look` again.");
210
+ if (screen.tree) {
211
+ console.log(formatTree(screen.tree, frame, compact));
212
+ if (screen.tree.truncated)
213
+ console.log(" More elements exist; pass --tree full or read `tree --max`.");
214
+ }
215
+ if (imageFile)
216
+ console.log(`\n Image ${frame.width}x${frame.height}: ${imageFile} (coordinates are its pixels)`);
217
+ if (screen.tree)
218
+ console.log(`\n Boxes are x,y and size in ${frame.width}x${frame.height} pixels.`);
219
+ if (display.held)
220
+ console.log(" A person at the device has taken it over; wait until they hand it back.");
221
+ else if (display.viewers?.some((viewer) => viewer.control))
222
+ console.log(" A person is watching with control; what you do here pauses while they act.");
223
+ }
224
+ const HANDOVER_ENDS = {
225
+ stopped: () => ["The person stopped this", "Do not go on with this Turn; ask them."],
226
+ gone: () => ["The display went away", "Ask the person what happened before going on."],
227
+ expired: (held) => held
228
+ ? ["Timed out while the person is still working", "Tell them to say so in the chat when done; they will not wake you by handing back."]
229
+ : ["Timed out; the person did not take over", "Tell them what is needed, and to say so in the chat when done."],
230
+ };
231
+ async function handover(spacesCmd, opts) {
232
+ try {
233
+ const timeoutMs = opts.timeoutMs ? parseNumber("timeout-ms", opts.timeoutMs) : DISPLAY_HANDOVER_DEFAULT_MS;
234
+ if (!Number.isInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > DISPLAY_HANDOVER_MAX_MS)
235
+ return error("Invalid timeout", `--timeout-ms must be between 1 and ${DISPLAY_HANDOVER_MAX_MS} milliseconds.`);
236
+ const { api, display } = await target(spacesCmd, opts);
237
+ const params = { id: randomUUID(), reason: opts.reason, timeoutMs, ...(ACTOR ? { actor: ACTOR } : {}) };
238
+ const giveUpAt = Date.now() + timeoutMs + HANDOVER_RETRY_MS;
239
+ let result = await api.handover(display.id, { ...params, waitMs: opts.wait ? DISPLAY_HANDOVER_WAIT_MS : 0 });
240
+ while (opts.wait && result.state === "waiting") {
241
+ try {
242
+ result = await api.handover(display.id, { ...params, waitMs: DISPLAY_HANDOVER_WAIT_MS });
243
+ }
244
+ catch (e) {
245
+ if (!isLostAnswer(e) || Date.now() > giveUpAt)
246
+ throw e;
247
+ await new Promise((done) => setTimeout(done, HANDOVER_RETRY_GAP_MS));
248
+ }
249
+ }
250
+ if (result.state === "waiting") {
251
+ if (jsonRequested(opts))
252
+ return outJson({ display: display.id, ...result });
253
+ return ok(`Asked the person: ${opts.reason}`);
254
+ }
255
+ if (result.state === "returned")
256
+ return act(spacesCmd, { display: display.id, json: opts.json }, []);
257
+ const [title, hint] = HANDOVER_ENDS[result.state](result.held);
258
+ if (jsonRequested(opts)) {
259
+ process.exitCode = 1;
260
+ return outJson({ display: display.id, ...result });
261
+ }
262
+ return error(title, hint);
263
+ }
264
+ catch (e) {
265
+ handleHttp(e);
266
+ }
267
+ }
268
+ function withAct(command) {
118
269
  return command
119
- .option("-d, --display <id>", "Display id; defaults to the only shared display")
120
- .option("--size <WxH>", "Coordinates are pixels of an image this size (default: the latest screenshot)")
121
- .option("--screenshot [path]", "Save a screenshot once the screen settles")
270
+ .option("-d, --display <id>", "Display id; defaults to the only one, or the agent screen")
271
+ .option("--size <WxH>", "Coordinates are pixels of an image this size (default: the latest image)")
272
+ .option("--until-text <text>", "Wait until this text shows")
273
+ .option("--until-gone <text>", "Wait until this text is gone, e.g. a loading label")
274
+ .option("--until-app <id>", "Wait until this app is in front")
275
+ .option("--until-stable", "Wait until the screen stops changing")
276
+ .option("--timeout-ms <ms>", "How long --until-* waits (default 8000)")
277
+ .option("--label <text>", "What this does, shown to people watching")
278
+ .option("--screenshot [path]", "Attach an image of the screen after (by default only when its tree misses much)")
279
+ .option("--tree <kind>", `The tree after: ${TREES.join(", ")}`)
122
280
  .option("--json", "Output as JSON");
123
281
  }
124
- function pointOrRef(x, y, command) {
282
+ function withTarget(command) {
283
+ return command
284
+ .option("--name <text>", "The element that says this: whole, then start, then part")
285
+ .option("--role <role>", `Its role: ${DISPLAY_ELEMENT_ROLES.join(", ")}`)
286
+ .option("--nth <n>", "The nth match, top to bottom")
287
+ .option("--scroll", "Scroll the list on until it shows");
288
+ }
289
+ function targetOf(x, y, opts, command) {
290
+ if (opts.name !== undefined || opts.role !== undefined) {
291
+ if (opts.role !== undefined && !DISPLAY_ELEMENT_ROLES.includes(opts.role)) {
292
+ return error("Invalid role", `Use one of: ${DISPLAY_ELEMENT_ROLES.join(", ")}`);
293
+ }
294
+ return {
295
+ ...(opts.name !== undefined ? { name: opts.name } : {}),
296
+ ...(opts.role !== undefined ? { role: opts.role } : {}),
297
+ ...(opts.nth !== undefined ? { nth: parseNumber("nth", opts.nth) } : {}),
298
+ ...(opts.scroll ? { scroll: true } : {}),
299
+ };
300
+ }
301
+ if (x === undefined)
302
+ return error("Missing target", `Pass --name, an element ref, or x and y, e.g. ${command} --name Send.`);
125
303
  if (y === undefined)
126
304
  return isRef(x) ? { ref: x } : error("Invalid target", `Pass x and y, or an element ref from \`cohub spaces displays tree\`, e.g. ${command} e3.12.`);
127
305
  return { x: parseNumber("x", x), y: parseNumber("y", y) };
128
306
  }
129
- function formatTree(tree, frame) {
130
- const scaleX = (frame.width - 1) || 1;
131
- const scaleY = (frame.height - 1) || 1;
132
- return tree.elements
133
- .map((element) => {
134
- const [x, y, width, height] = element.bounds;
135
- const box = `(${Math.round(x * scaleX)},${Math.round(y * scaleY)} ${Math.round(width * scaleX)}x${Math.round(height * scaleY)})`;
136
- const parts = [`${" ".repeat(element.depth)}${element.ref}`, element.role];
137
- if (element.name)
138
- parts.push(JSON.stringify(element.name));
139
- if (element.value)
140
- parts.push(`value=${JSON.stringify(element.value)}`);
141
- if (element.states?.length)
142
- parts.push(`[${element.states.join(", ")}]`);
143
- parts.push(box);
144
- return parts.join(" ");
145
- })
146
- .join("\n");
147
- }
148
307
  export function registerSpaceDisplays(spacesCmd) {
149
308
  const displays = spacesCmd
150
309
  .command("displays")
151
- .description("Screens shared with a Space: view, read and control")
310
+ .description("Screens shared with a Space: view, read and act on them")
152
311
  .hook("preAction", async () => { await resolveSpace(spacesCmd); });
153
312
  displays
154
313
  .command("ls")
155
314
  .alias("list")
156
- .description("List shared displays and who watches them")
315
+ .description("List shared displays, what they can do, and who uses them")
157
316
  .option("--json", "Output as JSON")
158
317
  .action(async (opts) => {
159
318
  try {
@@ -167,16 +326,20 @@ export function registerSpaceDisplays(spacesCmd) {
167
326
  table(result.displays.map((display) => ({
168
327
  ...display,
169
328
  size: `${display.width}x${display.height}`,
170
- platform: display.platform ?? "—",
171
- abilities: [display.stream && "view", display.capture && "capture", display.input && "control", display.tree && "tree", display.desktop && "desktop"].filter(Boolean).join(", "),
172
- watching: display.viewers?.length ? `${display.viewers.length}${display.viewers.some((viewer) => viewer.control) ? " (controlling)" : ""}` : "—",
329
+ kind: display.kind ?? "screen",
330
+ abilities: [display.stream && "view", display.capture && "capture", display.input && "control", display.tree && "tree", display.apps && "apps", display.desktop && "desktop"].filter(Boolean).join(", "),
331
+ using: [
332
+ display.held && "a person at the device",
333
+ display.activity && `an agent${display.activity.label ? `: ${display.activity.label}` : ""}`,
334
+ display.viewers?.length && `${display.viewers.length} watching${display.viewers.some((viewer) => viewer.control) ? " (controlling)" : ""}`,
335
+ ].filter(Boolean).join("; ") || "—",
173
336
  })), [
174
337
  { key: "id", label: "ID" },
175
338
  { key: "name", label: "Name" },
339
+ { key: "kind", label: "Kind" },
176
340
  { key: "size", label: "Size" },
177
- { key: "platform", label: "Platform" },
178
341
  { key: "abilities", label: "Abilities" },
179
- { key: "watching", label: "Watching" },
342
+ { key: "using", label: "In use by" },
180
343
  ]);
181
344
  for (const display of result.displays) {
182
345
  const hint = needsHint(display);
@@ -193,8 +356,8 @@ export function registerSpaceDisplays(spacesCmd) {
193
356
  });
194
357
  displays
195
358
  .command("start")
196
- .description("Start the machine's virtual screen, e.g. in a cloud sandbox")
197
- .option("--size <WxH>", "Screen size (default 1280x800)")
359
+ .description("Start the machine's virtual screen: a phone's agent screen, or a sandbox's")
360
+ .option("--size <WxH>", "Screen size where the machine lets you choose (default 1280x800)")
198
361
  .option("--json", "Output as JSON")
199
362
  .action(async (opts) => {
200
363
  try {
@@ -202,10 +365,11 @@ export function registerSpaceDisplays(spacesCmd) {
202
365
  if (opts.size)
203
366
  parseSize(opts.size);
204
367
  const result = await createClient().space(spaceId).displays.startVirtual(opts.size ? { size: opts.size } : {});
205
- const url = result.displays[0] ? displayUrl(spaceId, result.displays[0].id) : null;
368
+ const started = result.displays.find((display) => display.kind === "agent") ?? result.displays[0];
369
+ const url = started ? displayUrl(spaceId, started.id) : null;
206
370
  if (jsonRequested(opts))
207
371
  return outJson({ ...result, url });
208
- ok("Virtual screen running; programs started from now on draw on it");
372
+ ok("Virtual screen running; apps and programs you start from now on run on it");
209
373
  if (url)
210
374
  console.log(` Watch and take over: ${url}`);
211
375
  }
@@ -232,7 +396,7 @@ export function registerSpaceDisplays(spacesCmd) {
232
396
  displays
233
397
  .command("capture")
234
398
  .description("Save a screenshot; later coordinates are its pixels")
235
- .option("-d, --display <id>", "Display id; defaults to the only shared display")
399
+ .option("-d, --display <id>", "Display id; defaults to the only one, or the agent screen")
236
400
  .option("-o, --output <path>", "Where to write the image (default: <display>-<time>.jpg)")
237
401
  .option("--max-size <px>", `Longest edge in pixels (default ${DEFAULT_MAX_SIZE})`)
238
402
  .option("--json", "Output as JSON")
@@ -252,9 +416,9 @@ export function registerSpaceDisplays(spacesCmd) {
252
416
  displays
253
417
  .command("tree")
254
418
  .description("Read the interface as elements: refs to act on, roles, names and boxes")
255
- .option("-d, --display <id>", "Display id; defaults to the only shared display")
419
+ .option("-d, --display <id>", "Display id; defaults to the only one, or the agent screen")
256
420
  .option("--max <n>", "Most elements to return (default 300)")
257
- .option("--actionable", "Only elements that take an action, flat: fewer tokens when looking for a target")
421
+ .option("--compact", "What can be acted on or read, flat: fewer tokens")
258
422
  .option("--json", "Output as JSON")
259
423
  .action(async (opts) => {
260
424
  try {
@@ -263,38 +427,63 @@ export function registerSpaceDisplays(spacesCmd) {
263
427
  return error("This display has no element tree", "Use capture and coordinates instead.");
264
428
  const tree = await api.tree(display.id, {
265
429
  ...(opts.max ? { maxElements: parseNumber("max", opts.max) } : {}),
266
- ...(opts.actionable ? { actionable: true } : {}),
430
+ ...(opts.compact ? { compact: true } : {}),
267
431
  });
268
432
  if (jsonRequested(opts))
269
433
  return outJson(tree);
270
434
  const frame = await frameOf(spaceId, display);
271
- console.log(formatTree(tree, frame));
272
- console.log(`\n Boxes are x,y and size in ${frame.width}x${frame.height} pixels. Act with: tap <ref> · type <text> --into <ref> · scroll <direction> --in <ref>`);
435
+ if (tree.app)
436
+ console.log(tree.app);
437
+ console.log(formatTree(tree, frame, opts.compact));
438
+ console.log(`\n Boxes are x,y and size in ${frame.width}x${frame.height} pixels. Act with: tap --name <text> · type <text> --into <ref> · act < plan.json`);
273
439
  if (tree.truncated)
274
440
  console.log(" More elements exist; pass --max to read more.");
275
- if (display.viewers?.some((viewer) => viewer.control))
276
- console.log(" A person is watching with control; what you do here pauses while they act.");
277
441
  }
278
442
  catch (e) {
279
443
  handleHttp(e);
280
444
  }
281
445
  });
282
- withTarget(displays.command("tap <x> [y]").description("Tap or click a point, or an element by ref"))
283
- .option("--count <n>", "2 to double-click, 3 to triple-click")
446
+ displays
447
+ .command("apps")
448
+ .description("List the apps a display can launch")
449
+ .option("-d, --display <id>", "Display id; defaults to the only one, or the agent screen")
450
+ .option("--json", "Output as JSON")
451
+ .action(async (opts) => {
452
+ try {
453
+ const { api, display } = await target(spacesCmd, opts);
454
+ if (!display.apps)
455
+ return error("This display does not launch apps");
456
+ const { apps } = await api.apps(display.id);
457
+ if (jsonRequested(opts))
458
+ return outJson({ apps });
459
+ table(apps, [{ key: "id", label: "ID" }, { key: "name", label: "Name" }]);
460
+ }
461
+ catch (e) {
462
+ handleHttp(e);
463
+ }
464
+ });
465
+ withAct(displays.command("look").description("Read the screen as `act` reports it, without acting"))
466
+ .action((opts) => act(spacesCmd, opts, []));
467
+ withAct(displays.command("launch <app>").description("Launch an app by id (see `apps`) and wait until it is in front"))
468
+ .action((app, opts) => act(spacesCmd, { untilApp: app, ...opts }, [{ type: "launch", app }]));
469
+ withAct(displays.command("open <url>").description("Open a link, or a page of the system's settings"))
470
+ .action((url, opts) => act(spacesCmd, opts, [{ type: "open", url }]));
471
+ withAct(withTarget(displays.command("tap [x] [y]").description("Tap an element (--name, or a ref) or a point")))
472
+ .option("--count <n>", "2 to double-tap, 3 to triple-tap")
284
473
  .option("--button <button>", `Mouse button on a desktop: ${BUTTONS.join(", ")}`)
285
474
  .action((x, y, opts) => {
286
475
  const button = BUTTONS.find((candidate) => candidate === (opts.button ?? "primary"));
287
476
  if (!button)
288
477
  return error("Invalid button", `Use one of: ${BUTTONS.join(", ")}`);
289
478
  return act(spacesCmd, opts, [{
290
- type: "tap", ...pointOrRef(x, y, "tap"), button,
479
+ type: "tap", ...targetOf(x, y, opts, "tap"), button,
291
480
  ...(opts.count ? { count: parseNumber("count", opts.count) } : {}),
292
481
  }]);
293
482
  });
294
- withTarget(displays.command("long-press <x> [y]").description("Touch and hold a point, or an element by ref"))
483
+ withAct(withTarget(displays.command("long-press [x] [y]").description("Touch and hold an element or a point")))
295
484
  .option("--duration <ms>", "Hold time in milliseconds", "800")
296
- .action((x, y, opts) => act(spacesCmd, opts, [{ type: "long_press", ...pointOrRef(x, y, "long-press"), duration_ms: parseNumber("duration", opts.duration) }]));
297
- withTarget(displays.command("swipe <x1> <y1> <x2> <y2>").description("Swipe from one point to another"))
485
+ .action((x, y, opts) => act(spacesCmd, opts, [{ type: "long_press", ...targetOf(x, y, opts, "long-press"), duration_ms: parseNumber("duration", opts.duration) }]));
486
+ withAct(displays.command("swipe <x1> <y1> <x2> <y2>").description("Swipe from one point to another"))
298
487
  .option("--duration <ms>", "Swipe time in milliseconds", "300")
299
488
  .action((x1, y1, x2, y2, opts) => act(spacesCmd, opts, [{
300
489
  type: "swipe",
@@ -302,7 +491,7 @@ export function registerSpaceDisplays(spacesCmd) {
302
491
  to_x: parseNumber("x2", x2), to_y: parseNumber("y2", y2),
303
492
  duration_ms: parseNumber("duration", opts.duration),
304
493
  }]));
305
- withTarget(displays.command("scroll <direction>").description("Scroll content up, down, left or right"))
494
+ withAct(displays.command("scroll <direction>").description("Scroll content up, down, left or right"))
306
495
  .option("--amount <fraction>", "Fraction of the display, 0.1–1", "0.5")
307
496
  .option("--in <ref>", "Scroll this element of the tree")
308
497
  .action((direction, opts) => {
@@ -313,20 +502,50 @@ export function registerSpaceDisplays(spacesCmd) {
313
502
  ...(opts.in ? { ref: opts.in } : {}),
314
503
  }]);
315
504
  });
316
- withTarget(displays.command("type <text>").description("Type into the focused field, or replace an element's text"))
505
+ withAct(withTarget(displays.command("type <text>").description("Type into the focused field, or replace a field's text")))
317
506
  .option("--into <ref>", "Replace the text of this element of the tree")
318
- .action((text, opts) => act(spacesCmd, opts, [{ type: "type", text, ...(opts.into ? { ref: opts.into } : {}) }]));
319
- withTarget(displays.command("key <key>").description("Press a key or shortcut, e.g. Enter or Control+a"))
507
+ .action((text, opts) => {
508
+ const field = opts.into ? { ref: opts.into } : opts.name !== undefined || opts.role !== undefined ? targetOf(undefined, undefined, opts, "type") : {};
509
+ return act(spacesCmd, opts, [{ type: "type", text, ...field }]);
510
+ });
511
+ withAct(displays.command("key <key>").description("Press a key or shortcut, e.g. Enter or Control+a"))
320
512
  .action((key, opts) => act(spacesCmd, opts, [{ type: "key", key }]));
321
- withTarget(displays.command("press <button>").description(`Press a system button: ${DISPLAY_SYSTEM_ACTIONS.join(", ")}`))
513
+ withAct(displays.command("press <button>").description(`Press a system button: ${DISPLAY_SYSTEM_ACTIONS.join(", ")}`))
322
514
  .action((button, opts) => {
323
515
  const action = DISPLAY_SYSTEM_ACTIONS.find((candidate) => candidate === button);
324
516
  if (!action)
325
517
  return error("Invalid button", `Use one of: ${DISPLAY_SYSTEM_ACTIONS.join(", ")}`);
326
518
  return act(spacesCmd, opts, [{ type: "system", action }]);
327
519
  });
328
- withTarget(displays.command("act").description("Run a JSON array of actions from --actions or stdin"))
329
- .option("--actions <json>", "Actions, e.g. '[{\"type\":\"tap\",\"x\":540,\"y\":1200},{\"type\":\"tap\",\"ref\":\"e3.12\"}]'")
520
+ withAct(displays.command("wait").description("Wait for --until-* (or a pause) without acting"))
521
+ .option("--ms <ms>", "Pause this long first")
522
+ .action((opts) => act(spacesCmd, opts, [{ type: "wait", ...(opts.ms ? { duration_ms: parseNumber("ms", opts.ms) } : {}) }]));
523
+ displays
524
+ .command("handover")
525
+ .description("Ask the person to do what you must not (sign in, pay), and wait until they hand back")
526
+ .requiredOption("--reason <text>", "What they need to do, in their language; shown on the device")
527
+ .option("-d, --display <id>", "Display id; defaults to the only one, or the agent screen")
528
+ .option("--timeout-ms <ms>", `How long to wait (default: ${DISPLAY_HANDOVER_DEFAULT_MS}; max: ${DISPLAY_HANDOVER_MAX_MS})`)
529
+ .option("--no-wait", "Ask and exit without waiting")
530
+ .option("--json", "Output as JSON")
531
+ .action(async (opts) => {
532
+ await handover(spacesCmd, opts);
533
+ });
534
+ withAct(displays.command("act").description("Run a plan: a JSON array of actions from --actions or stdin"))
535
+ .option("--actions <json>", 'Actions, e.g. \'[{"type":"launch","app":"com.android.settings"},{"type":"tap","name":"Wi-Fi"}]\'')
536
+ .addHelpText("after", `
537
+ Each action is one step; the plan runs on the device and stops at the first
538
+ surprise (a target missing or ambiguous, an until that never came, a dialog,
539
+ another app, a person taking over), then reports the screen.
540
+
541
+ {"type":"launch","app":"<id>"} {"type":"open","url":"<link>"}
542
+ {"type":"tap","name":"Send"} {"type":"tap","name":"Ada","scroll":true} {"type":"tap","ref":"e3.12"} {"type":"tap","x":540,"y":1200}
543
+ {"type":"type","text":"hi","name":"Message","role":"textField"} {"type":"type","text":"hi"}
544
+ {"type":"scroll","direction":"down"} {"type":"swipe",...} {"type":"key","key":"Enter"}
545
+ {"type":"system","action":"back"} {"type":"long_press",...} {"type":"wait","duration_ms":500}
546
+
547
+ Any action takes "until": {"text"|"gone"|"app": …, "stable": true, "timeoutMs": 8000}
548
+ and "label": what it does, shown to people watching; name matches take "role" and "nth".`)
330
549
  .action(async (opts) => {
331
550
  let raw = opts.actions ?? "";
332
551
  if (!raw && !process.stdin.isTTY) {
@@ -1,6 +1,6 @@
1
1
  import { readdir, stat } from "node:fs/promises";
2
2
  import { basename, dirname, relative, resolve, sep } from "node:path";
3
- import { harnessSchema, isModelThinkingLevel, MODEL_THINKING_LEVELS, readSentTurns, readSessionTurnOrigin, resolveCohubEnvironment } from "@neta-art/cohub";
3
+ import { APPROVAL_MODES, harnessSchema, isApprovalMode, isModelThinkingLevel, MODEL_THINKING_LEVELS, readSentTurns, readSessionTurnOrigin, resolveCohubEnvironment } from "@neta-art/cohub";
4
4
  import { uploadAvatarAsset, uploadChatImageAsset } from "../avatar.js";
5
5
  import { createClient } from "../client.js";
6
6
  import { collectOption, readPromptContent } from "../command-input.js";
@@ -202,7 +202,8 @@ async function sendPrompt(command, words, opts) {
202
202
  if (opts.note) {
203
203
  const agentOnly = Object.entries({
204
204
  "--delay-ms": opts.delayMs, "--at": opts.at, "--cron": opts.cron, "--model": opts.model, "--provider": opts.provider,
205
- "--thinking-level": opts.thinkingLevel, "--read-only": opts.readOnly, "--steer": opts.steer, "--env": opts.env?.length,
205
+ "--thinking-level": opts.thinkingLevel, "--approval-mode": opts.approvalMode, "--service-tier": opts.serviceTier,
206
+ "--read-only": opts.readOnly, "--steer": opts.steer, "--env": opts.env?.length,
206
207
  }).flatMap(([flag, value]) => value ? [flag] : []);
207
208
  if (agentOnly.length)
208
209
  return error("Conflicting options", `--note cannot be combined with ${agentOnly.join(", ")}`);
@@ -215,6 +216,8 @@ async function sendPrompt(command, words, opts) {
215
216
  if (requestedThinkingLevel && !thinkingLevel) {
216
217
  return error("Invalid thinking level", `Use ${MODEL_THINKING_LEVELS.join("|")}`);
217
218
  }
219
+ if (opts.approvalMode !== undefined && !isApprovalMode(opts.approvalMode))
220
+ return error("Invalid approval mode", `Use ${APPROVAL_MODES.join("|")}`);
218
221
  const spaceId = await resolveSpace(command, { home: true });
219
222
  const client = createClient();
220
223
  try {
@@ -262,6 +265,8 @@ async function sendPrompt(command, words, opts) {
262
265
  provider: opts.provider,
263
266
  harness: opts.harness,
264
267
  thinkingLevel,
268
+ ...(isApprovalMode(opts.approvalMode) ? { approvalMode: opts.approvalMode } : {}),
269
+ ...(opts.serviceTier ? { serviceTier: opts.serviceTier } : {}),
265
270
  accessMode: opts.readOnly ? "read_only" : "full_access",
266
271
  intent: opts.steer ? "steer" : undefined,
267
272
  env: parseEnvOptions(opts.env),
@@ -372,6 +377,8 @@ export function registerPrompt(program) {
372
377
  .option("-p, --provider <provider>", "Provider name")
373
378
  .option("--harness <harness>", `Harness: ${harnessSchema.options.join(", ")}`, "cohub")
374
379
  .option("--thinking-level <level>", `Thinking level: ${MODEL_THINKING_LEVELS.join("|")}`)
380
+ .option("--approval-mode <mode>", `Local Harness approval mode: ${APPROVAL_MODES.join("|")}`)
381
+ .option("--service-tier <tier>", "Local Harness service tier, e.g. priority for Codex Fast")
375
382
  .option("--read-only", "Use read-only tools")
376
383
  .option("--steer", "Interrupt the current turn and run immediately")
377
384
  .option("--delay-ms <ms>", "Delay sending by milliseconds")
@@ -747,6 +754,8 @@ export function registerSpaces(program) {
747
754
  .option("-m, --model <model>", "Model name")
748
755
  .option("-p, --provider <provider>", "Provider name")
749
756
  .option("--thinking-level <level>", `Thinking level: ${MODEL_THINKING_LEVELS.join("|")}`)
757
+ .option("--approval-mode <mode>", `Local Harness approval mode: ${APPROVAL_MODES.join("|")}`)
758
+ .option("--service-tier <tier>", "Local Harness service tier, e.g. priority for Codex Fast")
750
759
  .option("--read-only", "Use read-only tools")
751
760
  .option("--steer", "Interrupt the current turn and run immediately")
752
761
  .option("--delay-ms <ms>", "Delay sending by milliseconds")
@@ -1,6 +1,7 @@
1
- import type { DEVICE_RUNTIME_REFUSALS, DeviceFolderListing, DeviceRuntime } from "@neta-art/cohub/host-bridge";
1
+ import type { DEVICE_RUNTIME_REFUSALS, DEVICE_RUNTIME_STOP_REFUSALS, DeviceDisplayStatus, DeviceFolderListing, DeviceRuntime } from "@neta-art/cohub/host-bridge";
2
2
  import type { HostEmit } from "./serve.js";
3
3
  type Refusal = (typeof DEVICE_RUNTIME_REFUSALS)[number];
4
+ type StopRefusal = (typeof DEVICE_RUNTIME_STOP_REFUSALS)[number];
4
5
  export type DeviceRuntimeHostOptions = {
5
6
  cli: string;
6
7
  env: () => Promise<NodeJS.ProcessEnv>;
@@ -19,7 +20,16 @@ export declare class DeviceRuntimeHost {
19
20
  connect(spaceId: string, root: string): Promise<DeviceRuntime & {
20
21
  refused: Refusal | null;
21
22
  }>;
22
- disconnect(spaceId: string): Promise<DeviceRuntime>;
23
+ disconnect(spaceId: string, force?: boolean): Promise<DeviceRuntime & {
24
+ refused: StopRefusal | null;
25
+ }>;
26
+ stopAccount(): Promise<void>;
27
+ accountChanged(): void;
28
+ displayStatus(): Promise<DeviceDisplayStatus>;
29
+ share(spaceId: string): Promise<DeviceDisplayStatus>;
30
+ stopSharing(): Promise<DeviceDisplayStatus>;
31
+ stopOwnSharing(): Promise<void>;
32
+ keepAwake(enabled?: boolean): Promise<boolean>;
23
33
  restore(): Promise<void>;
24
34
  }
25
35
  export {};