@bivy/bivy 0.17.3-staging.9 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/bivy.mjs +157 -12
- package/dist/apps/annotate.js +94 -0
- package/dist/apps/display-viewer.js +80 -7
- package/dist/apps/display.js +11 -1
- package/dist/apps/gateway.js +106 -14
- package/dist/apps/input.js +96 -0
- package/dist/apps/inspector.js +84 -29
- package/dist/apps/macos-display.js +170 -0
- package/dist/apps/macos-display.swift +873 -0
- package/dist/apps/menu.js +76 -0
- package/dist/apps/preview-shell.js +232 -16
- package/dist/apps/preview-stream.js +17 -3
- package/dist/apps/registry.js +76 -5
- package/dist/apps/review.js +111 -0
- package/dist/apps/rfb.js +69 -20
- package/dist/apps/screenshot.js +8 -2
- package/dist/apps/service.js +343 -13
- package/dist/apps/styles.css +162 -10
- package/dist/apps/tokens.css +6 -0
- package/dist/apps/types.js +17 -0
- package/dist/attach.js +37 -4
- package/dist/controllers/app-commands.js +16 -4
- package/dist/controllers/artifact-commands.js +42 -0
- package/dist/controllers/presence-commands.js +19 -0
- package/dist/protocol/client-command-routes.js +3 -1
- package/dist/protocol/client-command-schemas.js +19 -3
- package/dist/server.js +200 -21
- package/dist/session/attachment-store.js +15 -0
- package/dist/session/event-log.js +25 -3
- package/dist/session/presence.js +82 -0
- package/dist/session/run-terminal.js +113 -69
- package/dist/session/session-new-dedupe.js +1 -1
- package/dist/terminal.js +23 -0
- package/dist/worktree.js +40 -3
- package/package.json +1 -1
package/dist/apps/service.js
CHANGED
|
@@ -1,11 +1,21 @@
|
|
|
1
|
+
import { randomBytes } from "node:crypto";
|
|
2
|
+
import { REVIEW_MODES, shouldReview, visualChange } from "./review.js";
|
|
1
3
|
import fs from "node:fs";
|
|
2
4
|
import os from "node:os";
|
|
3
5
|
import path from "node:path";
|
|
4
6
|
import { scanListeners } from "./listeners.js";
|
|
5
7
|
import { takeShots } from "./screenshot.js";
|
|
8
|
+
import { approximate, composite, readStrokes } from "./annotate.js";
|
|
9
|
+
import { captureFrame, encodePng, sendInput } from "./rfb.js";
|
|
10
|
+
import { inputEvents, readAction } from "./input.js";
|
|
11
|
+
import { pressMenu, readMenuPath, readMenus } from "./menu.js";
|
|
12
|
+
import { pngSize } from "./review.js";
|
|
13
|
+
const NO_REVIEWS = { publish: (review) => review, expire: () => { } };
|
|
6
14
|
const NO_DISPLAYS = { unavailable: () => "Desktop app views aren't available on this machine.", ensure: () => Promise.reject(new Error("Desktop app views aren't available on this machine.")), stop: () => { } };
|
|
7
15
|
/** How long a screenshot waits for a desktop app's first window. */
|
|
8
16
|
const WINDOW_WAIT_MS = 15_000;
|
|
17
|
+
/** How long an app gets to react to input before its picture is taken. */
|
|
18
|
+
const AFTER_INPUT_MS = 400;
|
|
9
19
|
/** Composes view providers; registry never spawns processes and the gateway
|
|
10
20
|
* never knows about terminals. New providers can extend open/remove here. */
|
|
11
21
|
/** A managed server that keeps exiting is left down after this many restarts
|
|
@@ -26,6 +36,15 @@ export class AppService {
|
|
|
26
36
|
screenshots;
|
|
27
37
|
displays;
|
|
28
38
|
shooting = Promise.resolve();
|
|
39
|
+
reviews;
|
|
40
|
+
settleMs;
|
|
41
|
+
runs = new Map();
|
|
42
|
+
/** The latest card per view: a newer one expires its images. */
|
|
43
|
+
latest = new Map();
|
|
44
|
+
/** Per view: the newest reviewer note a run-end card has already counted. */
|
|
45
|
+
handedOver = new Map();
|
|
46
|
+
/** Screenshots in progress, so a card, Compare and a baseline share one browser run. */
|
|
47
|
+
inflight = new Map();
|
|
29
48
|
constructor(registry, gateway, terminals, options = {}) {
|
|
30
49
|
this.registry = registry;
|
|
31
50
|
this.gateway = gateway;
|
|
@@ -34,7 +53,10 @@ export class AppService {
|
|
|
34
53
|
this.serverWatchMs = options.serverWatchMs ?? 5_000;
|
|
35
54
|
this.screenshots = { enabled: options.screenshots?.enabled ?? (() => false), take: options.screenshots?.take ?? takeShots };
|
|
36
55
|
this.displays = options.displays ?? NO_DISPLAYS;
|
|
56
|
+
this.reviews = options.reviews ?? NO_REVIEWS;
|
|
57
|
+
this.settleMs = options.settleMs ?? COMPARE_SETTLE_MS;
|
|
37
58
|
}
|
|
59
|
+
/** A session's apps, or every app on this machine when no session is given. */
|
|
38
60
|
list(sessionId) {
|
|
39
61
|
const available = Boolean(this.gateway) && this.gateway?.available !== false;
|
|
40
62
|
const apps = this.registry.list(sessionId);
|
|
@@ -49,6 +71,8 @@ export class AppService {
|
|
|
49
71
|
view.notes = structuredClone(entry.notes);
|
|
50
72
|
if (entry?.stats)
|
|
51
73
|
view.stats = structuredClone(entry.stats);
|
|
74
|
+
if (entry?.lastPath)
|
|
75
|
+
view.lastPath = entry.lastPath;
|
|
52
76
|
}
|
|
53
77
|
return { apps, previewAvailable: available };
|
|
54
78
|
}
|
|
@@ -74,29 +98,200 @@ export class AppService {
|
|
|
74
98
|
void this.ensureServer(entry).catch(() => { });
|
|
75
99
|
}
|
|
76
100
|
if (bumped.length && this.screenshots.enabled())
|
|
77
|
-
setTimeout(() => void this.capture(bumped),
|
|
101
|
+
setTimeout(() => void this.capture(bumped), this.settleMs).unref?.();
|
|
78
102
|
return bumped;
|
|
79
103
|
}
|
|
80
104
|
/** One phone-width shot per view of the page last viewed, kept for Compare. */
|
|
81
105
|
async capture(viewIds) {
|
|
82
106
|
for (const id of viewIds) {
|
|
83
107
|
const entry = this.registry.getView(id);
|
|
84
|
-
if (
|
|
85
|
-
|
|
108
|
+
if (entry?.view.kind === "web")
|
|
109
|
+
await this.shotOf(entry);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
/** A phone-width screenshot of a page at the view's current revision: one
|
|
113
|
+
* already taken (Compare's after-turn shot), one in progress, or a new one.
|
|
114
|
+
* Undefined when it fails; a card then goes without a picture. */
|
|
115
|
+
shotOf(entry, page = entry.lastPath ?? "/") {
|
|
116
|
+
const revision = entry.revision;
|
|
117
|
+
const taken = [...entry.shots ?? []].reverse().find((shot) => shot.revision === revision && (shot.path ?? "/") === page);
|
|
118
|
+
if (taken)
|
|
119
|
+
return Promise.resolve(taken.png);
|
|
120
|
+
const key = `${entry.view.id}:${revision}:${page}`;
|
|
121
|
+
let pending = this.inflight.get(key);
|
|
122
|
+
if (pending)
|
|
123
|
+
return pending;
|
|
124
|
+
pending = (async () => {
|
|
86
125
|
await this.windowShown(entry).catch(() => { });
|
|
87
|
-
const outDir = path.join(os.tmpdir(), "bivy-shots", "compare", id);
|
|
88
|
-
const run = this.shooting.catch(() => { }).then(() => this.screenshots.take([entry], { widths: [390], themes: ["light"], path:
|
|
126
|
+
const outDir = path.join(os.tmpdir(), "bivy-shots", "compare", entry.view.id);
|
|
127
|
+
const run = this.shooting.catch(() => { }).then(() => this.screenshots.take([entry], { widths: [390], themes: ["light"], path: page }, outDir));
|
|
89
128
|
this.shooting = run;
|
|
90
129
|
try {
|
|
91
130
|
const [shot] = await run;
|
|
92
131
|
if (!shot)
|
|
93
|
-
|
|
132
|
+
return undefined;
|
|
94
133
|
const png = fs.readFileSync(shot.file);
|
|
95
134
|
fs.rmSync(outDir, { recursive: true, force: true });
|
|
96
|
-
entry.shots = [...(entry.shots ?? []), { revision
|
|
135
|
+
entry.shots = [...(entry.shots ?? []), { revision, at: Date.now(), png, path: page }].slice(-COMPARE_KEEP);
|
|
136
|
+
return png;
|
|
97
137
|
}
|
|
98
|
-
catch {
|
|
138
|
+
catch {
|
|
139
|
+
return undefined; /* a failed shot just leaves Compare without this turn */
|
|
140
|
+
}
|
|
141
|
+
finally {
|
|
142
|
+
this.inflight.delete(key);
|
|
143
|
+
}
|
|
144
|
+
})();
|
|
145
|
+
this.inflight.set(key, pending);
|
|
146
|
+
return pending;
|
|
147
|
+
}
|
|
148
|
+
/** Web views of a session whose app wants review cards on their own. */
|
|
149
|
+
reviewable(sessionId) {
|
|
150
|
+
return this.registry.list(sessionId).filter((app) => REVIEW_MODES[this.registry.reviewMode(app.id)].minChange !== undefined)
|
|
151
|
+
.flatMap((app) => app.views.filter((view) => view.kind === "web").map((view) => this.registry.getView(view.id))).filter(Boolean);
|
|
152
|
+
}
|
|
153
|
+
/** An agent run started. Remembers each view's revision, and the page as
|
|
154
|
+
* it is now, so the end of the run can tell whether anything visibly
|
|
155
|
+
* changed. A baseline is only taken when no screenshot of this revision
|
|
156
|
+
* exists yet — after the first run, the last run's screenshot serves. */
|
|
157
|
+
runStarted(sessionId) {
|
|
158
|
+
const views = this.reviewable(sessionId);
|
|
159
|
+
const run = { start: new Map(views.map((entry) => [entry.view.id, entry.revision])), baseline: new Map(), reviews: new Map(), muted: false, ended: false };
|
|
160
|
+
this.runs.set(sessionId, run);
|
|
161
|
+
if (!this.screenshots.enabled())
|
|
162
|
+
return;
|
|
163
|
+
for (const entry of views)
|
|
164
|
+
void this.shotOf(entry).then((png) => { if (png && entry.revision === run.start.get(entry.view.id))
|
|
165
|
+
run.baseline.set(entry.view.id, png); });
|
|
166
|
+
}
|
|
167
|
+
/** The run ended (done, or waiting for the user). Views whose revision
|
|
168
|
+
* changed are screenshotted and compared with the page before the run; a
|
|
169
|
+
* visible change makes (or updates) this run's card. Returns the run's
|
|
170
|
+
* latest card, for the "session finished" notification. */
|
|
171
|
+
async runEnded(sessionId) {
|
|
172
|
+
const run = this.runs.get(sessionId);
|
|
173
|
+
if (!run || run.ended)
|
|
174
|
+
return undefined;
|
|
175
|
+
run.ended = true;
|
|
176
|
+
let latest;
|
|
177
|
+
const changed = [...run.start].map(([id, revision]) => ({ entry: this.registry.getView(id), revision })).filter(({ entry, revision }) => entry && entry.revision !== revision);
|
|
178
|
+
if (changed.length && this.screenshots.enabled() && !run.muted)
|
|
179
|
+
await new Promise((r) => setTimeout(r, this.settleMs));
|
|
180
|
+
for (const { entry } of changed) {
|
|
181
|
+
if (!entry || !this.screenshots.enabled())
|
|
182
|
+
continue;
|
|
183
|
+
const mode = this.registry.reviewMode(entry.app.id);
|
|
184
|
+
if (run.muted || REVIEW_MODES[mode].minChange === undefined)
|
|
185
|
+
continue;
|
|
186
|
+
const before = run.baseline.get(entry.view.id);
|
|
187
|
+
const after = await this.shotOf(entry);
|
|
188
|
+
const change = before && after ? visualChange(before, after) : undefined;
|
|
189
|
+
// A card the agent presented earlier in the run is refreshed if the page changed since.
|
|
190
|
+
const current = this.latest.get(entry.view.id);
|
|
191
|
+
const presented = current && run.reviews.get(entry.app.id) === current.id;
|
|
192
|
+
if (!shouldReview({ trigger: "run", mode, muted: run.muted, revisionChanged: true, change }) && !(presented && after))
|
|
193
|
+
continue;
|
|
194
|
+
latest = this.review(entry, run, { trigger: presented ? current.trigger : "run", note: presented ? current.note : undefined, path: entry.lastPath ?? "/", shot: after, before });
|
|
195
|
+
}
|
|
196
|
+
// Reviewer notes that arrived since the last hand-over ride on the view's
|
|
197
|
+
// card for this run, or get a card of their own. A count only: the notes
|
|
198
|
+
// stay on the machine until the owner drafts them into a message.
|
|
199
|
+
for (const entry of this.webViews(sessionId)) {
|
|
200
|
+
const waiting = (entry.notes ?? []).filter((note) => note.at > (this.handedOver.get(entry.view.id) ?? 0));
|
|
201
|
+
if (!waiting.length || run.muted || this.registry.reviewMode(entry.app.id) === "off")
|
|
202
|
+
continue;
|
|
203
|
+
this.handedOver.set(entry.view.id, Math.max(...waiting.map((note) => note.at)));
|
|
204
|
+
const notes = entry.notes.length;
|
|
205
|
+
const current = this.latest.get(entry.view.id);
|
|
206
|
+
if (current && run.reviews.get(entry.app.id) === current.id) {
|
|
207
|
+
const card = this.reviews.publish({ ...current, notes, at: Date.now() }, {});
|
|
208
|
+
this.latest.set(entry.view.id, card);
|
|
209
|
+
if (!latest || latest.id === card.id)
|
|
210
|
+
latest = card;
|
|
211
|
+
}
|
|
212
|
+
else {
|
|
213
|
+
// Cards are one per app per run: another view's card keeps its own; these notes get a new one.
|
|
214
|
+
const card = this.review(entry, run.reviews.has(entry.app.id) ? undefined : run, { trigger: "notes", path: entry.lastPath ?? "/", notes });
|
|
215
|
+
// A visible change stays what "finished" talks about; notes fill in when there's nothing else.
|
|
216
|
+
latest ??= card;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
if (!latest) {
|
|
220
|
+
const id = [...run.reviews.values()].at(-1);
|
|
221
|
+
latest = [...this.latest.values()].find((review) => review.id === id);
|
|
222
|
+
}
|
|
223
|
+
return latest;
|
|
224
|
+
}
|
|
225
|
+
/** `bivy app present`, or Show me ("asked"): a card for one view now. */
|
|
226
|
+
async present(sessionId, input) {
|
|
227
|
+
const trigger = input.trigger ?? "present";
|
|
228
|
+
const page = input.path;
|
|
229
|
+
if (page !== undefined && (typeof page !== "string" || !page.startsWith("/") || page.startsWith("//") || page.length > 2048))
|
|
230
|
+
throw new Error("Path must start with /.");
|
|
231
|
+
const note = typeof input.note === "string" ? input.note.trim().slice(0, 500) : undefined;
|
|
232
|
+
const entry = this.pickView(sessionId, input.target);
|
|
233
|
+
const mode = this.registry.reviewMode(entry.app.id);
|
|
234
|
+
const active = this.runs.get(sessionId);
|
|
235
|
+
const run = active && !active.ended ? active : undefined;
|
|
236
|
+
if (!shouldReview({ trigger, mode, muted: run?.muted ?? false, revisionChanged: true })) {
|
|
237
|
+
return { message: run?.muted ? "The user muted preview cards for this run. They can still open the preview from the chat." : `Preview cards are off for ${entry.app.name}. The user can still open the preview from the chat.` };
|
|
238
|
+
}
|
|
239
|
+
// Presenting means the change is ready: pick up files written since the
|
|
240
|
+
// turn began, so the picture (and any open preview) shows them.
|
|
241
|
+
if (trigger === "present" && this.turnChanged(sessionId).includes(entry.view.id) && this.screenshots.enabled())
|
|
242
|
+
await new Promise((r) => setTimeout(r, this.settleMs));
|
|
243
|
+
const enabled = this.screenshots.enabled();
|
|
244
|
+
const path = page ?? entry.lastPath ?? "/";
|
|
245
|
+
const shot = enabled ? await this.shotOf(entry, path) : undefined;
|
|
246
|
+
const baseline = run?.baseline.get(entry.view.id);
|
|
247
|
+
const before = baseline && shot && !baseline.equals(shot) && path === (entry.lastPath ?? "/") ? baseline : undefined;
|
|
248
|
+
const review = this.review(entry, run, { trigger, note, path, shot, before, screenshotsOff: !enabled });
|
|
249
|
+
return { review, message: enabled ? (shot ? "The user sees it in the chat." : "The user sees a card in the chat, but the screenshot failed.") : "The user sees a card in the chat. Screenshots are off on this machine, so it has no picture." };
|
|
250
|
+
}
|
|
251
|
+
/** No more cards for the rest of the session's current run. */
|
|
252
|
+
mute(sessionId) {
|
|
253
|
+
const run = this.runs.get(sessionId);
|
|
254
|
+
if (run && !run.ended)
|
|
255
|
+
run.muted = true;
|
|
256
|
+
return { ok: true };
|
|
257
|
+
}
|
|
258
|
+
setReviewMode(sessionId, appId, mode) {
|
|
259
|
+
this.registry.setReviewMode(sessionId, appId, mode);
|
|
260
|
+
return { ok: true };
|
|
261
|
+
}
|
|
262
|
+
/** A view by app or view ID or name; by default the one opened last, else the newest app's first. */
|
|
263
|
+
webViews(sessionId) {
|
|
264
|
+
return this.registry.list(sessionId).flatMap((app) => app.views.filter((view) => view.kind === "web").map((view) => this.registry.getView(view.id))).filter(Boolean);
|
|
265
|
+
}
|
|
266
|
+
pickView(sessionId, target, desktop = false) {
|
|
267
|
+
const views = this.webViews(sessionId).filter((entry) => !desktop || entry.target.kind === "display");
|
|
268
|
+
const noun = desktop ? "desktop app" : "web view";
|
|
269
|
+
if (!views.length)
|
|
270
|
+
throw new Error(desktop ? "This session has no desktop apps. Publish one with bivy app run -- <command>." : "This session has no web views to present. Publish one with bivy app publish, or preview a detected server.");
|
|
271
|
+
if (target) {
|
|
272
|
+
const wanted = target.trim().toLowerCase();
|
|
273
|
+
const found = views.find((entry) => [entry.app.id, entry.view.id].includes(wanted))
|
|
274
|
+
?? views.find((entry) => entry.view.name.toLowerCase() === wanted) ?? views.find((entry) => entry.app.name.toLowerCase() === wanted);
|
|
275
|
+
if (!found)
|
|
276
|
+
throw new Error(`No ${noun} called "${target}" in this session. Run bivy app list to see them.`);
|
|
277
|
+
return found;
|
|
99
278
|
}
|
|
279
|
+
return views.reduce((best, entry) => (entry.openedAt ?? 0) > (best.openedAt ?? 0) || ((entry.openedAt ?? 0) === (best.openedAt ?? 0) && entry.app.createdAt > best.app.createdAt) ? entry : best);
|
|
280
|
+
}
|
|
281
|
+
/** Makes or updates a card, keeping each view's images to the latest card. */
|
|
282
|
+
review(entry, run, fields) {
|
|
283
|
+
const id = (run && run.reviews.get(entry.app.id)) ?? `review-${randomBytes(8).toString("hex")}`;
|
|
284
|
+
const { shot, before, ...rest } = fields;
|
|
285
|
+
const review = this.reviews.publish({
|
|
286
|
+
id, sessionId: entry.app.sessionId, appId: entry.app.id, viewId: entry.view.id, name: entry.app.name, view: entry.view.name,
|
|
287
|
+
at: Date.now(), ...Object.fromEntries(Object.entries(rest).filter(([, value]) => value !== undefined && value !== false)),
|
|
288
|
+
}, { shot, before });
|
|
289
|
+
const previous = this.latest.get(entry.view.id);
|
|
290
|
+
if (previous && previous.id !== review.id)
|
|
291
|
+
this.reviews.expire(previous, new Set([review.shot?.hash, review.before?.hash].filter((hash) => Boolean(hash))));
|
|
292
|
+
this.latest.set(entry.view.id, review);
|
|
293
|
+
run?.reviews.set(entry.app.id, review.id);
|
|
294
|
+
return review;
|
|
100
295
|
}
|
|
101
296
|
/** Servers running in the workspace that this session doesn't preview yet. */
|
|
102
297
|
async offers(sessionId, workspace) {
|
|
@@ -113,7 +308,8 @@ export class AppService {
|
|
|
113
308
|
/** `direct` opens the app origin itself (no shell): a signed-in device
|
|
114
309
|
* returning to a view's stable address. */
|
|
115
310
|
/** `scale`: the opening device's pixel density, used if this starts a display. */
|
|
116
|
-
|
|
311
|
+
/** `page`: start on this page instead of the app's root (a review card's page). */
|
|
312
|
+
async open(sessionId, appId, viewId, returnTo, direct = false, scale, page) {
|
|
117
313
|
const entry = this.registry.requireView(sessionId, appId, viewId);
|
|
118
314
|
if (entry.target.kind === "display" && !entry.display)
|
|
119
315
|
entry.displayScale = scale === 2 ? 2 : 1;
|
|
@@ -122,15 +318,18 @@ export class AppService {
|
|
|
122
318
|
throw new Error("Bivy's preview service is unavailable on this connection.");
|
|
123
319
|
if (direct && !this.gateway.openDirect)
|
|
124
320
|
throw new Error("This machine can't open previews directly.");
|
|
125
|
-
|
|
321
|
+
let url = direct ? this.gateway.openDirect(viewId) : this.gateway.open(viewId, returnTo);
|
|
322
|
+
if (page && page.startsWith("/") && !page.startsWith("//") && page.length <= 2048)
|
|
323
|
+
url += `~${encodeURIComponent(page).replace(/~/g, "%7E")}`;
|
|
126
324
|
// Don't wait for a managed server to boot: the preview shows it starting
|
|
127
325
|
// and reloads itself once it answers.
|
|
128
326
|
if (this.servers.get(viewId))
|
|
129
327
|
this.servers.get(viewId).restarts = [];
|
|
130
328
|
void this.ensureServer(entry).catch(() => { });
|
|
329
|
+
entry.openedAt = Date.now();
|
|
131
330
|
// Compare needs a "before": take a baseline the first time it's opened.
|
|
132
331
|
if (!entry.shots?.length && this.screenshots.enabled())
|
|
133
|
-
setTimeout(() => void this.capture([viewId]),
|
|
332
|
+
setTimeout(() => void this.capture([viewId]), this.settleMs).unref?.();
|
|
134
333
|
return { kind: "web", url };
|
|
135
334
|
}
|
|
136
335
|
if (entry.target.kind !== "terminal")
|
|
@@ -186,6 +385,99 @@ export class AppService {
|
|
|
186
385
|
this.shooting = run;
|
|
187
386
|
return { shots: await run };
|
|
188
387
|
}
|
|
388
|
+
/** Computer use: an agent clicks, types, presses keys and scrolls in one of
|
|
389
|
+
* its desktop apps, in the pixels of the app's screenshot, the way a viewer
|
|
390
|
+
* would. Starts the app if needed. With screenshots on, returns the app as
|
|
391
|
+
* it looks afterwards, so the agent sees what its action did. */
|
|
392
|
+
async act(sessionId, target, input) {
|
|
393
|
+
const action = readAction(input);
|
|
394
|
+
const entry = this.pickView(sessionId, target, true);
|
|
395
|
+
if (!entry.display)
|
|
396
|
+
await this.windowShown(entry);
|
|
397
|
+
if (!entry.display)
|
|
398
|
+
throw new Error("The app isn't running. Check its Logs in the Apps sheet.");
|
|
399
|
+
const { width, height } = await sendInput(entry.display, inputEvents(action));
|
|
400
|
+
if (!this.screenshots.enabled())
|
|
401
|
+
return { width, height, message: "Done. Agent screenshots are off on this machine, so there's no picture of the result: bivy config set sessions.appScreenshots true" };
|
|
402
|
+
return { width, height, shot: await this.after(entry, sessionId), message: "Done. The shot is the app now." };
|
|
403
|
+
}
|
|
404
|
+
/** A desktop app's menu bar: listed, or one item chosen by its path of
|
|
405
|
+
* titles ("File > Save"). Choosing returns the app afterwards, like `act`. */
|
|
406
|
+
async menu(sessionId, target, path) {
|
|
407
|
+
const item = path === undefined ? undefined : readMenuPath(path);
|
|
408
|
+
const entry = this.pickView(sessionId, target, true);
|
|
409
|
+
if (!entry.display)
|
|
410
|
+
await this.windowShown(entry);
|
|
411
|
+
if (!entry.displayControl)
|
|
412
|
+
throw new Error("This machine's desktop apps don't expose their menus (macOS only). Use keyboard shortcuts: bivy app key.");
|
|
413
|
+
if (!item)
|
|
414
|
+
return { menus: await readMenus(entry.displayControl) };
|
|
415
|
+
await pressMenu(entry.displayControl, item);
|
|
416
|
+
if (!this.screenshots.enabled())
|
|
417
|
+
return { message: "Chosen. Agent screenshots are off on this machine, so there's no picture of the result." };
|
|
418
|
+
return { shot: await this.after(entry, sessionId), message: "Chosen. The shot is the app now." };
|
|
419
|
+
}
|
|
420
|
+
/** The app a moment after an action, as a screenshot. */
|
|
421
|
+
async after(entry, sessionId) {
|
|
422
|
+
await new Promise((r) => setTimeout(r, AFTER_INPUT_MS));
|
|
423
|
+
const outDir = path.join(os.tmpdir(), "bivy-shots", sessionId.replace(/[^A-Za-z0-9_-]/g, "_"), String(Date.now()));
|
|
424
|
+
const run = this.shooting.catch(() => { }).then(() => this.screenshots.take([entry], { widths: [], themes: [], path: "/" }, outDir));
|
|
425
|
+
this.shooting = run;
|
|
426
|
+
return (await run)[0];
|
|
427
|
+
}
|
|
428
|
+
/** Draw on the preview: the user's marks on a picture of what they saw.
|
|
429
|
+
* The picture is the Compare screenshot they drew on, a desktop app's
|
|
430
|
+
* current frame, or a web page retaken at their viewport, pixel ratio and
|
|
431
|
+
* scroll — "approximate" when the page may hold state a fresh browser
|
|
432
|
+
* doesn't. Returned to the client (over the session channel) as a PNG. */
|
|
433
|
+
async annotate(sessionId, input) {
|
|
434
|
+
const entry = this.registry.requireView(sessionId, input.appId, input.viewId);
|
|
435
|
+
if (entry.view.kind !== "web")
|
|
436
|
+
throw new Error("Only previews can be drawn on.");
|
|
437
|
+
const strokes = readStrokes(input.strokes);
|
|
438
|
+
const whole = (n, min, max) => typeof n === "number" && Number.isFinite(n) && n >= min && n <= max;
|
|
439
|
+
// Compare's shot is drawn on at its on-screen size, which can be small.
|
|
440
|
+
if (!whole(input.viewport?.width, 40, 4000) || !whole(input.viewport?.height, 40, 8000))
|
|
441
|
+
throw new Error("Invalid viewport.");
|
|
442
|
+
const viewport = { width: Math.round(input.viewport.width), height: Math.round(input.viewport.height) };
|
|
443
|
+
const page = typeof input.path === "string" && input.path.startsWith("/") && !input.path.startsWith("//") && input.path.length <= 2048 ? input.path : entry.lastPath ?? "/";
|
|
444
|
+
const scroll = { x: whole(input.scroll?.x, 0, 1e6) ? input.scroll.x : 0, y: whole(input.scroll?.y, 0, 1e6) ? input.scroll.y : 0 };
|
|
445
|
+
const name = `${entry.app.name} ${page === "/" ? "" : page} marked.png`.replace(/[^\w .()-]+/g, "-").replace(/\s+/g, " ").trim();
|
|
446
|
+
const finish = (base, offset, approx) => {
|
|
447
|
+
const png = composite(base, strokes, { scale: pngSize(base).width / viewport.width, offset });
|
|
448
|
+
return { image: { data: png.toString("base64"), mimeType: "image/png", name, ...pngSize(png) }, approximate: approx };
|
|
449
|
+
};
|
|
450
|
+
// Compare: the exact frame they drew on, in its own coordinates.
|
|
451
|
+
if (input.compare !== undefined) {
|
|
452
|
+
const shot = Number.isInteger(input.compare) ? entry.shots?.[input.compare] : undefined;
|
|
453
|
+
if (!shot)
|
|
454
|
+
throw new Error("That Compare screenshot is no longer kept. Open Compare again.");
|
|
455
|
+
return finish(shot.png, { x: 0, y: 0 }, approximate("exact"));
|
|
456
|
+
}
|
|
457
|
+
if (!this.screenshots.enabled())
|
|
458
|
+
return { approximate: false, screenshotsOff: true };
|
|
459
|
+
// A desktop app: its display's current frame is exactly what they saw.
|
|
460
|
+
if (entry.target.kind === "display") {
|
|
461
|
+
await this.windowShown(entry).catch(() => { });
|
|
462
|
+
if (!entry.display)
|
|
463
|
+
throw new Error("The app isn't running, so there's nothing to draw on.");
|
|
464
|
+
const frame = await captureFrame(entry.display);
|
|
465
|
+
return finish(encodePng(frame.width, frame.height, frame.rgb), { x: 0, y: 0 }, approximate("exact"));
|
|
466
|
+
}
|
|
467
|
+
const outDir = path.join(os.tmpdir(), "bivy-shots", "annotate", entry.view.id);
|
|
468
|
+
const dpr = whole(input.dpr, 1, 3) ? Math.round(input.dpr * 4) / 4 : 2;
|
|
469
|
+
const run = this.shooting.catch(() => { }).then(() => this.screenshots.take([entry], { widths: [viewport.width], themes: [input.theme === "dark" ? "dark" : "light"], path: page, height: viewport.height, scale: dpr, scroll }, outDir));
|
|
470
|
+
this.shooting = run;
|
|
471
|
+
const [shot] = await run;
|
|
472
|
+
if (!shot)
|
|
473
|
+
throw new Error("Couldn't take a picture of the page.");
|
|
474
|
+
const base = fs.readFileSync(shot.file);
|
|
475
|
+
fs.rmSync(outDir, { recursive: true, force: true });
|
|
476
|
+
const landed = shot.scroll ?? scroll;
|
|
477
|
+
const scrolledTo = Math.abs(landed.x - scroll.x) <= 2 && Math.abs(landed.y - scroll.y) <= 2;
|
|
478
|
+
// Marks stay on the content they were drawn on, wherever the retake scrolled.
|
|
479
|
+
return finish(base, landed, approximate("retake", input.signals, scrolledTo));
|
|
480
|
+
}
|
|
189
481
|
/** The managed server's output, as an attachable terminal (starts it if needed). */
|
|
190
482
|
async logs(sessionId, appId, viewId) {
|
|
191
483
|
const entry = this.registry.requireView(sessionId, appId, viewId);
|
|
@@ -215,7 +507,9 @@ export class AppService {
|
|
|
215
507
|
throw new Error("This view has no program.");
|
|
216
508
|
return this.displays.ensure(entry.view.id, entry.app.name, entry.displayScale).then((display) => {
|
|
217
509
|
entry.display = display.socket;
|
|
218
|
-
|
|
510
|
+
entry.displayControl = display.control;
|
|
511
|
+
const [command, ...args] = [...display.launch ?? [], target.command, ...target.args];
|
|
512
|
+
return { command: command, args, workspace: target.workspace, env: display.env, name: `${entry.app.name} · ${entry.view.name}` };
|
|
219
513
|
});
|
|
220
514
|
}
|
|
221
515
|
/** Services start synchronously; a desktop app waits for its display. */
|
|
@@ -270,6 +564,7 @@ export class AppService {
|
|
|
270
564
|
const entry = this.registry.getView(id);
|
|
271
565
|
if (entry) {
|
|
272
566
|
entry.display = undefined;
|
|
567
|
+
entry.displayControl = undefined;
|
|
273
568
|
entry.displayScale = undefined;
|
|
274
569
|
}
|
|
275
570
|
const server = this.servers.get(id);
|
|
@@ -288,10 +583,45 @@ export class AppService {
|
|
|
288
583
|
throw new Error("Bivy's preview service is unavailable on this connection.");
|
|
289
584
|
return this.gateway.share(viewId);
|
|
290
585
|
}
|
|
586
|
+
/** `bivy app share`: mints a share link for a web view picked by app and/or
|
|
587
|
+
* view ID or name, like `present` (default: the one opened last). */
|
|
588
|
+
shareView(sessionId, input) {
|
|
589
|
+
const entry = this.pickTarget(sessionId, input);
|
|
590
|
+
return { ...this.share(sessionId, entry.app.id, entry.view.id), appId: entry.app.id, viewId: entry.view.id, app: entry.app.name, view: entry.view.name };
|
|
591
|
+
}
|
|
291
592
|
clearNotes(sessionId, appId, viewId) {
|
|
292
|
-
|
|
593
|
+
this.registry.requireView(sessionId, appId, viewId);
|
|
594
|
+
this.registry.clearNotes(viewId);
|
|
595
|
+
return { ok: true };
|
|
596
|
+
}
|
|
597
|
+
/** The owner's choice (never the agent's): may agents read this app's notes? */
|
|
598
|
+
setAgentNotes(sessionId, appId, enabled) {
|
|
599
|
+
this.registry.setAgentNotes(sessionId, appId, enabled);
|
|
293
600
|
return { ok: true };
|
|
294
601
|
}
|
|
602
|
+
/** `bivy app notes`: reviewer notes for an agent, only on apps whose owner
|
|
603
|
+
* allowed it. Picked like `bivy app share`. Untrusted text, and labelled so. */
|
|
604
|
+
notes(sessionId, input) {
|
|
605
|
+
const entry = this.pickTarget(sessionId, input);
|
|
606
|
+
if (!this.registry.agentNotes(entry.app.id))
|
|
607
|
+
throw new Error(`Reviewer notes on ${entry.app.name} are private to its owner. They can allow agents to read them in Apps → ${entry.app.name} → ⋯ → Agents can read notes.`);
|
|
608
|
+
const since = typeof input.since === "number" && Number.isFinite(input.since) ? input.since : 0;
|
|
609
|
+
return { app: entry.app.name, view: entry.view.name, appId: entry.app.id, viewId: entry.view.id, untrusted: true, notes: structuredClone((entry.notes ?? []).filter((note) => note.at > since)) };
|
|
610
|
+
}
|
|
611
|
+
/** An app and/or view by ID or name (`bivy app share`, `bivy app notes`); by default the view opened last. */
|
|
612
|
+
pickTarget(sessionId, input) {
|
|
613
|
+
if (!input.view)
|
|
614
|
+
return this.pickView(sessionId, input.app);
|
|
615
|
+
const wantedApp = input.app?.trim().toLowerCase();
|
|
616
|
+
const scope = this.webViews(sessionId).filter((candidate) => !wantedApp || candidate.app.id === wantedApp || candidate.app.name.toLowerCase() === wantedApp);
|
|
617
|
+
if (wantedApp && !scope.length)
|
|
618
|
+
throw new Error(`No app called "${input.app}" with web views in this session. Run bivy app list to see them.`);
|
|
619
|
+
const wanted = input.view.trim().toLowerCase();
|
|
620
|
+
const found = scope.find((candidate) => candidate.view.id === wanted) ?? scope.find((candidate) => candidate.view.name.toLowerCase() === wanted);
|
|
621
|
+
if (!found)
|
|
622
|
+
throw new Error(`No web view called "${input.view}"${input.app ? ` in "${input.app}"` : ""}. Run bivy app list to see them.`);
|
|
623
|
+
return found;
|
|
624
|
+
}
|
|
295
625
|
/** Ends every link, browser session and open connection for one view; the app stays. */
|
|
296
626
|
revoke(sessionId, appId, viewId) {
|
|
297
627
|
this.registry.requireView(sessionId, appId, viewId);
|