@bivy/bivy 0.19.3 → 0.19.4-staging.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps/annotate.js +85 -1
- package/dist/apps/inspector.js +82 -6
- package/dist/apps/preview-shell.js +385 -152
- package/dist/apps/review.js +30 -0
- package/dist/apps/screenshot.js +16 -2
- package/dist/apps/service.js +134 -10
- package/dist/apps/styles.css +5 -0
- package/dist/apps/types.js +16 -0
- package/dist/controllers/app-commands.js +4 -0
- package/dist/server.js +32 -2
- package/dist/session/event-log.js +22 -3
- package/package.json +1 -1
package/dist/apps/review.js
CHANGED
|
@@ -105,6 +105,36 @@ export function visualChange(before, after) {
|
|
|
105
105
|
}
|
|
106
106
|
return changed / (a.width * a.height);
|
|
107
107
|
}
|
|
108
|
+
/** Share of a pin's marked region that must differ between the page before a
|
|
109
|
+
* run and the page after it before the pin is called "changed". Higher than a
|
|
110
|
+
* whole-page threshold: a region is small, so a handful of antialiased pixels
|
|
111
|
+
* is a larger share of it, and calling a pin answered when it isn't is worse
|
|
112
|
+
* than leaving it open. */
|
|
113
|
+
export const PIN_CHANGE = 0.01;
|
|
114
|
+
/** Share of pixels that visibly differ inside one region (0–1). The region is
|
|
115
|
+
* in page CSS pixels; `scale` maps those to image pixels. Returns undefined
|
|
116
|
+
* when the region isn't inside both images — a page that grew or shrank says
|
|
117
|
+
* nothing about the marked spot. */
|
|
118
|
+
export function regionChange(before, after, region, scale) {
|
|
119
|
+
const a = decodePng(before), b = decodePng(after);
|
|
120
|
+
if (a.width !== b.width || a.height !== b.height)
|
|
121
|
+
return undefined;
|
|
122
|
+
const left = Math.max(0, Math.floor(region.x * scale)), top = Math.max(0, Math.floor(region.y * scale));
|
|
123
|
+
const right = Math.min(a.width, Math.ceil((region.x + region.width) * scale)), bottom = Math.min(a.height, Math.ceil((region.y + region.height) * scale));
|
|
124
|
+
const pixels = (right - left) * (bottom - top);
|
|
125
|
+
if (pixels <= 0)
|
|
126
|
+
return undefined;
|
|
127
|
+
let changed = 0;
|
|
128
|
+
for (let y = top; y < bottom; y++) {
|
|
129
|
+
for (let x = left; x < right; x++) {
|
|
130
|
+
const i = (y * a.width + x) * a.channels, j = (y * b.width + x) * b.channels;
|
|
131
|
+
const delta = Math.abs(a.pixels[i] - b.pixels[j]) + Math.abs(a.pixels[i + 1] - b.pixels[j + 1]) + Math.abs(a.pixels[i + 2] - b.pixels[j + 2]);
|
|
132
|
+
if (delta > 12)
|
|
133
|
+
changed++;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return changed / pixels;
|
|
137
|
+
}
|
|
108
138
|
/** PNG dimensions, from the header. */
|
|
109
139
|
export function pngSize(png) {
|
|
110
140
|
return { width: png.readUInt32BE(16), height: png.readUInt32BE(20) };
|
package/dist/apps/screenshot.js
CHANGED
|
@@ -122,6 +122,12 @@ export async function takeShots(views, request, outDir, chrome = findChrome()) {
|
|
|
122
122
|
let ws;
|
|
123
123
|
let browser;
|
|
124
124
|
const deadline = setTimeout(() => browser?.kill("SIGKILL"), 120_000);
|
|
125
|
+
// How long Chrome gets to print its DevTools line. Generous on purpose: this
|
|
126
|
+
// only bounds a browser that never comes up, and it costs nothing when one
|
|
127
|
+
// starts quickly. A tighter bound turned a loaded machine — a shared CI
|
|
128
|
+
// runner, a laptop mid-build — into "the browser didn't start in time" and a
|
|
129
|
+
// missing screenshot, which is a worse answer than waiting.
|
|
130
|
+
const START_MS = 60_000;
|
|
125
131
|
// Chromium's sandbox is unavailable as root and where AppArmor blocks user
|
|
126
132
|
// namespaces (Ubuntu 23.10+). The pages are the session's own apps, already
|
|
127
133
|
// running as this user, so retrying without it doesn't widen what they can do.
|
|
@@ -130,7 +136,7 @@ export async function takeShots(views, request, outDir, chrome = findChrome()) {
|
|
|
130
136
|
const child = spawn(chrome, args, { stdio: ["ignore", "ignore", "pipe"] });
|
|
131
137
|
browser = child;
|
|
132
138
|
let text = "";
|
|
133
|
-
const timer = setTimeout(() => reject(new Error("The browser didn't start in time.")),
|
|
139
|
+
const timer = setTimeout(() => reject(new Error("The browser didn't start in time.")), START_MS);
|
|
134
140
|
child.stderr.on("data", (chunk) => { text += chunk; const m = /DevTools listening on (ws:\/\/\S+)/.exec(text); if (m) {
|
|
135
141
|
clearTimeout(timer);
|
|
136
142
|
resolve(m[1]);
|
|
@@ -188,10 +194,18 @@ export async function takeShots(views, request, outDir, chrome = findChrome()) {
|
|
|
188
194
|
const { result } = await cdp.send("Runtime.evaluate", { expression: "[scrollX, scrollY]", returnByValue: true }, sessionId);
|
|
189
195
|
scroll = { x: Number(result?.value?.[0]) || 0, y: Number(result?.value?.[1]) || 0 };
|
|
190
196
|
}
|
|
197
|
+
let missing;
|
|
198
|
+
if (request.selectors?.length) {
|
|
199
|
+
// The page's own selectors, evaluated as data: a list of strings in,
|
|
200
|
+
// a list of strings out, never anything the page chooses to run.
|
|
201
|
+
const wanted = JSON.stringify(request.selectors.slice(0, 50));
|
|
202
|
+
const { result } = await cdp.send("Runtime.evaluate", { expression: `(${wanted}).filter(s => { try { return !document.querySelector(s); } catch { return false; } })`, returnByValue: true }, sessionId);
|
|
203
|
+
missing = Array.isArray(result?.value) ? result.value.filter((s) => typeof s === "string") : undefined;
|
|
204
|
+
}
|
|
191
205
|
const { data } = await cdp.send("Page.captureScreenshot", { format: "png" }, sessionId);
|
|
192
206
|
const file = path.join(outDir, `${entry.view.id.slice(0, 8)}-${width}-${theme}.png`);
|
|
193
207
|
fs.writeFileSync(file, Buffer.from(data, "base64"));
|
|
194
|
-
shots.push({ viewId: entry.view.id, view: entry.view.name, width, theme, file, ...(scroll ? { scroll } : {}), ...(elementScrollsRestored !== undefined ? { elementScrollsRestored } : {}) });
|
|
208
|
+
shots.push({ viewId: entry.view.id, view: entry.view.name, width, theme, file, ...(scroll ? { scroll } : {}), ...(elementScrollsRestored !== undefined ? { elementScrollsRestored } : {}), ...(missing ? { missing } : {}) });
|
|
195
209
|
}
|
|
196
210
|
}
|
|
197
211
|
return shots;
|
package/dist/apps/service.js
CHANGED
|
@@ -2,18 +2,27 @@
|
|
|
2
2
|
// Copyright (c) 2026 Petter André Sjulstad
|
|
3
3
|
import { SHARE_DURATIONS } from "./types.js";
|
|
4
4
|
import { randomBytes } from "node:crypto";
|
|
5
|
-
import { REVIEW_MODES, shouldReview, visualChange } from "./review.js";
|
|
5
|
+
import { PIN_CHANGE, REVIEW_MODES, regionChange, shouldReview, visualChange } from "./review.js";
|
|
6
6
|
import fs from "node:fs";
|
|
7
7
|
import os from "node:os";
|
|
8
8
|
import path from "node:path";
|
|
9
9
|
import { scanListeners } from "./listeners.js";
|
|
10
10
|
import { takeShots } from "./screenshot.js";
|
|
11
|
-
import { approximate, composite, readStrokes, readElementScrolls } from "./annotate.js";
|
|
11
|
+
import { approximate, composite, crop, noteAnchor, readNotes, readStrokes, readElementScrolls } from "./annotate.js";
|
|
12
12
|
import { captureFrame, encodePng, sendInput } from "./rfb.js";
|
|
13
13
|
import { inputEvents, readAction } from "./input.js";
|
|
14
14
|
import { pressMenu, readMenuPath, readMenus } from "./menu.js";
|
|
15
15
|
import { pngSize } from "./review.js";
|
|
16
16
|
const NO_REVIEWS = { publish: (review) => review, expire: () => { } };
|
|
17
|
+
const NO_PINS = { publish: (pin) => pin };
|
|
18
|
+
/** A pin's region is in the page CSS pixels of the viewport it was marked in.
|
|
19
|
+
* Run screenshots are taken at this width, so a pin marked at a very different
|
|
20
|
+
* one sat on a different layout: its region says nothing about these pixels,
|
|
21
|
+
* and it stays open rather than being answered wrongly. */
|
|
22
|
+
const PIN_SHOT_WIDTH = 390;
|
|
23
|
+
const PIN_WIDTH_TOLERANCE = 0.1;
|
|
24
|
+
/** How long a drawn picture is kept for the pin that may follow it. */
|
|
25
|
+
const MARK_TTL_MS = 10 * 60_000;
|
|
17
26
|
const NO_DISPLAYS = { unavailable: () => "Desktop app views aren't available on this machine.", ensure: () => Promise.reject(new Error("Desktop app views aren't available on this machine.")), stop: () => { } };
|
|
18
27
|
/** How long a screenshot waits for a desktop app's first window. */
|
|
19
28
|
const WINDOW_WAIT_MS = 15_000;
|
|
@@ -40,6 +49,12 @@ export class AppService {
|
|
|
40
49
|
displays;
|
|
41
50
|
shooting = Promise.resolve();
|
|
42
51
|
reviews;
|
|
52
|
+
pinSink;
|
|
53
|
+
/** Pins made in this node's lifetime, newest last, by view. */
|
|
54
|
+
pins = [];
|
|
55
|
+
/** The last picture drawn on a view, kept briefly for the pin that may
|
|
56
|
+
* follow it: the person is still typing their words when it is made. */
|
|
57
|
+
lastMark = new Map();
|
|
43
58
|
settleMs;
|
|
44
59
|
runs = new Map();
|
|
45
60
|
/** The latest card per view: a newer one expires its images. */
|
|
@@ -57,6 +72,7 @@ export class AppService {
|
|
|
57
72
|
this.screenshots = { enabled: options.screenshots?.enabled ?? (() => false), take: options.screenshots?.take ?? takeShots };
|
|
58
73
|
this.displays = options.displays ?? NO_DISPLAYS;
|
|
59
74
|
this.reviews = options.reviews ?? NO_REVIEWS;
|
|
75
|
+
this.pinSink = options.pins ?? NO_PINS;
|
|
60
76
|
this.settleMs = options.settleMs ?? COMPARE_SETTLE_MS;
|
|
61
77
|
}
|
|
62
78
|
/** A session's apps, or every app on this machine when no session is given. */
|
|
@@ -151,10 +167,15 @@ export class AppService {
|
|
|
151
167
|
this.inflight.set(key, pending);
|
|
152
168
|
return pending;
|
|
153
169
|
}
|
|
154
|
-
|
|
170
|
+
hasOpenPins(viewId) {
|
|
171
|
+
return this.pins.some((pin) => pin.viewId === viewId && pin.state === "open");
|
|
172
|
+
}
|
|
173
|
+
/** Web views a run watches: those whose app wants review cards, and those
|
|
174
|
+
* carrying a pin still waiting for an answer. */
|
|
155
175
|
reviewable(sessionId) {
|
|
156
|
-
return this.registry.list(sessionId)
|
|
157
|
-
.flatMap((app) => app.views.filter((view) => view.kind === "web").map((view) => this.registry.getView(view.id))).filter(Boolean)
|
|
176
|
+
return this.registry.list(sessionId)
|
|
177
|
+
.flatMap((app) => app.views.filter((view) => view.kind === "web").map((view) => this.registry.getView(view.id))).filter(Boolean)
|
|
178
|
+
.filter((entry) => REVIEW_MODES[this.registry.reviewMode(entry.app.id)].minChange !== undefined || this.hasOpenPins(entry.view.id));
|
|
158
179
|
}
|
|
159
180
|
/** An agent run started. Remembers each view's revision, and the page as
|
|
160
181
|
* it is now, so the end of the run can tell whether anything visibly
|
|
@@ -187,10 +208,15 @@ export class AppService {
|
|
|
187
208
|
if (!entry || !this.screenshots.enabled())
|
|
188
209
|
continue;
|
|
189
210
|
const mode = this.registry.reviewMode(entry.app.id);
|
|
190
|
-
|
|
191
|
-
continue;
|
|
211
|
+
const cards = !run.muted && REVIEW_MODES[mode].minChange !== undefined;
|
|
192
212
|
const before = run.baseline.get(entry.view.id);
|
|
193
|
-
|
|
213
|
+
// Pins are answered from evidence even where cards are off or muted:
|
|
214
|
+
// turning cards off means "stop showing me the app", not "forget what I
|
|
215
|
+
// asked for".
|
|
216
|
+
const after = cards || this.hasOpenPins(entry.view.id) ? await this.shotOf(entry) : undefined;
|
|
217
|
+
await this.resolvePins(entry, before, after);
|
|
218
|
+
if (!cards)
|
|
219
|
+
continue;
|
|
194
220
|
const change = before && after ? visualChange(before, after) : undefined;
|
|
195
221
|
// A card the agent presented earlier in the run is refreshed if the page changed since.
|
|
196
222
|
const current = this.latest.get(entry.view.id);
|
|
@@ -440,7 +466,10 @@ export class AppService {
|
|
|
440
466
|
const entry = this.registry.requireView(sessionId, input.appId, input.viewId);
|
|
441
467
|
if (entry.view.kind !== "web")
|
|
442
468
|
throw new Error("Only previews can be drawn on.");
|
|
443
|
-
|
|
469
|
+
// Several notes, each owning its strokes; one unnumbered mark is the same
|
|
470
|
+
// thing with a single entry, so there is one path through here.
|
|
471
|
+
const grouped = readNotes(input.notes);
|
|
472
|
+
const strokes = grouped.length ? grouped.flatMap((note) => note.strokes) : readStrokes(input.strokes);
|
|
444
473
|
const elementScrolls = readElementScrolls(input.elementScrolls);
|
|
445
474
|
const whole = (n, min, max) => typeof n === "number" && Number.isFinite(n) && n >= min && n <= max;
|
|
446
475
|
// Compare's shot is drawn on at its on-screen size, which can be small.
|
|
@@ -450,8 +479,22 @@ export class AppService {
|
|
|
450
479
|
const page = typeof input.path === "string" && input.path.startsWith("/") && !input.path.startsWith("//") && input.path.length <= 2048 ? input.path : entry.lastPath ?? "/";
|
|
451
480
|
const scroll = { x: whole(input.scroll?.x, 0, 1e6) ? input.scroll.x : 0, y: whole(input.scroll?.y, 0, 1e6) ? input.scroll.y : 0 };
|
|
452
481
|
const name = `${entry.app.name} ${page === "/" ? "" : page} marked.png`.replace(/[^\w .()-]+/g, "-").replace(/\s+/g, " ").trim();
|
|
482
|
+
const box = (of) => {
|
|
483
|
+
const xs = of.flatMap((stroke) => stroke.points.map(([x]) => x));
|
|
484
|
+
const ys = of.flatMap((stroke) => stroke.points.map(([, y]) => y));
|
|
485
|
+
return { x: Math.min(...xs), y: Math.min(...ys), width: Math.max(...xs) - Math.min(...xs), height: Math.max(...ys) - Math.min(...ys) };
|
|
486
|
+
};
|
|
487
|
+
const loose = (Array.isArray(input.selectors) ? input.selectors : []).filter((value) => typeof value === "string" && value.length > 0 && value.length <= 300).slice(0, 8);
|
|
488
|
+
// What each note covers, in the page pixels it was drawn in, so a later
|
|
489
|
+
// screenshot can be compared exactly there (see resolvePins).
|
|
490
|
+
const held = grouped.length
|
|
491
|
+
? grouped.map((note) => ({ n: note.n, words: note.words, selectors: note.selectors, region: box(note.strokes) }))
|
|
492
|
+
: [{ n: 1, words: "", selectors: loose, region: box(strokes) }];
|
|
493
|
+
const badges = grouped.length > 1 ? grouped.map((note) => ({ n: note.n, ...noteAnchor(note) })) : [];
|
|
453
494
|
const finish = (base, offset, approx) => {
|
|
454
|
-
const png = composite(base, strokes, { scale: pngSize(base).width / viewport.width, offset });
|
|
495
|
+
const png = composite(base, strokes, { scale: pngSize(base).width / viewport.width, offset }, badges);
|
|
496
|
+
// Keep it for the pins that may follow: the person is still writing.
|
|
497
|
+
this.lastMark.set(entry.view.id, { at: Date.now(), png, path: page, viewport, notes: held });
|
|
455
498
|
return { image: { data: png.toString("base64"), mimeType: "image/png", name, ...pngSize(png) }, approximate: approx };
|
|
456
499
|
};
|
|
457
500
|
// Compare: the exact frame they drew on, in its own coordinates.
|
|
@@ -485,6 +528,87 @@ export class AppService {
|
|
|
485
528
|
// Marks stay on the content they were drawn on, wherever the retake scrolled.
|
|
486
529
|
return finish(base, landed, approximate("retake", input.signals, scrolledTo && shot.elementScrollsRestored !== false));
|
|
487
530
|
}
|
|
531
|
+
/** A pin: the marks the person just sent, kept as a thing with a state
|
|
532
|
+
* rather than a paragraph in a message. Made only when the message is
|
|
533
|
+
* actually sent, so nothing appears in the chat that the person didn't send.
|
|
534
|
+
* The picture is the one `annotate` already made for the message. */
|
|
535
|
+
pin(sessionId, input) {
|
|
536
|
+
const entry = this.registry.requireView(sessionId, input.appId, input.viewId);
|
|
537
|
+
const mark = this.lastMark.get(input.viewId);
|
|
538
|
+
if (!mark || Date.now() - mark.at > MARK_TTL_MS)
|
|
539
|
+
throw new Error("That mark is no longer held. Mark the preview again.");
|
|
540
|
+
this.lastMark.delete(input.viewId);
|
|
541
|
+
const scale = pngSize(mark.png).width / mark.viewport.width;
|
|
542
|
+
const fallback = String(input.words ?? "").trim().slice(0, 2000);
|
|
543
|
+
const made = mark.notes.map((note) => {
|
|
544
|
+
// The card shows what that note marked, not the whole page it sat on.
|
|
545
|
+
// The message keeps the full picture; this is the pin's own thumbnail.
|
|
546
|
+
let picture = mark.png;
|
|
547
|
+
try {
|
|
548
|
+
picture = crop(mark.png, note.region, scale);
|
|
549
|
+
}
|
|
550
|
+
catch { /* the whole picture still tells the story */ }
|
|
551
|
+
const pin = this.pinSink.publish({
|
|
552
|
+
id: `pin-${randomBytes(8).toString("hex")}`, sessionId, appId: entry.app.id, viewId: entry.view.id,
|
|
553
|
+
name: entry.app.name, view: entry.view.name, path: mark.path,
|
|
554
|
+
words: note.words || fallback, at: Date.now(),
|
|
555
|
+
...(mark.notes.length > 1 ? { number: note.n } : {}),
|
|
556
|
+
selectors: note.selectors, region: note.region, viewport: mark.viewport, state: "open",
|
|
557
|
+
}, picture);
|
|
558
|
+
this.pins = [...this.pins, pin].slice(-200);
|
|
559
|
+
return pin;
|
|
560
|
+
});
|
|
561
|
+
return { pins: made };
|
|
562
|
+
}
|
|
563
|
+
/** The person's own verdict on a pin. Only they can say "done". */
|
|
564
|
+
setPinState(sessionId, pinId, state) {
|
|
565
|
+
const current = this.pins.find((pin) => pin.id === pinId && pin.sessionId === sessionId);
|
|
566
|
+
if (!current)
|
|
567
|
+
throw new Error("That pin is no longer held on this machine.");
|
|
568
|
+
return { pin: this.updatePin(current, state) };
|
|
569
|
+
}
|
|
570
|
+
updatePin(pin, state) {
|
|
571
|
+
const next = this.pinSink.publish({ ...pin, state, stateAt: Date.now() });
|
|
572
|
+
this.pins = this.pins.map((item) => (item.id === pin.id ? next : item));
|
|
573
|
+
return next;
|
|
574
|
+
}
|
|
575
|
+
/** After a run, say what became of the pins on a view — from evidence only.
|
|
576
|
+
* "gone" when what a pin named is no longer on the page; "changed" when the
|
|
577
|
+
* pixels it marked differ from before the run. A run that changed nothing
|
|
578
|
+
* there leaves the pin open: it has not been answered. */
|
|
579
|
+
async resolvePins(entry, before, after) {
|
|
580
|
+
const open = this.pins.filter((pin) => pin.viewId === entry.view.id && pin.state === "open");
|
|
581
|
+
if (!open.length)
|
|
582
|
+
return;
|
|
583
|
+
const selectors = [...new Set(open.flatMap((pin) => pin.selectors))];
|
|
584
|
+
let missing;
|
|
585
|
+
if (selectors.length && this.screenshots.enabled()) {
|
|
586
|
+
const outDir = path.join(os.tmpdir(), "bivy-shots", "pins", entry.view.id);
|
|
587
|
+
try {
|
|
588
|
+
const run = this.shooting.catch(() => { }).then(() => this.screenshots.take([entry], { widths: [PIN_SHOT_WIDTH], themes: ["light"], path: entry.lastPath ?? "/", selectors }, outDir));
|
|
589
|
+
this.shooting = run;
|
|
590
|
+
missing = (await run)[0]?.missing;
|
|
591
|
+
}
|
|
592
|
+
catch { /* no answer is not an answer: the pins stay open */ }
|
|
593
|
+
finally {
|
|
594
|
+
fs.rmSync(outDir, { recursive: true, force: true });
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
for (const pin of open) {
|
|
598
|
+
if (missing && pin.selectors.length && pin.selectors.every((selector) => missing.includes(selector))) {
|
|
599
|
+
this.updatePin(pin, "gone");
|
|
600
|
+
continue;
|
|
601
|
+
}
|
|
602
|
+
if (!before || !after || pin.path !== (entry.lastPath ?? "/"))
|
|
603
|
+
continue;
|
|
604
|
+
// A pin marked at a very different width sat on a different layout.
|
|
605
|
+
if (Math.abs(pin.viewport.width - PIN_SHOT_WIDTH) > PIN_SHOT_WIDTH * PIN_WIDTH_TOLERANCE)
|
|
606
|
+
continue;
|
|
607
|
+
const change = regionChange(before, after, pin.region, pngSize(after).width / pin.viewport.width);
|
|
608
|
+
if (change !== undefined && change > PIN_CHANGE)
|
|
609
|
+
this.updatePin(pin, "changed");
|
|
610
|
+
}
|
|
611
|
+
}
|
|
488
612
|
/** The managed server's output, as an attachable terminal (starts it if needed). */
|
|
489
613
|
async logs(sessionId, appId, viewId) {
|
|
490
614
|
const entry = this.registry.requireView(sessionId, appId, viewId);
|
package/dist/apps/styles.css
CHANGED
|
@@ -3161,6 +3161,11 @@ p.setup-note .btn.link { min-height: 44px; margin-block: calc((1lh - 44px) / 2);
|
|
|
3161
3161
|
.review-actions { display: flex; flex-wrap: wrap; gap: var(--space-1); padding: var(--space-3); }
|
|
3162
3162
|
.review-actions .btn { min-block-size: var(--space-7); }
|
|
3163
3163
|
|
|
3164
|
+
/* Pin card (PinCard.tsx): the review card's layout, carrying what someone
|
|
3165
|
+
marked instead of what a run produced. The state is one .badge, and the
|
|
3166
|
+
words are the same quoted block a review note uses. */
|
|
3167
|
+
.pin-card .review-note { margin-block-start: 0; }
|
|
3168
|
+
|
|
3164
3169
|
/* Apps sheet (AppsSheet.tsx): one card per app, its views nested, one primary
|
|
3165
3170
|
action per view; secondary tools as quiet icon buttons; rare or destructive
|
|
3166
3171
|
actions behind ⋯ (MoreMenu, the canonical .menu). Status uses .banner. */
|
package/dist/apps/types.js
CHANGED
|
@@ -17,6 +17,22 @@ export function isAppReview(value) {
|
|
|
17
17
|
&& (review.note === undefined || typeof review.note === "string")
|
|
18
18
|
&& (review.notes === undefined || (Number.isInteger(review.notes) && review.notes >= 0));
|
|
19
19
|
}
|
|
20
|
+
export const APP_PIN_STATES = ["open", "changed", "gone", "done"];
|
|
21
|
+
export const APP_PIN_BLOCK = "bivy_app_pin";
|
|
22
|
+
export function isAppPin(value) {
|
|
23
|
+
if (!value || typeof value !== "object")
|
|
24
|
+
return false;
|
|
25
|
+
const pin = value;
|
|
26
|
+
const box = (v, keys) => !!v && typeof v === "object" && keys.every((k) => typeof v[k] === "number");
|
|
27
|
+
return typeof pin.id === "string" && typeof pin.sessionId === "string" && typeof pin.appId === "string" && /^[a-f0-9]{32}$/.test(pin.appId)
|
|
28
|
+
&& typeof pin.viewId === "string" && typeof pin.name === "string" && typeof pin.view === "string" && typeof pin.path === "string"
|
|
29
|
+
&& typeof pin.words === "string" && typeof pin.at === "number"
|
|
30
|
+
&& (pin.number === undefined || (Number.isInteger(pin.number) && pin.number >= 1))
|
|
31
|
+
&& Array.isArray(pin.selectors) && pin.selectors.every((s) => typeof s === "string")
|
|
32
|
+
&& box(pin.region, ["x", "y", "width", "height"]) && box(pin.viewport, ["width", "height"])
|
|
33
|
+
&& APP_PIN_STATES.includes(pin.state)
|
|
34
|
+
&& (pin.shot === undefined || isShot(pin.shot));
|
|
35
|
+
}
|
|
20
36
|
export const APP_PUBLICATION_BLOCK = "bivy_app";
|
|
21
37
|
export function isAppReference(value) {
|
|
22
38
|
if (!value || typeof value !== "object")
|
|
@@ -29,6 +29,10 @@ export function createAppCommands(service, workspaceFor, published = () => { })
|
|
|
29
29
|
"apps.mute": (msg) => service.mute(String(msg.sessionId)),
|
|
30
30
|
"apps.reviewMode": (msg) => service.setReviewMode(String(msg.sessionId), String(msg.appId), msg.mode),
|
|
31
31
|
"apps.annotate": (msg) => service.annotate(String(msg.sessionId), msg),
|
|
32
|
+
// A pin is made only once the person has actually sent the message the
|
|
33
|
+
// marks went into, so nothing appears in the chat that they didn't send.
|
|
34
|
+
"apps.pin": (msg) => service.pin(String(msg.sessionId), { appId: String(msg.appId), viewId: String(msg.viewId), words: typeof msg.words === "string" ? msg.words : "" }),
|
|
35
|
+
"apps.pinState": (msg) => service.setPinState(String(msg.sessionId), String(msg.pinId), msg.state),
|
|
32
36
|
"apps.clearNotes": (msg) => service.clearNotes(String(msg.sessionId), String(msg.appId), String(msg.viewId)),
|
|
33
37
|
"apps.agentNotes": (msg) => service.setAgentNotes(String(msg.sessionId), String(msg.appId), msg.enabled === true),
|
|
34
38
|
"apps.notes": (msg) => service.notes(String(msg.sessionId), { app: typeof msg.appId === "string" ? msg.appId : undefined, view: typeof msg.view === "string" ? msg.view : undefined, since: typeof msg.since === "number" ? msg.since : undefined }),
|
package/dist/server.js
CHANGED
|
@@ -1856,7 +1856,7 @@ function referencedAttachmentHashes() {
|
|
|
1856
1856
|
// `scan`, not `entries`: this visits every session ever created, and must not pin
|
|
1857
1857
|
// each one's full log in the event-log cache.
|
|
1858
1858
|
for (const id of ids) {
|
|
1859
|
-
for (const entry of eventLog.scan(id, ["attachment", "outbound-attachment", "inline-image", "app-review"])) {
|
|
1859
|
+
for (const entry of eventLog.scan(id, ["attachment", "outbound-attachment", "inline-image", "app-review", "app-pin"])) {
|
|
1860
1860
|
if (entry.bivyKind === "attachment")
|
|
1861
1861
|
for (const ref of entry.refs)
|
|
1862
1862
|
hashes.add(ref.hash);
|
|
@@ -1865,6 +1865,10 @@ function referencedAttachmentHashes() {
|
|
|
1865
1865
|
if (shot)
|
|
1866
1866
|
hashes.add(shot.hash);
|
|
1867
1867
|
}
|
|
1868
|
+
else if (entry.bivyKind === "app-pin") {
|
|
1869
|
+
if (entry.pin.shot)
|
|
1870
|
+
hashes.add(entry.pin.shot.hash);
|
|
1871
|
+
}
|
|
1868
1872
|
else if (entry.bivyKind === "outbound-attachment" || entry.bivyKind === "inline-image")
|
|
1869
1873
|
hashes.add(entry.ref.hash);
|
|
1870
1874
|
}
|
|
@@ -2146,6 +2150,32 @@ function createReviewSink() {
|
|
|
2146
2150
|
},
|
|
2147
2151
|
};
|
|
2148
2152
|
}
|
|
2153
|
+
/** Pins: the picture becomes an encrypted attachment like a review card's, and
|
|
2154
|
+
* the pin is logged where it was made, so it keeps its place in the chat while
|
|
2155
|
+
* only its state moves. */
|
|
2156
|
+
function createPinSink() {
|
|
2157
|
+
const anchors = new Map();
|
|
2158
|
+
return {
|
|
2159
|
+
publish(pin, image) {
|
|
2160
|
+
const record = resolveSession(pin.sessionId);
|
|
2161
|
+
if (!record)
|
|
2162
|
+
return pin;
|
|
2163
|
+
const card = { ...pin };
|
|
2164
|
+
if (image && !card.shot) {
|
|
2165
|
+
try {
|
|
2166
|
+
const ref = attachmentStore.put(image, { name: `${pin.name} ${pin.path} pinned.png`, mimeType: "image/png", kind: "image" });
|
|
2167
|
+
card.shot = { hash: ref.hash, size: ref.size, ...pngSize(image) };
|
|
2168
|
+
}
|
|
2169
|
+
catch { /* a pin without a picture still carries the words and the marks */ }
|
|
2170
|
+
}
|
|
2171
|
+
const at = anchors.get(card.id) ?? { afterMessageCount: record.session.getMessages().length, createdAt: Date.now() };
|
|
2172
|
+
anchors.set(card.id, at);
|
|
2173
|
+
eventLog.appendAppPin(record.id, { ...at, pin: card });
|
|
2174
|
+
broadcast(stampSessionEvent({ type: "session.event", sessionId: record.id, event: { type: "app_pin", id: card.id, pin: card } }));
|
|
2175
|
+
return card;
|
|
2176
|
+
},
|
|
2177
|
+
};
|
|
2178
|
+
}
|
|
2149
2179
|
const appService = new AppService(appRegistry, appGateway ?? remotePreview, {
|
|
2150
2180
|
start: async (spec) => {
|
|
2151
2181
|
let failure = "Could not start the app terminal.";
|
|
@@ -2160,7 +2190,7 @@ const appService = new AppService(appRegistry, appGateway ?? remotePreview, {
|
|
|
2160
2190
|
},
|
|
2161
2191
|
has: (id) => terminals.has(id),
|
|
2162
2192
|
close: (id) => { terminals.close(id); },
|
|
2163
|
-
}, { screenshots: { enabled: appScreenshotsEnabled }, displays: appDisplays, reviews: createReviewSink() });
|
|
2193
|
+
}, { screenshots: { enabled: appScreenshotsEnabled }, displays: appDisplays, reviews: createReviewSink(), pins: createPinSink() });
|
|
2164
2194
|
// A reviewer's note shows up in an open Apps sheet without reopening it, and
|
|
2165
2195
|
// the owner gets one push per burst of notes on a view ("2 notes on
|
|
2166
2196
|
// Storefront"), which opens the Apps sheet at that app. Counts, names and IDs
|
|
@@ -30,7 +30,7 @@ import fs from "node:fs";
|
|
|
30
30
|
import path from "node:path";
|
|
31
31
|
import { normalizedIntermediateText, thinkingTextFromContent, mergeTranscript } from "./transcript-merge.js";
|
|
32
32
|
import { DELEGATION_BLOCK, isDelegationCard } from "./delegations.js";
|
|
33
|
-
import { APP_PUBLICATION_BLOCK, APP_REVIEW_BLOCK, isAppReference, isAppReview } from "../apps/types.js";
|
|
33
|
+
import { APP_PIN_BLOCK, APP_PUBLICATION_BLOCK, APP_REVIEW_BLOCK, isAppPin, isAppReference, isAppReview } from "../apps/types.js";
|
|
34
34
|
import { SUGGESTION_BLOCK, isTaskSuggestion } from "./suggestions.js";
|
|
35
35
|
/**
|
|
36
36
|
* Decompose a transcript message into serialization-independent "atoms": the
|
|
@@ -267,8 +267,14 @@ function isAppReviewEntry(value) {
|
|
|
267
267
|
const entry = value;
|
|
268
268
|
return entry.bivyKind === "app-review" && typeof entry.id === "string" && typeof entry.createdAt === "number" && typeof entry.afterMessageCount === "number" && isAppReview(entry.review);
|
|
269
269
|
}
|
|
270
|
+
function isAppPinEntry(value) {
|
|
271
|
+
if (!value || typeof value !== "object")
|
|
272
|
+
return false;
|
|
273
|
+
const entry = value;
|
|
274
|
+
return entry.bivyKind === "app-pin" && typeof entry.id === "string" && typeof entry.createdAt === "number" && typeof entry.afterMessageCount === "number" && isAppPin(entry.pin);
|
|
275
|
+
}
|
|
270
276
|
function isRecord(value) {
|
|
271
|
-
return isDelegationEntry(value) || isOverlay(value) || isBase(value) || isForkDisplay(value) || isAttachment(value) || isOutboundAttachment(value) || isInlineImage(value) || isAppPublication(value) || isAppReviewEntry(value) || isSuggestionEntry(value);
|
|
277
|
+
return isDelegationEntry(value) || isOverlay(value) || isBase(value) || isForkDisplay(value) || isAttachment(value) || isOutboundAttachment(value) || isInlineImage(value) || isAppPublication(value) || isAppReviewEntry(value) || isAppPinEntry(value) || isSuggestionEntry(value);
|
|
272
278
|
}
|
|
273
279
|
/**
|
|
274
280
|
* Fold attachment records into a text→refs list: last write wins per text (a
|
|
@@ -390,6 +396,15 @@ export function replayExtras(entries) {
|
|
|
390
396
|
role: "assistant", content: [{ type: APP_REVIEW_BLOCK, review: entry.review }],
|
|
391
397
|
id: entry.id, afterMessageCount: entry.afterMessageCount, createdAt: entry.createdAt,
|
|
392
398
|
}));
|
|
399
|
+
// A pin keeps where it was made and shows its latest state.
|
|
400
|
+
const pins = new Map();
|
|
401
|
+
for (const entry of entries)
|
|
402
|
+
if (entry.bivyKind === "app-pin")
|
|
403
|
+
pins.set(entry.id, entry);
|
|
404
|
+
const pinCards = [...pins.values()].map((entry) => ({
|
|
405
|
+
role: "assistant", content: [{ type: APP_PIN_BLOCK, pin: entry.pin }],
|
|
406
|
+
id: entry.id, afterMessageCount: entry.afterMessageCount, createdAt: entry.createdAt,
|
|
407
|
+
}));
|
|
393
408
|
// A delegation card keeps the place it was created and shows its latest state.
|
|
394
409
|
const delegations = new Map();
|
|
395
410
|
for (const entry of entries) {
|
|
@@ -406,7 +421,7 @@ export function replayExtras(entries) {
|
|
|
406
421
|
role: "assistant", content: [{ type: SUGGESTION_BLOCK, suggestion: entry.suggestion }],
|
|
407
422
|
id: entry.id, afterMessageCount: entry.afterMessageCount, createdAt: entry.createdAt,
|
|
408
423
|
}));
|
|
409
|
-
return [...foldIntermediate(intermediate), ...foldTool(tool), ...replayOutboundAttachments(entries), ...publications, ...cards, ...delegationCards, ...suggestions];
|
|
424
|
+
return [...foldIntermediate(intermediate), ...foldTool(tool), ...replayOutboundAttachments(entries), ...publications, ...cards, ...pinCards, ...delegationCards, ...suggestions];
|
|
410
425
|
}
|
|
411
426
|
/**
|
|
412
427
|
* Fold the outbound (agent-sent) attachment records into time-anchored synthetic
|
|
@@ -719,6 +734,10 @@ export class EventLog {
|
|
|
719
734
|
this.load(id);
|
|
720
735
|
this.enqueue(id, `review:${entry.review.id}`, { bivyKind: "app-review", createdAt: entry.createdAt ?? Date.now(), afterMessageCount: entry.afterMessageCount, id: entry.review.id, review: structuredClone(entry.review) });
|
|
721
736
|
}
|
|
737
|
+
appendAppPin(id, entry) {
|
|
738
|
+
this.load(id);
|
|
739
|
+
this.enqueue(id, `pin:${entry.pin.id}`, { bivyKind: "app-pin", createdAt: entry.createdAt ?? Date.now(), afterMessageCount: entry.afterMessageCount, id: entry.pin.id, pin: structuredClone(entry.pin) });
|
|
740
|
+
}
|
|
722
741
|
appendSuggestion(id, entry) {
|
|
723
742
|
this.load(id);
|
|
724
743
|
this.enqueue(id, `suggestion:${entry.suggestion.id}`, { bivyKind: "suggestion", createdAt: Date.now(), afterMessageCount: entry.afterMessageCount, id: entry.suggestion.id, suggestion: { ...entry.suggestion } });
|