@bivy/bivy 0.17.3-staging.24 → 0.17.3-staging.25

package/dist/apps/annotate.js

@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (c) 2026 Petter André Sjulstad
+import { readFileSync } from "node:fs";
+import { decodePng } from "./review.js";
+import { encodePng } from "./rfb.js";
+const MAX_STROKES = 50;
+const MAX_POINTS = 10_000;
+/** Validates strokes from the client (untrusted shape): bounded, finite. */
+export function readStrokes(input) {
+    if (!Array.isArray(input) || !input.length || input.length > MAX_STROKES)
+        throw new Error("Draw at least one mark (at most 50).");
+    let total = 0;
+    return input.map((raw) => {
+        const stroke = raw;
+        if ((stroke?.tool !== "pen" && stroke?.tool !== "box") || !Array.isArray(stroke.points) || !stroke.points.length)
+            throw new Error("Invalid mark.");
+        total += stroke.points.length;
+        if (total > MAX_POINTS)
+            throw new Error("Too many points in the marks.");
+        const points = stroke.points.map((p) => {
+            if (!Array.isArray(p) || p.length !== 2 || !p.every((n) => typeof n === "number" && Number.isFinite(n) && Math.abs(n) < 1e6))
+                throw new Error("Invalid mark point.");
+            return [p[0], p[1]];
+        });
+        return { tool: stroke.tool, points };
+    });
+}
+/** Whether a picture retaken on the machine may not match what the user saw:
+ * the page kept state a fresh browser doesn't have (storage, cookies, typed
+ * input, an open menu or dialog, anything the user did since it loaded), or
+ * the retake couldn't scroll to where they were. A Compare screenshot or a
+ * desktop app's frame is the exact picture, so never approximate. */
+export function approximate(source, signals = {}, scrolledTo = true) {
+    if (source === "exact")
+        return false;
+    return !scrolledTo || Object.values(signals).some(Boolean);
+}
+/** The annotation colours, read from the design tokens (single source). */
+function tokenColor(name, fallback) {
+    let hex = fallback;
+    try {
+        hex = new RegExp(`--${name}:\\s*(#[0-9a-fA-F]{6})`).exec(readFileSync(new URL("./tokens.css", import.meta.url), "utf8"))?.[1] ?? fallback;
+    }
+    catch { /* fallback */ }
+    return [1, 3, 5].map((i) => parseInt(hex.slice(i, i + 2), 16));
+}
+const INK = tokenColor("annotate", "#ff2d78");
+const HALO = tokenColor("annotate-halo", "#ffffff");
+/** Mark width in CSS pixels, and its halo on each side. */
+const WIDTH = 4;
+const HALO_WIDTH = 1.5;
+/** Draws marks onto a PNG. `scale` maps CSS pixels to image pixels;
+ * `offset` (CSS px) is subtracted first — the scroll position the picture
+ * was taken at. Halos go under every mark first, so crossings stay clean. */
+export function composite(png, strokes, map) {
+    const image = decodePng(png);
+    const { width, height, channels } = image;
+    const rgb = Buffer.alloc(width * height * 3);
+    for (let p = 0; p < width * height; p++)
+        image.pixels.copy(rgb, p * 3, p * channels, p * channels + 3);
+    const off = map.offset ?? { x: 0, y: 0 };
+    const toImage = ([x, y]) => [(x - off.x) * map.scale, (y - off.y) * map.scale];
+    const paths = strokes.map((stroke) => {
+        const points = stroke.points.map(toImage);
+        if (stroke.tool === "pen")
+            return points;
+        const [a, b] = [points[0], points[points.length - 1]];
+        return [a, [b[0], a[1]], b, [a[0], b[1]], a];
+    });
+    const disc = (cx, cy, r, color) => {
+        for (let y = Math.max(0, Math.floor(cy - r)); y <= Math.min(height - 1, Math.ceil(cy + r)); y++) {
+            for (let x = Math.max(0, Math.floor(cx - r)); x <= Math.min(width - 1, Math.ceil(cx + r)); x++) {
+                if ((x - cx) ** 2 + (y - cy) ** 2 <= r * r)
+                    rgb.set(color, (y * width + x) * 3);
+            }
+        }
+    };
+    const trace = (path, r, color) => {
+        const step = Math.max(0.5, r / 2);
+        for (let i = 0; i < path.length; i++) {
+            const [x0, y0] = path[i];
+            const [x1, y1] = path[i + 1] ?? path[i];
+            const n = Math.max(1, Math.ceil(Math.hypot(x1 - x0, y1 - y0) / step));
+            for (let k = 0; k <= n; k++)
+                disc(x0 + ((x1 - x0) * k) / n, y0 + ((y1 - y0) * k) / n, r, color);
+        }
+    };
+    const r = (WIDTH / 2) * map.scale;
+    for (const path of paths)
+        trace(path, r + HALO_WIDTH * map.scale, HALO);
+    for (const path of paths)
+        trace(path, r, INK);
+    return encodePng(width, height, rgb);
+}

package/dist/apps/inspector.js

@@ -16,7 +16,9 @@ const log=(level,text)=>post({type:'console',level,text:String(text).slice(0,500
 for(const level of ['error','warn']){const original=console[level];console[level]=function(...args){log(level,args.map(fmt).join(' '));return original.apply(this,args);};}
 addEventListener('error',e=>{if(e.message)log('error',e.message+(e.filename?' ('+e.filename.split('/').pop()+':'+e.lineno+')':''));},true);
 addEventListener('unhandledrejection',e=>log('error','Unhandled rejection: '+fmt(e.reason)));
-const route=()=>post({type:'route',path:location.pathname+location.search+location.hash});
+// Taps and keys since this page (or route) loaded: state a retaken picture may lack.
+let interacted=false;
+const route=()=>{interacted=false;post({type:'route',path:location.pathname+location.search+location.hash});};
 for(const name of ['pushState','replaceState']){const original=history[name];history[name]=function(...args){const result=original.apply(this,args);route();return result;};}
 addEventListener('popstate',route);addEventListener('hashchange',route);route();
 const selector=el=>{
@@ -78,6 +80,50 @@ function startPointing(){
   document.documentElement.append(layer,box);
   addEventListener('keydown',key,true);
 }
+for(const type of ['pointerdown','keydown'])addEventListener(type,e=>{if(!layer&&!(host&&e.composedPath?.().includes(host)))interacted=true;},true);
+// Draw (in the shell, over this page): where the page is, what state it holds
+// that a fresh browser on the machine wouldn't, and what's under each mark.
+const signals=()=>{
+  let storage=false;try{storage=localStorage.length+sessionStorage.length>0;}catch{}
+  const edited=[...document.querySelectorAll('input,textarea,select')].some(el=>el.type==='checkbox'||el.type==='radio'?el.checked!==el.defaultChecked:el.localName==='select'?[...el.options].some(o=>o.selected!==o.defaultSelected):el.type!=='hidden'&&el.value!==el.defaultValue);
+  const open=Boolean(document.querySelector('dialog[open],details[open],[aria-expanded="true"],[aria-modal="true"]'));
+  return {storage,cookies:Boolean(document.cookie),interacted,open,edited};
+};
+const drawState=()=>({type:'draw-state',scroll:{x:Math.round(scrollX),y:Math.round(scrollY)},viewport:{width:innerWidth,height:innerHeight},dpr:devicePixelRatio,
+  theme:matchMedia('(prefers-color-scheme: dark)').matches?'dark':'light',path:location.pathname+location.search+location.hash,signals:signals()});
+/** Elements inside a mark (viewport rect): a grid sample, then the largest
+ * element under each hit that sits mostly inside the mark — circling a
+ * button names the button, not the bar it's in. A line through something
+ * falls back to what it crossed. Outermost first, at most eight. */
+const marked=r=>{
+  const hits=new Set();
+  for(let i=0;i<8;i++)for(let j=0;j<8;j++){
+    const x=Math.min(innerWidth-1,Math.max(0,r.x+r.width*(i+0.5)/8)),y=Math.min(innerHeight-1,Math.max(0,r.y+r.height*(j+0.5)/8));
+    const el=document.elementsFromPoint(x,y).find(n=>n!==layer&&n!==box&&n!==host);if(el)hits.add(el);
+  }
+  const big=innerWidth*innerHeight*0.6,page=n=>n===document.body||n===document.documentElement;
+  // Where an element's content is: a full-width block around "Total $102"
+  // is judged by its text, which is what a circle goes around.
+  const range=document.createRange();
+  const rectOf=n=>{const b=n.getBoundingClientRect();if(!n.firstChild||/^(img|svg|video|canvas|input|select|textarea|button)$/.test(n.localName))return b;range.selectNodeContents(n);const c=range.getBoundingClientRect();return c.width&&c.height?c:b;};
+  const inside=n=>{const b=rectOf(n),a=b.width*b.height;if(!a)return 0;const w=Math.max(0,Math.min(b.right,r.x+r.width)-Math.max(b.left,r.x)),h=Math.max(0,Math.min(b.bottom,r.y+r.height)-Math.max(b.top,r.y));return w*h/a;};
+  const area=n=>{const b=rectOf(n);return b.width*b.height;};
+  const picked=new Set();
+  for(const hit of hits){let best=null;for(let n=hit;n&&n.nodeType===1&&!page(n);n=n.parentElement){if(inside(n)>=0.6&&area(n)<big)best=n;else if(best)break;}if(best)picked.add(best);}
+  if(!picked.size)for(const hit of hits)if(!page(hit)&&area(hit)<big&&(hit.innerText||'').trim())picked.add(hit);
+  const list=[...picked].filter(n=>![...picked].some(o=>o!==n&&o.contains(n)))
+    .sort((a,b)=>{const p=a.getBoundingClientRect(),q=b.getBoundingClientRect();return p.top-q.top||p.left-q.left;});
+  return list.slice(0,8).map(el=>{const b=el.getBoundingClientRect();return {selector:selector(el).slice(0,300),tag:el.localName,
+    text:(el.innerText||el.getAttribute('aria-label')||el.getAttribute('alt')||'').trim().replace(/\\s+/g,' ').slice(0,120),
+    rect:{x:Math.round(b.left+scrollX),y:Math.round(b.top+scrollY),width:Math.round(b.width),height:Math.round(b.height)}};});
+};
+addEventListener('message',e=>{
+  if(!framed||e.origin!==SHELL||e.source!==parent)return;
+  const d=e.data||{},n=v=>Number.isFinite(+v)?+v:0;
+  if(d.type==='bivy:draw')post(drawState());
+  else if(d.type==='bivy:scroll'){scrollBy({left:n(d.dx),top:n(d.dy),behavior:'instant'});post({type:'scrolled',scroll:{x:Math.round(scrollX),y:Math.round(scrollY)}});}
+  else if(d.type==='bivy:marks'&&d.rect)post({type:'marked',id:n(d.id),elements:marked({x:n(d.rect.x),y:n(d.rect.y),width:n(d.rect.width),height:n(d.rect.height)})});
+});
 addEventListener('message',e=>{
   if(!framed||e.origin!==SHELL||e.source!==parent||e.data?.type!=='bivy:point')return;
   // Held: the layer goes when the finger lifts (see up).

package/dist/apps/preview-shell.js

@@ -22,8 +22,8 @@ iframe { width:100%; height:100%; border:0; background:var(--bg); }
 #dock > * { pointer-events:auto; }
 /* The hint sits over the app; a tap on it is meant for the app underneath. */
 #dock > #pointing { pointer-events:none; }
-nav { display:flex; align-items:center; gap:var(--space-1); max-width:100%; padding:var(--space-1); background:var(--surface); border:thin solid var(--line); border-radius:var(--radius-full); box-shadow:var(--shadow-lg); }
-nav .btn { flex-shrink:0; border-radius:var(--radius-full); }
+nav, #draw-bar { display:flex; align-items:center; gap:var(--space-1); max-width:100%; padding:var(--space-1); background:var(--surface); border:thin solid var(--line); border-radius:var(--radius-full); box-shadow:var(--shadow-lg); }
+nav .btn, #draw-bar .btn { flex-shrink:0; border-radius:var(--radius-full); }
 nav .btn[aria-pressed="true"] { background:var(--accent-soft); color:var(--accent); }
 #name { display:flex; flex-direction:column; min-width:0; padding:0 var(--space-2); }
 #title { font-size:var(--text-sm); font-weight:var(--weight-semibold); overflow:hidden; text-overflow:ellipsis; white-space:nowrap; max-width:14em; }
@@ -55,6 +55,22 @@ nav .btn[aria-pressed="true"] { background:var(--accent-soft); color:var(--accen
 #draft-hint { font-size:var(--text-xs); margin:var(--space-2) 0 var(--space-1); }
 #draft-context { margin:0; padding:var(--space-2); background:var(--surface-2); border-radius:var(--radius-md); font-family:var(--font-mono); font-size:var(--text-xs); white-space:pre-wrap; overflow-wrap:anywhere; color:var(--muted); }
 #show { position:fixed; right:var(--space-3); bottom:calc(var(--space-3) + env(safe-area-inset-bottom)); z-index:var(--z-sticky); border-radius:var(--radius-full); box-shadow:var(--shadow-lg); }
+/* Draw: marks over the frozen app (or Compare's "after" shot). The layer takes
+   every touch, so nothing reaches the app; two fingers scroll the page. */
+#ink { position:fixed; z-index:var(--z-sticky); touch-action:none; cursor:crosshair; user-select:none; -webkit-user-select:none; -webkit-touch-callout:none; }
+#ink.frozen { pointer-events:none; cursor:default; }
+#ink .halo { fill:none; stroke:var(--annotate-halo); stroke-width:7; stroke-linecap:round; stroke-linejoin:round; }
+#ink .mark { fill:none; stroke:var(--annotate); stroke-width:4; stroke-linecap:round; stroke-linejoin:round; }
+#dock > #drawing { pointer-events:none; }
+/* Hints float over the app, whose page may be any colour: back them solidly. */
+#dock > .banner.inline[data-tone="accent"] { background:color-mix(in srgb, var(--accent) 14%, var(--surface)); box-shadow:var(--shadow-sm); }
+#draw-bar .btn { min-block-size:44px; }
+#draw-bar .seg-btn { min-block-size:36px; min-inline-size:48px; }
+#draw-bar .segmented { flex-shrink:0; }
+#compare-draw { margin-top:var(--space-2); }
+/* Marking a Compare shot: make it big enough to draw on with a thumb. */
+#compare[data-drawing] #compare-stage img { max-height:min(60vh, 560px); }
+#compare[data-drawing] #compare-slider, #compare[data-drawing] #compare-hint, #compare[data-drawing] #compare-draw { display:none; }
 .label-narrow { display:none; }
 @media (max-width: 699px) { #lens { display:none; } #name { display:none; } .label-wide { display:none; } .label-narrow { display:inline; } }
 [hidden] { display:none !important; }
@@ -62,6 +78,7 @@ nav .btn[aria-pressed="true"] { background:var(--accent-soft); color:var(--accen
 <div class="banner" data-tone="warn" id="down" hidden><span class="banner-text" id="down-text" role="status"></span><span class="banner-actions"><button class="btn sm" id="ask" hidden>Ask agent to fix</button></span></div>
 <p id="status" role="status">Opening app…</p>
 <div id="stage" data-lens="full"><iframe id="app" title="App preview" hidden sandbox="allow-scripts allow-same-origin allow-forms allow-downloads allow-popups" referrerpolicy="no-referrer"></iframe></div>
+<svg id="ink" hidden aria-hidden="true"></svg>
 <div id="dock">
   <section class="panel" id="console" hidden aria-labelledby="console-title">
     <h2><span id="console-title">Console</span><button class="btn sm ghost" id="clear">Clear</button></h2>
@@ -73,6 +90,7 @@ nav .btn[aria-pressed="true"] { background:var(--accent-soft); color:var(--accen
     <h2 id="compare-title">Before and after the agent’s last change</h2>
     <div id="compare-stage"><img id="compare-after" alt="After the agent’s last change"><img id="compare-before" alt="Before the agent’s last change"><input type="range" id="compare-slider" min="0" max="100" value="50" aria-label="Show more of before or after"></div>
     <p class="muted" id="compare-hint">Left of the handle: before. Right: now. Screenshots at phone width.</p>
+    <button class="btn sm" id="compare-draw" type="button" hidden>Draw on “now”</button>
   </section>
   <section class="panel" id="draft" hidden aria-labelledby="draft-title">
     <h2 id="draft-title">Draft for the agent</h2>
@@ -84,15 +102,24 @@ nav .btn[aria-pressed="true"] { background:var(--accent-soft); color:var(--accen
     <div class="panel-actions"><button class="btn sm ghost" id="draft-cancel">Cancel</button><button class="btn sm primary" id="draft-add">Add to chat</button></div>
   </section>
   <p class="banner inline" data-tone="accent" id="pointing" role="status" hidden>Tap anything in the app to point at it. <span id="pointing-voice" hidden>Hold to point and speak. </span>Press Escape or Point again to stop.</p>
+  <p class="banner inline" data-tone="accent" id="drawing" role="status" hidden>Circle or box what’s wrong, then Done. Two fingers scroll the page.</p>
+  <div id="draw-bar" role="toolbar" aria-label="Drawing tools" hidden>
+    <div class="segmented" role="radiogroup" aria-label="Mark with"><button class="seg-btn" type="button" role="radio" aria-checked="true" data-tool="pen">Pen</button><button class="seg-btn" type="button" role="radio" aria-checked="false" data-tool="box">Box</button></div>
+    <button class="btn sm ghost" id="draw-undo" type="button" disabled>Undo</button>
+    <button class="btn sm ghost" id="draw-clear" type="button" disabled>Clear</button>
+    <button class="btn sm ghost" id="draw-cancel" type="button" aria-label="Stop drawing">✕</button>
+    <button class="btn sm primary" id="draw-done" type="button" disabled>Done</button>
+  </div>
   <nav aria-label="Bivy preview controls">
     <button class="btn sm ghost" id="back" aria-label="Back to chat">‹ <span class="label-wide">Back to chat</span><span class="label-narrow">Chat</span></button>
     <span id="name"><span id="title">App preview</span><span id="stamp" class="muted" role="status"></span></span>
     <button class="btn sm ghost" id="hide" aria-label="Hide Bivy controls" title="Hide controls">⌄</button>
     <button class="btn sm ghost" id="point" aria-pressed="false" disabled>Point</button>
+    <button class="btn sm ghost" id="draw" type="button" hidden>Draw</button>
     <button class="btn sm ghost" id="compare-btn" aria-pressed="false" hidden>Compare</button>
     <button class="btn sm ghost" id="errors" aria-pressed="false" aria-label="Console" disabled>Console <span class="badge" data-variant="solid" data-tone="danger" id="error-count" hidden></span></button>
     <div class="segmented" id="lens" role="radiogroup" aria-label="Preview width"><button class="seg-btn" role="radio" aria-selected="true" data-lens="full">Full</button><button class="seg-btn" role="radio" aria-selected="false" data-lens="tablet">Tablet</button><button class="seg-btn" role="radio" aria-selected="false" data-lens="phone">Phone</button></div>
-    <button class="btn sm ghost" id="reload" disabled>Reload</button>
+    <button class="btn sm ghost" id="reload" aria-label="Reload" disabled><span class="label-wide">Reload</span><span class="label-narrow" aria-hidden="true">↻</span></button>
   </nav>
 </div>
 <button class="btn sm" id="show" hidden aria-label="Show Bivy controls">Bivy</button>
@@ -192,14 +219,17 @@ mic.onpointerdown=e=>{
 mic.onpointerup=mic.onpointercancel=()=>{if(heldAt&&Date.now()-heldAt>400)stopListening();heldAt=0;};
 mic.onclick=e=>{if(e.detail===0){if(listening)stopListening();else startListening();}};
 function fromBivy(d){
-  if(d.type==='voice'){voice=d.available===true;mic.hidden=!voice;$('pointing-voice').hidden=!voice;if(!voice)cancelListening();}
+  if(d.type==='draw'){drawOk=d.available===true;$('draw').hidden=!drawOk;$('compare-draw').hidden=!drawOk;}
+  else if(d.type==='voice'){voice=d.available===true;mic.hidden=!voice;$('pointing-voice').hidden=!voice;if(!voice)cancelListening();}
   else if(d.type==='transcript'){setListening(false);if(typeof d.error==='string')sayVoice(d.error.slice(0,200));else addSpoken(d.text);}
   else if(d.type==='listening'&&d.on===false&&listening){setListening(false);if(voiceStatus.textContent.startsWith('Listening'))sayVoice('');}
 }
-$('draft-cancel').onclick=()=>{cancelListening();$('draft').hidden=true;};
+$('draft-cancel').onclick=()=>{cancelListening();$('draft').hidden=true;endDraw();};
 $('draft-add').onclick=async()=>{
   cancelListening();
   const note=$('draft-text').value.trim(),text=(note?note+'\\n\\n':'')+$('draft-context').textContent;
+  // Marks go to the Bivy client, which adds a picture made on the machine.
+  if(marks){const mark=marks;endDraw();return toBivy({type:'annotation',text,mark});}
   if(metadata.returnTo)return toChat(text);
   try{await navigator.clipboard.writeText(text);$('draft-add').textContent='Copied';}catch{$('draft-text').select();}
 };
@@ -232,20 +262,146 @@ function picked(d){
     +'Element: '+String(d.selector).slice(0,300)+(d.text?' ("'+String(d.text).slice(0,200)+'")':'')+', '+r.width+'×'+r.height+' at '+r.x+','+r.y
     +(errors.length?'\\nRecent errors:\\n'+errors.map(e=>'- '+e.text).join('\\n'):''),d.hold===true&&voice);
 }
+// Draw: freeze the app and mark what's wrong. Marks are kept in page
+// coordinates (so they stay on the content while two fingers scroll), and
+// the app reports what's under each one. On Done they go into the draft box
+// with that context; Add to chat hands them to the Bivy client, which gets a
+// picture from the machine. Nothing about them goes through this origin's
+// network. Compare's "after" shot can be marked the same way.
+const ink=$('ink'),SVG='http://www.w3.org/2000/svg';
+let draw=null,marks=null,drawOk=false,nextStroke=0;
+const pos=v=>({x:Number(v?.x)||0,y:Number(v?.y)||0});
+function startDraw(target){
+  endDraw();if(point.getAttribute('aria-pressed')==='true')pointing(false);
+  if(target==='frame')panels(null);
+  if(target==='compare'){$('compare-slider').value='0';split();$('compare').dataset.drawing='';}
+  // Tools first: they change the layout the marks are measured against.
+  $('drawing').hidden=false;$('draw-bar').hidden=false;document.querySelector('nav').hidden=true;
+  const r=placeInk(target);
+  draw={target,rect:r,scroll:{x:0,y:0},strokes:[],elements:{},tool:'pen',current:null,pointers:new Map(),pan:null,done:false,
+    state:{viewport:{width:Math.round(r.width),height:Math.round(r.height)},dpr:devicePixelRatio,path:currentPath,signals:{}}};
+  ink.style.zIndex=target==='compare'?'calc(var(--z-sticky) + 1)':'';
+  ink.classList.remove('frozen');ink.toggleAttribute('hidden',false);
+  // Where the page is and what state it holds. Without an inspector (a
+  // desktop app, or a page whose policy kept it out) state is unknown.
+  if(target==='frame'&&metadata.inspect!==false){
+    const current=draw;
+    current.state.signals={unknown:true};
+    current.waiting=d=>{current.waiting=null;current.scroll=pos(d.scroll);current.state={viewport:{width:Math.round(Number(d.viewport?.width))||current.state.viewport.width,height:Math.round(Number(d.viewport?.height))||current.state.viewport.height},dpr:Number(d.dpr)||devicePixelRatio,theme:d.theme==='dark'?'dark':'light',path:safePath(d.path),signals:Object.fromEntries(Object.entries(d.signals||{}).map(([k,v])=>[k,v===true]))};renderInk();};
+    frame.contentWindow?.postMessage({type:'bivy:draw'},metadata.origin);
+  }
+  $('draw-bar').querySelector('[aria-checked="true"]').focus();
+  updateDrawBar();renderInk();
+}
+/** Lays the marking layer over what's marked; returns where that is. */
+function placeInk(target){
+  const r=(target==='compare'?$('compare-after'):frame).getBoundingClientRect();
+  Object.assign(ink.style,{left:r.left+'px',top:r.top+'px',width:r.width+'px',height:r.height+'px'});
+  ink.setAttribute('viewBox','0 0 '+r.width+' '+r.height);
+  return {left:r.left,top:r.top,width:r.width,height:r.height};
+}
+// A rotation or a keyboard moves things: keep the layer on them. (Marks on
+// the frame are in page pixels, so they follow; the size in use is kept.)
+addEventListener('resize',()=>{if(draw&&!draw.done)draw.rect=Object.assign(placeInk(draw.target),{});});
+function endDraw(){
+  if(!draw)return;
+  delete $('compare').dataset.drawing;
+  draw=null;marks=null;ink.toggleAttribute('hidden',true);ink.replaceChildren();
+  $('drawing').hidden=true;$('draw-bar').hidden=true;document.querySelector('nav').hidden=false;
+}
+function updateDrawBar(){const has=Boolean(draw?.strokes.length);for(const id of ['draw-undo','draw-clear','draw-done'])$(id).disabled=!has;}
+const onScreen=([x,y])=>draw.target==='frame'?[x-draw.scroll.x,y-draw.scroll.y]:[x,y];
+function shape(stroke){
+  const pts=stroke.points.map(onScreen);
+  if(stroke.tool==='box'){const [a,b]=[pts[0],pts[pts.length-1]];const el=document.createElementNS(SVG,'rect');el.setAttribute('x',Math.min(a[0],b[0]));el.setAttribute('y',Math.min(a[1],b[1]));el.setAttribute('width',Math.abs(b[0]-a[0]));el.setAttribute('height',Math.abs(b[1]-a[1]));el.setAttribute('rx','3');return el;}
+  const el=document.createElementNS(SVG,'polyline');el.setAttribute('points',pts.map(p=>p.join(',')).join(' '));return el;
+}
+function renderInk(){
+  if(!draw)return;
+  const all=[...draw.strokes,...(draw.current?[draw.current]:[])];
+  ink.replaceChildren(...all.flatMap(s=>['halo','mark'].map(c=>{const el=shape(s);el.setAttribute('class',c);return el;})));
+}
+function local(e){const x=e.clientX-draw.rect.left,y=e.clientY-draw.rect.top;return draw.target==='frame'?[Math.round(x+draw.scroll.x),Math.round(y+draw.scroll.y)]:[Math.round(x),Math.round(y)];}
+let scrollAsk=null;
+function scrollPage(dx,dy){
+  if(draw?.target!=='frame'||metadata.inspect===false)return;
+  scrollAsk=scrollAsk?{dx:scrollAsk.dx+dx,dy:scrollAsk.dy+dy}:{dx,dy};
+  requestAnimationFrame(()=>{if(!scrollAsk)return;frame.contentWindow?.postMessage(Object.assign({type:'bivy:scroll'},scrollAsk),metadata.origin);scrollAsk=null;});
+}
+const middle=()=>{const p=[...draw.pointers.values()];return {x:(p[0].x+p[1].x)/2,y:(p[0].y+p[1].y)/2};};
+ink.onpointerdown=e=>{
+  if(!draw||draw.done)return;e.preventDefault();try{ink.setPointerCapture(e.pointerId);}catch{}
+  draw.pointers.set(e.pointerId,{x:e.clientX,y:e.clientY});
+  // A second finger scrolls instead: the stroke it interrupted is dropped.
+  if(draw.pointers.size>=2){draw.current=null;draw.pan=middle();renderInk();return;}
+  draw.current={id:nextStroke++,tool:draw.tool,points:[local(e)]};renderInk();
+};
+ink.onpointermove=e=>{
+  if(!draw||!draw.pointers.has(e.pointerId))return;
+  draw.pointers.set(e.pointerId,{x:e.clientX,y:e.clientY});
+  if(draw.pan&&draw.pointers.size>=2){const m=middle();scrollPage(draw.pan.x-m.x,draw.pan.y-m.y);draw.pan=m;return;}
+  const s=draw.current;if(!s)return;const p=local(e);
+  if(s.tool==='box')s.points=[s.points[0],p];
+  else{const last=s.points[s.points.length-1];if(Math.hypot(p[0]-last[0],p[1]-last[1])<2||s.points.length>=2000)return;s.points.push(p);}
+  renderInk();
+};
+ink.onpointerup=ink.onpointercancel=e=>{
+  if(!draw)return;draw.pointers.delete(e.pointerId);if(draw.pointers.size<2)draw.pan=null;
+  const s=draw.current;draw.current=null;
+  if(s&&e.type==='pointerup'){
+    const xs=s.points.map(p=>p[0]),ys=s.points.map(p=>p[1]),w=Math.max(...xs)-Math.min(...xs),h=Math.max(...ys)-Math.min(...ys);
+    if(w>=4||h>=4){
+      draw.strokes.push(s);
+      // Ask what's under it while it's on screen (viewport coordinates).
+      if(draw.target==='frame'&&metadata.inspect!==false){const [x,y]=onScreen([Math.min(...xs),Math.min(...ys)]);frame.contentWindow?.postMessage({type:'bivy:marks',id:s.id,rect:{x,y,width:w,height:h}},metadata.origin);}
+    }
+  }
+  updateDrawBar();renderInk();
+};
+ink.onwheel=e=>{if(!draw||draw.done)return;e.preventDefault();scrollPage(e.deltaX,e.deltaY);};
+for(const b of $('draw-bar').querySelectorAll('[data-tool]'))b.onclick=()=>{if(!draw)return;draw.tool=b.dataset.tool;for(const o of $('draw-bar').querySelectorAll('[data-tool]'))o.setAttribute('aria-checked',String(o===b));};
+$('draw-undo').onclick=()=>{if(!draw)return;const s=draw.strokes.pop();if(s)delete draw.elements[s.id];updateDrawBar();renderInk();};
+$('draw-clear').onclick=()=>{if(!draw)return;draw.strokes=[];draw.elements={};updateDrawBar();renderInk();};
+$('draw-cancel').onclick=()=>endDraw();
+$('draw').onclick=()=>startDraw('frame');
+$('compare-draw').onclick=()=>startDraw('compare');
+$('draw-done').onclick=()=>{
+  if(!draw||!draw.strokes.length)return;
+  const d=draw,st=d.state,seen=new Set(),els=[];
+  for(const s of d.strokes)for(const el of d.elements[s.id]||[]){const key=String(el.selector);if(!seen.has(key)){seen.add(key);els.push(el);}}
+  const bounds=s=>{const xs=s.points.map(p=>p[0]),ys=s.points.map(p=>p[1]);return s.tool+' '+Math.min(...xs)+','+Math.min(...ys)+'–'+Math.max(...xs)+','+Math.max(...ys);};
+  const where=d.target==='compare'?'On the Compare screenshot after the agent’s last change in "'+metadata.name+'"':metadata.inspect===false?'On the desktop app "'+metadata.name+'"':'In the app preview "'+metadata.name+'"';
+  const scrolled=d.target==='frame'&&(d.scroll.x||d.scroll.y)?', scrolled to '+d.scroll.x+','+d.scroll.y:'';
+  const context=where+' (page '+st.path+', viewport '+st.viewport.width+'×'+st.viewport.height+scrolled+'), marked:\\n'
+    +(els.length?els.map(el=>'- '+String(el.selector).slice(0,300)+(el.text?' ("'+String(el.text).slice(0,120)+'")':'')).join('\\n')+'\\n':'')
+    +'Marks ('+(d.target==='compare'?'screenshot':'page')+' px): '+d.strokes.map(bounds).join('; ');
+  const mark=Object.assign({path:st.path,viewport:st.viewport,dpr:st.dpr,strokes:d.strokes.map(s=>({tool:s.tool,points:s.points})),signals:st.signals},
+    d.target==='frame'?{scroll:d.scroll,theme:st.theme}:{compare:compareAfter});
+  // The marks stay on screen, frozen, while you add your words.
+  d.done=true;draw=null;
+  draft(context,false);
+  draw=d;marks=mark;ink.classList.add('frozen');
+  // Compare closed for the draft box: its marks go with it (they're in the context).
+  if(d.target==='compare')ink.toggleAttribute('hidden',true);$('drawing').hidden=true;$('draw-bar').hidden=true;document.querySelector('nav').hidden=false;
+};
 // Compare: screenshots around the agent's last change (agent screenshots on).
-const compareBtn=$('compare-btn');let shots=[],urls=[];
+const compareBtn=$('compare-btn');let shots=[],urls=[],compareAfter=-1;
 async function loadCompare(){
   try{const r=await fetch(metadata.origin+'/__bivy/compare',{credentials:'include',cache:'no-store'});if(!r.ok)return;shots=(await r.json()).shots||[];compareBtn.hidden=shots.length<2;}catch{}
 }
-function panels(open){if(listening)cancelListening();for(const [id,btn] of [['console','errors'],['compare','compare-btn']]){$(id).hidden=id!==open;$(btn).setAttribute('aria-pressed',String(id===open));}$('draft').hidden=true;}
+function panels(open){if(listening)cancelListening();if(draw&&!(open==='compare'&&draw.target==='compare'))endDraw();for(const [id,btn] of [['console','errors'],['compare','compare-btn']]){$(id).hidden=id!==open;$(btn).setAttribute('aria-pressed',String(id===open));}$('draft').hidden=true;}
 compareBtn.onclick=async()=>{
   if(!$('compare').hidden)return panels(null);
   panels('compare');
   for(const u of urls)URL.revokeObjectURL(u);
   const pick=[shots[shots.length-2],shots[shots.length-1]];
+  compareAfter=pick[1].index;
   urls=await Promise.all(pick.map(async s=>URL.createObjectURL(await (await fetch(metadata.origin+'/__bivy/compare/'+s.index,{credentials:'include'})).blob())));
   const before=$('compare-before'),after=$('compare-after');
   before.src=urls[0];after.src=urls[1];
+  // Drawing on "now" measures the picture, so wait until it's there.
+  $('compare-draw').disabled=true;
+  void Promise.all([before.decode(),after.decode()]).catch(()=>{}).then(()=>{$('compare-draw').disabled=false;});
   split();
 };
 const split=()=>{$('compare-before').style.clipPath='inset(0 '+(100-Number($('compare-slider').value))+'% 0 0)';};
@@ -277,10 +433,13 @@ addEventListener('message',e=>{
     else if(d.type==='console'&&(d.level==='error'||d.level==='warn')){entries.push({level:d.level,text:String(d.text).slice(0,500)});if(entries.length>50)entries.shift();renderConsole();}
     else if(d.type==='picked')picked(d);
     else if(d.type==='release'&&heldAt){heldAt=0;stopListening();}
+    else if(d.type==='draw-state'&&draw&&draw.waiting){draw.waiting(d);}
+    else if(d.type==='scrolled'&&draw&&draw.target==='frame'){draw.scroll=pos(d.scroll);renderInk();}
+    else if(d.type==='marked'&&draw){draw.elements[Number(d.id)]=Array.isArray(d.elements)?d.elements.slice(0,8):[];}
   }
 });
 ask.onclick=()=>toChat('The app preview "'+metadata.name+'" isn’t loading: nothing is answering on port '+downPort+'. Please find out why the server stopped, restart it, and tell me when it’s back.');
-addEventListener('keydown',e=>{if(e.key==='Escape'){if(listening)cancelListening();else if(point.getAttribute('aria-pressed')==='true')pointing(false);else panels(null);}});
+addEventListener('keydown',e=>{if(e.key==='Escape'){if(listening)cancelListening();else if(draw&&!draw.done)endDraw();else if(point.getAttribute('aria-pressed')==='true')pointing(false);else panels(null);}});
 back.onclick=()=>{if(metadata?.returnTo){window.close();setTimeout(()=>location.replace(metadata.returnTo),100);}else{window.close();status.hidden=false;status.textContent='You can close this tab to return to Bivy.';}};
 reload.onclick=()=>{if(metadata)open(currentPath,'Reloading app…');};
 (async()=>{

package/dist/apps/screenshot.js

@@ -160,16 +160,22 @@ export async function takeShots(views, request, outDir, chrome = findChrome()) {
             for (const width of request.widths)
                 for (const theme of request.themes) {
                     const mobile = width < 600;
-                    await cdp.send("Emulation.setDeviceMetricsOverride", { width, height: mobile ? 844 : 800, deviceScaleFactor: mobile ? 2 : 1, mobile }, sessionId);
+                    await cdp.send("Emulation.setDeviceMetricsOverride", { width, height: request.height ?? (mobile ? 844 : 800), deviceScaleFactor: request.scale ?? (mobile ? 2 : 1), mobile }, sessionId);
                     await cdp.send("Emulation.setEmulatedMedia", { features: [{ name: "prefers-color-scheme", value: theme }] }, sessionId);
                     const loaded = cdp.once("Page.loadEventFired", sessionId, 15_000);
                     await cdp.send("Page.navigate", { url: origin + request.path }, sessionId);
                     await loaded;
                     await new Promise((r) => setTimeout(r, 400)); // let fonts and first effects settle
+                    let scroll;
+                    if (request.scroll) {
+                        const { result } = await cdp.send("Runtime.evaluate", { expression: `scrollTo(${Number(request.scroll.x) || 0}, ${Number(request.scroll.y) || 0}); [scrollX, scrollY]`, returnByValue: true }, sessionId);
+                        scroll = { x: Number(result?.value?.[0]) || 0, y: Number(result?.value?.[1]) || 0 };
+                        await new Promise((r) => setTimeout(r, 150));
+                    }
                     const { data } = await cdp.send("Page.captureScreenshot", { format: "png" }, sessionId);
                     const file = path.join(outDir, `${entry.view.id.slice(0, 8)}-${width}-${theme}.png`);
                     fs.writeFileSync(file, Buffer.from(data, "base64"));
-                    shots.push({ viewId: entry.view.id, view: entry.view.name, width, theme, file });
+                    shots.push({ viewId: entry.view.id, view: entry.view.name, width, theme, file, ...(scroll ? { scroll } : {}) });
                 }
         }
         return shots;

package/dist/apps/service.js

@@ -5,6 +5,9 @@ import os from "node:os";
 import path from "node:path";
 import { scanListeners } from "./listeners.js";
 import { takeShots } from "./screenshot.js";
+import { approximate, composite, readStrokes } from "./annotate.js";
+import { captureFrame, encodePng } from "./rfb.js";
+import { pngSize } from "./review.js";
 const NO_REVIEWS = { publish: (review) => review, expire: () => { } };
 const NO_DISPLAYS = { unavailable: () => "Desktop app views aren't available on this machine.", ensure: () => Promise.reject(new Error("Desktop app views aren't available on this machine.")), stop: () => { } };
 /** How long a screenshot waits for a desktop app's first window. */
@@ -347,6 +350,59 @@ export class AppService {
         this.shooting = run;
         return { shots: await run };
     }
+    /** Draw on the preview: the user's marks on a picture of what they saw.
+     * The picture is the Compare screenshot they drew on, a desktop app's
+     * current frame, or a web page retaken at their viewport, pixel ratio and
+     * scroll — "approximate" when the page may hold state a fresh browser
+     * doesn't. Returned to the client (over the session channel) as a PNG. */
+    async annotate(sessionId, input) {
+        const entry = this.registry.requireView(sessionId, input.appId, input.viewId);
+        if (entry.view.kind !== "web")
+            throw new Error("Only previews can be drawn on.");
+        const strokes = readStrokes(input.strokes);
+        const whole = (n, min, max) => typeof n === "number" && Number.isFinite(n) && n >= min && n <= max;
+        // Compare's shot is drawn on at its on-screen size, which can be small.
+        if (!whole(input.viewport?.width, 40, 4000) || !whole(input.viewport?.height, 40, 8000))
+            throw new Error("Invalid viewport.");
+        const viewport = { width: Math.round(input.viewport.width), height: Math.round(input.viewport.height) };
+        const page = typeof input.path === "string" && input.path.startsWith("/") && !input.path.startsWith("//") && input.path.length <= 2048 ? input.path : entry.lastPath ?? "/";
+        const scroll = { x: whole(input.scroll?.x, 0, 1e6) ? input.scroll.x : 0, y: whole(input.scroll?.y, 0, 1e6) ? input.scroll.y : 0 };
+        const name = `${entry.app.name} ${page === "/" ? "" : page} marked.png`.replace(/[^\w .()-]+/g, "-").replace(/\s+/g, " ").trim();
+        const finish = (base, offset, approx) => {
+            const png = composite(base, strokes, { scale: pngSize(base).width / viewport.width, offset });
+            return { image: { data: png.toString("base64"), mimeType: "image/png", name, ...pngSize(png) }, approximate: approx };
+        };
+        // Compare: the exact frame they drew on, in its own coordinates.
+        if (input.compare !== undefined) {
+            const shot = Number.isInteger(input.compare) ? entry.shots?.[input.compare] : undefined;
+            if (!shot)
+                throw new Error("That Compare screenshot is no longer kept. Open Compare again.");
+            return finish(shot.png, { x: 0, y: 0 }, approximate("exact"));
+        }
+        if (!this.screenshots.enabled())
+            return { approximate: false, screenshotsOff: true };
+        // A desktop app: its display's current frame is exactly what they saw.
+        if (entry.target.kind === "display") {
+            await this.windowShown(entry).catch(() => { });
+            if (!entry.display)
+                throw new Error("The app isn't running, so there's nothing to draw on.");
+            const frame = await captureFrame(entry.display);
+            return finish(encodePng(frame.width, frame.height, frame.rgb), { x: 0, y: 0 }, approximate("exact"));
+        }
+        const outDir = path.join(os.tmpdir(), "bivy-shots", "annotate", entry.view.id);
+        const dpr = whole(input.dpr, 1, 3) ? Math.round(input.dpr * 4) / 4 : 2;
+        const run = this.shooting.catch(() => { }).then(() => this.screenshots.take([entry], { widths: [viewport.width], themes: [input.theme === "dark" ? "dark" : "light"], path: page, height: viewport.height, scale: dpr, scroll }, outDir));
+        this.shooting = run;
+        const [shot] = await run;
+        if (!shot)
+            throw new Error("Couldn't take a picture of the page.");
+        const base = fs.readFileSync(shot.file);
+        fs.rmSync(outDir, { recursive: true, force: true });
+        const landed = shot.scroll ?? scroll;
+        const scrolledTo = Math.abs(landed.x - scroll.x) <= 2 && Math.abs(landed.y - scroll.y) <= 2;
+        // Marks stay on the content they were drawn on, wherever the retake scrolled.
+        return finish(base, landed, approximate("retake", input.signals, scrolledTo));
+    }
     /** The managed server's output, as an attachable terminal (starts it if needed). */
     async logs(sessionId, appId, viewId) {
         const entry = this.registry.requireView(sessionId, appId, viewId);

package/dist/apps/styles.css

@@ -3091,6 +3091,10 @@ select.picker-search { appearance: auto; }
 .preview-peek-stage { position: relative; block-size: 100%; }
 /* Point and speak: the recording bar floats over the top of the preview while
    the finger stays on the shell's mic, so nothing under it moves. */
+/* Draw: making the marked-up picture, or the choice when screenshots are off. */
+.preview-marking { position: absolute; inset-inline: var(--space-2); bottom: var(--space-2); z-index: var(--z-sticky); box-shadow: var(--shadow-lg); border-radius: var(--radius-md); background: var(--surface); }
+.preview-marking .banner { margin: 0; }
+.preview-marking .banner-actions { flex-wrap: wrap; }
 .preview-voice { position: absolute; inset-inline: var(--space-2); top: var(--space-2); z-index: var(--z-sticky); padding: var(--space-1) var(--space-2); border: 1px solid var(--line-strong); border-radius: var(--radius-full); background: var(--surface); box-shadow: var(--shadow-lg); }
 .app-terminal-status { padding: var(--space-2) var(--space-4); margin: 0; color: var(--muted); font-size: var(--text-sm); }
 .artifacts-group { display: flex; flex-direction: column; gap: 8px; margin-bottom: 16px; }

package/dist/apps/tokens.css

@@ -62,6 +62,12 @@
   --s-github: #24292f; --s-slack: #7c2d92; --s-linear: #5e6ad2; --s-schedule: #0e7490;
   --s-webhook: #6d28d9; --s-manual: #b45309; --s-cli: #3f6212; --s-app: #64748b;
 
+  /* ---- Annotation: marks drawn over an app preview. The same in both themes
+     (the app underneath has its own), vivid and rare in real UIs, with a white
+     halo so it reads on light and dark pages. The node composites it too. ---- */
+  --annotate: #ff2d78;
+  --annotate-halo: #ffffff;
+
   /* ---- Terminal text selection ---- */
   --term-selection: rgba(63, 99, 119, 0.30);
   --term-selection-inactive: rgba(63, 99, 119, 0.20);

package/dist/controllers/app-commands.js

@@ -26,6 +26,7 @@ export function createAppCommands(service, workspaceFor, published = () => { })
         "apps.showMe": (msg) => service.present(String(msg.sessionId), { target: typeof msg.appId === "string" ? msg.appId : undefined, trigger: "asked" }),
         "apps.mute": (msg) => service.mute(String(msg.sessionId)),
         "apps.reviewMode": (msg) => service.setReviewMode(String(msg.sessionId), String(msg.appId), msg.mode),
+        "apps.annotate": (msg) => service.annotate(String(msg.sessionId), msg),
         "apps.clearNotes": (msg) => service.clearNotes(String(msg.sessionId), String(msg.appId), String(msg.viewId)),
         "apps.logs": (msg) => service.logs(String(msg.sessionId), String(msg.appId), String(msg.viewId)),
         "apps.share": (msg) => service.share(String(msg.sessionId), String(msg.appId), String(msg.viewId)),

package/dist/protocol/client-command-routes.js

@@ -3,7 +3,7 @@
 /** Direct-mode adapters for operations whose canonical implementation is the
  * client command registry. Relay and HTTP now differ only in framing. */
 export const CLIENT_COMMAND_ROUTES = [
-    ...["list", "publish", "offers", "adopt", "open", "logs", "shot", "present", "showMe", "mute", "reviewMode", "clearNotes", "share", "revoke", "remove"].map((action) => ({ method: "post", path: `/api/apps/${action}`, kind: `apps.${action}` })),
+    ...["list", "publish", "offers", "adopt", "open", "logs", "shot", "present", "showMe", "mute", "reviewMode", "annotate", "clearNotes", "share", "revoke", "remove"].map((action) => ({ method: "post", path: `/api/apps/${action}`, kind: `apps.${action}` })),
     { method: "post", path: "/api/artifacts/list", kind: "artifacts.list" },
     { method: "post", path: "/api/auth/credentials/native-preview", kind: "credentials.native.preview" },
     { method: "post", path: "/api/auth/credentials/native-import", kind: "credentials.native.import" },

package/dist/protocol/client-command-schemas.js

@@ -19,6 +19,11 @@ export const CLIENT_COMMAND_SCHEMAS = {
     "apps.showMe": Type.Object({ ...request, ...session, appId: Type.Optional(Type.String()) }),
     "apps.mute": Type.Object({ ...request, ...session }),
     "apps.reviewMode": Type.Object({ ...request, ...session, appId: Type.String(), mode: Type.Union([Type.Literal("ready"), Type.Literal("every"), Type.Literal("off")]) }),
+    "apps.annotate": Type.Object({ ...request, ...session, appId: Type.String(), viewId: Type.String(), path: Type.Optional(Type.String({ maxLength: 2048 })),
+        viewport: Type.Object({ width: Type.Number(), height: Type.Number() }), scroll: Type.Optional(Type.Object({ x: Type.Number(), y: Type.Number() })),
+        dpr: Type.Optional(Type.Number()), theme: Type.Optional(Type.Union([Type.Literal("light"), Type.Literal("dark")])), compare: Type.Optional(Type.Integer({ minimum: 0 })),
+        strokes: Type.Array(Type.Object({ tool: Type.Union([Type.Literal("pen"), Type.Literal("box")]), points: Type.Array(Type.Array(Type.Number(), { minItems: 2, maxItems: 2 }), { maxItems: 10000 }) }), { minItems: 1, maxItems: 50 }),
+        signals: Type.Optional(Type.Record(Type.String(), Type.Boolean())) }),
     "apps.clearNotes": Type.Object({ ...request, ...session, appId: Type.String(), viewId: Type.String() }),
     "apps.logs": Type.Object({ ...request, ...session, appId: Type.String(), viewId: Type.String() }),
     "apps.share": Type.Object({ ...request, ...session, appId: Type.String(), viewId: Type.String() }),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bivy/bivy",
-  "version": "0.17.3-staging.24",
+  "version": "0.17.3-staging.25",
   "type": "module",
   "license": "AGPL-3.0-only",
   "description": "Run coding agents on machines you own. Open-source, self-hostable agent workspace.",