agent-device-lab 0.3.1 → 0.4.1

package/README.md

@@ -11,7 +11,7 @@ The same commands work from a terminal (`agentlab`), from an MCP server for Clau
 
 ![The dashboard after a run on the demo app: the live viewport in the centre, the timeline of actions on the left, and the findings on the right with one selected to show its evidence and reproduction steps](docs/images/dashboard.png)
 
-**Status:** Web V1 (`0.3.1`). It is tested on Linux x64 (Debian 12 and Ubuntu 24.04) with Node.js 22 and 24. macOS, Windows and arm64 are untested, and Alpine does not work. See [Limitations](#limitations).
+**Status:** Web V1 (`0.4.1`). It is tested on Linux x64 (Debian 12 and Ubuntu 24.04) with Node.js 22 and 24. macOS, Windows and arm64 are untested, and Alpine does not work. See [Limitations](#limitations).
 
 ## Install
 

package/assets/dashboard/app.css

@@ -57,8 +57,27 @@ body { margin: 0; background: var(--bg); color: var(--ink); font: 14px/1.45 syst
 .findings-panel { order: 3; }
 .vp-now { margin: 0 0 10px; text-align: center; font-size: 13px; }
 .vp-now strong { font-weight: 600; }
+.vp-toolbar { display: flex; align-items: center; justify-content: space-between; flex-wrap: wrap; gap: 8px; margin-bottom: 10px; }
+.vp-toolbar h2 { margin: 0; }
+.vp-actions { display: flex; gap: 6px; flex-wrap: wrap; }
+.vp-actions button { font: inherit; font-size: 12px; min-height: 32px; padding: 4px 10px; border: 1px solid var(--line); border-radius: 6px; color: var(--ink); background: var(--panel); cursor: pointer; }
+.vp-actions button:hover { border-color: var(--accent); }
+.vp-actions button:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; }
+.vp-dialog { width: calc(100vw - 32px); height: calc(100dvh - 32px); max-width: none; max-height: none; padding: 0; border: 1px solid var(--line); border-radius: 10px; background: var(--panel); color: var(--ink); }
+.vp-dialog::backdrop { background: rgb(0 0 0 / .65); }
+.vp-dialog[open] { display: flex; flex-direction: column; }
+.vp-dialog .supervision { margin: 0; border: 0; border-bottom: 1px solid var(--line); border-radius: 0; flex: none; max-height: 30%; overflow: auto; }
+:root:fullscreen .vp-dialog { width: 100vw; height: 100dvh; border: 0; border-radius: 0; }
+.vp-dialog .viewport-panel { position: static; display: flex; flex-direction: column; flex: 1; min-height: 0; border: 0; overflow: auto; }
+.vp-dialog .vp-stage { flex: 1; min-height: 0; display: grid; place-items: center; }
+.vp-dialog .device { width: min(calc(100% - 16px), calc(var(--vp-w, 390) * 1px), calc((var(--stage-height, 600px) - 16px) * var(--vp-w, 390) / var(--vp-h, 844))); transition: none; }
+.vp-dialog .sweep { display: none; }
+.vp-dialog .human-panel { flex: none; max-height: 35%; overflow: auto; }
+.vp-dialog .vp-now, .vp-dialog .vp-toolbar { flex: none; }
+@media (max-width: 720px) { .vp-dialog { width: calc(100vw - 12px); height: calc(100dvh - 12px); } }
+@media (max-height: 600px) { .vp-dialog #vp-display-note { display: none; } .vp-dialog .vp-toolbar { margin-bottom: 6px; } .vp-dialog .vp-now { margin-bottom: 6px; font-size: 12px; } }
 /* Sized to the device on screen (--vp-w × --vp-h in CSS pixels), shrunk to fit the column and the window. */
-.device { position: relative; box-sizing: border-box; margin: 0 auto; width: min(100%, calc(var(--vp-w, 390) * 1px + 16px), calc(max(420px, 100vh - 330px) * var(--vp-w, 390) / var(--vp-h, 844))); border: 8px solid #20252c; border-radius: 22px; overflow: hidden; background: #000; aspect-ratio: var(--vp-w, 390) / var(--vp-h, 844); transition: width .25s ease; }
+.device { position: relative; box-sizing: content-box; margin: 0 auto; width: min(calc(100% - 16px), calc(var(--vp-w, 390) * 1px), calc(max(404px, 100vh - 346px) * var(--vp-w, 390) / var(--vp-h, 844))); border: 8px solid #20252c; border-radius: 22px; overflow: hidden; background: #000; aspect-ratio: var(--vp-w, 390) / var(--vp-h, 844); transition: width .25s ease; }
 @media (prefers-reduced-motion: reduce) { .device { transition: none; } }
 .device img { position: absolute; inset: 0; display: block; width: 100%; height: 100%; object-fit: contain; background: #fff; }
 .device img:not([src]) { visibility: hidden; }

package/assets/dashboard/app.js

@@ -53,6 +53,16 @@ function connect() {
   loadCapabilities();
   source = new EventSource(api('/api/events'));
   source.addEventListener('open', () => setConn('live', 'live'));
+  source.addEventListener('session-replaced', (e) => {
+    const next = JSON.parse(e.data).token;
+    if (typeof next !== 'string' || !/^[\w-]{32}$/.test(next)) return;
+    source.close();
+    setConn('reconnecting', 'new session…');
+    // Replace the fragment without keeping the expired link in history, then reload to discard
+    // the previous session's state, pending requests and cached evidence frames.
+    history.replaceState(null, '', `#token=${encodeURIComponent(next)}`);
+    location.reload();
+  });
   source.addEventListener('snapshot', (e) => {
     const snap = JSON.parse(e.data);
     model.timeline = snap.timeline;
@@ -72,9 +82,9 @@ function connect() {
   source.addEventListener('feed', (e) => apply(JSON.parse(e.data).event));
   source.addEventListener('error', () => {
     const s = model.status && model.status.state;
+    if (s === 'ended' || s === 'failed') return setConn('closed', 'session over');
     if (source.readyState === EventSource.CLOSED) {
       // The server refused the stream (e.g. 401 after a new session rotated the token) or went away.
-      if (s === 'ended' || s === 'failed') return setConn('closed', 'session over');
       setConn('denied', 'disconnected');
       checkAccess();
     } else {
@@ -270,7 +280,8 @@ function setStatus(s) {
   }
   if (s.state === 'ended' || s.state === 'failed') {
     setConn('closed', 'session over');
-    if (source) source.close();
+    // Keep the event stream attached: an MCP server can start another session and hand us its link.
+    // Frame capture still stops when the session ends.
   }
   if (s.control) { model.control = s.control; model.controlEpoch++; }
   renderControl();
@@ -327,6 +338,74 @@ setInterval(() => {
 }, 200);
 document.addEventListener('visibilitychange', updateViewport);
 
+// Move the existing viewport into a modal stage: expanding never opens a second stream.
+const viewportPanel = $('viewport-panel');
+const viewportHome = document.createComment('viewport position');
+viewportPanel.before(viewportHome);
+const viewportDialog = $('vp-dialog');
+const supervisionHome = document.createComment('supervision position');
+$('supervision').before(supervisionHome);
+let viewportOpener;
+new ResizeObserver(([entry]) => {
+  $('device').style.setProperty('--stage-height', `${entry.contentRect.height}px`);
+}).observe($('vp-stage'));
+
+function expandViewport(opener) {
+  if (viewportDialog.open) return;
+  viewportOpener = opener;
+  viewportDialog.append($('supervision'), viewportPanel);
+  $('vp-expand').hidden = true;
+  $('vp-close').hidden = false;
+  $('vp-display-note').textContent = 'Press Escape or close this view to return to the timeline and findings.';
+  $('vp-display-note').hidden = false;
+  viewportDialog.showModal();
+  $('vp-close').focus();
+}
+
+async function closeViewport() {
+  if (document.fullscreenElement === document.documentElement) await document.exitFullscreen().catch(() => {});
+  viewportDialog.close();
+}
+
+$('vp-expand').addEventListener('click', (e) => expandViewport(e.currentTarget));
+$('vp-close').addEventListener('click', closeViewport);
+$('vp-fullscreen').addEventListener('click', async (e) => {
+  if (document.fullscreenElement === document.documentElement) {
+    await document.exitFullscreen().catch(() => {});
+    return;
+  }
+  expandViewport(e.currentTarget);
+  try {
+    await document.documentElement.requestFullscreen();
+  } catch {
+    $('vp-display-note').textContent = 'Full screen is unavailable in this browser. You can still use the expanded view.';
+  }
+});
+document.addEventListener('fullscreenchange', () => {
+  $('vp-fullscreen').textContent = document.fullscreenElement === document.documentElement ? 'Exit full screen' : 'Full screen';
+});
+viewportDialog.addEventListener('cancel', (e) => {
+  e.preventDefault();
+  if (document.fullscreenElement === document.documentElement) void document.exitFullscreen().catch(() => {});
+  else void closeViewport();
+});
+viewportDialog.addEventListener('keydown', (e) => {
+  if (e.key !== 'Tab') return;
+  const controls = [...viewportDialog.querySelectorAll('button, input, [tabindex="0"]')]
+    .filter((el) => !el.disabled && el.getClientRects().length);
+  const first = controls[0], last = controls[controls.length - 1];
+  if (e.shiftKey && document.activeElement === first) { e.preventDefault(); last?.focus(); }
+  else if (!e.shiftKey && document.activeElement === last) { e.preventDefault(); first?.focus(); }
+});
+viewportDialog.addEventListener('close', () => {
+  supervisionHome.after($('supervision'));
+  viewportHome.after(viewportPanel);
+  $('vp-expand').hidden = false;
+  $('vp-close').hidden = true;
+  $('vp-display-note').hidden = true;
+  viewportOpener?.focus();
+});
+
 // ---------- timeline ----------
 
 function renderTimeline() {

package/assets/dashboard/index.html

@@ -56,13 +56,23 @@
 </section>
 
 <main class="grid">
-  <section class="panel viewport-panel" aria-label="Live viewport">
-    <h2>Viewport <span id="vp-caption" class="muted small"></span></h2>
+  <section id="viewport-panel" class="panel viewport-panel" aria-label="Live viewport">
+    <div class="vp-toolbar">
+      <h2 id="vp-title">Viewport <span id="vp-caption" class="muted small"></span></h2>
+      <div class="vp-actions" role="group" aria-label="Viewport display">
+        <button id="vp-expand" type="button" aria-haspopup="dialog">Expand view</button>
+        <button id="vp-fullscreen" type="button">Full screen</button>
+        <button id="vp-close" type="button" hidden>Close view</button>
+      </div>
+    </div>
+    <p id="vp-display-note" class="small muted" role="status" hidden></p>
     <p id="vp-now" class="vp-now" role="status"></p>
+    <div id="vp-stage" class="vp-stage">
     <div class="device" id="device">
       <img id="viewport" alt="Live view of the device viewport">
       <div id="vp-empty" class="vp-empty">No frames yet</div>
     </div>
+    </div>
     <section id="human-panel" class="human-panel" hidden aria-label="Take over the browser">
       <p class="human-banner"><strong>You have control of the browser.</strong> The agent is waiting. Your actions are recorded as descriptions only (no typed text).<span id="human-headed" hidden> You can also use the browser window directly.</span></p>
       <p class="small muted">Click the viewport to tap; scroll over it to scroll.</p>
@@ -116,5 +126,6 @@
     <article id="finding-detail" class="finding-detail" hidden></article>
   </section>
 </main>
+<dialog id="vp-dialog" class="vp-dialog" aria-labelledby="vp-title"></dialog>
 </body>
 </html>

package/dist/core/format.js

@@ -14,6 +14,8 @@ export function formatControl(c) {
         parts.push(c.checked ? 'checked' : 'unchecked');
     if (c.pressed !== undefined)
         parts.push(c.pressed ? 'pressed' : 'not-pressed');
+    if (c.selected !== undefined)
+        parts.push(c.selected ? 'selected' : 'not-selected');
     if (c.expanded !== undefined)
         parts.push(c.expanded ? 'expanded' : 'collapsed');
     for (const flag of ['disabled', 'required', 'invalid', 'focused'])
@@ -98,6 +100,8 @@ export function formatChanges(c) {
     }
     if (c.route)
         items.push(`route ${c.route.from} → ${c.route.to}`);
+    if (c.title)
+        items.push(`title ${q(c.title.from)} → ${q(c.title.to)}`);
     if (c.dialog) {
         if (c.dialog.to !== undefined && c.dialog.from === undefined)
             items.push(`+ dialog ${q(c.dialog.to)}`);
@@ -136,7 +140,7 @@ export function formatChanges(c) {
         items.push(`- layout ${f.kind} resolved`);
     if (items.length > MAX_CHANGE_ITEMS) {
         const rest = items.length - MAX_CHANGE_ITEMS;
-        return [...items.slice(0, MAX_CHANGE_ITEMS), `… ${rest} more (use --json for all)`];
+        return [...items.slice(0, MAX_CHANGE_ITEMS), `… ${rest} more changes (observe for current state; CLI --json for all changes)`];
     }
     return items;
 }
@@ -144,7 +148,7 @@ export function formatAction(r) {
     const target = r.target ? ` ${r.target.role} ${q(r.target.name)}` : '';
     const head = `${r.action}${r.ref ? ` ${r.ref}` : ''}${target}`;
     if (r.outcome === 'error') {
-        return [`${head} → ERROR ${formatError(r.error)}`, ...r.notes.map((n) => `note: ${n}`)].join('\n');
+        return [`${head} → ERROR ${formatError(r.error)}`, ...r.notes.map((n) => `note: ${n}`), formatActionDiagnostics(r)].join('\n');
     }
     const lines = [`${head} → ok (${r.method}, ${r.elapsedMs}ms, ${r.settle ? formatSettle(r.settle) : 'settle unknown'})`];
     for (const n of r.notes)
@@ -166,15 +170,20 @@ export function formatAction(r) {
     }
     if (!c.reset) {
         const o = r.observation;
+        if (o.omitted)
+            lines.push(`controls: ${o.omitted} omitted by budget (observe with a larger limit for current controls)`);
         if (c.removed.length)
             lines.push(`refs: ${c.removed.length <= 8 ? c.removed.map((x) => x.ref).join(', ') : `${c.removed.length} earlier refs`} no longer exist; act on refs from this result or a fresh observe`);
         if (o.layout.length && !c.layoutAdded.length)
             lines.push(`layout: ${o.layout.length} existing flag(s) still present`);
     }
+    lines.push(formatActionDiagnostics(r));
+    return lines.join('\n');
+}
+function formatActionDiagnostics(r) {
     const errs = r.newConsoleErrors.map((e) => q(e.text)).join('; ');
     const fails = r.newFailedRequests.map((f) => `${f.method} ${f.url} ${f.status ?? f.failure}`).join('; ');
-    lines.push(`console: ${r.newConsoleErrors.length} new errors${errs ? ` (${errs})` : ''}  network: ${r.newFailedRequests.length} new failures${fails ? ` (${fails})` : ''}`);
-    return lines.join('\n');
+    return `console: ${r.newConsoleErrors.length} new errors${errs ? ` (${errs})` : ''}  network: ${r.newFailedRequests.length} new failures${fails ? ` (${fails})` : ''}`;
 }
 export function formatStart(r) {
     const s = r.session;
@@ -255,8 +264,37 @@ export function formatTabs(tabs) {
 }
 export function formatError(e) {
     const lines = [`${e.code}: ${e.message}`];
+    lines.push(`recoverable: ${e.recoverable}`);
     if (e.hint)
         lines.push(`hint: ${e.hint}`);
+    const control = e.details?.control;
+    if (control && typeof control === 'object') {
+        const s = control;
+        const parts = [];
+        if (typeof s.mode === 'string')
+            parts.push(s.mode);
+        if (typeof s.pending === 'string')
+            parts.push(`pending=${s.pending}`);
+        if (typeof s.by === 'string')
+            parts.push(`by=${q(s.by)}`);
+        if (typeof s.since === 'string')
+            parts.push(`since=${q(s.since)}`);
+        if (typeof s.observeRequired === 'boolean')
+            parts.push(`observeRequired=${s.observeRequired}`);
+        if (typeof s.interrupt === 'boolean')
+            parts.push(`interrupt=${s.interrupt}`);
+        if (typeof s.humanInteractions === 'number')
+            parts.push(`humanInteractions=${s.humanInteractions}`);
+        if (s.busy && typeof s.busy === 'object') {
+            const busy = s.busy;
+            if (typeof busy.command === 'string')
+                parts.push(`busy=${q(busy.command)}`);
+            if (typeof busy.since === 'string')
+                parts.push(`busySince=${q(busy.since)}`);
+        }
+        if (parts.length)
+            lines.push(`control: ${parts.join(' ')}`);
+    }
     const tail = e.details?.logTail;
     if (Array.isArray(tail) && tail.length)
         lines.push('server log tail:', ...tail.map((l) => `  ${l}`));
@@ -274,6 +312,9 @@ export function formatError(e) {
     const candidates = e.details?.candidates;
     if (Array.isArray(candidates))
         lines.push(`candidates: ${candidates.map((c) => `${c.ref} ${c.role} ${q(c.name)}`).join('; ')}`);
+    const sameRole = e.details?.sameRole;
+    if (Array.isArray(sameRole))
+        lines.push(`same role: ${sameRole.map((c) => q(String(c))).join('; ')}`);
     return lines.join('\n');
 }
 /** A person's interaction in words, e.g. `tapped button "Save"`, `typed into textbox "Password" (hidden)`. */

package/dist/dashboard/server.js

@@ -10,7 +10,9 @@ import { readFileSync } from 'node:fs';
 import { readFile } from 'node:fs/promises';
 import { createServer } from 'node:http';
 import { LabError } from '../core/schema.js';
-export const DEFAULT_SCREENCAST = { maxFps: 5, maxWidth: 800, quality: 60 };
+// Preserve desktop text at 1× CSS resolution; Lab never upscales smaller devices.
+// Keep the rate conservative: clearer frames need more encoding and bandwidth.
+export const DEFAULT_SCREENCAST = { maxFps: 5, maxWidth: 1920, quality: 85 };
 const BOUNDARY = 'agentlab-frame';
 const PART = `\r\n--${BOUNDARY}\r\nContent-Type: image/jpeg\r\n\r\n`;
 /** A slow viewer skips frames rather than buffering them without limit. */
@@ -36,7 +38,8 @@ export class Dashboard {
     token = newToken();
     /** Read-only access; this is the URL placed in start and status results that agents see. */
     viewToken = newToken();
-    events = new Set();
+    /** Each authenticated stream keeps its access level for the next session's handoff. */
+    events = new Map();
     viewers = new Set();
     lastFrame;
     cast;
@@ -61,7 +64,7 @@ export class Dashboard {
             d.server.listen(opts.port ?? 0, '127.0.0.1', () => resolve());
         });
         d.port = d.server.address().port;
-        d.heartbeat = setInterval(() => { for (const res of d.events)
+        d.heartbeat = setInterval(() => { for (const res of d.events.keys())
             res.write(': ping\n\n'); }, HEARTBEAT_MS);
         d.heartbeat.unref();
         return d;
@@ -96,16 +99,20 @@ export class Dashboard {
     // ---------- feed ----------
     onMessage(m) {
         if (m.event.type === 'reset') {
-            // A new session: previous viewers and the old tokens lose access.
+            // Rotate credentials, but let already authenticated, connected dashboards follow the new
+            // session. Send only the matching access level, outside the shared/replayable feed.
             this.token = newToken();
             this.viewToken = newToken();
+            for (const [res, control] of this.events) {
+                res.write(`event: session-replaced\ndata: ${JSON.stringify({ token: control ? this.token : this.viewToken })}\n\n`);
+            }
             this.lastFrame = undefined;
             this.endStreams();
             void this.stopCast();
             return;
         }
         const frame = `id: ${m.seq}\nevent: feed\ndata: ${JSON.stringify(m)}\n\n`;
-        for (const res of this.events)
+        for (const res of this.events.keys())
             res.write(frame);
         if (m.event.type === 'status') {
             const state = m.event.status.state;
@@ -118,7 +125,7 @@ export class Dashboard {
         }
     }
     endStreams() {
-        for (const res of [...this.events, ...this.viewers])
+        for (const res of [...this.events.keys(), ...this.viewers])
             res.end();
         this.events.clear();
         this.viewers.clear();
@@ -219,15 +226,17 @@ export class Dashboard {
             return void this.sendFile(res, this.opts.feed.sweepFrameFile(sweep[1], sweep[2]));
         res.writeHead(404).end();
     }
-    authorized(req, url) {
+    requestToken(req, url) {
         const header = req.headers.authorization;
-        const given = header?.startsWith('Bearer ') ? header.slice(7) : url.searchParams.get('token');
+        return header?.startsWith('Bearer ') ? header.slice(7) : url.searchParams.get('token');
+    }
+    authorized(req, url) {
+        const given = this.requestToken(req, url);
         return !!given && (same(given, this.token) || same(given, this.viewToken));
     }
     /** Whether this request carries the control token (the page shows control buttons only then). */
     canControl(req, url) {
-        const header = req.headers.authorization;
-        const given = header?.startsWith('Bearer ') ? header.slice(7) : url.searchParams.get('token');
+        const given = this.requestToken(req, url);
         return !!given && !!this.opts.control && same(given, this.token);
     }
     /**
@@ -319,7 +328,7 @@ export class Dashboard {
             const snap = this.opts.feed.snapshot();
             res.write(`id: ${snap.seq}\nevent: snapshot\ndata: ${JSON.stringify(snap)}\n\n`);
         }
-        this.events.add(res);
+        this.events.set(res, same(this.requestToken(req, url), this.token));
         res.on('close', () => { this.events.delete(res); });
     }
     /** MJPEG stream of the live viewport. Capture runs only while at least one of these is open. */

package/dist/mcp/server.js

@@ -22,8 +22,8 @@ export async function runMcpServer(opts) {
     const version = productVersion();
     const ui = opts.ui ?? true;
     const feed = new SessionFeed();
-    // A new session rotates the dashboard's tokens (old viewers lose access), so the person's control URL
-    // is re-published whenever one starts; it would otherwise stop working after the agent's second start.
+    // A new session rotates the dashboard's tokens. Connected tabs receive a handoff; re-publish the
+    // person's control URL too, so `agentlab ui` and newly opened tabs use the current credentials.
     let announce;
     const host = new SessionHost({
         stateDir: opts.stateDir, evidenceFrames: ui,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-device-lab",
-  "version": "0.3.1",
+  "version": "0.4.1",
   "description": "Lets a coding agent start your web app, use it at phone, tablet or desktop size, and report what breaks, while you watch.",
   "keywords": [
     "agent",