agent-device-lab 0.0.0-stage → 0.3.1

package/dist/dashboard/server.js

@@ -0,0 +1,366 @@
+// Local live dashboard. Serves a monitoring page, the session's typed event stream (SSE) and the
+// active device's viewport (MJPEG), bound to 127.0.0.1 and gated by per-session access tokens.
+// Contains no session logic: state comes from SessionFeed, frames from the Lab's screencast, and
+// supervision requests (pause, takeover, stop, a person's input) are forwarded to the session owner.
+//
+// Two tokens: the view token (in the URL agents receive) reads; only the control token (the URL a
+// person gets from `agentlab ui`) may POST control requests and input.
+import { randomBytes, timingSafeEqual } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import { readFile } from 'node:fs/promises';
+import { createServer } from 'node:http';
+import { LabError } from '../core/schema.js';
+export const DEFAULT_SCREENCAST = { maxFps: 5, maxWidth: 800, quality: 60 };
+const BOUNDARY = 'agentlab-frame';
+const PART = `\r\n--${BOUNDARY}\r\nContent-Type: image/jpeg\r\n\r\n`;
+/** A slow viewer skips frames rather than buffering them without limit. */
+const MAX_BUFFERED = 512 * 1024;
+const HEARTBEAT_MS = 15_000;
+/** Open streams per dashboard; more are refused (503) rather than served without limit. */
+const MAX_EVENT_STREAMS = 32;
+const MAX_VIEWERS = 8;
+const MAX_BODY = 8 * 1024;
+const STATIC_DIR = new URL('../../assets/dashboard/', import.meta.url);
+const STATIC = {
+    '/': { file: 'index.html', type: 'text/html; charset=utf-8' },
+    '/app.js': { file: 'app.js', type: 'text/javascript; charset=utf-8' },
+    '/app.css': { file: 'app.css', type: 'text/css; charset=utf-8' },
+};
+const CSP = "default-src 'none'; script-src 'self'; style-src 'self'; img-src 'self' blob:; connect-src 'self'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'";
+const newToken = () => randomBytes(24).toString('base64url');
+export class Dashboard {
+    opts;
+    server;
+    assets = new Map();
+    /** Full access, including control; only ever given to the person (`agentlab ui`). */
+    token = newToken();
+    /** Read-only access; this is the URL placed in start and status results that agents see. */
+    viewToken = newToken();
+    events = new Set();
+    viewers = new Set();
+    lastFrame;
+    cast;
+    castOpts;
+    stats = { running: false, starts: 0, stops: 0, framesCaptured: 0, framesSent: 0, bytesSent: 0 };
+    unsubscribe;
+    heartbeat;
+    closed = false;
+    port = 0;
+    constructor(opts) {
+        this.opts = opts;
+        this.castOpts = { ...DEFAULT_SCREENCAST, ...opts.screencast };
+        for (const { file } of Object.values(STATIC))
+            this.assets.set(file, readFileSync(new URL(file, STATIC_DIR)));
+        this.server = createServer((req, res) => this.route(req, res));
+        this.unsubscribe = opts.feed.subscribe((m) => this.onMessage(m));
+    }
+    static async listen(opts) {
+        const d = new Dashboard(opts);
+        await new Promise((resolve, reject) => {
+            d.server.once('error', reject);
+            d.server.listen(opts.port ?? 0, '127.0.0.1', () => resolve());
+        });
+        d.port = d.server.address().port;
+        d.heartbeat = setInterval(() => { for (const res of d.events)
+            res.write(': ping\n\n'); }, HEARTBEAT_MS);
+        d.heartbeat.unref();
+        return d;
+    }
+    /** The person's URL (control and view). The token is in the fragment, so it is never sent in requests or Referer headers. */
+    get url() {
+        return `http://127.0.0.1:${this.port}/#token=${this.token}`;
+    }
+    /** A view-only URL: monitoring works, control requests are refused. This is what agents are given. */
+    get viewUrl() {
+        return `http://127.0.0.1:${this.port}/#token=${this.viewToken}`;
+    }
+    get origin() {
+        return `http://127.0.0.1:${this.port}`;
+    }
+    getStats() {
+        return { viewers: { events: this.events.size, frames: this.viewers.size }, screencast: { ...this.stats } };
+    }
+    /** Stop frame capture, end every stream and close the listening socket. Idempotent. */
+    async close() {
+        if (this.closed)
+            return;
+        this.closed = true;
+        this.unsubscribe();
+        if (this.heartbeat)
+            clearInterval(this.heartbeat);
+        this.endStreams();
+        await this.stopCast();
+        this.server.closeAllConnections();
+        await new Promise((resolve) => this.server.close(() => resolve()));
+    }
+    // ---------- feed ----------
+    onMessage(m) {
+        if (m.event.type === 'reset') {
+            // A new session: previous viewers and the old tokens lose access.
+            this.token = newToken();
+            this.viewToken = newToken();
+            this.lastFrame = undefined;
+            this.endStreams();
+            void this.stopCast();
+            return;
+        }
+        const frame = `id: ${m.seq}\nevent: feed\ndata: ${JSON.stringify(m)}\n\n`;
+        for (const res of this.events)
+            res.write(frame);
+        if (m.event.type === 'status') {
+            const state = m.event.status.state;
+            if (state === 'active')
+                void this.ensureCast();
+            if (state === 'ended' || state === 'failed') {
+                void this.stopCast().then(() => { for (const res of this.viewers)
+                    res.end(); this.viewers.clear(); });
+            }
+        }
+    }
+    endStreams() {
+        for (const res of [...this.events, ...this.viewers])
+            res.end();
+        this.events.clear();
+        this.viewers.clear();
+    }
+    // ---------- screencast ----------
+    async ensureCast() {
+        if (this.cast || this.closed || !this.viewers.size)
+            return;
+        const source = this.opts.source();
+        if (!source?.active)
+            return;
+        const pending = source.screencast((jpeg) => this.onFrame(jpeg), this.castOpts).catch(() => undefined);
+        this.cast = pending;
+        const stop = await pending;
+        if (!stop) {
+            if (this.cast === pending)
+                this.cast = undefined;
+            return;
+        }
+        this.stats.starts++;
+        this.stats.running = true;
+        // Every viewer left while capture was starting (a concurrent stopCast already handled a replaced one).
+        if (this.cast === pending && !this.viewers.size)
+            await this.stopCast(pending);
+    }
+    async stopCast(which = this.cast) {
+        if (!which)
+            return;
+        if (this.cast === which)
+            this.cast = undefined;
+        const stop = await which;
+        if (!stop)
+            return;
+        await stop();
+        this.stats.stops++;
+        this.stats.running = !!this.cast;
+    }
+    onFrame(jpeg) {
+        this.lastFrame = jpeg;
+        this.stats.framesCaptured++;
+        for (const res of this.viewers)
+            this.sendFrame(res, jpeg);
+    }
+    /**
+     * Chromium shows a multipart part only once the next part has begun, so every frame is followed at
+     * once by the next part's boundary and headers (otherwise an idle page's last frame never shows).
+     */
+    sendFrame(res, jpeg) {
+        if (res.writableLength > MAX_BUFFERED)
+            return;
+        res.write(jpeg);
+        res.write(PART);
+        this.stats.framesSent++;
+        this.stats.bytesSent += jpeg.length;
+    }
+    // ---------- HTTP ----------
+    route(req, res) {
+        res.setHeader('Cache-Control', 'no-store');
+        res.setHeader('Referrer-Policy', 'no-referrer');
+        res.setHeader('X-Content-Type-Options', 'nosniff');
+        // DNS rebinding and cross-site requests: only this exact loopback origin may talk to the dashboard.
+        const hosts = [`127.0.0.1:${this.port}`, `localhost:${this.port}`];
+        if (!hosts.includes(req.headers.host ?? ''))
+            return void res.writeHead(403).end('forbidden host\n');
+        const origin = req.headers.origin;
+        if (origin !== undefined && !hosts.some((h) => origin === `http://${h}`))
+            return void res.writeHead(403).end('forbidden origin\n');
+        const url = new URL(req.url ?? '/', this.origin);
+        if (req.method === 'POST' && (url.pathname === '/api/control' || url.pathname === '/api/input'))
+            return void this.post(req, res, url.pathname);
+        if (req.method !== 'GET')
+            return void res.writeHead(405, { allow: 'GET' }).end();
+        const asset = STATIC[url.pathname];
+        if (asset) {
+            // Static code only; no session data is served without the token.
+            res.writeHead(200, { 'content-type': asset.type, 'content-security-policy': CSP, 'x-frame-options': 'DENY' });
+            return void res.end(this.assets.get(asset.file));
+        }
+        if (!url.pathname.startsWith('/api/'))
+            return void res.writeHead(404).end();
+        if (!this.authorized(req, url)) {
+            return void res.writeHead(401, { 'content-type': 'application/json' }).end(JSON.stringify({ error: 'missing or invalid dashboard token; open the URL printed by `agentlab ui`' }));
+        }
+        if (url.pathname === '/api/state')
+            return this.json(res, { ...this.opts.feed.snapshot(), stats: this.getStats(), canControl: this.canControl(req, url) });
+        if (url.pathname === '/api/events')
+            return this.openEvents(req, res, url);
+        if (url.pathname === '/api/viewport')
+            return this.openViewport(res);
+        const frame = /^\/api\/frames\/(F\d{1,6})(?:-(\d{1,2}))?\.jpg$/.exec(url.pathname);
+        if (frame)
+            return void this.sendFile(res, frame[2] === undefined ? this.opts.feed.frameFile(frame[1]) : this.opts.feed.extraFrameFile(frame[1], Number(frame[2])));
+        const scanFrame = /^\/api\/scans\/(R\d{1,6})\/(\d{1,4})\/(s\d{1,4})\.jpg$/.exec(url.pathname);
+        if (scanFrame)
+            return void this.sendFile(res, this.opts.feed.scanFrameFile(scanFrame[1], Number(scanFrame[2]), scanFrame[3]));
+        const sweep = /^\/api\/sweeps\/(S\d{1,6})\/([a-z0-9-]{1,40})\.jpg$/.exec(url.pathname);
+        if (sweep)
+            return void this.sendFile(res, this.opts.feed.sweepFrameFile(sweep[1], sweep[2]));
+        res.writeHead(404).end();
+    }
+    authorized(req, url) {
+        const header = req.headers.authorization;
+        const given = header?.startsWith('Bearer ') ? header.slice(7) : url.searchParams.get('token');
+        return !!given && (same(given, this.token) || same(given, this.viewToken));
+    }
+    /** Whether this request carries the control token (the page shows control buttons only then). */
+    canControl(req, url) {
+        const header = req.headers.authorization;
+        const given = header?.startsWith('Bearer ') ? header.slice(7) : url.searchParams.get('token');
+        return !!given && !!this.opts.control && same(given, this.token);
+    }
+    /**
+     * POST /api/control {op} and /api/input {type, …}. Only the control token, only in an Authorization
+     * header (never a query string), only from this exact origin, only JSON, and bodies of at most 8 KB.
+     * A cross-site page cannot send the header without a CORS preflight, which is never answered.
+     */
+    post(req, res, path) {
+        const reply = (status, body) => res.writeHead(status, { 'content-type': 'application/json' }).end(JSON.stringify(body));
+        const header = req.headers.authorization;
+        if (!header?.startsWith('Bearer ') || !same(header.slice(7), this.token))
+            return void reply(401, { error: 'control needs the dashboard URL from `agentlab ui` (this one is view-only or expired)' });
+        if (req.headers.origin !== this.origin && req.headers.origin !== `http://localhost:${this.port}`)
+            return void reply(403, { error: 'forbidden origin' });
+        const site = req.headers['sec-fetch-site'];
+        if (site !== undefined && site !== 'same-origin')
+            return void reply(403, { error: 'forbidden site' });
+        if (!/^application\/json\b/.test(req.headers['content-type'] ?? ''))
+            return void reply(415, { error: 'send application/json' });
+        const handler = path === '/api/control' ? this.opts.control : this.opts.input;
+        if (!handler)
+            return void reply(404, { error: 'this dashboard has no session to control' });
+        let size = 0;
+        const chunks = [];
+        let aborted = false;
+        req.on('data', (c) => {
+            size += c.length;
+            if (size > MAX_BODY && !aborted) {
+                aborted = true;
+                res.setHeader('connection', 'close');
+                res.once('finish', () => req.destroy());
+                reply(413, { error: 'request too large' });
+                return;
+            }
+            if (!aborted)
+                chunks.push(c);
+        });
+        req.on('end', () => {
+            if (aborted)
+                return;
+            let body;
+            try {
+                const parsed = JSON.parse(Buffer.concat(chunks).toString('utf8'));
+                if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed))
+                    throw new Error('not an object');
+                body = parsed;
+            }
+            catch {
+                return void reply(400, { error: 'body must be a JSON object' });
+            }
+            void (async () => {
+                try {
+                    if (path === '/api/control') {
+                        if (typeof body.op !== 'string')
+                            return reply(400, { error: 'op is required' });
+                        reply(200, { control: await this.opts.control(body.op, 'dashboard') });
+                    }
+                    else {
+                        await this.opts.input(body);
+                        reply(200, { ok: true });
+                    }
+                }
+                catch (err) {
+                    const e = LabError.from(err).toJSON();
+                    reply(e.code === 'invalid_control' ? 409 : e.code === 'invalid_request' ? 400 : e.code === 'no_session' || e.code === 'browser_closed' ? 410 : 500, { error: e.message, code: e.code });
+                }
+            })();
+        });
+    }
+    json(res, body) {
+        res.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify(body));
+    }
+    /**
+     * Server-sent events. A new connection gets a snapshot; a reconnect with Last-Event-ID gets only the
+     * messages it missed, or a fresh snapshot when those are no longer buffered.
+     */
+    openEvents(req, res, url) {
+        if (this.events.size >= MAX_EVENT_STREAMS)
+            return void res.writeHead(503, { 'retry-after': '5' }).end('too many open dashboards\n');
+        res.writeHead(200, { 'content-type': 'text/event-stream', connection: 'keep-alive', 'x-accel-buffering': 'no' });
+        res.write('retry: 1000\n\n');
+        const last = req.headers['last-event-id'] ?? url.searchParams.get('since');
+        const missed = last !== undefined && last !== null && /^\d+$/.test(String(last)) ? this.opts.feed.since(Number(last)) : undefined;
+        if (missed) {
+            for (const m of missed)
+                res.write(`id: ${m.seq}\nevent: feed\ndata: ${JSON.stringify(m)}\n\n`);
+        }
+        else {
+            const snap = this.opts.feed.snapshot();
+            res.write(`id: ${snap.seq}\nevent: snapshot\ndata: ${JSON.stringify(snap)}\n\n`);
+        }
+        this.events.add(res);
+        res.on('close', () => { this.events.delete(res); });
+    }
+    /** MJPEG stream of the live viewport. Capture runs only while at least one of these is open. */
+    openViewport(res) {
+        if (this.viewers.size >= MAX_VIEWERS)
+            return void res.writeHead(503, { 'retry-after': '5' }).end('too many viewport viewers\n');
+        res.writeHead(200, { 'content-type': `multipart/x-mixed-replace; boundary=${BOUNDARY}`, connection: 'close' });
+        res.write(PART);
+        this.viewers.add(res);
+        res.on('close', () => {
+            this.viewers.delete(res);
+            if (!this.viewers.size)
+                void this.stopCast();
+        });
+        if (this.lastFrame)
+            this.sendFrame(res, this.lastFrame);
+        const state = this.opts.feed.state;
+        if (state === 'ended' || state === 'failed') {
+            // Nothing more will come: show the final frame and finish.
+            this.viewers.delete(res);
+            res.end();
+            return;
+        }
+        void this.ensureCast();
+    }
+    /** Only files the feed recorded (evidence, sweep and scan frames in the run directory) are ever served. */
+    async sendFile(res, file) {
+        if (!file)
+            return void res.writeHead(404).end();
+        try {
+            const data = await readFile(file);
+            res.writeHead(200, { 'content-type': 'image/jpeg', 'content-length': data.length }).end(data);
+        }
+        catch {
+            res.writeHead(404).end();
+        }
+    }
+}
+function same(given, token) {
+    const a = Buffer.from(given);
+    const b = Buffer.from(token);
+    return a.length === b.length && timingSafeEqual(a, b);
+}
+//# sourceMappingURL=server.js.map

package/dist/mcp/server.js

@@ -0,0 +1,89 @@
+// Local stdio MCP server. A thin adapter: tools, schemas and behaviour all come from the command
+// table in src/core/commands.ts; this file only translates to and from MCP messages.
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
+import { COMMANDS, SessionHost, dispatch } from '../core/commands.js';
+import { SessionFeed } from '../core/feed.js';
+import { formatError } from '../core/format.js';
+import { productVersion } from '../core/versions.js';
+import { Dashboard } from '../dashboard/server.js';
+import { clearDashboardRecord, writeDashboardRecord } from '../daemon/state.js';
+const INSTRUCTIONS = `Agent Device Lab drives a local web app in a Chromium mobile profile.
+1. start {project}: starts or reuses the services declared in the project's agentlab.json (in dependency order) and returns the first observation. auth "fresh" ignores saved sign-in state.
+2. observe returns visible controls with refs (e.g. e12). Actions take a ref, or an exact role + name: click, fill, press, select, check, uncheck, hover, upload, drag; scroll and swipe (the page or a region); back and forward.
+3. Every action returns only what changed ("changed"), plus notes, settling and new usability findings. Pop-ups and new tabs open as tabs: the new tab becomes active; tabs / switch_tab / close_tab manage them.
+4. sweep {route?} checks one route at 320, 390, 768 and 1440 px in isolated contexts. scan runs the project's declared UI states (drawers, menus, dialogs, tabs; scan.scenarios in agentlab.json), optionally exploring safe ones, and returns a verdict, problem groups and report paths. Confirmed findings rest on a measurement (hit test, clipping, a WCAG rule); heuristic ones are warnings.
+5. inspect lists findings (overflow, clipping, obstruction, tap targets, wrapping, layout shift…) with evidence and reproduction steps; inspect {id} for one.
+6. auth_save stores the session's sign-in state for later sessions (never returned). bundle {note?} writes a secret-free failure bundle (action log, findings, frames) that "agentlab replay" can re-run. stop closes the browser and stops only the services the lab started.
+Refs from a previous page or another tab are stale; observe again rather than guessing. Upload files must be inside the project's uploads.allow directories.`;
+const DASHBOARD_NOTE = '\nThe start result includes a local dashboard URL (live viewport, timeline, findings) for a person to watch; share it with the user if they want to follow along.';
+export async function runMcpServer(opts) {
+    const version = productVersion();
+    const ui = opts.ui ?? true;
+    const feed = new SessionFeed();
+    // A new session rotates the dashboard's tokens (old viewers lose access), so the person's control URL
+    // is re-published whenever one starts; it would otherwise stop working after the agent's second start.
+    let announce;
+    const host = new SessionHost({
+        stateDir: opts.stateDir, evidenceFrames: ui,
+        onEvent: (e) => { feed.apply(e); if (e.kind === 'starting')
+            announce?.(); },
+    }, { headless: opts.headless });
+    // Frames and events go to the dashboard only; tool results carry at most its URL.
+    const dashboard = ui ? await Dashboard.listen({
+        feed, source: () => host.lab,
+        control: (op, by) => host.supervise(op, by),
+        input: (i) => host.lab.humanInput(i),
+    }) : undefined;
+    if (dashboard) {
+        // The agent's results carry the view-only URL. The person's control URL goes to stderr (the MCP
+        // client's log) and to an owner-only record that `agentlab ui` reads; it never reaches tool results.
+        host.dashboardUrl = () => dashboard.viewUrl;
+        let published = '';
+        announce = () => {
+            if (dashboard.url === published)
+                return;
+            published = dashboard.url;
+            writeDashboardRecord(opts.stateDir, dashboard.url);
+            process.stderr.write(`agentlab: dashboard with controls: ${dashboard.url}\n  (or run \`agentlab ui\` in ${process.cwd()})\n`);
+        };
+        announce();
+    }
+    const tools = COMMANDS.filter((c) => c.surfaces.includes('mcp'));
+    const server = new Server({ name: 'agent-device-lab', version }, { capabilities: { tools: {} }, instructions: INSTRUCTIONS + (dashboard ? DASHBOARD_NOTE : '') });
+    server.setRequestHandler(ListToolsRequestSchema, async () => ({
+        tools: tools.map((c) => ({ name: c.name, description: c.description, inputSchema: c.inputSchema })),
+    }));
+    // One page, one session: tool calls run strictly in order even if a client pipelines them.
+    let queue = Promise.resolve();
+    server.setRequestHandler(CallToolRequestSchema, (request) => {
+        const task = queue.then(async () => {
+            const out = await dispatch(host, 'mcp', request.params.name, request.params.arguments ?? {});
+            if (!out.ok) {
+                return { isError: true, content: [{ type: 'text', text: `error ${formatError(out.error)}` }], structuredContent: { error: out.error } };
+            }
+            const failed = out.output.result.outcome === 'error';
+            return { content: [{ type: 'text', text: out.output.text }], structuredContent: out.output.result, ...(failed ? { isError: true } : {}) };
+        });
+        queue = task.catch(() => undefined);
+        return task;
+    });
+    let closing = false;
+    const shutdown = async (reason) => {
+        if (closing)
+            return;
+        closing = true;
+        await host.lab.close(reason).catch(() => undefined);
+        await dashboard?.close().catch(() => undefined);
+        if (dashboard)
+            clearDashboardRecord(opts.stateDir);
+        process.exit(0);
+    };
+    server.onclose = () => void shutdown('mcp client disconnected');
+    process.stdin.on('end', () => void shutdown('mcp client disconnected'));
+    for (const sig of ['SIGINT', 'SIGTERM', 'SIGHUP'])
+        process.on(sig, () => void shutdown(`mcp server received ${sig}`));
+    await server.connect(new StdioServerTransport());
+}
+//# sourceMappingURL=server.js.map

package/package.json

@@ -1,6 +1,65 @@
 {
   "name": "agent-device-lab",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.3.1",
+  "description": "Lets a coding agent start your web app, use it at phone, tablet or desktop size, and report what breaks, while you watch.",
+  "keywords": [
+    "agent",
+    "coding-agent",
+    "mcp",
+    "playwright",
+    "responsive",
+    "mobile",
+    "desktop",
+    "testing",
+    "dev-server",
+    "claude-code",
+    "codex"
+  ],
+  "license": "Apache-2.0",
+  "author": "Colony Innovations (https://github.com/Colony-Innovations)",
+  "homepage": "https://github.com/Colony-Innovations/agent-device-lab#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Colony-Innovations/agent-device-lab.git"
+  },
+  "bugs": {
+    "url": "https://github.com/Colony-Innovations/agent-device-lab/issues"
+  },
+  "funding": {
+    "type": "patreon",
+    "url": "https://www.patreon.com/Ikafa"
+  },
+  "type": "module",
+  "bin": {
+    "agentlab": "bin/agentlab.js"
+  },
+  "files": [
+    "bin/",
+    "dist/**/*.js",
+    "assets/dashboard/",
+    "README.md",
+    "LICENSE",
+    "NOTICE"
+  ],
+  "engines": {
+    "node": ">=22"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "test": "npm run build && node --test --test-concurrency=1 \"test/**/*.test.mjs\"",
+    "flow:clean": "npm run build && node bin/agentlab.js run flows/clean.flow.json",
+    "flow:defect": "npm run build && node bin/agentlab.js run flows/mobile-defect.flow.json",
+    "prepack": "npm run build",
+    "verify:package": "node scripts/verify-package.mjs"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "1.32.1",
+    "playwright": "1.63.0"
+  },
+  "devDependencies": {
+    "@playwright/mcp": "0.0.82",
+    "@types/node": "22.20.4",
+    "typescript": "7.0.2"
+  }
+}