@petukhovart/agent-view 0.15.0 → 0.17.0

package/dist/server/server.js

@@ -1,7 +1,7 @@
 import { createServer } from 'node:net';
 import { writeFile, unlink } from 'node:fs/promises';
 import { randomBytes } from 'node:crypto';
-import { homedir, tmpdir } from 'node:os';
+import { tmpdir } from 'node:os';
 import { join, resolve } from 'node:path';
 import { existsSync, mkdirSync, statSync, writeFileSync } from 'node:fs';
 import { getAdapter } from '../adapters/registry.js';
@@ -28,8 +28,10 @@ import { AxTreeCache } from '../cdp/ax-cache.js';
 import { WatchSession } from './watch-session.js';
 import { StopReason, WATCH_MIN_INTERVAL_MS } from '../inspectors/watch/index.js';
 import { buildMatcher } from './pattern.js';
+import { findByLocator, locatorFromArgs, testIdAttributes } from './locator.js';
+import { SERVER_PORT, AGENT_VIEW_DIR, TOKEN_PATH } from './port.js';
+import { runAct } from './act-session.js';
 import { DEFAULT_TAIL_LINES, LogRecorder, filterLogLines, localStamp, parseSinceToken, readFeedLines, resolveLogFile, } from './log-recorder.js';
-const SERVER_PORT = 47922;
 /**
  * Every non-streaming command answers within this budget (plus its own `--timeout`,
  * for the commands that poll). A CDP call that never returns used to leave the CLI
@@ -43,8 +45,6 @@ const VALID_ENGINES = new Set(Object.values(WebGLEngine));
 const IDLE_TIMEOUT_MS = 5 * 60 * 1000;
 const DELIMITER = '\n';
 const MAX_BUFFER_SIZE = 1_048_576; // 1 MB
-const TOKEN_DIR = join(homedir(), '.agent-view');
-const TOKEN_PATH = join(TOKEN_DIR, 'token');
 const EVAL_OUTPUT_CAP = 64 * 1024;
 const DEFAULT_CONSOLE_TARGETS = [TargetType.Page, TargetType.SharedWorker, TargetType.ServiceWorker];
 const DEFAULT_NETWORK_BUFFER = 200;
@@ -194,13 +194,14 @@ export class AgentViewServer {
         coverage: (req) => this.handleCoverage(req),
         listeners: (req) => this.handleListeners(req),
         heap: (req) => this.handleHeap(req),
+        act: (req) => this.handleAct(req),
         stop: () => this.handleStop(),
     };
     streamingCommands = new Set(['watch']);
     validCommands = new Set([...Object.keys(this.handlers), ...this.streamingCommands]);
     async start() {
         installCDPErrorGuard();
-        mkdirSync(TOKEN_DIR, { recursive: true });
+        mkdirSync(AGENT_VIEW_DIR, { recursive: true });
         this.token = randomBytes(32).toString('hex');
         // Publish the token only after winning the port. A server that loses the bind race
         // must not overwrite the live server's token — that leaves every CLI call
@@ -225,6 +226,7 @@ export class AgentViewServer {
                 networkRefs: new Map(),
                 networkNextRef: 1,
                 logRecorder: null,
+                act: null,
             };
             this.portStates.set(port, state);
         }
@@ -509,6 +511,7 @@ export class AgentViewServer {
             startRef: this.refStore.getNextRef(),
             compact,
             maxLines: argNum(req.args, 'maxLines'),
+            testIds: await conn.getTestIds(testIdAttributes(argStr(req.args, 'testIdAttribute'))),
         });
         this.refStore.store(refs, req.port, targetId, nextRef);
         if (isDiff) {
@@ -634,6 +637,15 @@ export class AgentViewServer {
             this.axTreeCache.invalidate(cacheKey);
             return { ok: true, data: `${verb} at (${x}, ${y})` };
         }
+        const locator = locatorFromArgs(req.args);
+        if (locator) {
+            const found = await findByLocator(conn, locator);
+            if ('error' in found)
+                return { ok: false, error: found.error };
+            await conn.clickByNodeId(found.backendDOMNodeId, clickOpts);
+            this.axTreeCache.invalidate(cacheKey);
+            return { ok: true, data: `${verb} ${found.label}` };
+        }
         const clickFilter = argStr(req.args, 'filter');
         if (clickFilter) {
             const filter = clickFilter;
@@ -648,7 +660,7 @@ export class AgentViewServer {
         }
         const ref = argNum(req.args, 'ref');
         if (ref === undefined) {
-            return { ok: false, error: 'click requires --ref, --filter, or --pos' };
+            return { ok: false, error: 'click requires --ref, --filter, --testid, --selector, or --pos' };
         }
         const entry = this.refStore.get(ref);
         if (!entry) {
@@ -658,6 +670,19 @@ export class AgentViewServer {
         this.axTreeCache.invalidate(cacheKey);
         return { ok: true, data: `${verb} ref ${ref}` };
     }
+    async handleAct(req) {
+        const { targetId } = await this.resolveWindow(req);
+        const cacheKey = `${req.port}:${targetId}`;
+        return runAct(this.stateFor(req.port), req.args, {
+            conn: await this.getPageSession(req, targetId),
+            invalidateAxCache: () => this.axTreeCache.invalidate(cacheKey),
+            // A login that reloads the window kills the cached socket before its disconnect event lands.
+            reconnect: async () => {
+                this.dropSessionsForPort(req.port);
+                return this.getPageSession(req, (await this.resolveWindow(req)).targetId);
+            },
+        });
+    }
     async handleDrag(req) {
         const { targetId } = await this.resolveWindow(req);
         const conn = await this.getPageSession(req, targetId);
@@ -672,13 +697,20 @@ export class AgentViewServer {
             steps: argNum(req.args, 'steps'),
             button: parseMouseButton(argStr(req.args, 'button')),
             holdMs: argNum(req.args, 'holdMs'),
+            mode: argStr(req.args, 'mode'),
+            cancel: argBool(req.args, 'cancel'),
+            mask: argNum(req.args, 'mask'),
         };
-        await conn.dragBetweenPositions(from.point, to.point, opts);
+        const result = await conn.dragBetweenPositions(from.point, to.point, opts);
         this.axTreeCache.invalidate(cacheKey);
-        return {
-            ok: true,
-            data: `Dragged (${from.point.x.toFixed(0)}, ${from.point.y.toFixed(0)}) → (${to.point.x.toFixed(0)}, ${to.point.y.toFixed(0)})`,
-        };
+        const head = `Dragged (${from.point.x.toFixed(0)}, ${from.point.y.toFixed(0)}) → (${to.point.x.toFixed(0)}, ${to.point.y.toFixed(0)})`;
+        if (result.path === 'pointer') {
+            const warning = result.warning ? `\n  warning: ${result.warning}` : '';
+            return { ok: true, data: `${head} via pointer${warning}` };
+        }
+        const items = result.data.items.map(i => `  ${i.mimeType}: ${JSON.stringify(i.data.length > 200 ? i.data.slice(0, 200) + '…' : i.data)}`);
+        const summary = `${head} via html5 ${result.ended} (dragOperationsMask=${result.data.dragOperationsMask})`;
+        return { ok: true, data: [summary, ...items].join('\n') };
     }
     async resolveDragPoint(req, conn, side, opts) {
         const ref = argNum(req.args, `${side}Ref`);
@@ -705,6 +737,15 @@ export class AgentViewServer {
         if (value === undefined) {
             return { ok: false, error: 'fill requires --value' };
         }
+        const locator = locatorFromArgs(req.args);
+        if (locator) {
+            const found = await findByLocator(conn, locator);
+            if ('error' in found)
+                return { ok: false, error: found.error };
+            await conn.fillByNodeId(found.backendDOMNodeId, value);
+            this.axTreeCache.invalidate(cacheKey);
+            return { ok: true, data: `Filled ${found.label} with "${value}"` };
+        }
         const fillFilter = argStr(req.args, 'filter');
         if (fillFilter) {
             const filter = fillFilter;
@@ -719,7 +760,7 @@ export class AgentViewServer {
         }
         const fillRef = argNum(req.args, 'ref');
         if (fillRef === undefined) {
-            return { ok: false, error: 'fill requires --ref or --filter' };
+            return { ok: false, error: 'fill requires --ref, --filter, --testid, or --selector' };
         }
         const entry = this.refStore.get(fillRef);
         if (!entry) {
@@ -1028,14 +1069,25 @@ export class AgentViewServer {
     }
     async handleWait(req) {
         const filter = argStr(req.args, 'filter');
-        if (!filter) {
-            return { ok: false, error: 'wait requires --filter' };
+        const locator = locatorFromArgs(req.args);
+        if (!filter && !locator) {
+            return { ok: false, error: 'wait requires --filter, --testid, or --selector' };
         }
         const timeout = argNum(req.args, 'timeout') ?? 10;
         const interval = 500;
         const maxAttempts = Math.ceil((timeout * 1000) / interval);
         const { targetId } = await this.resolveWindow(req);
         const conn = await this.getPageSession(req, targetId);
+        if (locator) {
+            let found = await findByLocator(conn, locator);
+            for (let i = 1; i < maxAttempts && 'error' in found; i++) {
+                await new Promise(resolve => setTimeout(resolve, interval));
+                found = await findByLocator(conn, locator);
+            }
+            return 'error' in found
+                ? { ok: false, error: `Timeout after ${timeout}s: ${found.error}` }
+                : { ok: true, data: `Found ${found.label}` };
+        }
         for (let i = 0; i < maxAttempts; i++) {
             const nodes = await conn.getAccessibilityTree();
             const { refs, text, nextRef } = formatAccessibilityTree(nodes, {
@@ -1052,8 +1104,7 @@ export class AgentViewServer {
     }
     async captureScreenshotToFile(conn, opts = {}) {
         const { buffer, format } = await conn.captureScreenshot(opts);
-        const ext = format === 'jpeg' ? 'jpg' : format;
-        const filename = `agent-view-screenshot-${Date.now()}.${ext}`;
+        const filename = `agent-view-screenshot-${Date.now()}.${format}`;
         const filepath = join(tmpdir(), filename);
         await writeFile(filepath, buffer);
         return filepath;
@@ -1063,20 +1114,33 @@ export class AgentViewServer {
         const conn = await this.getPageSession(req, targetId);
         const scale = argNum(req.args, 'scale');
         const cropFilter = argStr(req.args, 'crop');
+        const locator = locatorFromArgs(req.args);
         let warning;
         let clip;
-        if (cropFilter !== undefined) {
+        let cropNode;
+        if (locator) {
+            // Unlike a text filter, an exact locator that misses is a wrong locator: a
+            // full-window fallback would spend ~19k vision tokens on the wrong question.
+            const found = await findByLocator(conn, locator);
+            if ('error' in found)
+                return { ok: false, error: found.error };
+            cropNode = found.backendDOMNodeId;
+        }
+        else if (cropFilter !== undefined) {
             const found = await this.findByFilter(conn, cropFilter, req, targetId);
-            if (!found) {
-                warning = `crop filter '${cropFilter}' matched nothing — capturing full window`;
+            if (found) {
+                cropNode = found.backendDOMNodeId;
             }
             else {
-                const cropUp = argNum(req.args, 'cropUp') ?? 0;
-                clip = await conn.getBoxRect(found.backendDOMNodeId, { scrollIntoView: true, ancestorLevels: cropUp });
-                if (clip.height < TEXT_LINE_HEIGHT_PX && cropUp === 0) {
-                    warning = `crop matched a text-sized box (${Math.round(clip.width)}×${Math.round(clip.height)}) — `
-                        + `pass --crop-up 1 (or 2) to capture the surrounding container instead`;
-                }
+                warning = `crop filter '${cropFilter}' matched nothing — capturing full window`;
+            }
+        }
+        if (cropNode !== undefined) {
+            const cropUp = argNum(req.args, 'cropUp') ?? 0;
+            clip = await conn.getBoxRect(cropNode, { scrollIntoView: true, ancestorLevels: cropUp });
+            if (clip.height < TEXT_LINE_HEIGHT_PX && cropUp === 0) {
+                warning = `crop matched a text-sized box (${Math.round(clip.width)}×${Math.round(clip.height)}) — `
+                    + `pass --crop-up 1 (or 2) to capture the surrounding container instead`;
             }
         }
         const filepath = await this.captureScreenshotToFile(conn, { scale, clip });