@e2edev/mobile 0.8.0-canary-20260921180210

package/dist/surface.js

@@ -0,0 +1,837 @@
+/**
+ * The agent-device surface: one simulator or emulator session, driven through
+ * agent-device's typed client, exposed to the runner as the contract's
+ * observe/locate/perform members. It owns the id space (one fresh generation
+ * per observation), the attempt state (artifact directory, screenshot
+ * counter), and every translation between the contract's vocabulary and
+ * agent-device's commands. Its device and session come from the target's
+ * `DevicePool`, by worker slot. The runner owns everything else.
+ *
+ * Every observation is reported under one root of a stable id (`ROOT_ID`)
+ * wrapping the device's top-level elements; a `swipe` performed on that root
+ * scrolls the whole screen.
+ */
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { EngineError, KEY_NAMES, parseKey, raceAbort, withinCleanupBudget, ConfigurationError, } from 'e2e/engine';
+import { runCommand, staleOr } from './errors.js';
+import { pointerInteraction } from './actions.js';
+import { resolveExpression } from './locate.js';
+import { isWithin, projectSnapshot, ROOT_ID, screenRoot, screenTitle, } from './nodes.js';
+import { maskPng } from './png.js';
+import { pinnedApp } from './bindings.js';
+import { DevicePool, deviceSelection } from './pool.js';
+import { invalidState, notActionable, logicalScreenSize, readPngSize, sanitizeFilename, screenLocation, swipeWithin, unsupported, } from './support.js';
+/**
+ * How long a control that appeared or moved with the last action is given to
+ * finish arriving before a test acts on it. Accessibility frames come from
+ * the model layer, which jumps to the final position the moment a transition
+ * starts, so the tree cannot tell a sliding control from a landed one; a
+ * modal or pushed screen keeps sliding for about half a second after the
+ * action that opened it, and a tap at a point the control has not reached
+ * yet lands on whatever is behind it. Controls that were already on screen
+ * at the same place before the action are acted on at once.
+ */
+const DEFAULT_TRANSITION_MS = 500;
+/** The centre of a rect in logical pixels. */
+function centreOf(rect) {
+    return { x: rect.x + rect.width / 2, y: rect.y + rect.height / 2 };
+}
+/** Whether two rects are the same to the pixel; a missing rect never matches, so a target must be measurable to be stable. */
+function sameRect(a, b) {
+    if (a === undefined || b === undefined)
+        return false;
+    return Math.abs(a.x - b.x) < 1 && Math.abs(a.y - b.y) < 1 && Math.abs(a.width - b.width) < 1 && Math.abs(a.height - b.height) < 1;
+}
+const MAX_LOCATED_REFS = 2048;
+/**
+ * The Apple runner defers a full snapshot after slow accessibility work and
+ * returns a sparse one-node tree. Retrying device-side costs under a second;
+ * returning the sparse tree costs the model a confused turn.
+ */
+const SPARSE_RETRY_BACKOFF_MS = [0, 1_200, 3_000];
+/** Number of parent hops from `entry` up to `ancestor`. */
+function depthBelow(entry, ancestor) {
+    let hops = 0;
+    for (let current = entry.parent; current !== undefined; current = current.parent) {
+        hops += 1;
+        if (current === ancestor)
+            return hops;
+    }
+    return hops;
+}
+function sleep(ms, signal) {
+    return new Promise((resolve) => {
+        const timer = setTimeout(done, ms);
+        function done() {
+            clearTimeout(timer);
+            signal.removeEventListener('abort', done);
+            resolve();
+        }
+        signal.addEventListener('abort', done, { once: true });
+    });
+}
+const DEFAULT_SETTLE_QUIET_MS = 150;
+/** Resolves the `transition` option: the default budget or a custom one. */
+function transitionMs(transition) {
+    const budget = transition ?? DEFAULT_TRANSITION_MS;
+    if (!Number.isInteger(budget) || budget < 0) {
+        throw new ConfigurationError('INVALID_CONFIG', 'mobile: `transition` must be a non-negative integer of milliseconds');
+    }
+    return budget;
+}
+/** Resolves the `settle` option: the default window, a custom one, or no wait at all. */
+function settleOptions(settle) {
+    if (settle === false)
+        return {};
+    const quietMs = settle ?? DEFAULT_SETTLE_QUIET_MS;
+    if (!Number.isInteger(quietMs) || quietMs < 0) {
+        throw new ConfigurationError('INVALID_CONFIG', 'mobile: `settle` must be a non-negative integer of milliseconds, or false');
+    }
+    return { settle: true, settleQuietMs: quietMs };
+}
+export class AgentDeviceSurface {
+    options;
+    createClient;
+    client;
+    attempt;
+    /** The device this worker drives, the pool's entry for its slot; undefined leaves the choice to agent-device. */
+    device;
+    generation = new Map();
+    located = new Map();
+    idCounter = 0;
+    appIdentity;
+    /** The app the build `appPath` installed, once `init` has, itself or through a lease. */
+    installedApp;
+    /** Where relative build paths resolve; the run's project root once init has told us. */
+    projectRoot = process.cwd();
+    /**
+     * Commands still running on the device. agent-device takes no abort
+     * signal, so a cancelled or timed-out call is only abandoned by its
+     * caller; it keeps executing. The next attempt waits for these to settle
+     * before it opens anything, so a ghost tap can never land in a retry.
+     */
+    inflight = new Set();
+    /** The target's devices, one per worker slot; `init` takes this worker's from it. */
+    pool;
+    /** The settle wait every action carries: quiet window from the `settle` option, or nothing when it is `false`. */
+    settleOptions;
+    /** When this surface last acted on the device (an input or an app launch); the screen may be in transition for a while after. */
+    lastActionAt = 0;
+    /** The screen as last projected before that action, to tell controls that were already there from ones that came with it. */
+    indexBeforeAction;
+    /** The most recent projection of the screen, from any observe or locate. */
+    latestIndex;
+    /** Budget a control that came with the last action gets to finish arriving; see DEFAULT_TRANSITION_MS. */
+    transitionMs;
+    /**
+     * The screen's logical size as last learned from a snapshot with geometry
+     * or from the device itself, so a snapshot without geometry (an empty
+     * screen before any app is open, a sparse tree) still reports the viewport
+     * every observation must carry.
+     */
+    knownViewport;
+    constructor(options, createClient) {
+        this.options = options;
+        this.createClient = createClient;
+        this.pool = new DevicePool(options, createClient);
+        this.settleOptions = settleOptions(options.settle);
+        this.transitionMs = transitionMs(options.transition);
+    }
+    /** Whether the manifest declares app restart and state clearing. */
+    get managesApp() {
+        return this.options.app !== undefined || this.options.appPath !== undefined;
+    }
+    /** The app opened fresh per attempt: the `app` option, else the build `appPath` installed. */
+    get pinnedApp() {
+        return pinnedApp(this.options, this.installedApp);
+    }
+    /** Whether an attempt is running on this surface right now. */
+    get attemptRunning() {
+        return this.attempt !== undefined;
+    }
+    /** The live client; INVALID_STATE before init or after dispose. */
+    requireClient() {
+        if (this.client === undefined)
+            throw invalidState('the agent-device engine is not initialized');
+        return this.client;
+    }
+    /**
+     * Runs one agent-device command under an operation budget and translates
+     * its failure. Contributed-fixture methods route through here too, so the
+     * device fixture never carries its own error mapping.
+     */
+    async command(label, run, signal) {
+        const client = this.requireClient();
+        return runCommand(label, () => this.track(run(client)), signal ?? new AbortController().signal);
+    }
+    /** Registers one device command as in flight until it settles. */
+    track(pending) {
+        this.inflight.add(pending);
+        pending.then(() => this.inflight.delete(pending), () => this.inflight.delete(pending));
+        return pending;
+    }
+    /**
+     * Waits for every abandoned command to settle, within the caller's
+     * budget. A command that never settles fails the attempt launch instead of
+     * racing it: the launch timeout is the honest bound on a stuck device.
+     */
+    async settleInflight(signal) {
+        while (this.inflight.size > 0) {
+            await raceAbort(Promise.allSettled(this.inflight), signal, 'settling in-flight device commands');
+        }
+    }
+    /** The device selection this worker's commands take. */
+    selection() {
+        return deviceSelection(this.options.platform, this.device);
+    }
+    async init(info) {
+        this.projectRoot = info.projectRoot;
+        const binding = this.pool.binding(info.targetName, info.workerSlot, info.env);
+        this.device = binding?.device;
+        this.client ??= this.createClient(this.pool.session(info.targetName, info.workerSlot), binding?.daemon);
+        await this.command('boot', (client) => client.devices.boot(this.selection()), info.signal);
+        if (this.options.appPath === undefined)
+            return;
+        // A provider that installed the build itself says so on the binding; the worker then installs nothing.
+        this.installedApp =
+            binding?.installedApp ??
+                (await this.installApp(this.options.appPath, this.options.app === undefined ? {} : { app: this.options.app }, info.signal)).app;
+    }
+    async startAttempt(context) {
+        if (this.attempt !== undefined) {
+            throw invalidState('an attempt is already running on this agent-device engine');
+        }
+        await this.settleInflight(context.signal);
+        this.attempt = { artifactsDir: context.artifactsDir, screenshots: 0, video: undefined };
+        this.generation = new Map();
+        this.located.clear();
+        const app = this.pinnedApp;
+        if (app === undefined)
+            return;
+        await this.openApp(app, true, context.signal);
+    }
+    async endAttempt(context) {
+        const dangling = this.attempt?.video;
+        this.attempt = undefined;
+        this.generation = new Map();
+        this.located.clear();
+        // The harness stops the video before it ends the attempt; a recording still
+        // marked here belongs to an attempt cut short, or to a stop that failed,
+        // and the device must not keep recording into the next one. A start that
+        // outlived its budget may still be landing: it settles first, so the stop
+        // cannot overtake it.
+        if (dangling !== undefined) {
+            await this.settleInflight(context.signal).catch(() => undefined);
+            await this.command('stop video recording', (client) => client.recording.record({ action: 'stop' }), context.signal).catch(() => undefined);
+        }
+    }
+    // --- video ---
+    /**
+     * Asks the device to record its screen into the attempt directory. Taps stay
+     * visible in the recording (agent-device's touch indicator), which is the
+     * closest a phone comes to a cursor.
+     */
+    async startVideo(operation) {
+        const attempt = this.attempt;
+        if (attempt === undefined)
+            throw invalidState('startVideo outside an attempt');
+        if (attempt.video !== undefined)
+            throw invalidState('a video is already recording');
+        const relative = path.join('video', 'video.mp4');
+        const absolute = path.join(attempt.artifactsDir, relative);
+        mkdirSync(path.dirname(absolute), { recursive: true });
+        // Marked before the device is asked: a start that outlives its budget
+        // still records, and `endAttempt` must be able to stop it.
+        const recording = { relative, absolute, startedAt: new Date().toISOString() };
+        attempt.video = recording;
+        await this.command('start video recording', (client) => client.recording.record({ action: 'start', path: absolute, quality: 'medium' }), operation.signal);
+        // The device confirmed: it is recording from about now.
+        recording.startedAt = new Date().toISOString();
+    }
+    /** Stops the recording and returns its one segment, or none when the device wrote nothing. */
+    async stopVideo(operation) {
+        const attempt = this.attempt;
+        if (attempt === undefined)
+            throw invalidState('stopVideo outside an attempt');
+        const video = attempt.video;
+        if (video === undefined)
+            return [];
+        const result = await this.command('stop video recording', (client) => client.recording.record({ action: 'stop' }), operation.signal);
+        // Cleared only now: a stop that failed leaves the recording for `endAttempt`.
+        attempt.video = undefined;
+        // The device may finalize the file under a path of its own choosing; the
+        // artifact must live where the attempt directory expects it.
+        const written = typeof result.outPath === 'string' ? result.outPath : video.absolute;
+        if (written !== video.absolute && existsSync(written))
+            renameSync(written, video.absolute);
+        if (!existsSync(video.absolute))
+            return [];
+        return [{ path: video.relative, startedAt: video.startedAt }];
+    }
+    async dispose(context) {
+        const client = this.client;
+        this.client = undefined;
+        this.attempt = undefined;
+        this.generation = new Map();
+        this.located.clear();
+        this.appIdentity = undefined;
+        this.installedApp = undefined;
+        this.knownViewport = undefined;
+        if (client === undefined)
+            return;
+        await withinCleanupBudget(client.sessions.close().catch(() => undefined), context);
+    }
+    /** Opens an app in the session, remembering its identity for the path anchor. */
+    async openApp(app, relaunch, signal) {
+        const result = await this.command(`open ${app}`, (client) => client.apps.open({
+            app,
+            ...this.selection(),
+            ...(relaunch ? { relaunch: true } : {}),
+        }), signal);
+        // A launch has no screen before it worth matching against: every control
+        // of the new screen is arriving, whatever the previous one looked like.
+        this.markAction(undefined);
+        this.appIdentity = result.appBundleId ?? result.appName ?? app;
+        this.generation = new Map();
+        this.located.clear();
+    }
+    /**
+     * Installs a build on the session's device. `reinstall` removes the app
+     * named by `options.app` (else the pinned app) first, so the build starts
+     * with no data; a plain install replaces the binary and keeps its data.
+     */
+    async installApp(appPath, options, signal) {
+        const resolved = path.resolve(this.projectRoot, appPath);
+        const selection = this.selection();
+        const app = options.app ?? (options.reinstall === true ? this.pinnedApp : undefined);
+        if (options.reinstall === true && app === undefined) {
+            throw invalidState('reinstall needs an app: pass `app`, or pin one with the engine option `app` or `appPath`');
+        }
+        const result = (await this.command(`install ${resolved}`, (client) => options.reinstall === true && app !== undefined
+            ? client.apps.reinstall({ ...selection, app, appPath: resolved })
+            : client.apps.install({ ...selection, ...(app === undefined ? {} : { app }), appPath: resolved }), signal));
+        const identity = result.bundleId ?? result.package ?? result.appId;
+        return { app: identity ?? result.app, ...(identity === undefined ? {} : { bundleId: identity }) };
+    }
+    async snapshot(operation, interactiveOnly) {
+        let last = {};
+        for (const backoffMs of SPARSE_RETRY_BACKOFF_MS) {
+            if (backoffMs > 0)
+                await sleep(backoffMs, operation.signal);
+            if (operation.signal.aborted)
+                throw new EngineError('CANCELLED', 'snapshot cancelled', { retryable: false });
+            last = (await this.command('snapshot', (client) => client.capture.snapshot({ interactiveOnly }), operation.signal));
+            if (last.appBundleId !== undefined || last.appName !== undefined) {
+                this.appIdentity = last.appBundleId ?? last.appName;
+            }
+            if (last.snapshotQuality?.state !== 'sparse')
+                return last;
+        }
+        return last;
+    }
+    /**
+     * Snapshot for an observation. Without a pinned app, an observation before
+     * anything is open is an empty screen rather than a fault: the model's next
+     * move is the open tool, and failing the step would take that move away.
+     */
+    async snapshotOrEmpty(operation, interactiveOnly) {
+        try {
+            return await this.snapshot(operation, interactiveOnly);
+        }
+        catch (cause) {
+            if (!this.managesApp && cause instanceof EngineError && cause.code === 'INVALID_STATE')
+                return { nodes: [] };
+            throw cause;
+        }
+    }
+    project(raw) {
+        const projected = projectSnapshot(raw.nodes ?? [], {
+            mintId: () => {
+                this.idCounter += 1;
+                return `n${this.idCounter}`;
+            },
+        });
+        this.latestIndex = projected.index;
+        if (projected.viewport !== undefined)
+            this.knownViewport = projected.viewport;
+        return projected;
+    }
+    /**
+     * The viewport of a snapshot: its own geometry, else the last one this
+     * session learned, else the device's logical screen size read off one
+     * screenshot. A device whose size cannot be learned fails the observation:
+     * the harness measures every rect and point against the viewport, so an
+     * invented one would misplace every tap.
+     */
+    async viewportFor(projected, operation) {
+        const known = projected.viewport ?? this.knownViewport;
+        if (known !== undefined)
+            return known;
+        const probed = await this.probeViewport(operation.signal);
+        if (probed === undefined) {
+            throw new EngineError('ENGINE_FAILURE', 'the device reported no screen geometry: the snapshot has no bounds and the screenshot no logical size, so the viewport is unknown', { retryable: false });
+        }
+        this.knownViewport = probed;
+        return probed;
+    }
+    /** The device's logical screen size as its screenshot reports it; undefined when it reports none. */
+    async probeViewport(signal) {
+        const result = await this.captureScreenshot(signal, async (shot) => shot);
+        return logicalScreenSize(result);
+    }
+    async observe(operation, options) {
+        const raw = await this.snapshotOrEmpty(operation, this.options.snapshot === 'interactive');
+        const projected = this.project(raw);
+        this.generation = new Map(projected.index.map((entry) => [entry.id, this.bind(entry, projected.index)]));
+        const viewport = await this.viewportFor(projected, operation);
+        const location = screenLocation(raw.appBundleId ?? raw.appName ?? this.appIdentity, screenTitle(projected));
+        const capture = options?.pixels === true ? await this.capturePixels(operation, projected, viewport) : undefined;
+        return {
+            root: screenRoot(projected.roots, viewport),
+            viewport,
+            ...(location === undefined ? {} : { location }),
+            ...(capture === undefined ? {} : { pixels: capture.pixels, maskedRegionCount: capture.masked }),
+        };
+    }
+    /** Retains only action bindings for matches, independently of the observation generation. */
+    async locate(expression, operation) {
+        const raw = await this.snapshotOrEmpty(operation, false);
+        const projected = this.project(raw);
+        const matches = resolveExpression(expression, projected.index);
+        for (const entry of matches)
+            this.located.set(entry.id, this.bind(entry, projected.index));
+        for (const oldest of this.located.keys()) {
+            if (this.located.size <= MAX_LOCATED_REFS)
+                break;
+            this.located.delete(oldest);
+        }
+        return matches.map((entry) => entry.node);
+    }
+    resolveRef(ref) {
+        const entry = this.located.get(ref.id) ?? this.generation.get(ref.id);
+        if (entry === undefined) {
+            throw new EngineError('NODE_STALE', `node ${ref.id} is not part of the newest observation`, { retryable: true });
+        }
+        return entry;
+    }
+    actionTarget(entry, control = false) {
+        const ref = control ? entry.controlRef : entry.ref;
+        if (ref === '')
+            throw notActionable(`node ${entry.id} has no agent-device ref to act on`);
+        return { ref: `@${ref}` };
+    }
+    /**
+     * The node a toggle press must land on. UIKit reports a settings row as a
+     * labelled `Switch` spanning the whole row with the real control as an
+     * unlabelled `Switch` child at its trailing edge; a press at the row's
+     * centre hits the label and changes nothing. The innermost same-role
+     * descendant with a ref is the control; a node without one is its own.
+     */
+    controlOf(entry, snapshot) {
+        const role = entry.node.role;
+        if (role !== 'switch' && role !== 'checkbox')
+            return entry;
+        let control = entry;
+        let depth = 0;
+        for (const candidate of snapshot) {
+            if (candidate.ref === '' || candidate.node.role !== role || !isWithin(candidate, entry))
+                continue;
+            const candidateDepth = depthBelow(candidate, entry);
+            if (candidateDepth > depth) {
+                control = candidate;
+                depth = candidateDepth;
+            }
+        }
+        return control;
+    }
+    /** Copies the fields actions use and pre-resolves a toggle's inner control. */
+    bind(entry, snapshot) {
+        const { children: _children, ...node } = entry.node;
+        const control = this.controlOf(entry, snapshot);
+        return { id: entry.id, ref: entry.ref, node, controlRef: control.ref };
+    }
+    /**
+     * The node a binding stands for, in a fresh snapshot: same role, same test
+     * id, same name and text, and of those the one closest to where it was.
+     */
+    refind(entry, index) {
+        const candidates = index.filter((candidate) => candidate.node.role === entry.node.role &&
+            candidate.node.testId === entry.node.testId &&
+            candidate.node.name === entry.node.name &&
+            candidate.node.text === entry.node.text);
+        if (candidates.length <= 1 || entry.node.rect === undefined)
+            return candidates[0];
+        const was = centreOf(entry.node.rect);
+        let best = candidates[0];
+        let bestDistance = Number.POSITIVE_INFINITY;
+        for (const candidate of candidates) {
+            if (candidate.node.rect === undefined)
+                continue;
+            const at = centreOf(candidate.node.rect);
+            const distance = Math.hypot(at.x - was.x, at.y - was.y);
+            if (distance < bestDistance) {
+                best = candidate;
+                bestDistance = distance;
+            }
+        }
+        return best;
+    }
+    /**
+     * Lets a control that came with the last action finish arriving before a
+     * test acts on it. A control already present at the same place in the
+     * snapshot the last action was resolved from is not in transition and is
+     * acted on at once; anything else waits out the remainder of the
+     * transition budget since that action. Nothing is observed here: the
+     * budget is the only signal, because the tree reports final frames.
+     */
+    async awaitTransition(entry, operation) {
+        const since = Date.now() - this.lastActionAt;
+        const remaining = this.transitionMs - since;
+        if (remaining <= 0)
+            return;
+        const before = this.indexBeforeAction;
+        if (before !== undefined) {
+            const prior = this.refind(entry, before);
+            if (prior !== undefined && sameRect(prior.node.rect, entry.node.rect))
+                return;
+        }
+        await sleep(remaining, operation.signal);
+        // The sleep resolves on abort; the action behind it must not go out once
+        // the caller has already been told the operation was cancelled.
+        if (operation.signal.aborted) {
+            throw new EngineError('CANCELLED', 'transition wait cancelled', { retryable: false });
+        }
+    }
+    /**
+     * Records an action the device received: the screen as it was projected
+     * before it, and when it returned. The transition budget counts from the
+     * return, since the input lands late in the command's own round trip.
+     */
+    markAction(before) {
+        this.indexBeforeAction = before;
+        this.lastActionAt = Date.now();
+    }
+    async perform(ref, action, operation) {
+        // The observation root is the screen: a swipe on it scrolls the whole
+        // viewport, which is the one action a screen takes.
+        if (ref.id === ROOT_ID) {
+            if (action.kind !== 'swipe')
+                throw notActionable(`the screen root takes swipe only, not ${action.kind}; act on a node`);
+            const before = this.latestIndex;
+            await this.command('swipe', (client) => client.interactions.scroll({ direction: action.direction }), operation.signal);
+            this.markAction(before);
+            return;
+        }
+        const entry = this.resolveRef(ref);
+        const label = `perform ${action.kind}`;
+        const client = this.requireClient();
+        // A test's step verifies its outcome with `expect`, so it never waits for
+        // the screen to settle afterwards; it only lets a control that came with
+        // the last action finish arriving. The agent reads the screen right after
+        // acting, so its actions settle first.
+        const deterministic = operation.origin === 'test';
+        const settle = deterministic ? {} : this.settleOptions;
+        // A toggle already in the wanted state sends nothing, so it neither waits
+        // for a transition nor counts as an action the next control must wait on.
+        if ((action.kind === 'check' || action.kind === 'uncheck') && entry.node.states?.checked === (action.kind === 'check')) {
+            return;
+        }
+        const before = this.latestIndex;
+        const run = async () => {
+            if (deterministic)
+                await this.awaitTransition(entry, operation);
+            switch (action.kind) {
+                case 'tap':
+                    return client.interactions.press({ ...this.actionTarget(entry, true), ...settle });
+                case 'focus':
+                    // A touch surface focuses by tapping, and a tap on anything but an
+                    // editable field activates it; focus is offered for fields only.
+                    if (entry.node.role !== 'textbox') {
+                        throw unsupported(`agent-device can only focus editable fields; node ${entry.id} is ${entry.node.role ?? 'unknown'}`);
+                    }
+                    return client.interactions.press({ ...this.actionTarget(entry), ...settle });
+                case 'doubleTap':
+                    return client.interactions.press({ ...this.actionTarget(entry), doubleTap: true, ...settle });
+                case 'longPress':
+                    return client.interactions.longPress({
+                        ...this.actionTarget(entry),
+                        ...settle,
+                        ...(action.durationMs === undefined ? {} : { durationMs: action.durationMs }),
+                    });
+                case 'hover':
+                    return client.interactions.hover(this.actionTarget(entry));
+                case 'fill':
+                    // oxlint-disable-next-line unicorn/no-array-fill-with-reference-type -- agent-device fill, not Array#fill
+                    return client.interactions.fill({ ...this.actionTarget(entry), text: action.value, ...settle });
+                case 'clear':
+                    // oxlint-disable-next-line unicorn/no-array-fill-with-reference-type -- agent-device fill, not Array#fill
+                    return client.interactions.fill({ ...this.actionTarget(entry), text: '', ...settle });
+                case 'check':
+                case 'uncheck': {
+                    const wanted = action.kind === 'check';
+                    const checked = entry.node.states?.checked;
+                    // A toggle whose state the tree does not expose (Android switches)
+                    // cannot be set, only flipped; flipping blind could undo a correct state.
+                    if (checked === undefined) {
+                        throw unsupported(`agent-device cannot read whether node ${entry.id} is checked; tap it instead`);
+                    }
+                    if (checked === wanted)
+                        return undefined;
+                    return client.interactions.press({ ...this.actionTarget(entry, true), ...settle });
+                }
+                case 'press':
+                    return this.pressKey(client, entry, action.key, settle);
+                case 'swipe': {
+                    const rect = entry.node.rect;
+                    if (rect === undefined)
+                        throw notActionable(`node ${entry.id} has no bounds to swipe within`);
+                    return client.interactions.swipe(swipeWithin(rect, action.direction, action.momentum));
+                }
+                case 'dragTo': {
+                    const destination = this.resolveRef(action.target);
+                    return client.interactions.drag({
+                        source: this.actionTarget(entry).ref,
+                        destination: this.actionTarget(destination).ref,
+                    });
+                }
+                case 'scrollIntoView':
+                case 'selectOption':
+                case 'setInputFiles':
+                    // Not in DEVICE_ACTIONS (actions.ts), so the harness never sends them; kept exhaustive.
+                    throw unsupported(`agent-device cannot perform "${action.kind}" on a device surface`);
+            }
+        };
+        try {
+            await raceAbort(() => this.track(run()), operation.signal, label);
+        }
+        catch (cause) {
+            throw staleOr(cause, label);
+        }
+        this.markAction(before);
+    }
+    /**
+     * Keys on a touch surface, in the contract's key grammar: `Enter` submits
+     * through the soft keyboard, `Space` and any single character are typed
+     * into the focused field. agent-device exposes no key event bus, so
+     * modifiers and the other named keys (`Escape`, `Tab`, `Backspace`, the
+     * arrows) have nothing to land on and are refused.
+     */
+    async pressKey(client, entry, key, settle) {
+        const parsed = parseKey(key);
+        if (parsed === undefined) {
+            throw unsupported(`"${key}" is not a key: press takes one key in the form [Modifier+]...Key, a named key (${KEY_NAMES.join(', ')}) or one character`);
+        }
+        if (parsed.modifiers.length > 0) {
+            throw unsupported(`agent-device cannot hold ${parsed.modifiers.join('+')} on a device surface; press the key alone`);
+        }
+        if (parsed.key.kind === 'named' && parsed.key.name === 'Enter') {
+            return client.command.keyboard({ action: 'enter' });
+        }
+        if (parsed.key.kind === 'char' || parsed.key.name === 'Space') {
+            const text = parsed.key.kind === 'char' ? parsed.key.char : ' ';
+            if (entry.node.role === 'textbox' && entry.node.states?.focused !== true) {
+                await client.interactions.press({ ...this.actionTarget(entry), ...settle });
+            }
+            return client.interactions.type({ text });
+        }
+        throw unsupported(`agent-device cannot press ${parsed.key.name} on a device surface; only Enter, Space, and single characters reach the soft keyboard`);
+    }
+    /**
+     * One pointer action at a screen point in logical pixels, the space every
+     * node's bounds are in, with no element resolved behind it. `settle` waits
+     * for the UI to go quiet, as the node taps do.
+     */
+    async performAt(point, action, operation) {
+        const settle = operation.origin === 'test' ? {} : this.settleOptions;
+        const before = this.latestIndex;
+        await this.command(`${action.kind} at point`, (client) => pointerInteraction(client, point, action, settle), operation.signal);
+        this.markAction(before);
+    }
+    /**
+     * Types into whatever holds focus through the device's text-input path;
+     * agent-device's `type` lands on the focused field. `replace` is not a
+     * device primitive without a target, so it is refused rather than faked:
+     * the model clears a listed field by filling it by id.
+     */
+    async typeText(text, options, operation) {
+        if (options.replace) {
+            throw unsupported('agent-device cannot clear the focused field without a node; type into a listed field by id to replace its value');
+        }
+        const before = this.latestIndex;
+        await this.command('keyboard.type', (client) => client.interactions.type({ text }), operation.signal);
+        this.markAction(before);
+    }
+    /**
+     * One key to the focused field, with the same reach as a node press: Enter
+     * submits through the soft keyboard, Space and single characters are typed.
+     * Modifiers and the other named keys have no event bus to land on.
+     */
+    async pressFocusedKey(key, operation) {
+        const parsed = parseKey(key);
+        if (parsed === undefined) {
+            throw unsupported(`"${key}" is not a key: press takes one key in the form [Modifier+]...Key, a named key (${KEY_NAMES.join(', ')}) or one character`);
+        }
+        if (parsed.modifiers.length > 0) {
+            throw unsupported(`agent-device cannot hold ${parsed.modifiers.join('+')} on a device surface; press the key alone`);
+        }
+        const before = this.latestIndex;
+        if (parsed.key.kind === 'named' && parsed.key.name === 'Enter') {
+            await this.command('keyboard.press', (client) => client.command.keyboard({ action: 'enter' }), operation.signal);
+        }
+        else if (parsed.key.kind === 'char' || parsed.key.name === 'Space') {
+            const text = parsed.key.kind === 'char' ? parsed.key.char : ' ';
+            await this.command('keyboard.press', (client) => client.interactions.type({ text }), operation.signal);
+        }
+        else {
+            throw unsupported(`agent-device cannot press ${parsed.key.name} on a device surface; only Enter, Space, and single characters reach the soft keyboard`);
+        }
+        this.markAction(before);
+    }
+    /** Hides the soft keyboard, so a control it covered can be reached. */
+    async dismissKeyboard(operation) {
+        const before = this.latestIndex;
+        await this.command('keyboard.dismiss', (client) => client.command.keyboard({ action: 'dismiss' }), operation.signal);
+        this.markAction(before);
+    }
+    async back(operation) {
+        const settle = operation.origin === 'test' ? {} : this.settleOptions;
+        const before = this.latestIndex;
+        await this.command('back', (client) => client.command.back({ ...settle }), operation.signal);
+        this.markAction(before);
+    }
+    async restart(operation) {
+        const app = this.pinnedApp;
+        if (app === undefined)
+            throw unsupported('app.restart needs the engine option `app` or `appPath`');
+        await this.openApp(app, true, operation.signal);
+    }
+    /** Clears the pinned app's persisted state and relaunches it: the device equivalent of a fresh context. */
+    async reset(operation) {
+        const app = this.pinnedApp;
+        if (app === undefined)
+            throw unsupported('app.clearState needs the engine option `app` or `appPath`');
+        await this.command('clear app state', (client) => client.settings.update({ setting: 'clear-app-state', state: 'clear', app }), operation.signal);
+        await this.openApp(app, true, operation.signal);
+    }
+    /**
+     * A redacted screenshot artifact. The device paints secure fields as dots,
+     * but the last typed character shows in clear, so every secure node's
+     * bounds are painted over before the file is kept. A secure node without
+     * bounds cannot be masked, and an image that cannot be redacted is not
+     * written at all.
+     */
+    async screenshot(label, operation) {
+        const attempt = this.attempt;
+        if (attempt === undefined)
+            throw invalidState('screenshot outside an attempt');
+        const masked = await this.maskedScreenshot(operation.signal);
+        attempt.screenshots += 1;
+        const name = `${String(attempt.screenshots).padStart(3, '0')}-${sanitizeFilename(label ?? 'screenshot')}.png`;
+        const relative = path.join('screenshots', name);
+        mkdirSync(path.join(attempt.artifactsDir, 'screenshots'), { recursive: true });
+        writeFileSync(path.join(attempt.artifactsDir, relative), masked.data);
+        return relative;
+    }
+    /** Raw device pixels; cleanup follows the capture even when its caller abandons it. */
+    rawScreenshot(signal) {
+        return this.captureScreenshot(signal, async (shot, file) => new Uint8Array(readFileSync(shot.path ?? file)));
+    }
+    /**
+     * One screenshot into a temp directory that is removed once `read` has
+     * taken what it needs from the response or the file, even when the caller
+     * abandons the capture.
+     */
+    captureScreenshot(signal, read) {
+        return this.command('screenshot', async (client) => {
+            const directory = mkdtempSync(path.join(tmpdir(), 'e2e-agent-device-'));
+            const file = path.join(directory, 'screenshot.png');
+            try {
+                return await read((await client.capture.screenshot({ path: file })), file);
+            }
+            finally {
+                rmSync(directory, { recursive: true, force: true });
+            }
+        }, signal);
+    }
+    /**
+     * Screenshot with every secure node on the current screen painted over.
+     * Observes first so the regions describe the screen the pixels show;
+     * throws when a secure field cannot be covered, because an image that may
+     * hold a credential must not leave the engine.
+     */
+    async maskedScreenshot(signal) {
+        const operation = {
+            signal: signal ?? new AbortController().signal,
+            timeoutMs: 30_000,
+            runId: '',
+            attemptId: '',
+            origin: 'test',
+        };
+        const projected = this.project(await this.snapshotOrEmpty(operation, false));
+        const data = await this.rawScreenshot(signal);
+        const masked = redactSecure(data, projected, projected.viewport ?? this.knownViewport);
+        if (masked === undefined) {
+            throw new EngineError('ENGINE_FAILURE', 'a secure field on screen could not be masked; screenshot withheld', {
+                retryable: false,
+            });
+        }
+        return masked;
+    }
+    /**
+     * Viewport pixels for an observation. Best-effort: a screenshot that cannot
+     * be produced or redacted costs the observation its image, not the step.
+     */
+    async capturePixels(operation, projected, viewport) {
+        let raw;
+        try {
+            raw = await this.rawScreenshot(operation.signal);
+        }
+        catch {
+            return undefined;
+        }
+        const redacted = redactSecure(raw, projected, viewport);
+        if (redacted === undefined)
+            return undefined;
+        const size = readPngSize(redacted.data);
+        if (size === undefined)
+            return undefined;
+        const scale = viewport.width > 0 ? size.width / viewport.width : 1;
+        return {
+            pixels: { data: redacted.data, mediaType: 'image/png', width: size.width, height: size.height, scale },
+            masked: redacted.masked,
+        };
+    }
+}
+/**
+ * Paints every secure node's bounds black on a screenshot of the same
+ * screen. Returns the masked bytes and how many regions were covered, or
+ * undefined when a secure node has no bounds or the image format cannot be
+ * edited: the caller then withholds the image rather than ship one it could
+ * not prove redacted. Bounds are in logical points; the image may be at
+ * device scale, so they are scaled by the image-to-viewport ratio.
+ */
+function redactSecure(data, projected, viewport) {
+    const secure = projected.index.filter((entry) => entry.node.states?.secure === true);
+    if (secure.length === 0)
+        return { data, masked: 0 };
+    const size = readPngSize(data);
+    if (size === undefined)
+        return undefined;
+    const scale = viewport !== undefined && viewport.width > 0 ? size.width / viewport.width : 1;
+    const rects = [];
+    for (const entry of secure) {
+        const rect = entry.node.rect;
+        if (rect === undefined)
+            return undefined;
+        rects.push({ x: rect.x * scale, y: rect.y * scale, width: rect.width * scale, height: rect.height * scale });
+    }
+    try {
+        return { data: maskPng(data, rects), masked: rects.length };
+    }
+    catch {
+        return undefined;
+    }
+}
+//# sourceMappingURL=surface.js.map