@revyl/verify 0.4.1 → 0.6.0

package/dist/computer-session.js

@@ -6,13 +6,26 @@ import { edgeBlockedError, responseDetail, responseErrorCode, responseRefusalRea
 import { trackSession, untrackSession } from './cleanup.js';
 import { validateHostedTarget } from './config.js';
 import { VerifyError, isVerifyError } from './errors.js';
-import { DeviceSession, pngSize } from './session.js';
+import { WEB_KEYS, checkKey } from './agent/loop.js';
+import { DeviceSession, checkLocation, pngSize } from './session.js';
 const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 const STATES = new Set(['starting', 'active', 'closing', 'stopped', 'failed', 'owner_lost']);
 const MAX_OUTPUT_BYTES = 1_048_576;
 const COMPUTER_AVAILABILITY = 'Computers are early access. Ask support@revyl.ai about access.';
 const COMPUTER_CAPACITY = 'Computer admission was refused because of queued work (including iOS) or the concurrency limit. Verify will not retry the start.';
-const ERROR_CODES = new Set(['computer_not_enabled', 'run_closed', 'inaccessible', 'invalid_handle', 'operation_not_found', 'owner_unavailable', 'invalid_request', 'unauthenticated', 'forbidden', 'runtime_unavailable', 'start_failed', 'computer_limit', 'request_conflict', 'run_conflict', 'evidence_conflict', 'evidence_unavailable', 'authentication_unavailable', 'operation_limit', 'screen_starting', 'screen_unavailable']);
+const ERROR_CODES = new Set(['computer_not_enabled', 'run_closed', 'inaccessible', 'invalid_handle', 'operation_not_found', 'owner_unavailable', 'invalid_request', 'unauthenticated', 'forbidden', 'runtime_unavailable', 'start_failed', 'computer_limit', 'request_conflict', 'run_conflict', 'evidence_conflict', 'evidence_unavailable', 'authentication_unavailable', 'operation_limit', 'screen_starting', 'screen_unavailable', 'screen_control_held', 'screen_control_lost', 'screen_input_unavailable', 'screen_input_uncertain', 'screen_key_unsupported', 'screen_text_unsupported', 'screen_target_not_found', 'screen_target_unavailable']);
+const COMPUTER_KEYS = { enter: 'Return', backspace: 'BackSpace', tab: 'Tab', escape: 'Escape', delete: 'Delete', space: 'space', arrowup: 'Up', arrowdown: 'Down', arrowleft: 'Left', arrowright: 'Right', home: 'Home', end: 'End', pageup: 'Page_Up', pagedown: 'Page_Down' };
+const REFUSAL_NEXT_STEPS = {
+    AUTH_REJECTED: "Check that REVYL_API_KEY is a current key and that its owner has the Member role or above in this computer's organization.",
+    SESSION_ENDED: 'This computer has stopped; start a new computer to continue.',
+    NOT_FOUND: 'Check that the id is right and that REVYL_API_KEY belongs to the organization that started this computer.',
+    ACTION_UNCERTAIN: "The request may have taken effect. Verify will not repeat it; check the computer's state before you do.",
+    INVALID_ARGUMENT: "Check the call's arguments against the docs; if they match, update Verify to the latest version.",
+    SESSION_START_FAILED: 'Start a new computer.',
+    SESSION_BUSY: 'The computer is busy with other work; try again once it finishes.',
+    API_ERROR: 'Try again; if it keeps failing, send this message to support@revyl.ai.',
+};
+const SCREEN_CALLS = 'For the screen, use screenshot(), tap({ x, y }) or tap({ target }), typeFocused(text, { clear: false }) and pressKey(); for commands, use exec().';
 export class ComputerSession {
     client;
     options;
@@ -123,21 +136,115 @@ export class ComputerSession {
     navigate() {
         return this.unavailable('navigate');
     }
-    tap() {
-        return this.unavailable('tap');
+    /** Clicks a point in screenshot pixels, or an element Revyl finds from a description, with the left button. Returns the point clicked. */
+    async tap(at) {
+        if (at && typeof at === 'object' && 'target' in at && ('x' in at || 'y' in at))
+            throw new VerifyError('INVALID_ARGUMENT', 'tap() takes a point { x, y } or a description { target }, not both.');
+        const location = checkLocation(at);
+        if ('target' in location)
+            return this.tapTarget(location.target);
+        const { x, y } = location;
+        let response;
+        try {
+            response = await this.act({ action: 'click', x, y });
+        }
+        catch (error) {
+            if (isVerifyError(error) && error.code === 'INVALID_ARGUMENT' && error.httpStatus === 422)
+                throw new VerifyError('INVALID_ARGUMENT', `Nothing was clicked: the computer refused a click at (${x}, ${y}). That happens when the point is no longer on its screen, for example after its display size changed. Take a new screenshot() and pick the point from it. See ${this.viewerUrl}`, { cause: error, reason: error.reason, httpStatus: 422 });
+            throw error;
+        }
+        if (response.status === 422) {
+            const size = await this.screenSize().catch(() => undefined);
+            const range = size ? `x is 0 to ${size.width - 1} and y is 0 to ${size.height - 1}, in pixels of the image screenshot() returns.` : 'Use a point inside the image screenshot() returns.';
+            throw new VerifyError('INVALID_ARGUMENT', `Nothing was clicked: (${x}, ${y}) is outside the screen. ${range}`);
+        }
+        return { x, y };
+    }
+    async tapTarget(target) {
+        let response;
+        try {
+            response = await this.act({ action: 'click', target }, 90_000);
+        }
+        catch (error) {
+            if (isVerifyError(error) && error.code === 'INVALID_ARGUMENT' && error.httpStatus === 422)
+                throw new VerifyError('NOT_AVAILABLE_ON_PLATFORM', `Nothing was clicked: this Revyl backend can't click a described target on a computer yet (invalid_request). Read the point off screenshot() and pass it to tap({ x, y }). See ${this.viewerUrl}`, { cause: error, httpStatus: 422 });
+            throw error;
+        }
+        let point;
+        try {
+            point = JSON.parse(new TextDecoder().decode(response.body))?.point;
+        }
+        catch { }
+        const { x, y } = point ?? {};
+        if (!Number.isInteger(x) || !Number.isInteger(y) || x < 0 || y < 0)
+            throw new VerifyError('ACTION_UNCERTAIN', `Revyl answered the described click without the point it clicked, so where it landed is unknown. Take a screenshot() to check. See ${this.viewerUrl}`);
+        return { x: x, y: y };
     }
     type() {
         return this.unavailable('type');
     }
+    /** Types into whatever has keyboard focus. Computers can't clear a field, so pass { clear: false }. */
+    async typeFocused(text, options = {}) {
+        if (options.clear !== false)
+            throw new VerifyError('INVALID_ARGUMENT', "On a computer, typeFocused() can't clear the field first, so pass { clear: false } to type at the cursor: typeFocused(text, { clear: false }). To replace text, select it first, for example with pressKey('a', { modifiers: ['meta'] }) (Command+A on a Mac computer).");
+        if (typeof text !== 'string' || text.length > 16_384)
+            throw new VerifyError('INVALID_ARGUMENT', 'typeFocused() takes text of at most 16384 characters.');
+        const control = /[\u0000-\u0008\u000b-\u001f\u007f-\u009f]|[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/.exec(text);
+        if (control)
+            throw new VerifyError('INVALID_ARGUMENT', `Nothing was typed: typeFocused() on a computer takes newline and tab but no other control characters, and the text has one at index ${control.index}. Use pressKey() for keys such as 'escape'.`);
+        const chunks = [];
+        let chunk = '';
+        for (const character of text) {
+            if ([...chunk].length === 500 || Buffer.byteLength(chunk + character) > 1_500) {
+                chunks.push(chunk);
+                chunk = '';
+            }
+            chunk += character;
+        }
+        if (chunk)
+            chunks.push(chunk);
+        const total = [...text].length;
+        let typed = 0;
+        for (const part of chunks) {
+            const length = [...part].length;
+            try {
+                await this.act({ action: 'type', text: part });
+            }
+            catch (error) {
+                if (chunks.length === 1 || !isVerifyError(error))
+                    throw error;
+                const uncertain = error.code === 'ACTION_UNCERTAIN';
+                const done = typed === 0 ? `typed nothing${uncertain ? ' for certain' : ''}: the first` : `typed the first ${typed} of ${total} characters; the next`;
+                throw new VerifyError(error.code, `typeFocused() ${done} ${length}${typed === 0 ? ` of ${total}` : ''} characters ${uncertain ? 'may have been typed' : 'were not typed'}, and nothing after them was. ${error.message}`, { cause: error, reason: error.reason, httpStatus: error.httpStatus });
+            }
+            typed += length;
+        }
+    }
+    doubleTap() {
+        return this.unavailable('doubleTap');
+    }
+    longPress() {
+        return this.unavailable('longPress');
+    }
     swipe() {
         return this.unavailable('swipe');
     }
-    pressKey() {
-        return this.unavailable('pressKey');
+    drag() {
+        return this.unavailable('drag');
+    }
+    /** Presses a key with modifiers held. 'meta' is the Command key on a Mac computer, and 'alt' is Option. */
+    async pressKey(key, options = {}) {
+        const checked = checkKey('web', key, options.modifiers ?? []);
+        const name = checked.key in WEB_KEYS ? COMPUTER_KEYS[checked.key] : checked.key;
+        const modifiers = [...new Set(checked.modifiers)].map((modifier) => (modifier === 'meta' ? 'cmd' : modifier));
+        await this.act({ action: 'key', keys: [...modifiers, name].join('+') });
     }
     back() {
         return this.unavailable('back');
     }
+    goHome() {
+        return this.unavailable('goHome');
+    }
     openUrl() {
         return this.unavailable('openUrl');
     }
@@ -145,10 +252,12 @@ export class ComputerSession {
         return this.unavailable('launchApp');
     }
     locate() {
-        return this.unavailable('locate');
+        return Promise.reject(new VerifyError('NOT_AVAILABLE_ON_PLATFORM', "locate() isn't available on computers, because Revyl can't find a point there without clicking it. tap({ target }) clicks a description and returns the point it clicked; to choose the point yourself, read it off screenshot() and pass it to tap({ x, y })."));
     }
-    screenSize() {
-        return this.unavailable('screenSize');
+    /** The screenshot's size in pixels, the space tap() uses. */
+    async screenSize() {
+        const { width, height } = pngSize(await this.screenshot());
+        return { width, height };
     }
     get expiresAt() {
         return this.expiresAtMs === undefined ? undefined : new Date(this.expiresAtMs);
@@ -391,13 +500,15 @@ export class ComputerSession {
             throw error;
         }
     }
-    async request(method, suffix, body, signal) {
+    async request(method, suffix, body, signal, timeoutMs) {
         const path = `/api/v1/computer-runs${suffix === '' ? '' : `/${this.id}${suffix}`}`;
         let response;
         try {
-            response = await this.client.send(method, path, { body, signal, timeoutMs: suffix === '/stop' || suffix.startsWith('/operations') ? 310_000 : 30_000, retry: method === 'GET' && suffix !== '/screen/screenshot', headers: suffix === '/operations' ? { Prefer: 'respond-async' } : undefined });
+            response = await this.client.send(method, path, { body, signal, timeoutMs: timeoutMs ?? (suffix === '/stop' || suffix.startsWith('/operations') ? 310_000 : 30_000), retry: method === 'GET' && suffix !== '/screen/screenshot', headers: suffix === '/operations' ? { Prefer: 'respond-async' } : undefined });
         }
         catch (error) {
+            if (suffix === '/screen/actions')
+                throw new VerifyError('ACTION_UNCERTAIN', `The screen action may have happened: Revyl's answer never arrived. Verify will not repeat it; take a screenshot() to check before you do. See ${this.viewerUrl}`);
             if (method === 'POST' || suffix.startsWith('/operations/'))
                 throw new VerifyError('ACTION_UNCERTAIN', `Computer request outcome is unconfirmed. Verify will not repeat it. See ${this.viewerUrl}`);
             if (isVerifyError(error))
@@ -413,6 +524,7 @@ export class ComputerSession {
             return response;
         const detail = responseRefusalReason(response) ?? responseDetail(response);
         const refusal = { reason: responseRefusalReason(response), httpStatus: response.status };
+        const reason = detail && ERROR_CODES.has(detail) ? detail : `HTTP ${response.status}`;
         if (suffix === '/screen/screenshot') {
             let raw = null;
             try {
@@ -428,6 +540,30 @@ export class ComputerSession {
         }
         if (response.status === 429 && detail === 'computer_limit')
             throw new VerifyError('COMPUTER_LIMIT', `${COMPUTER_CAPACITY} See ${this.viewerUrl}`, refusal);
+        if (suffix === '/screen/actions') {
+            if (response.status === 422 && detail === 'screen_point_outside')
+                return response;
+            if (response.status === 409 && detail === 'screen_starting')
+                return response;
+            if (detail === 'screen_target_not_found')
+                throw new VerifyError('ACTION_FAILED', `Nothing was clicked: Revyl could not find the described target on the screen (screen_target_not_found). Take a screenshot() and pass a point to tap({ x, y }). See ${this.viewerUrl}`, refusal);
+            if (detail === 'screen_target_unavailable')
+                throw new VerifyError('API_ERROR', `Nothing was clicked: Revyl could not look up the described target just now (screen_target_unavailable). It is safe to try again. See ${this.viewerUrl}`, refusal);
+            if (detail === 'screen_control_held')
+                throw new VerifyError('SESSION_BUSY', `Nothing happened: someone else is controlling this computer's screen in the viewer (screen_control_held). Try again after they release it; control frees itself after 2 minutes without input. See ${this.viewerUrl}`, refusal);
+            if (detail === 'screen_input_uncertain' || detail === 'screen_unavailable' || detail === 'screen_input_unavailable')
+                throw new VerifyError('ACTION_UNCERTAIN', `The screen action may have happened (${detail}). Verify will not repeat it; take a screenshot() to check before you do. See ${this.viewerUrl}`, refusal);
+            if (response.status === 403)
+                throw new VerifyError('AUTH_REJECTED', `Revyl refused the screen action (forbidden). The API key's owner needs the Member role or above in the organization, and CI runner keys can't control the screen yet. See ${this.viewerUrl}`, refusal);
+            if (response.status === 503 && detail === 'authentication_unavailable')
+                throw new VerifyError('API_ERROR', `Nothing happened: Revyl could not confirm your access just now (authentication_unavailable). It is safe to try again. See ${this.viewerUrl}`, refusal);
+            if (detail === 'screen_text_unsupported')
+                throw new VerifyError('INVALID_ARGUMENT', `Nothing was typed: this computer can't type some of these characters (screen_text_unsupported). See ${this.viewerUrl}`, refusal);
+            if (detail === 'screen_control_lost')
+                throw new VerifyError('SESSION_BUSY', `Nothing happened: control of this computer's screen changed hands before the action was sent (screen_control_lost). Try again; if someone is controlling the screen in the viewer, wait until they release it, which happens on its own after 2 minutes without input. See ${this.viewerUrl}`, refusal);
+            if (detail === 'screen_key_unsupported')
+                throw new VerifyError('INVALID_ARGUMENT', `Nothing was pressed: this computer can't press that key (screen_key_unsupported). Use a key pressKey() lists in the docs, or typeFocused() for a printable character. See ${this.viewerUrl}`, refusal);
+        }
         let code = 'API_ERROR';
         if (response.status === 401 || response.status === 403)
             code = 'AUTH_REJECTED';
@@ -447,8 +583,38 @@ export class ComputerSession {
             code = 'ACTION_UNCERTAIN';
         else if (method === 'POST' && (response.status >= 500 || response.status < 400))
             code = 'ACTION_UNCERTAIN';
-        const reason = detail && ERROR_CODES.has(detail) ? detail : `HTTP ${response.status}`;
-        throw new VerifyError(code, `Revyl refused the computer request (${reason}). See ${this.viewerUrl}`, refusal);
+        let next = REFUSAL_NEXT_STEPS[code];
+        if (suffix === '/screen/actions' && code === 'ACTION_UNCERTAIN')
+            next = 'The screen action may have happened. Verify will not repeat it; take a screenshot() to check before you do.';
+        else if (suffix === '/screen/actions' && code === 'INVALID_ARGUMENT')
+            next = 'Part of the screen action may have run, so take a screenshot() to check the screen before you try again.';
+        else if (code === 'ACTION_UNCERTAIN' && method === 'GET' && !suffix.startsWith('/operations/'))
+            next = 'Nothing on the computer changed; it is safe to try again.';
+        throw new VerifyError(code, `Revyl refused the computer request (${reason}). ${next} See ${this.viewerUrl}`, refusal);
+    }
+    async act(action, timeoutMs = 30_000) {
+        this.assertOpen();
+        const signal = this.observationSignal(timeoutMs);
+        const deadlineMs = Date.now() + 30_000;
+        for (;;) {
+            const response = await this.request('POST', '/screen/actions', { action, screenshot: false }, signal, timeoutMs);
+            if (response.status !== 409)
+                return response;
+            const retryAfter = Number(response.headers.get('retry-after'));
+            const waitMs = Number.isFinite(retryAfter) && retryAfter > 0 ? Math.min(retryAfter, 60) * 1_000 : 1_000;
+            const seconds = Math.ceil(waitMs / 1_000);
+            const starting = new VerifyError('SESSION_BUSY', `Nothing happened: the computer's screen is still starting (screen_starting). Try again in ${seconds} second${seconds === 1 ? '' : 's'}. See ${this.viewerUrl}`, { httpStatus: 409 });
+            if (Date.now() + waitMs >= deadlineMs)
+                throw starting;
+            try {
+                await delay(waitMs, undefined, { signal });
+            }
+            catch {
+                this.assertOpen();
+                throw starting;
+            }
+            this.assertOpen();
+        }
     }
     parseReport(response) {
         const report = parseComputerJson(response);
@@ -467,7 +633,7 @@ export class ComputerSession {
         return AbortSignal.any([this.lifetime.signal, AbortSignal.timeout(Math.max(0, remainingMs))]);
     }
     unavailable(method) {
-        return Promise.reject(new VerifyError('NOT_AVAILABLE_ON_PLATFORM', `${method}() is not available on computers; use exec() and file operations. See ${this.viewerUrl}`));
+        return Promise.reject(new VerifyError('NOT_AVAILABLE_ON_PLATFORM', `${method}() is not available on computers. ${SCREEN_CALLS} See ${this.viewerUrl}`));
     }
     log(line) {
         this.options.log?.(`${this.target}: ${line}`);

package/dist/index.d.ts

@@ -7,7 +7,7 @@ export type { Platform, ComputerPlatform, MobilePlatform } from './api/types.js'
 export { VerifyError, isVerifyError } from './errors.js';
 export type { RefusalReason, VerifyErrorCode } from './errors.js';
 export { localAgent } from './agent/loop.js';
-export type { Grounding, LocalAgent, LocalAgentOptions, Location, ModelRun, ModelTurn, Point, SwipeDirection, Target } from './agent/loop.js';
+export type { Grounding, Key, KeyModifier, LocalAgent, LocalAgentOptions, Location, ModelRun, ModelTurn, Point, SwipeDirection, Target } from './agent/loop.js';
 export type { CustomModel, ModelOption } from './agent/providers.js';
 export { anthropic, openaiCompatible } from './agent/model.js';
 export type { AnthropicOptions, JsonSchema, ModelAdapter, ModelReply, ModelRequest, ModelTool, ModelToolCall, ModelUsage, OpenAICompatibleOptions } from './agent/model.js';

package/dist/session.d.ts

@@ -1,7 +1,7 @@
 import { RevylClient } from './api/client.js';
 import type { LiveStepType, Platform } from './api/types.js';
 import type { HostedTargetConfig, ResolvedDeviceTarget } from './config.js';
-import { type LocalAgent, type Location, type ModelRun, type Point, type SwipeDirection } from './agent/loop.js';
+import { type Key, type KeyModifier, type LocalAgent, type Location, type ModelRun, type Point, type SwipeDirection } from './agent/loop.js';
 import { type ModelOption } from './agent/providers.js';
 export type StepType = LiveStepType;
 /** The outcome of one instruction or validation, as Revyl's agent reported it. */
@@ -103,27 +103,43 @@ export interface OpenedSession extends Session {
     report(): Promise<Record<string, unknown>>;
     navigate(url: string): Promise<void>;
     open(target: HostedTargetConfig): Promise<OpenedSession>;
-    /** Taps a point in screenshot pixels, or an element Revyl locates from a description. Returns the point tapped, in screenshot pixels. Browsers and devices only. */
+    /** Taps a point in screenshot pixels, or an element Revyl locates from a description. Returns the point tapped, in screenshot pixels. */
     tap(at: Location): Promise<Point>;
     /** Taps a text field and types into it, clearing it first unless clear is false. Returns the point tapped. Browsers and devices only. */
     type(at: Location, text: string, options?: {
         clear?: boolean;
     }): Promise<Point>;
+    /** Types into the field that already has focus, without tapping, replacing its content unless clear is false. Web and Mac computers, where clear must be false; on iOS and Android, use type(location, text). */
+    typeFocused(text: string, options?: {
+        clear?: boolean;
+    }): Promise<void>;
+    /** Double-taps a point in screenshot pixels, or an element Revyl locates from a description. Returns the point, in screenshot pixels. Browsers and devices only. */
+    doubleTap(at: Location): Promise<Point>;
+    /** Presses and holds a point or element for duration milliseconds (1 to 5000, default 1500). Returns the point, in screenshot pixels. Browsers and devices only. */
+    longPress(at: Location, options?: {
+        duration?: number;
+    }): Promise<Point>;
     /** Swipes from a point or the screen's center. 'up' moves the finger up, which scrolls content down. Browsers and devices only. */
     swipe(direction: SwipeDirection, options?: {
         from?: Location;
     }): Promise<void>;
-    /** Presses Enter or Backspace on the focused field. Browsers and devices only. */
-    pressKey(key: 'enter' | 'backspace'): Promise<void>;
+    /** Presses at one point or element, moves to another and releases. Browsers and devices only. */
+    drag(from: Location, to: Location): Promise<void>;
+    /** Presses a key on the focused element. iOS and Android take 'enter' and 'backspace'; Web and Mac computers also take the other Key names, and modifiers, with which a lowercase letter or digit also works. On a Mac computer 'meta' is Command. */
+    pressKey(key: Key | string, options?: {
+        modifiers?: KeyModifier[];
+    }): Promise<void>;
     /** Goes back: browser history on Web, the back button on Android. Not on iOS. */
     back(): Promise<void>;
+    /** Goes to the home screen on iOS and Android. Not on Web. */
+    goHome(): Promise<void>;
     /** Opens a URL or deep link on a device, or loads a URL in a browser. */
     openUrl(url: string): Promise<void>;
     /** Launches an installed app by bundle ID or package name. Devices only. */
     launchApp(bundleId: string): Promise<void>;
     /** Asks Revyl's grounding model where an element is on the current screen, in screenshot pixels. Browsers and devices only. */
     locate(target: string): Promise<Point>;
-    /** The screenshot's size in pixels, the space every coordinate uses. Browsers and devices only. */
+    /** The screenshot's size in pixels, the space every coordinate uses. */
     screenSize(): Promise<{
         width: number;
         height: number;
@@ -203,14 +219,30 @@ export declare class DeviceSession implements Session {
     type(at: Location, text: string, options?: {
         clear?: boolean;
     }): Promise<Point>;
+    /** Types into the field that already has focus, without tapping, replacing its content unless clear is false. Web only. */
+    typeFocused(text: string, options?: {
+        clear?: boolean;
+    }): Promise<void>;
+    /** Double-taps a point in screenshot pixels, or an element Revyl locates from a description. Returns the point, in screenshot pixels. */
+    doubleTap(at: Location): Promise<Point>;
+    /** Presses and holds a point or element for duration milliseconds (1 to 5000, default 1500). Returns the point, in screenshot pixels. */
+    longPress(at: Location, options?: {
+        duration?: number;
+    }): Promise<Point>;
     /** Swipes from a point or the screen's center. 'up' moves the finger up, which scrolls content down. */
     swipe(direction: SwipeDirection, options?: {
         from?: Location;
     }): Promise<void>;
-    /** Presses Enter or Backspace on the focused field. */
-    pressKey(key: 'enter' | 'backspace'): Promise<void>;
+    /** Presses at one point or element, moves to another and releases. */
+    drag(from: Location, to: Location): Promise<void>;
+    /** Presses a key on the focused element, with modifiers held on Web. */
+    pressKey(key: Key | string, options?: {
+        modifiers?: KeyModifier[];
+    }): Promise<void>;
     /** Goes back: browser history on Web, the back button on Android. Not on iOS. */
     back(): Promise<void>;
+    /** Goes to the home screen on iOS and Android. */
+    goHome(): Promise<void>;
     /** Opens a URL or deep link on a device, or loads a URL in a browser. */
     openUrl(url: string): Promise<void>;
     /** Launches an installed app by bundle ID or package name. Devices only. */
@@ -245,3 +277,4 @@ export declare function pngSize(bytes: Uint8Array): {
     width: number;
     height: number;
 } | undefined;
+export declare function checkLocation(at: Location): Location;

package/dist/session.js

@@ -6,7 +6,7 @@ import { apiError, edgeBlockedError, parseJson, responseDetail, responseErrorCod
 import { trackSession, untrackSession } from './cleanup.js';
 import { VerifyError, errorMessage, isVerifyError } from './errors.js';
 import { screenHasContent } from './screen-readiness.js';
-import { runLocalStep } from './agent/loop.js';
+import { WEB_KEYS, checkKey, runLocalStep } from './agent/loop.js';
 import { chooseAgent } from './agent/providers.js';
 const POLL_INTERVAL_MS = 2_000;
 const DEVICE_CONNECT_TIMEOUT_MS = 90_000;
@@ -441,28 +441,61 @@ export class DeviceSession {
             await this.readScreenshotSize();
         return this.toScreenshot((await this.perform({ kind: 'type', at: location, text, clear: options.clear !== false })));
     }
+    /** Types into the field that already has focus, without tapping, replacing its content unless clear is false. Web only. */
+    async typeFocused(text, options = {}) {
+        if (this.platform !== 'web')
+            throw new VerifyError('NOT_AVAILABLE_ON_PLATFORM', 'typeFocused() is available only on web sessions; on iOS and Android, use type(location, text), which taps the field first.');
+        if (typeof text !== 'string' || text.length > 16_384)
+            throw new VerifyError('INVALID_ARGUMENT', 'typeFocused() takes text of at most 16384 characters.');
+        await this.perform({ kind: 'type_focused', text, clear: options.clear !== false });
+    }
+    /** Double-taps a point in screenshot pixels, or an element Revyl locates from a description. Returns the point, in screenshot pixels. */
+    async doubleTap(at) {
+        const location = await this.toDevice(checkLocation(at));
+        if (this.platform === 'ios')
+            await this.readScreenshotSize();
+        return this.toScreenshot((await this.perform({ kind: 'double_tap', at: location })));
+    }
+    /** Presses and holds a point or element for duration milliseconds (1 to 5000, default 1500). Returns the point, in screenshot pixels. */
+    async longPress(at, options = {}) {
+        const { duration } = options;
+        if (duration !== undefined && (!Number.isInteger(duration) || duration < 1 || duration > 5_000))
+            throw new VerifyError('INVALID_ARGUMENT', 'longPress() duration is whole milliseconds from 1 to 5000.');
+        const location = await this.toDevice(checkLocation(at));
+        if (this.platform === 'ios')
+            await this.readScreenshotSize();
+        return this.toScreenshot((await this.perform({ kind: 'long_press', at: location, ...(duration !== undefined ? { durationMs: duration } : {}) })));
+    }
     /** Swipes from a point or the screen's center. 'up' moves the finger up, which scrolls content down. */
     async swipe(direction, options = {}) {
         if (!['up', 'down', 'left', 'right'].includes(direction))
             throw new VerifyError('INVALID_ARGUMENT', "swipe() takes 'up', 'down', 'left' or 'right'.");
         await this.perform({ kind: 'swipe', direction, ...(options.from ? { from: await this.toDevice(checkLocation(options.from)) } : {}) });
     }
-    /** Presses Enter or Backspace on the focused field. */
-    async pressKey(key) {
-        if (key !== 'enter' && key !== 'backspace')
-            throw new VerifyError('INVALID_ARGUMENT', "pressKey() takes 'enter' or 'backspace'.");
-        await this.perform({ kind: 'key', key });
+    /** Presses at one point or element, moves to another and releases. */
+    async drag(from, to) {
+        const [start, end] = [checkLocation(from), checkLocation(to)];
+        await this.perform({ kind: 'drag', from: await this.toDevice(start), to: await this.toDevice(end) });
+    }
+    /** Presses a key on the focused element, with modifiers held on Web. */
+    async pressKey(key, options = {}) {
+        const checked = checkKey(this.platform, key, options.modifiers ?? []);
+        await this.perform({ kind: 'key', key: checked.key, ...(checked.modifiers.length ? { modifiers: checked.modifiers } : {}) });
     }
     /** Goes back: browser history on Web, the back button on Android. Not on iOS. */
     async back() {
         await this.perform({ kind: 'back' });
     }
+    /** Goes to the home screen on iOS and Android. */
+    async goHome() {
+        if (this.platform === 'web')
+            throw new VerifyError('NOT_AVAILABLE_ON_PLATFORM', 'goHome() is available only on iOS and Android sessions; on web, use openUrl() or navigate() to load a page.');
+        await this.perform({ kind: 'go_home' });
+    }
     /** Opens a URL or deep link on a device, or loads a URL in a browser. */
     async openUrl(url) {
         if (typeof url !== 'string' || url.length === 0 || url.length > 8_192)
             throw new VerifyError('INVALID_ARGUMENT', 'openUrl() takes a URL of 1 to 8192 characters.');
-        if (this.platform === 'web' && !/^https?:/i.test(url))
-            throw new VerifyError('INVALID_ARGUMENT', 'On web, openUrl() takes an http(s) URL; deep links such as myapp://orders/42 open only on iOS and Android.');
         await this.perform({ kind: 'open_url', url });
     }
     /** Launches an installed app by bundle ID or package name. Devices only. */
@@ -577,11 +610,30 @@ export class DeviceSession {
                 await this.act('tap', pointBody(point, action.at), signal);
                 return { x: point.x, y: point.y };
             }
+            case 'double_tap': {
+                const point = await this.resolve(action.at, screenToken, signal);
+                await this.act('double_tap', pointBody(point, action.at), signal);
+                return { x: point.x, y: point.y };
+            }
+            case 'long_press': {
+                const point = await this.resolve(action.at, screenToken, signal);
+                await this.act('longpress', { ...pointBody(point, action.at), ...(action.durationMs !== undefined ? { duration_ms: action.durationMs } : {}) }, signal);
+                return { x: point.x, y: point.y };
+            }
             case 'type': {
                 const point = await this.resolve(action.at, screenToken, signal);
                 await this.act('input', { ...pointBody(point, action.at), text: action.text, clear_first: action.clear }, signal);
                 return { x: point.x, y: point.y };
             }
+            case 'type_focused':
+                await this.act('input', { text: action.text, clear_first: action.clear }, signal);
+                return undefined;
+            case 'drag': {
+                const from = await this.resolve(action.from, screenToken, signal);
+                const to = await this.resolve(action.to, screenToken, signal);
+                await this.act('drag', { start_x: from.x, start_y: from.y, end_x: to.x, end_y: to.y }, signal);
+                return undefined;
+            }
             case 'swipe': {
                 const point = action.from ? await this.resolve(action.from, screenToken, signal) : await this.readScreenSize(signal).then((size) => ({ x: Math.floor(size.width / 2), y: Math.floor(size.height / 2) }));
                 const direction = this.platform === 'web' ? { up: 'down', down: 'up', left: 'right', right: 'left' }[action.direction] : action.direction;
@@ -589,7 +641,10 @@ export class DeviceSession {
                 return undefined;
             }
             case 'key':
-                await this.act('key', { key: this.platform === 'web' ? (action.key === 'enter' ? 'Enter' : 'Backspace') : action.key.toUpperCase() }, signal);
+                await this.act('key', { key: this.platform === 'web' ? (WEB_KEYS[action.key] ?? action.key) : action.key.toUpperCase(), ...(action.modifiers?.length ? { modifiers: action.modifiers } : {}) }, signal);
+                return undefined;
+            case 'go_home':
+                await this.act('go_home', undefined, signal);
                 return undefined;
             case 'back':
                 if (this.platform === 'ios')
@@ -598,6 +653,8 @@ export class DeviceSession {
                 return undefined;
             case 'open_url':
                 signal?.throwIfAborted();
+                if (this.platform === 'web' && !/^https?:/i.test(action.url))
+                    throw new VerifyError('INVALID_ARGUMENT', 'On web, openUrl() takes an http(s) URL; deep links such as myapp://orders/42 open only on iOS and Android.');
                 if (this.platform === 'web')
                     await this.navigateTo(action.url, signal);
                 else
@@ -822,7 +879,7 @@ export function pngSize(bytes) {
     const height = view.getUint32(20);
     return width > 0 && height > 0 ? { width, height } : undefined;
 }
-function checkLocation(at) {
+export function checkLocation(at) {
     if (at && typeof at === 'object' && 'target' in at) {
         if (typeof at.target !== 'string' || at.target.trim() === '' || at.target.length > 500)
             throw new VerifyError('INVALID_ARGUMENT', 'A target is a description of 1 to 500 characters.');

package/docs/README.md

@@ -10,7 +10,7 @@ These files ship inside the npm package at `node_modules/@revyl/verify/docs/`, s
 | [configuration.md](configuration.md) | Look up every option in `verify.config.ts`, the environment variables, and the CLI |
 | [web.md](web.md) | Open a web app in a Revyl cloud browser, and act on it with exact actions |
 | [mobile.md](mobile.md) | Open an iOS or Android app on a Revyl cloud device, and act on it with exact actions |
-| [computers.md](computers.md) | Open a Mac computer for commands, files, processes, screenshots, and optional recording metadata |
+| [computers.md](computers.md) | Open a Mac computer for commands, files, processes, screenshots, clicks and keys on its screen, and optional recording metadata |
 | [ci.md](ci.md) | Call Verify from CI or from a test runner you already use |
 | [own-model.md](own-model.md) | Run instructions and validations on your own model, such as Claude, GPT or a local model |
 | [errors.md](errors.md) | Understand an error code and fix it, or troubleshoot a common problem |

package/docs/agents.md

@@ -24,11 +24,11 @@ This page is for AI coding agents (Claude Code, Cursor, Codex, and others) that
 
 ## Exact actions and coordinates
 
-Use `instruction()` when the step needs judgement, and exact actions (`tap`, `type`, `swipe`, `pressKey`, `locate`, `screenSize`) when you know exactly what to press. They work on Web, iOS and Android, not on computers.
+Use `instruction()` when the step needs judgement, and exact actions (`tap`, `doubleTap`, `longPress`, `type`, `swipe`, `drag`, `pressKey`, `locate`, `screenSize`) when you know exactly what to press. They work on Web, iOS and Android. On a Mac computer's screen, `tap({ x, y })`, `tap({ target })`, `screenSize()`, `typeFocused(text, { clear: false })` and `pressKey` work; see [computers.md](computers.md#screen). On Web and Mac computers, `typeFocused(text)` types into the field that has focus, and `pressKey` also takes keys such as `'escape'`, `'tab'` and arrows, with `modifiers`. Select all with `pressKey('a', { modifiers: ['meta'] })` on a Mac computer, where `'meta'` is Command, and with `pressKey('a', { modifiers: ['ctrl'] })` on Web, where `'meta'` is not the shortcut key. On iOS and Android, `goHome()` goes to the home screen, `pressKey` takes only `'enter'` and `'backspace'`, and you type with `type(location, text)`. A call the platform lacks throws `NOT_AVAILABLE_ON_PLATFORM` before anything is sent, and the message names what to use instead.
 
-Prefer a description, such as `tap({ target: 'Sign in button' })`: Revyl finds the element on the current screen, and you never handle coordinates. Otherwise a point is `{ x, y }` in screenshot pixels on every platform: whole numbers from the top-left corner of the image `screenshot()` returns. So a position you read off a screenshot goes to `tap()`, `type()` and `swipe()` unchanged. `screenSize()` returns the screenshot's width and height, and `locate()` and the points `tap()` and `type()` return use the same pixels. A point outside the screenshot is refused with `INVALID_ARGUMENT` before any action is sent, and the message names the valid range. On iOS, Verify converts the pixels to the screen's points for you. On Web they equal CSS pixels at the default device scale factor of 1, and you still pass screenshot pixels at any other. This holds from TypeScript 0.4.1, Python 0.2.1 and Go v0.3.1; earlier versions took iOS points.
+Prefer a description, such as `tap({ target: 'Sign in button' })`: Revyl finds the element on the current screen, and you never handle coordinates. Otherwise a point is `{ x, y }` in screenshot pixels on every platform: whole numbers from the top-left corner of the image `screenshot()` returns. So a position you read off a screenshot goes to `tap()`, `type()`, `swipe()` and every other exact action unchanged. `screenSize()` returns the screenshot's width and height, and `locate()` and the points `tap()`, `doubleTap()`, `longPress()` and `type()` return use the same pixels. A point outside the screenshot is refused with `INVALID_ARGUMENT` before any action is sent, and the message names the valid range. On iOS, Verify converts the pixels to the screen's points for you. On Web they equal CSS pixels at the default device scale factor of 1, and you still pass screenshot pixels at any other. This holds from TypeScript 0.4.1, Python 0.2.1 and Go v0.3.1; earlier versions took iOS points. On a Mac computer a point is in screenshot pixels too, and `tap({ target })` returns the point it clicked in the same pixels. Verify refuses a negative or fractional point before sending it, and Revyl refuses one past the screen's edge without clicking; both throw `INVALID_ARGUMENT` naming the valid range.
 
-`swipe('up')` moves the finger up, so content scrolls down, on every platform. Actions return once Revyl accepts them, so after a swipe, `openUrl` or `launchApp`, wait for the screen to settle before you `locate` or tap by coordinates.
+`swipe('up')` moves the finger up, so content scrolls down, on every platform. Actions return once Revyl accepts them, so after a swipe, drag, `goHome`, `openUrl` or `launchApp`, wait for the screen to settle before you `locate` or tap by coordinates.
 
 ```ts
 import { open } from '@revyl/verify';
@@ -44,18 +44,22 @@ try {
   await session.swipe('up', { from: { x: 660, y: 2000 } }); // swipe from a screenshot position too
   const { width, height } = await session.screenSize();     // the screenshot's size
   console.log(`x is 0 to ${width - 1}, y is 0 to ${height - 1}`);
+
+  await session.longPress({ target: 'Wi-Fi row' });          // press and hold, 1.5 s by default
+  await session.drag({ x: 660, y: 2000 }, { x: 660, y: 800 }); // press, move, release: the list follows the finger
+  await session.goHome();                                   // the home screen, on iOS and Android
 } finally {
   await session.close();
 }
 ```
 
-The [Python](https://github.com/RevylAI/verify/blob/v0.4.1/python/README.md#exact-actions) and [Go](https://github.com/RevylAI/verify/blob/v0.4.1/go/README.md#exact-actions) READMEs state the same rule with each language's names. [api.md](api.md#exact-actions) lists every call and its errors.
+The [Python](https://github.com/RevylAI/verify/blob/v0.6.0/python/README.md#exact-actions) and [Go](https://github.com/RevylAI/verify/blob/v0.6.0/go/README.md#exact-actions) READMEs state the same rule with each language's names. [api.md](api.md#exact-actions) lists every call and its errors.
 
 ## Rules
 
 Mac computers are early access. Ask support@revyl.ai about access before starting a computer; do not repeat an uncertain start/stop to probe availability.
 
-For computer work, use structured `exec(['program', 'arg'])`, `readFile`, `writeFile`, `stat`, and run-owned handles from `processes()`/`terminate(id)`. Check command output with ordinary assertions and close in `finally`; screenshots are supported, but instruction/validation are not. Do not fall back to agent steps on computers or shell/file methods on mobile/web. See [computers.md](computers.md) for limits. Starting a computer is a paid external action and needs the same approval as starting a device.
+For computer work, use structured `exec(['program', 'arg'])`, `readFile`, `writeFile`, `stat`, and run-owned handles from `processes()`/`terminate(id)`. Check command output with ordinary assertions and close in `finally`; screenshots are supported, but instruction/validation are not. To use the screen, read a point off `screenshot()` and `tap({ x, y })` it, or describe it with `tap({ target })`, and type with `typeFocused(text, { clear: false })`. A described click takes several seconds, because Revyl takes a screenshot and finds the element first. It can pick a look-alike, and when the target isn't on the screen it can still click something else instead of failing. So check the point it returns or take a screenshot afterwards, and for anything that's hard to undo, read the point off `screenshot()` and use `tap({ x, y })`. After `ACTION_UNCERTAIN`, take a screenshot before you act again. Do not fall back to agent steps on computers or shell/file methods on mobile/web. See [computers.md](computers.md) for limits. Starting a computer is a paid external action and needs the same approval as starting a device.
 
 Computers have a hard lifetime from create (two hours today), not from readiness or activity; `expiresAt` is the authoritative deadline. `expiresAt` is optional on older APIs; expiry does not prevent close. After terminal state and confirmed release, `recording()` can return private signed metadata or null, including after close. Never log or share its signed URL/query. Do not automatically wait for recordings or retry `COMPUTER_LIMIT` starts: queued work (including iOS) or the concurrency limit may be responsible.