@moxxy/plugin-computer-control 0.41.2 → 0.42.0

package/src/contract/keys.test.ts

@@ -0,0 +1,103 @@
+import { describe, expect, it } from 'vitest';
+import { clipboardFlagsFor, formatChord, isSystemKeyCombo, parseKeyCombo } from './keys.js';
+
+describe('parseKeyCombo (xdotool syntax)', () => {
+  it.each([
+    ['Return', [], 'enter'],
+    ['Tab', [], 'tab'],
+    ['BackSpace', [], 'backspace'],
+    ['Delete', [], 'forward_delete'],
+    ['Up', [], 'up'],
+    ['Page_Down', [], 'page_down'],
+    ['ArrowDown', [], 'down'],
+    ['shift+ArrowLeft', ['shift'], 'left'],
+    ['Next', [], 'page_down'],
+    ['F12', [], 'f12'],
+    ['KP_0', [], 'numpad_0'],
+    ['KP_Enter', [], 'numpad_enter'],
+    ['space', [], 'space'],
+    ['comma', [], ','],
+    ['a', [], 'a'],
+    ['A', [], 'A'],
+    ['+', [], '+'],
+  ])('%s is one neutral key', (text, modifiers, key) => {
+    expect(parseKeyCombo(text)).toEqual({ modifiers, key });
+  });
+
+  it('maps super, cmd and windows to the platform meta key and orders modifiers', () => {
+    expect(parseKeyCombo('super+c')).toEqual({ modifiers: ['meta'], key: 'c' });
+    expect(parseKeyCombo('cmd+shift+z')).toEqual({ modifiers: ['shift', 'meta'], key: 'z' });
+    expect(parseKeyCombo('shift+Control_L+alt+Tab')).toEqual({ modifiers: ['ctrl', 'alt', 'shift'], key: 'tab' });
+    expect(parseKeyCombo('win+r')).toEqual({ modifiers: ['meta'], key: 'r' });
+  });
+
+  it('treats a letter under a modifier as the key, not its capital', () => {
+    expect(parseKeyCombo('super+C')).toEqual({ modifiers: ['meta'], key: 'c' });
+  });
+
+  it('reads a trailing plus as the plus key', () => {
+    expect(parseKeyCombo('ctrl++')).toEqual({ modifiers: ['ctrl'], key: '+' });
+  });
+
+  it('accepts modifiers alone for holding', () => {
+    expect(parseKeyCombo('shift')).toEqual({ modifiers: ['shift'], key: null });
+  });
+
+  it.each([
+    ['', /empty/],
+    ['ctrl+a+b', /one key/],
+    ['Foo_Bar', /unknown key "Foo_Bar"/],
+    ['ctrl+', /empty/],
+  ])('rejects %j', (text, message) => {
+    expect(() => parseKeyCombo(text)).toThrow(message);
+  });
+
+  it('carries the invalid_key code', () => {
+    expect(() => parseKeyCombo('Nope_Key')).toThrow(expect.objectContaining({ code: 'invalid_key' }));
+  });
+});
+
+describe('formatChord', () => {
+  it('writes the canonical modifier order', () => {
+    expect(formatChord(parseKeyCombo('shift+ctrl+Tab'))).toBe('ctrl+shift+tab');
+    expect(formatChord(parseKeyCombo('alt'))).toBe('alt');
+  });
+});
+
+describe('isSystemKeyCombo', () => {
+  it.each([
+    ['super+q', 'darwin', true],
+    ['cmd+Tab', 'darwin', true],
+    ['super+space', 'darwin', true],
+    ['super+c', 'darwin', false],
+    ['Return', 'darwin', false],
+    ['alt+F4', 'win32', true],
+    ['ctrl+alt+Delete', 'win32', true],
+    ['super+l', 'win32', true],
+    ['ctrl+c', 'win32', false],
+    ['alt+F4', 'linux', true],
+    ['alt+Tab', 'linux', true],
+    ['ctrl+alt+F2', 'linux', true],
+    ['super+l', 'linux', true],
+    ['ctrl+c', 'linux', false],
+    ['ctrl+t', 'linux', false],
+  ] as const)('%s on %s → %s', (text, platform, expected) => {
+    expect(isSystemKeyCombo(parseKeyCombo(text), platform)).toBe(expected);
+  });
+});
+
+describe('clipboardFlagsFor', () => {
+  it.each([
+    ['super+v', ['clipboardRead']],
+    ['ctrl+v', ['clipboardRead']],
+    ['super+c', ['clipboardWrite']],
+    ['ctrl+x', ['clipboardWrite']],
+    ['shift+Insert', ['clipboardRead']],
+    ['ctrl+Insert', ['clipboardWrite']],
+    ['shift+Delete', ['clipboardWrite']],
+    ['Return', []],
+    ['v', []],
+  ])('%s needs %j', (text, flags) => {
+    expect(clipboardFlagsFor(parseKeyCombo(text))).toEqual(flags);
+  });
+});

package/src/contract/keys.ts

@@ -0,0 +1,117 @@
+import { ComputerUseError } from './outcome.js';
+
+export type Modifier = 'ctrl' | 'alt' | 'shift' | 'meta';
+/** One chord: modifiers in canonical order and at most one key (none when only modifiers are held). */
+export interface KeyCombo { readonly modifiers: readonly Modifier[]; readonly key: string | null }
+export type KeyPlatform = 'darwin' | 'win32' | 'linux';
+
+const MODIFIER_ORDER: readonly Modifier[] = ['ctrl', 'alt', 'shift', 'meta'];
+
+// `meta` is Command on macOS, the Windows key on Windows and Super on Linux.
+const MODIFIERS: Readonly<Record<string, Modifier>> = {
+  ctrl: 'ctrl', control: 'ctrl', control_l: 'ctrl', control_r: 'ctrl',
+  alt: 'alt', option: 'alt', opt: 'alt', alt_l: 'alt', alt_r: 'alt',
+  shift: 'shift', shift_l: 'shift', shift_r: 'shift',
+  super: 'meta', super_l: 'meta', super_r: 'meta', meta: 'meta', meta_l: 'meta', meta_r: 'meta',
+  cmd: 'meta', command: 'meta', win: 'meta', windows: 'meta',
+};
+
+// xdotool keysym names (case-insensitive) → neutral key names the helpers map to virtual keys.
+// Delete is forward delete, as in xdotool; BackSpace removes the character before the caret.
+const NAMED_KEYS: Readonly<Record<string, string>> = {
+  return: 'enter', enter: 'enter', tab: 'tab', iso_left_tab: 'tab', escape: 'escape', esc: 'escape',
+  backspace: 'backspace', delete: 'forward_delete', del: 'forward_delete', insert: 'insert',
+  space: 'space', up: 'up', down: 'down', left: 'left', right: 'right', arrowup: 'up', arrowdown: 'down', arrowleft: 'left', arrowright: 'right', home: 'home', end: 'end',
+  page_up: 'page_up', prior: 'page_up', pageup: 'page_up', page_down: 'page_down', next: 'page_down', pagedown: 'page_down',
+  caps_lock: 'caps_lock', help: 'help', menu: 'menu',
+  kp_enter: 'numpad_enter', kp_add: 'numpad_add', kp_subtract: 'numpad_subtract', kp_multiply: 'numpad_multiply',
+  kp_divide: 'numpad_divide', kp_decimal: 'numpad_decimal', kp_equal: 'numpad_equal',
+  comma: ',', period: '.', slash: '/', backslash: '\\', minus: '-', plus: '+', equal: '=', semicolon: ';',
+  apostrophe: "'", grave: '`', bracketleft: '[', bracketright: ']', parenleft: '(', parenright: ')',
+  exclam: '!', at: '@', numbersign: '#', dollar: '$', percent: '%', asciicircum: '^', ampersand: '&',
+  asterisk: '*', underscore: '_', colon: ':', quotedbl: '"', less: '<', greater: '>', question: '?',
+  braceleft: '{', braceright: '}', bar: '|', asciitilde: '~',
+};
+
+function namedKey(token: string): string | undefined {
+  const lower = token.toLowerCase();
+  const named = NAMED_KEYS[lower];
+  if (named) return named;
+  const numpad = /^kp_(\d)$/.exec(lower);
+  if (numpad) return `numpad_${numpad[1]}`;
+  if (/^f([1-9]|1\d|2[0-4])$/.test(lower)) return lower;
+  return undefined;
+}
+
+function splitTokens(text: string): string[] {
+  if (text === '+') return ['+'];
+  // "ctrl++" holds ctrl and presses plus.
+  if (text.endsWith('++')) return [...text.slice(0, -2).split('+'), '+'];
+  return text.split('+');
+}
+
+/** Parse one xdotool-style chord ("Return", "super+c", "ctrl+shift+Tab", "KP_0"). */
+export function parseKeyCombo(text: string): KeyCombo {
+  const trimmed = text.trim();
+  if (trimmed === '') throw new ComputerUseError('invalid_key', 'Key combo is empty');
+  const modifiers = new Set<Modifier>();
+  const keys: string[] = [];
+  for (const raw of splitTokens(trimmed)) {
+    const token = raw.trim();
+    if (token === '') throw new ComputerUseError('invalid_key', `Key combo "${text}" has an empty part`);
+    const modifier = MODIFIERS[token.toLowerCase()];
+    if (modifier) { modifiers.add(modifier); continue; }
+    const key = namedKey(token) ?? ([...token].length === 1 ? token : undefined);
+    if (key === undefined) throw new ComputerUseError('invalid_key', `Key combo has an unknown key "${token}"`);
+    keys.push(key);
+  }
+  if (keys.length > 1) throw new ComputerUseError('invalid_key', `Key combo "${text}" presses more than one key; send one key per call`);
+  const ordered = MODIFIER_ORDER.filter((modifier) => modifiers.has(modifier));
+  const key = keys[0] ?? null;
+  // Under a modifier a letter names the physical key: cmd+C is cmd+c.
+  const normalized = key !== null && ordered.length > 0 && /^[A-Z]$/.test(key) ? key.toLowerCase() : key;
+  return { modifiers: ordered, key: normalized };
+}
+
+export function formatChord(combo: KeyCombo): string {
+  return [...combo.modifiers, ...(combo.key === null ? [] : [combo.key])].join('+');
+}
+
+// Chords that act on the whole system rather than the target app (quit, switch, lock, Spaces).
+const SYSTEM_COMBOS: Readonly<Record<KeyPlatform, ReadonlySet<string>>> = {
+  darwin: new Set([
+    'meta+q', 'shift+meta+q', 'alt+shift+meta+q', 'alt+meta+escape', 'meta+tab', 'shift+meta+tab', 'meta+space',
+    'ctrl+meta+q', 'ctrl+f1', 'ctrl+f2', 'ctrl+f3', 'ctrl+f7', 'ctrl+f8', 'ctrl+up', 'ctrl+down',
+  ]),
+  win32: new Set([
+    'ctrl+alt+forward_delete', 'alt+f4', 'alt+tab', 'alt+shift+tab', 'ctrl+alt+tab', 'ctrl+alt+shift+tab',
+    'meta+l', 'meta+d', 'meta+r', 'meta+e', 'meta+s', 'meta+q', 'ctrl+escape', 'meta+i', 'meta+u', 'meta+x',
+  ]),
+  // Window manager and session chords of the common desktops, and the switch to a text console.
+  linux: new Set([
+    'alt+f4', 'alt+tab', 'alt+shift+tab', 'meta+tab', 'shift+meta+tab', 'alt+f2', 'ctrl+alt+forward_delete', 'ctrl+alt+backspace',
+    'ctrl+alt+l', 'ctrl+alt+t', 'meta+l', 'meta+d', 'meta+a', 'meta+s', 'meta',
+    ...Array.from({ length: 12 }, (_, index) => `ctrl+alt+f${index + 1}`),
+  ]),
+};
+
+export function isSystemKeyCombo(combo: KeyCombo, platform: KeyPlatform): boolean {
+  return SYSTEM_COMBOS[platform].has(formatChord(combo).toLowerCase());
+}
+
+export type ClipboardFlag = 'clipboardRead' | 'clipboardWrite';
+
+/** Clipboard grants a chord needs: paste reads the clipboard, copy and cut write it. */
+export function clipboardFlagsFor(combo: KeyCombo): ClipboardFlag[] {
+  const has = (modifier: Modifier) => combo.modifiers.includes(modifier);
+  const key = combo.key?.toLowerCase();
+  const flags = new Set<ClipboardFlag>();
+  if (has('meta') || has('ctrl')) {
+    if (key === 'v') flags.add('clipboardRead');
+    if (key === 'c' || key === 'x') flags.add('clipboardWrite');
+  }
+  if (key === 'insert' && has('shift')) flags.add('clipboardRead');
+  if (key === 'insert' && has('ctrl')) flags.add('clipboardWrite');
+  if (key === 'forward_delete' && has('shift')) flags.add('clipboardWrite');
+  return [...flags];
+}

package/src/contract/outcome.test.ts

@@ -0,0 +1,71 @@
+import { describe, expect, it } from 'vitest';
+import { ComputerUseError, actionResultSchema, describeResult, errorCodes, hintFor } from './outcome.js';
+
+describe('actionResultSchema', () => {
+  it('accepts the four technical outcomes', () => {
+    for (const outcome of ['delivered', 'ineffective', 'unsupported', 'blocked'] as const) {
+      expect(actionResultSchema.parse({ outcome })).toEqual({ outcome });
+    }
+    expect(actionResultSchema.parse({ outcome: 'blocked', code: 'not_frontmost', method: 'input' }))
+      .toEqual({ outcome: 'blocked', code: 'not_frontmost', method: 'input' });
+  });
+
+  it('names how a delivered action was sent', () => {
+    for (const method of ['ax', 'input', 'background'] as const) {
+      expect(actionResultSchema.parse({ outcome: 'delivered', method })).toEqual({ outcome: 'delivered', method });
+    }
+  });
+
+  it('rejects an outcome or code the contract does not know', () => {
+    expect(() => actionResultSchema.parse({ outcome: 'verified' })).toThrow();
+    expect(() => actionResultSchema.parse({ outcome: 'blocked', code: 'mystery' })).toThrow();
+  });
+});
+
+describe('hintFor', () => {
+  it('has a next step for every code', () => {
+    for (const code of errorCodes) expect(hintFor(code).length).toBeGreaterThan(10);
+  });
+
+  it('sends a stale index back to a fresh observation', () => {
+    expect(hintFor('stale_state')).toMatch(/computer_get_app_state/);
+  });
+
+  it('points an ambiguous name at computer_list_apps', () => {
+    expect(hintFor('ambiguous_app')).toMatch(/computer_list_apps/);
+  });
+});
+
+describe('describeResult', () => {
+  it('says a delivered action is not yet verified', () => {
+    expect(describeResult({ outcome: 'delivered' })).toMatch(/delivered.*verify/i);
+  });
+
+  it('says when a followed link has not brought its page yet, so the model looks before clicking again', () => {
+    const text = describeResult({ outcome: 'delivered', code: 'page_loading', method: 'ax' });
+    expect(text).toMatch(/delivered/i);
+    expect(text).toContain(hintFor('page_loading'));
+    expect(hintFor('page_loading')).toMatch(/still loading[\s\S]*computer_get_app_state[\s\S]*twice/);
+  });
+
+  it('names the code and its hint when the action did not go through', () => {
+    const text = describeResult({ outcome: 'blocked', code: 'tier_insufficient' });
+    expect(text).toContain('blocked');
+    expect(text).toContain('tier_insufficient');
+    expect(text).toContain(hintFor('tier_insufficient'));
+  });
+
+  it('prefers the helper hint when it has one', () => {
+    expect(describeResult({ outcome: 'unsupported', code: 'canvas', hint: 'Use computer_drag.' }))
+      .toContain('Use computer_drag.');
+  });
+});
+
+describe('ComputerUseError', () => {
+  it('carries a known code and its hint', () => {
+    const error = new ComputerUseError('stale_state', 'Element 4 is gone');
+    expect(error.code).toBe('stale_state');
+    expect(error.message).toContain('Element 4 is gone');
+    expect(error.message).toContain(hintFor('stale_state'));
+  });
+});

package/src/contract/outcome.ts

@@ -0,0 +1,72 @@
+import { z } from 'zod';
+
+/** Why an action did not go through, with the model's next step. One table for every platform. */
+const hints = {
+  app_not_allowed: 'Ask for this app with computer_request_access, or finish the task without it.',
+  app_not_found: 'Check the name with computer_list_apps and use the bundle identifier or app id it returns.',
+  ambiguous_app: 'Several apps match that name; pick one id from computer_list_apps.',
+  tier_insufficient: 'The grant for this app does not allow this action; ask the user for a higher level with computer_request_access or use a read-only path.',
+  permissions_not_granted: 'The operating system has not granted Accessibility or Screen Recording; ask the user to allow it in System Settings, then try again.',
+  permissions_pending: 'The user has not answered the permission prompt yet; wait for the answer before acting.',
+  user_stopped: 'The user stopped Computer Use; do not act again until they ask.',
+  user_intervened: 'The user took over the mouse or keyboard; the action was paused. Observe again once they resume.',
+  screen_locked: 'The screen is locked; ask the user to unlock it.',
+  not_frontmost: 'Another app came to the front; call computer_get_app_state for the target app and retry only if it is still needed.',
+  target_blocked: 'A dialog or sheet covers the target; handle it first, based on a fresh computer_get_app_state.',
+  hit_test_mismatch: 'Something else is at that point now; call computer_get_app_state and target by element_index.',
+  point_outside_frame: 'Use coordinates inside the latest screenshot of this window.',
+  screen_changed: 'The pixels at the target changed since the last screenshot; call computer_get_app_state before clicking.',
+  stale_state: 'That index or point belongs to an older state; call computer_get_app_state and use its indices.',
+  no_state: 'Call computer_get_app_state for this app first; actions use its indices and screenshot coordinates.',
+  unsupported_action: 'This element does not support that action; use an action listed for it or click by coordinates.',
+  canvas: 'The point is on a canvas without accessibility; use computer_click or computer_drag on screenshot coordinates, or keyboard shortcuts.',
+  would_replace_content: 'Typing would replace the whole field; select the insertion point first or confirm with the user.',
+  system_key_combo: 'System key combinations need the systemKeyCombos grant from computer_request_access.',
+  clipboard_not_granted: 'Clipboard access needs the clipboardRead or clipboardWrite grant from computer_request_access.',
+  own_window: 'That target belongs to Moxxy itself; act only on the granted app.',
+  protected_path: 'A save dialog points at a protected location; choose a different folder or ask the user.',
+  no_progress: 'The same action left the app unchanged twice; try another method (element action, keyboard shortcut, menu or coordinates).',
+  invalid_key: 'Use xdotool key syntax such as "Return", "Tab", "super+c", "ctrl+shift+Tab", "Page_Down" or "KP_0".',
+  timeout: 'The app did not respond in time; the action was not retried. Observe again before deciding.',
+  helper_failed: 'The native helper failed or answered unexpectedly; the action was not retried. Observe again before deciding.',
+  page_loading: 'The click followed a link to another page, and after several seconds the page has not changed: it is probably still loading. Look again with computer_get_app_state before you act; clicking the link again, double-clicking it or pressing Return on it can start the same thing twice (a second new design, a second order).',
+} as const;
+
+export type ErrorCode = keyof typeof hints;
+export const errorCodes = Object.keys(hints) as ErrorCode[];
+const errorCodeSchema = z.enum(errorCodes as [ErrorCode, ...ErrorCode[]]);
+
+export function isErrorCode(value: string): value is ErrorCode {
+  return Object.hasOwn(hints, value);
+}
+
+export function hintFor(code: ErrorCode): string {
+  return hints[code];
+}
+
+/** Technical result of one action; `delivered` means sent, never verified. */
+export const actionResultSchema = z.object({
+  outcome: z.enum(['delivered', 'ineffective', 'unsupported', 'blocked']),
+  code: errorCodeSchema.optional(),
+  hint: z.string().max(1000).optional(),
+  method: z.enum(['ax', 'input', 'background']).optional(),
+}).strict();
+export type ActionResult = z.infer<typeof actionResultSchema>;
+
+export function describeResult(result: ActionResult): string {
+  if (result.outcome === 'delivered') {
+    return ['Action delivered; verify its effect in the new state before relying on it.', result.code ? hintFor(result.code) : undefined].filter(Boolean).join(' ');
+  }
+  const hint = result.hint ?? (result.code ? hintFor(result.code) : undefined);
+  return [`Action ${result.outcome}${result.code ? ` (${result.code})` : ''}.`, hint].filter(Boolean).join(' ');
+}
+
+export class ComputerUseError extends Error {
+  readonly code: ErrorCode;
+
+  constructor(code: ErrorCode, message: string) {
+    super(`${message}. ${hintFor(code)}`);
+    this.name = 'ComputerUseError';
+    this.code = code;
+  }
+}

package/src/contract/progress.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it } from 'vitest';
+import { ProgressTracker, fingerprint, looksDifferent } from './progress.js';
+import type { AppTree } from '@moxxy/jev';
+
+const tree = (value: string): AppTree => ({ app: 'Notes', window: 'Untitled', elements: [{ key: 'w/text', index: 1, depth: 1, role: 'text area', value }] });
+
+describe('fingerprint', () => {
+  it('changes with the tree or the screenshot, and only with them', () => {
+    const image = { base64: 'AAAA' };
+    expect(fingerprint(tree('a'), image)).toBe(fingerprint(tree('a'), { base64: 'AAAA' }));
+    expect(fingerprint(tree('a'), image)).not.toBe(fingerprint(tree('b'), image));
+    // A canvas changes only in pixels; that is progress too.
+    expect(fingerprint(tree('a'), image)).not.toBe(fingerprint(tree('a'), { base64: 'BBBB' }));
+    expect(fingerprint(tree('a'))).not.toBe(fingerprint(tree('a'), image));
+  });
+});
+
+describe('looksDifferent', () => {
+  const framed = (value: string): AppTree => ({ ...tree(value), elements: tree(value).elements.map((element) => ({ ...element, frame: { x: 1, y: 2, width: 3, height: 4 } })) });
+
+  it('compares pictures only when both looks have one, and otherwise the elements without their places', () => {
+    const image = { mediaType: 'image/png' as const, base64: 'AAAA', width: 1, height: 1 };
+    expect(looksDifferent({ tree: framed('a'), screenshot: image }, { tree: tree('a') })).toBe(false);
+    expect(looksDifferent({ tree: framed('a'), screenshot: image }, { tree: tree('b') })).toBe(true);
+    expect(looksDifferent({ tree: framed('a'), screenshot: image }, { tree: framed('a'), screenshot: { ...image, base64: 'BBBB' } })).toBe(true);
+    expect(looksDifferent({ tree: tree('a') }, { tree: tree('a') })).toBe(false);
+  });
+});
+
+describe('ProgressTracker', () => {
+  it('counts how many times in a row the same action left the app unchanged', () => {
+    const progress = new ProgressTracker();
+    expect(progress.record('notes', 'click 2', false)).toBe(1);
+    expect(progress.record('notes', 'click 2', false)).toBe(2);
+    expect(() => progress.check('notes', 'click 2')).toThrow(expect.objectContaining({ code: 'no_progress' }));
+  });
+
+  it('starts over after a change, another action or another app', () => {
+    const progress = new ProgressTracker();
+    progress.record('notes', 'click 2', false);
+    expect(progress.record('notes', 'click 2', true)).toBe(0);
+    expect(progress.record('notes', 'click 2', false)).toBe(1);
+    expect(progress.record('notes', 'press Return', false)).toBe(1);
+    progress.record('notes', 'press Return', false);
+    expect(() => progress.check('notes', 'click 2')).not.toThrow();
+    expect(() => progress.check('mail', 'press Return')).not.toThrow();
+  });
+
+  it('forgets an app whose action did not go through', () => {
+    const progress = new ProgressTracker();
+    progress.record('notes', 'click 2', false);
+    progress.record('notes', 'click 2', false);
+    progress.forget('notes');
+    expect(() => progress.check('notes', 'click 2')).not.toThrow();
+  });
+});

package/src/contract/progress.ts

@@ -0,0 +1,43 @@
+import { createHash } from 'node:crypto';
+import { ComputerUseError } from './outcome.js';
+import type { AppTree } from '@moxxy/jev';
+
+/** What the model can see of an app: its tree and its screenshot. Pixels count, so canvas work is progress. */
+export function fingerprint(tree: AppTree, image?: { base64: string }): string {
+  return createHash('sha256').update(JSON.stringify(tree)).update('\0').update(image ? image.base64 : '-').digest('hex');
+}
+
+type Look = { readonly tree: AppTree; readonly screenshot?: { readonly base64: string } };
+
+/**
+ * Whether the window changed between two looks: by elements and picture when both have a picture, else by the
+ * elements alone, without their places (which come with a picture).
+ */
+export function looksDifferent(a: Look, b: Look): boolean {
+  if (a.screenshot && b.screenshot) return fingerprint(a.tree, a.screenshot) !== fingerprint(b.tree, b.screenshot);
+  const placeless = (tree: AppTree): AppTree => ({ ...tree, elements: tree.elements.map(({ frame: _, ...element }) => element) });
+  return fingerprint(placeless(a.tree)) !== fingerprint(placeless(b.tree));
+}
+
+/** Each app's last action, and how many times in a row it left the app looking the same. */
+export class ProgressTracker {
+  private readonly last = new Map<string, { signature: string; unchanged: number }>();
+
+  /** The same action a third time after two that changed nothing is refused before it is sent. */
+  check(app: string, signature: string): void {
+    const last = this.last.get(app);
+    if (last && last.signature === signature && last.unchanged >= 2) {
+      throw new ComputerUseError('no_progress', 'This exact action already left the app unchanged twice and was not sent again');
+    }
+  }
+
+  /** Returns how many times in a row `signature` has now left `app` unchanged (0 after a change). */
+  record(app: string, signature: string, changed: boolean): number {
+    const last = this.last.get(app);
+    const unchanged = changed ? 0 : (last?.signature === signature ? last.unchanged : 0) + 1;
+    this.last.set(app, { signature, unchanged });
+    return unchanged;
+  }
+
+  forget(app: string): void { this.last.delete(app); }
+}