@moxxy/plugin-computer-control 0.41.2 → 0.42.0

package/src/index.test.ts

@@ -0,0 +1,112 @@
+import { mkdtempSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { ToolDef } from '@moxxy/sdk';
+import { afterEach, describe, expect, it } from 'vitest';
+import type { PlatformProfile } from './backend/backend.js';
+import { computerTools } from './contract/tools.js';
+import { createComputerControlPlugin } from './index.js';
+import { linuxProfile } from './linux/profile.js';
+import { macosProfile } from './macos/profile.js';
+import { windowsProfile } from './windows/profile.js';
+import { removeDirSync } from '@moxxy/vitest-preset/fs';
+
+const directories: string[] = [];
+afterEach(() => { for (const made of directories.splice(0)) removeDirSync(made); });
+
+/** A profile whose helper files exist (or not) in a scratch directory; the helper is never started. */
+function profile(manifest?: Record<string, unknown>, base: PlatformProfile = macosProfile): PlatformProfile {
+  const directory = mkdtempSync(join(tmpdir(), 'moxxy-computer-index-'));
+  directories.push(directory);
+  const helperPath = join(directory, 'moxxy-computer');
+  if (manifest) {
+    writeFileSync(helperPath, 'binary');
+    writeFileSync(`${helperPath}.json`, JSON.stringify(manifest));
+  }
+  return { ...base, helperPath };
+}
+const names = (tools: ReadonlyArray<ToolDef> | undefined) => (tools ?? []).map((tool) => tool.name).sort();
+const ready = { protocolVersion: macosProfile.protocolVersion, architecture: 'universal', sha256: '0'.repeat(64) };
+const context = { signal: new AbortController().signal } as never;
+
+describe('createComputerControlPlugin', () => {
+  it('runs macOS on the native helper with the shared tool set and human controls', () => {
+    const plugin = createComputerControlPlugin('darwin', 'arm64', profile(ready));
+    expect(names(plugin.tools)).toEqual(Object.keys(computerTools).sort());
+    expect(names(plugin.tools)).not.toContain('computer_applescript');
+    expect(plugin.hooks?.onInit).toBeTypeOf('function');
+    expect(plugin.hooks?.onTurnEnd).toBeTypeOf('function');
+    expect(plugin.hooks?.onBeforeProviderCall).toBeTypeOf('function');
+    expect(plugin.surfaces?.map((surface) => surface.kind)).toEqual(['computer-preview']);
+  });
+
+  it('offers only computer_status, with the reason, when the macOS helper is missing', async () => {
+    const plugin = createComputerControlPlugin('darwin', 'arm64', profile());
+    expect(names(plugin.tools)).toEqual(['computer_status']);
+    expect(plugin.hooks).toBeUndefined();
+    const [status] = plugin.tools ?? [];
+    expect(await status?.handler({}, context)).toEqual({
+      platform: 'darwin', architecture: 'arm64', ready: false,
+      limitations: [expect.stringMatching(/helper is missing/)],
+    });
+  });
+
+  it('offers only computer_status when the helper speaks another protocol', async () => {
+    const plugin = createComputerControlPlugin('darwin', 'arm64', profile({ ...ready, protocolVersion: macosProfile.protocolVersion - 1 }));
+    expect(names(plugin.tools)).toEqual(['computer_status']);
+    const [status] = plugin.tools ?? [];
+    expect(await status?.handler({}, context)).toMatchObject({ ready: false, limitations: [expect.stringMatching(/protocol/)] });
+  });
+
+  it('runs Windows x64 on the same tool set, human controls and live view as macOS', () => {
+    const plugin = createComputerControlPlugin('win32', 'x64', profile({ ...ready, architecture: 'x64' }, windowsProfile));
+    expect(names(plugin.tools)).toEqual(Object.keys(computerTools).sort());
+    expect(names(plugin.tools)).not.toContain('computer_app_catalog');
+    expect(plugin.hooks?.onBeforeProviderCall).toBeTypeOf('function');
+    expect(plugin.surfaces?.map((surface) => surface.kind)).toEqual(['computer-preview']);
+  });
+
+  it('offers only computer_status, with the reason, when the Windows helper is missing', async () => {
+    const plugin = createComputerControlPlugin('win32', 'x64', profile(undefined, windowsProfile));
+    expect(names(plugin.tools)).toEqual(['computer_status']);
+    const [status] = plugin.tools ?? [];
+    expect(await status?.handler({}, context)).toEqual({
+      platform: 'win32', architecture: 'x64', ready: false,
+      limitations: [expect.stringMatching(/helper is missing.*full installer/s)],
+    });
+  });
+
+  it('speaks one protocol on both platforms', () => {
+    expect(windowsProfile).toMatchObject({ platform: 'win32', protocolVersion: macosProfile.protocolVersion });
+    expect(windowsProfile.helperPath.replaceAll('\\', '/')).toMatch(/bin\/win32-x64\/moxxy-computer\.exe$/);
+  });
+
+  it('runs Linux on the same tool set with a live view of pictures', () => {
+    const plugin = createComputerControlPlugin('linux', 'x64', profile({ ...ready, os: 'linux', architecture: 'x64' }, linuxProfile('x64')));
+    expect(names(plugin.tools)).toEqual(Object.keys(computerTools).sort());
+    expect(plugin.surfaces?.map((surface) => surface.kind)).toEqual(['computer-preview']);
+    expect(linuxProfile('x64')).toMatchObject({ platform: 'linux', protocolVersion: macosProfile.protocolVersion, previewCodecs: ['jpeg'] });
+    expect(linuxProfile('x64').helperPath.replaceAll('\\', '/')).toMatch(/bin\/linux-x64\/moxxy-computer$/);
+    expect(linuxProfile('arm64').helperPath.replaceAll('\\', '/')).toMatch(/bin\/linux-arm64\/moxxy-computer$/);
+  });
+
+  it('offers only computer_status, with the reason, when the Linux helper is missing', async () => {
+    const plugin = createComputerControlPlugin('linux', 'arm64', profile(undefined, linuxProfile('arm64')));
+    expect(names(plugin.tools)).toEqual(['computer_status']);
+    const [status] = plugin.tools ?? [];
+    expect(await status?.handler({}, context)).toMatchObject({ platform: 'linux', ready: false, limitations: [expect.stringMatching(/helper is missing/)] });
+  });
+
+  it.each([['linux', 'ia32'], ['freebsd', 'x64'], ['win32', 'arm64']] as const)('reports %s %s as unsupported through computer_status alone', async (platform, arch) => {
+    const plugin = createComputerControlPlugin(platform, arch);
+    expect(names(plugin.tools)).toEqual(['computer_status']);
+    const [status] = plugin.tools ?? [];
+    expect(await status?.handler({}, context)).toEqual({ platform, architecture: arch, ready: false, limitations: ['Unsupported platform or architecture'] });
+  });
+
+  it('asks before every tool on every platform', () => {
+    for (const plugin of [createComputerControlPlugin('darwin', 'arm64', profile(ready)), createComputerControlPlugin('win32', 'x64', profile({ ...ready, architecture: 'x64' }, windowsProfile)), createComputerControlPlugin('linux', 'ia32')]) {
+      for (const tool of plugin.tools ?? []) expect(tool.permission?.action, tool.name).toBe('prompt');
+    }
+  });
+});

package/src/index.ts

@@ -1,64 +1,47 @@
-import { definePlugin, defineTool, z, type Plugin, type ToolDef } from '@moxxy/sdk';
-import { WindowsBackend } from './windows/backend.js';
-import { IS_DARWIN } from './shell.js';
-import { applescriptTool } from './tools/applescript.js';
-import { clickTool } from './tools/click.js';
-import { clipboardTool } from './tools/clipboard.js';
-import { keyTool } from './tools/key.js';
-import { openTool } from './tools/open.js';
-import { screenshotTool } from './tools/screenshot.js';
-import { typeTool } from './tools/type.js';
+import { definePlugin, defineTool, z, type Plugin } from '@moxxy/sdk';
+import { ComputerBackend, type PlatformProfile } from './backend/backend.js';
+import { helperProblem } from './helper/artifact.js';
+import { linuxProfile } from './linux/profile.js';
+import { macosProfile } from './macos/profile.js';
+import { windowsProfile } from './windows/profile.js';
 
-export {
-  applescriptTool,
-  clickTool,
-  clipboardTool,
-  keyTool,
-  openTool,
-  screenshotTool,
-  typeTool,
-};
+const name = '@moxxy/plugin-computer-control';
 
-export const computerControlTools: ReadonlyArray<ToolDef> = [
-  screenshotTool,
-  clickTool,
-  typeTool,
-  keyTool,
-  openTool,
-  clipboardTool,
-  applescriptTool,
-];
-
-/**
- * `@moxxy/plugin-computer-control` — programmatic control of the host
- * computer (mouse, keyboard, screenshot, clipboard, app launching,
- * AppleScript escape hatch).
- *
- * macOS retains its system-binary backend; Windows x64 uses our bundled
- * native helper. Unsupported hosts expose status only.
- *
- * Every tool is `permission: 'prompt'`. There is intentionally no
- * "allow always" shortcut for these — granting blanket permission to
- * drive the user's screen + keyboard is exactly the wrong default.
- */
-export function createComputerControlPlugin(platform: NodeJS.Platform = process.platform, arch: string = process.arch): Plugin {
-  const backend = platform === 'win32' && arch === 'x64' ? new WindowsBackend() : undefined;
+/** The only tool of a host that cannot run Computer Use: it says why. */
+function statusOnly(platform: NodeJS.Platform, architecture: string, limitation: string): Plugin {
   const status = defineTool({
     name: 'computer_status', description: 'Report Computer Use platform capabilities and limitations.',
     inputSchema: z.object({}).strict(), permission: { action: 'prompt' },
-    handler: () => ({ platform, architecture: arch, ready: platform === 'darwin',
-      limitations: platform === 'darwin' ? ['Requires Screen Recording and Accessibility permissions'] : ['Unsupported platform or architecture'] }),
-  });
-  return definePlugin({
-    name: '@moxxy/plugin-computer-control', version: '0.0.0',
-    tools: backend ? backend.tools() : platform === 'darwin' ? [...computerControlTools, status] : [status],
-    ...(backend ? { hooks: backend.hooks } : {}),
+    handler: () => ({ platform, architecture, ready: false, limitations: [limitation] }),
   });
+  return definePlugin({ name, version: '0.0.0', tools: [status] });
+}
+
+function profileFor(platform: NodeJS.Platform, arch: string): PlatformProfile | undefined {
+  if (platform === 'darwin') return macosProfile;
+  if (platform === 'linux') return arch === 'x64' || arch === 'arm64' ? linuxProfile(arch) : undefined;
+  return platform === 'win32' && arch === 'x64' ? windowsProfile : undefined;
+}
+
+/**
+ * `@moxxy/plugin-computer-control` — operates the user's desktop applications
+ * through a bundled native helper: macOS (universal), Windows x64 and Linux
+ * (x64, arm64; X11 sessions). Other
+ * hosts, and a host whose helper is missing, expose `computer_status` only.
+ *
+ * Every tool is `permission: 'prompt'`; which apps may be controlled, and how
+ * far, is a separate grant recorded in the session log.
+ */
+export function createComputerControlPlugin(
+  platform: NodeJS.Platform = process.platform, arch: string = process.arch, profile: PlatformProfile | undefined = profileFor(platform, arch),
+): Plugin {
+  if (!profile) return statusOnly(platform, arch, 'Unsupported platform or architecture');
+  const problem = helperProblem(profile.helperPath, profile.protocolVersion);
+  if (problem) return statusOnly(platform, arch, `${problem} ${profile.unavailableMessage}`);
+  const backend = new ComputerBackend(profile);
+  return definePlugin({ name, version: '0.0.0', tools: backend.tools(), hooks: backend.hooks, surfaces: backend.surfaces() });
 }
 
 export const computerControlPlugin = createComputerControlPlugin();
 
 export default computerControlPlugin;
-
-// Re-export for callers that want a runtime gate.
-export { IS_DARWIN };

package/src/jev/ladder.test.ts

@@ -0,0 +1,113 @@
+import { describe, expect, it } from 'vitest';
+import type { AppElement } from '@moxxy/jev';
+import { judge, rungs } from './ladder.js';
+
+const field: AppElement = { key: 'w/name', index: 7, depth: 1, role: 'text field', title: 'Name', frame: { x: 100, y: 40, width: 200, height: 20 } };
+const bare: AppElement = { key: 'w/bare', index: 8, depth: 1, role: 'button' };
+
+describe('rungs', () => {
+  it('clicks the element, then the point in its middle', () => {
+    expect(rungs({ do: 'click', target: 'Name' }, field, 'super+a')).toEqual([
+      [{ action: 'click', element_index: 7, mouse_button: 'left', click_count: 1 }],
+      [{ action: 'click', x: 200, y: 50, mouse_button: 'left', click_count: 1 }],
+    ]);
+    expect(rungs({ do: 'click', target: 'x' }, bare, 'super+a')).toHaveLength(1);
+  });
+
+  it('sets a value directly, then by clicking the field, selecting what it holds and typing', () => {
+    expect(rungs({ do: 'set_value', target: 'Name', text: 'clip' }, field, 'ctrl+a')).toEqual([
+      [{ action: 'set_value', element_index: 7, value: 'clip' }],
+      [
+        { action: 'click', element_index: 7, mouse_button: 'left', click_count: 1 },
+        { action: 'press_key', key: 'ctrl+a', repeat: 1 },
+        { action: 'type_text', text: 'clip' },
+      ],
+    ]);
+  });
+
+  it('types into a described element, then into whatever a click on it focuses', () => {
+    expect(rungs({ do: 'type', target: 'Name', text: 'hi' }, field, 'super+a')).toEqual([
+      [{ action: 'type_text', element_index: 7, text: 'hi' }],
+      [{ action: 'click', element_index: 7, mouse_button: 'left', click_count: 1 }, { action: 'type_text', text: 'hi' }],
+    ]);
+  });
+
+  it('has one way to type into the focus, press a key or scroll', () => {
+    expect(rungs({ do: 'type', text: 'hi' }, undefined, 'super+a')).toEqual([[{ action: 'type_text', text: 'hi' }]]);
+    expect(rungs({ do: 'key', key: 'Return' }, undefined, 'super+a')).toEqual([[{ action: 'press_key', key: 'Return', repeat: 1 }]]);
+    expect(rungs({ do: 'scroll', target: 'list', direction: 'down' }, field, 'super+a')).toEqual([[{ action: 'scroll', element_index: 7, direction: 'down', pages: 1 }]]);
+  });
+});
+
+describe('judge', () => {
+  const delivered = { outcome: 'delivered' } as const;
+
+  it('goes on after an action that was delivered and changed the window', () => {
+    expect(judge({ step: { do: 'click', target: 'Save' }, result: delivered, changed: true })).toEqual({ verdict: 'done', verified: false });
+  });
+
+  it('trusts Jev on what the step was expected to show', () => {
+    const step = { do: 'click', target: 'Save', expect: 'a save dialog is open' } as const;
+    expect(judge({ step, result: delivered, changed: true, expected: 0.9 })).toEqual({ verdict: 'done', verified: true });
+    expect(judge({ step, result: delivered, changed: true, expected: 0.1 })).toMatchObject({ verdict: 'retry' });
+    // Jev undecided: a changed window is taken as progress, an unchanged one is not.
+    expect(judge({ step, result: delivered, changed: true, expected: 0.45 })).toEqual({ verdict: 'done', verified: false });
+    expect(judge({ step, result: delivered, changed: false, expected: 0.45 })).toMatchObject({ verdict: 'retry' });
+  });
+
+  it('takes typing for done once its text shows in the field: typing again would append it once more', () => {
+    const step = { do: 'type', target: 'the address field', text: 'olx.pl', expect: 'OLX opens' } as const;
+    expect(judge({ step, result: delivered, changed: true, expected: 0.1, typed: true })).toEqual({ verdict: 'done', verified: false });
+    expect(judge({ step, result: delivered, changed: false, typed: true })).toEqual({ verdict: 'done', verified: false });
+    expect(judge({ step, result: delivered, changed: true, expected: 0.9, typed: true })).toEqual({ verdict: 'done', verified: true });
+    expect(judge({ step, result: delivered, changed: true, expected: 0.1, typed: false })).toMatchObject({ verdict: 'retry' });
+  });
+
+  it('takes a key for done when another window came to the front: Jev cannot tell a new window from an old one', () => {
+    const step = { do: 'key', key: 'super+n', expect: 'a new window is open' } as const;
+    expect(judge({ step, result: delivered, changed: true, expected: 0.2, moved: true })).toEqual({ verdict: 'done', verified: false });
+    expect(judge({ step, result: delivered, changed: true, expected: 0.2, moved: false })).toMatchObject({ verdict: 'retry' });
+    expect(judge({ step: { do: 'click', target: 'Tab', expect: 'a new window is open' }, result: delivered, changed: true, expected: 0.2, moved: true })).toMatchObject({ verdict: 'retry' });
+  });
+
+  it('stops when only the screenshot changed after a click at a point: Jev cannot see what the click did, and another click may undo it', () => {
+    const step = { do: 'click', target: 'Sort', expect: 'the sort options show' } as const;
+    expect(judge({ step, result: delivered, changed: true, expected: 0.1, unseen: true })).toMatchObject({ verdict: 'stop', why: expect.stringMatching(/screenshot/) });
+    expect(judge({ step, result: delivered, changed: true, expected: 0.9, unseen: true })).toEqual({ verdict: 'done', verified: true });
+  });
+
+  // Canva creates the design before its editor loads: a second click would start a second one.
+  it('stops after a link was followed whose page has not come yet, instead of clicking again', () => {
+    const step = { do: 'click', target: 'YouTube thumbnail', expect: 'the editor opens' } as const;
+    const loading = { outcome: 'delivered', code: 'page_loading' } as const;
+    expect(judge({ step, result: loading, changed: false, expected: 0.1 })).toMatchObject({ verdict: 'stop', why: expect.stringMatching(/loading/) });
+    expect(judge({ step: { do: 'click', target: 'YouTube thumbnail' }, result: loading, changed: false })).toMatchObject({ verdict: 'stop' });
+    expect(judge({ step, result: loading, changed: true, expected: 0.9 })).toEqual({ verdict: 'done', verified: true });
+  });
+
+  it('tries another way after a click or a value that changed nothing', () => {
+    expect(judge({ step: { do: 'click', target: 'Save' }, result: delivered, changed: false })).toMatchObject({ verdict: 'retry' });
+    expect(judge({ step: { do: 'set_value', target: 'Name', text: 'a' }, result: delivered, changed: false })).toMatchObject({ verdict: 'retry' });
+  });
+
+  it('accepts a key or a scroll that changed nothing visible', () => {
+    expect(judge({ step: { do: 'key', key: 'Escape' }, result: delivered, changed: false })).toEqual({ verdict: 'done', verified: false });
+    expect(judge({ step: { do: 'scroll', target: 'list', direction: 'down' }, result: delivered, changed: false })).toEqual({ verdict: 'done', verified: false });
+  });
+
+  it('tries another way when the action did not go through', () => {
+    expect(judge({ step: { do: 'click', target: 'Save' }, result: { outcome: 'unsupported', code: 'unsupported_action' }, changed: false })).toMatchObject({ verdict: 'retry' });
+    expect(judge({ step: { do: 'click', target: 'Save' }, result: { outcome: 'blocked', code: 'hit_test_mismatch' }, changed: false })).toMatchObject({ verdict: 'retry' });
+  });
+
+  it('stops at once for what only the user or the main model can resolve', () => {
+    for (const code of ['user_stopped', 'user_intervened', 'screen_locked', 'permissions_not_granted', 'tier_insufficient', 'app_not_allowed', 'protected_path'] as const) {
+      expect(judge({ step: { do: 'click', target: 'Save' }, result: { outcome: 'blocked', code }, changed: false })).toMatchObject({ verdict: 'stop' });
+    }
+  });
+
+  it('takes another way when the element would not take the input: focus left the field, or something covers it', () => {
+    const step = { do: 'type' as const, target: 'the search field', text: 'x' };
+    expect(judge({ step, result: { outcome: 'blocked', code: 'target_blocked' }, changed: false })).toEqual({ verdict: 'retry', why: 'blocked (target_blocked)' });
+  });
+});

package/src/jev/ladder.ts

@@ -0,0 +1,88 @@
+import { assertDefined } from '@moxxy/sdk';
+import type { ActionResult, ErrorCode } from '../contract/outcome.js';
+import type { ComputerAction, RunStep } from '../contract/tools.js';
+import type { AppElement } from '@moxxy/jev';
+
+/** How sure Jev must be that the expected result shows, and below what it certainly does not. */
+const HOLDS = 0.6;
+const FAILS = 0.3;
+
+/** What only the user or the main model can resolve; trying another way would not help. */
+const FINAL: ReadonlySet<ErrorCode> = new Set<ErrorCode>([
+  'user_stopped', 'user_intervened', 'screen_locked', 'permissions_not_granted', 'permissions_pending', 'tier_insufficient',
+  'app_not_allowed', 'protected_path', 'system_key_combo', 'clipboard_not_granted', 'own_window',
+]);
+
+const click = (target: { element_index: number } | { x: number; y: number }): ComputerAction => ({ action: 'click', ...target, mouse_button: 'left', click_count: 1 });
+
+/** The ways to carry out a step on its element, best first. Each way is a short run of actions. */
+export function rungs(step: RunStep, element: AppElement | undefined, selectAll: string): ComputerAction[][] {
+  const { key, text, direction } = step;
+  if (step.do === 'key') {
+    assertDefined(key, 'a key step has a key');
+    return [[{ action: 'press_key', key, repeat: 1 }]];
+  }
+  if (step.do === 'type' && !element) {
+    assertDefined(text, 'a type step has text');
+    return [[{ action: 'type_text', text }]];
+  }
+  assertDefined(element, `a ${step.do} step has an element`);
+  const { index, frame } = element;
+  const onElement = click({ element_index: index });
+  if (step.do === 'click') {
+    return [[onElement], ...(frame ? [[click({ x: Math.round(frame.x + frame.width / 2), y: Math.round(frame.y + frame.height / 2) })]] : [])];
+  }
+  if (step.do === 'scroll') {
+    assertDefined(direction, 'a scroll step has a direction');
+    return [[{ action: 'scroll', element_index: index, direction, pages: 1 }]];
+  }
+  assertDefined(text, `a ${step.do} step has text`);
+  if (step.do === 'type') return [[{ action: 'type_text', element_index: index, text }], [onElement, { action: 'type_text', text }]];
+  const typed: ComputerAction = text === '' ? { action: 'press_key', key: 'BackSpace', repeat: 1 } : { action: 'type_text', text };
+  return [[{ action: 'set_value', element_index: index, value: text }], [onElement, { action: 'press_key', key: selectAll, repeat: 1 }, typed]];
+}
+
+export type Verdict =
+  | { readonly verdict: 'done'; readonly verified: boolean }
+  | { readonly verdict: 'retry'; readonly why: string }
+  | { readonly verdict: 'stop'; readonly why: string };
+
+export interface Attempt {
+  readonly step: RunStep;
+  readonly result: ActionResult;
+  /** Whether the window looks different than before the attempt. */
+  readonly changed: boolean;
+  /** Jev's probability that `step.expect` holds now; absent when the step expects nothing. */
+  readonly expected?: number;
+  /** A click at a point changed the screenshot and no element: what it did is outside what Jev is shown. */
+  readonly unseen?: boolean;
+  /** A type step's text shows in its element now: typing it again would append it once more. */
+  readonly typed?: boolean;
+  /** Another window is in front than before the attempt. */
+  readonly moved?: boolean;
+}
+
+const UNSEEN = 'the click changed the screenshot but no element, so its result cannot be checked here and it was not clicked again (another click may undo it). Read the screenshot: when it shows the result, continue with the single tools by x and y';
+
+const LOADING = 'the click followed a link and the new page is still loading, so it was not clicked again (a second click can start the same thing twice). Look at the window again before the next action';
+
+/** What one attempt at a step means for the run. */
+export function judge({ step, result, changed, expected, unseen, typed, moved }: Attempt): Verdict {
+  if (result.outcome !== 'delivered') {
+    const why = `${result.outcome}${result.code ? ` (${result.code})` : ''}`;
+    return result.code && FINAL.has(result.code) ? { verdict: 'stop', why } : { verdict: 'retry', why };
+  }
+  // Doing these again would do them twice. A key that brought another window forward opened or switched one, which Jev,
+  // reading a single window, cannot tell from one that was there before.
+  if (typed || (moved && step.do === 'key')) return { verdict: 'done', verified: expected !== undefined && expected >= HOLDS };
+  if (expected !== undefined && expected >= HOLDS) return { verdict: 'done', verified: true };
+  if (result.code === 'page_loading') return { verdict: 'stop', why: LOADING };
+  if (expected !== undefined) {
+    if (unseen) return { verdict: 'stop', why: UNSEEN };
+    if (expected <= FAILS || !changed) return { verdict: 'retry', why: 'the expected result does not show' };
+    return { verdict: 'done', verified: false };
+  }
+  // A key or a scroll can be right and still change nothing visible; a click or a value cannot.
+  if (!changed && step.do !== 'key' && step.do !== 'scroll') return { verdict: 'retry', why: 'nothing changed' };
+  return { verdict: 'done', verified: false };
+}

package/src/jev/memory.ts

@@ -0,0 +1,20 @@
+import { fileURLToPath } from 'node:url';
+import { RunMemory as SharedMemory, describeRoutes as describeShared, promote as promoteShared, type AppMemory } from '@moxxy/jev';
+import { moxxyPath } from '@moxxy/sdk/server';
+import { runStepShape } from '../contract/tools.js';
+
+export { guess, labelOf, recall, sameWords, targetOf, type AppMemory, type LearnedRoute, type LearnedTarget, type Lesson } from '@moxxy/jev';
+
+/** What the moxxy team learned and ships with the plugin, so a new install does not start from nothing. */
+export const shippedLearned = fileURLToPath(new URL('../../learned', import.meta.url));
+
+/** Computer Use's lessons: one file per app under `~/.moxxy/computer-use/learned`, routes of computer_run steps. */
+export class RunMemory extends SharedMemory {
+  constructor(directory: string = moxxyPath('computer-use', 'learned'), now: () => number = Date.now, shipped?: string) {
+    super(directory, now, shipped, runStepShape);
+  }
+}
+
+export const describeRoutes = (memory: AppMemory): string | undefined => describeShared(memory, 'computer_run');
+
+export const promote = (from: string, to: string): Promise<string[]> => promoteShared(from, to, runStepShape);