@deeeed/metamask-harness 0.32.0 → 0.33.0

package/CHANGELOG.md

@@ -2,6 +2,16 @@
 
 ## Unreleased
 
+## 0.33.0 - 2026-08-03
+
+### Added
+
+- Expose standard full-surface capture and continuous UI gestures for Mobile and Extension by delegating to the shared native and browser transports.
+
+### Changed
+
+- Upgrade `@farmslot/agent-runtime` to 0.7.0, `@farmslot/protocol` to 0.18.0, and `@farmslot/recipe-harness` to 0.14.0, and add `@farmslot/expo-recipe` 0.7.1 with `agent-device` 0.19.3 for the native UI provider.
+
 ## 0.32.0 - 2026-08-02
 
 ### Changed

package/dist/adapters.js

@@ -6,6 +6,13 @@ import {
 import { withExtensionPage } from "../library/actions/extension/platform/cdp.mjs";
 import { bridgeCommand, evalSync, MOBILE_BRIDGE_ERROR_CODES, selectBridgeStatusEntry, simulatorScreenshot } from "../library/actions/mobile/platform/bridge.mjs";
 import { observeNativeUi } from "../library/actions/mobile/platform/observe-ui.mjs";
+const NATIVE_PROVIDER_UI_ACTIONS = /* @__PURE__ */ new Set([
+  "ui.swipe",
+  "ui.pan",
+  "ui.drag",
+  "ui.long_press",
+  "ui.capture_surface"
+]);
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
@@ -480,7 +487,7 @@ function createMetaMaskUiTransport(platform, harness, preparedLiveAdapters) {
     return {
       async execute(action) {
         throw new Error(
-          `core adapter is headless and cannot execute UI action ${action}. Use a perps read action or run the recipe against the mobile/extension adapter.`
+          `core adapter is headless and cannot execute UI action ${action}. Next: mm-harness actions --matrix`
         );
       }
     };
@@ -516,10 +523,75 @@ function createMetaMaskUiTransport(platform, harness, preparedLiveAdapters) {
         if (live) return live.output;
         throw new Error(`ui.navigate requires library/actions/${platform}/ui/navigate.mjs.`);
       }
+      if (platform === "mobile" && NATIVE_PROVIDER_UI_ACTIONS.has(action)) {
+        return executeNativeProviderAction(action, node, context, harness.createNativeUiTransport);
+      }
       return base.execute(action, action === "ui.wait_for" ? normalizeUiWaitNode(node) : node, context);
     }
   };
 }
+async function executeNativeProviderAction(action, node, context, createTransport) {
+  if (!createTransport) {
+    throw new Error(
+      `${action} requires the MetaMask native UI provider. Next: npm install -g @deeeed/metamask-harness@latest`
+    );
+  }
+  const platform = mobileNativePlatform(context.env);
+  const device = platform === "android" ? context.env.ADB_SERIAL ?? context.env.ANDROID_SERIAL ?? process.env.ADB_SERIAL ?? process.env.ANDROID_SERIAL : context.env.SIM_UDID ?? context.env.IOS_SIMULATOR ?? process.env.SIM_UDID ?? process.env.IOS_SIMULATOR;
+  if (!device) {
+    throw new Error(
+      `${action} requires one selected ${platform} device. Next: mm-harness status --adapter mobile --all-devices --json`
+    );
+  }
+  const app = platform === "android" ? context.env.ANDROID_PACKAGE_ID ?? process.env.ANDROID_PACKAGE_ID : context.env.IOS_BUNDLE_ID ?? process.env.IOS_BUNDLE_ID;
+  if (!app) {
+    const variable = platform === "android" ? "ANDROID_PACKAGE_ID" : "IOS_BUNDLE_ID";
+    throw new Error(
+      `${action} requires the selected MetaMask app id. Next: export ${variable}=<installed-app-id>`
+    );
+  }
+  const session = `mm-harness-${process.pid}-${context.nodeId}`.replace(/[^a-zA-Z0-9._-]/gu, "-");
+  let transport;
+  try {
+    transport = await createTransport({ platform, device, app, session });
+  } catch (error) {
+    throw new Error(
+      `${action} could not start the MetaMask native UI provider. Next: npm install -g @deeeed/metamask-harness@latest`,
+      { cause: error }
+    );
+  }
+  let executionError;
+  let output;
+  try {
+    output = await transport.execute(action, node, context);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    executionError = /^Native recipe actions require the optional agent-device provider:/u.test(message) ? new Error(
+      `${action} could not start the MetaMask native UI provider. Next: npm install -g @deeeed/metamask-harness@latest`,
+      { cause: error }
+    ) : error;
+  }
+  let closeError;
+  try {
+    await transport.close?.();
+  } catch (error) {
+    closeError = error;
+  }
+  if (executionError && closeError) {
+    const message = executionError instanceof Error ? executionError.message : String(executionError);
+    throw new AggregateError([executionError, closeError], message);
+  }
+  if (executionError) throw executionError;
+  if (closeError) throw closeError;
+  return output;
+}
+function mobileNativePlatform(env) {
+  if (env.PLATFORM === "android" || env.PLATFORM === "ios") return env.PLATFORM;
+  if (env.ADB_SERIAL || env.ANDROID_SERIAL) return "android";
+  if (env.SIM_UDID || env.IOS_SIMULATOR) return "ios";
+  if (process.env.PLATFORM === "android" || process.env.PLATFORM === "ios") return process.env.PLATFORM;
+  return process.env.ADB_SERIAL || process.env.ANDROID_SERIAL ? "android" : "ios";
+}
 function normalizeUiWaitNode(node) {
   if (node.expected !== void 0 || typeof node.visible !== "boolean") return node;
   return { ...node, expected: node.visible ? "visible" : "absent" };

package/dist/paths.js

@@ -128,6 +128,12 @@ async function importRecipeHarnessRuntimeReactNativeBridge() {
     "packages/recipe-harness/src/runtime/react-native-bridge.ts"
   );
 }
+async function importMetaMaskNativeUiProvider() {
+  return importProtocolPackage(
+    "@farmslot/expo-recipe",
+    "packages/expo-recipe/src/index.ts"
+  );
+}
 async function importRecipeHarnessCli() {
   return importProtocolPackage(
     "@farmslot/recipe-harness/cli",
@@ -164,6 +170,7 @@ export {
   DEFAULT_RECIPE_RUNTIME_DIR,
   assertAdapter,
   extensionIdPath,
+  importMetaMaskNativeUiProvider,
   importRecipeHarness,
   importRecipeHarnessAppLifecycle,
   importRecipeHarnessCli,

package/dist/runner.js

@@ -12,7 +12,14 @@ import {
   prepareLiveAdapterScript
 } from "./live-adapter-contract.js";
 import { createMetaMaskRecordingTargetProvider } from "./recording-target.js";
-import { importRecipeHarness, importRecipeHarnessAppLifecycle, importRecipeHarnessRuntimeCdp, importRecipeHarnessRuntimeReactNativeBridge, runnerDir } from "./paths.js";
+import {
+  importMetaMaskNativeUiProvider,
+  importRecipeHarness,
+  importRecipeHarnessAppLifecycle,
+  importRecipeHarnessRuntimeCdp,
+  importRecipeHarnessRuntimeReactNativeBridge,
+  runnerDir
+} from "./paths.js";
 const OFFICIAL_ACTIONS = new Set(OFFICIAL_RECIPE_ACTIONS);
 async function createMetaMaskMobileRunner(options = {}) {
   return createMetaMaskRunner(
@@ -51,7 +58,11 @@ async function createMetaMaskRunner(adapter, actionManifest, options = {}) {
     actions: actions.filter((action) => !projectOwnedOfficialActions.has(action)),
     transport: createMetaMaskUiTransport(adapter, {
       createCdpWebUiTransport,
-      createReactNativeBridgeUiTransport
+      createReactNativeBridgeUiTransport,
+      async createNativeUiTransport(options2) {
+        const { createAgentDeviceUiTransport } = await importMetaMaskNativeUiProvider();
+        return createAgentDeviceUiTransport(options2);
+      }
     }, preparedLiveAdapters)
   });
   const existing = new Set([...core, ...ui].map((entry) => entry.action));

package/library/manifests/extension.action-manifest.json

@@ -664,6 +664,183 @@
         "additionalProperties": false
       }
     },
+    "ui.swipe": {
+      "description": "Swipe from a test-id target or screen coordinates in one cardinal direction through the shared browser UI transport.",
+      "schema": {
+        "type": "object",
+        "properties": {
+          "target": {
+            "type": ["string", "object"],
+            "properties": {
+              "x": { "type": "number" },
+              "y": { "type": "number" }
+            },
+            "required": ["x", "y"],
+            "additionalProperties": false
+          },
+          "direction": {
+            "type": "string",
+            "enum": ["up", "down", "left", "right"]
+          },
+          "distance": { "type": "number", "minimum": 1 },
+          "duration_ms": { "type": "integer", "minimum": 1 },
+          "timeout_ms": { "type": "integer", "minimum": 1 },
+          "settle": { "type": "boolean" }
+        },
+        "required": ["target", "direction", "distance", "duration_ms"],
+        "additionalProperties": false
+      },
+      "examples": [
+        {
+          "action": "ui.swipe",
+          "target": "example-scrollable",
+          "direction": "up",
+          "distance": 240,
+          "duration_ms": 300,
+          "intent": "Swipe the requested visible surface.",
+          "next": "done"
+        }
+      ]
+    },
+    "ui.pan": {
+      "description": "Pan through a continuous pointer path from a test-id target or screen coordinates through the shared browser UI transport.",
+      "schema": {
+        "type": "object",
+        "properties": {
+          "target": {
+            "type": ["string", "object"],
+            "properties": {
+              "x": { "type": "number" },
+              "y": { "type": "number" }
+            },
+            "required": ["x", "y"],
+            "additionalProperties": false
+          },
+          "path": {
+            "type": "array",
+            "minItems": 1,
+            "items": {
+              "type": "object",
+              "properties": {
+                "x": { "type": "number" },
+                "y": { "type": "number" }
+              },
+              "required": ["x", "y"],
+              "additionalProperties": false
+            }
+          },
+          "delta": {
+            "type": "object",
+            "properties": {
+              "x": { "type": "number" },
+              "y": { "type": "number" }
+            },
+            "required": ["x", "y"],
+            "additionalProperties": false
+          },
+          "duration_ms": { "type": "integer", "minimum": 1 },
+          "timeout_ms": { "type": "integer", "minimum": 1 },
+          "settle": { "type": "boolean" }
+        },
+        "required": ["target", "duration_ms"],
+        "additionalProperties": false
+      },
+      "examples": [
+        {
+          "action": "ui.pan",
+          "target": "example-surface",
+          "path": [{ "x": 40, "y": 10 }, { "x": 80, "y": 20 }],
+          "duration_ms": 400,
+          "intent": "Pan through the requested pointer path.",
+          "next": "done"
+        }
+      ]
+    },
+    "ui.drag": {
+      "description": "Drag a test-id target or screen coordinate through the shared browser UI transport.",
+      "schema": {
+        "type": "object",
+        "properties": {
+          "target": {
+            "type": ["string", "object"],
+            "properties": {
+              "x": { "type": "number" },
+              "y": { "type": "number" }
+            },
+            "required": ["x", "y"],
+            "additionalProperties": false
+          },
+          "path": {
+            "type": "array",
+            "minItems": 1,
+            "items": {
+              "type": "object",
+              "properties": {
+                "x": { "type": "number" },
+                "y": { "type": "number" }
+              },
+              "required": ["x", "y"],
+              "additionalProperties": false
+            }
+          },
+          "delta": {
+            "type": "object",
+            "properties": {
+              "x": { "type": "number" },
+              "y": { "type": "number" }
+            },
+            "required": ["x", "y"],
+            "additionalProperties": false
+          },
+          "duration_ms": { "type": "integer", "minimum": 1 },
+          "timeout_ms": { "type": "integer", "minimum": 1 },
+          "settle": { "type": "boolean" }
+        },
+        "required": ["target", "duration_ms"],
+        "additionalProperties": false
+      },
+      "examples": [
+        {
+          "action": "ui.drag",
+          "target": "example-slider",
+          "delta": { "x": 120, "y": 0 },
+          "duration_ms": 500,
+          "intent": "Drag the requested control to a new value.",
+          "next": "done"
+        }
+      ]
+    },
+    "ui.long_press": {
+      "description": "Hold a test-id target or screen coordinate through the shared browser UI transport.",
+      "schema": {
+        "type": "object",
+        "properties": {
+          "target": {
+            "type": ["string", "object"],
+            "properties": {
+              "x": { "type": "number" },
+              "y": { "type": "number" }
+            },
+            "required": ["x", "y"],
+            "additionalProperties": false
+          },
+          "hold_ms": { "type": "integer", "minimum": 1 },
+          "timeout_ms": { "type": "integer", "minimum": 1 },
+          "settle": { "type": "boolean" }
+        },
+        "required": ["target", "hold_ms"],
+        "additionalProperties": false
+      },
+      "examples": [
+        {
+          "action": "ui.long_press",
+          "target": "example-control",
+          "hold_ms": 700,
+          "intent": "Hold the requested visible control.",
+          "next": "done"
+        }
+      ]
+    },
     "ui.wait_for": {
       "description": "Wait for a UI condition through the Extension CDP adapter.",
       "schema": {
@@ -763,6 +940,27 @@
         "additionalProperties": false
       }
     },
+    "ui.capture_surface": {
+      "description": "Capture the complete current browser page as one registered PNG while leaving ui.screenshot viewport-only.",
+      "schema": {
+        "type": "object",
+        "properties": {
+          "path": { "type": "string" },
+          "label": { "type": "string" },
+          "category": { "type": "string" },
+          "timeout_ms": { "type": "number" }
+        },
+        "additionalProperties": false
+      },
+      "examples": [
+        {
+          "action": "ui.capture_surface",
+          "path": "screenshots/full-page.png",
+          "intent": "Preserve the complete page context for human review.",
+          "next": "done"
+        }
+      ]
+    },
     "app.status": {
       "description": "Report the adapter's static status — platform, project root, resolved checkout shape, and headless compatibility mode (no live route or account).",
       "examples": [
@@ -2982,8 +3180,13 @@
         "ui.key_press",
         "ui.set_input",
         "ui.scroll",
+        "ui.swipe",
+        "ui.pan",
+        "ui.drag",
+        "ui.long_press",
         "ui.wait_for",
-        "ui.screenshot"
+        "ui.screenshot",
+        "ui.capture_surface"
       ]
     },
     {
@@ -2995,8 +3198,13 @@
         "ui.key_press",
         "ui.set_input",
         "ui.scroll",
+        "ui.swipe",
+        "ui.pan",
+        "ui.drag",
+        "ui.long_press",
         "ui.wait_for",
-        "ui.screenshot"
+        "ui.screenshot",
+        "ui.capture_surface"
       ]
     }
   ]

package/library/manifests/mobile.action-manifest.json

@@ -675,6 +675,185 @@
         "additionalProperties": false
       }
     },
+    "ui.swipe": {
+      "description": "Swipe from a test-id target or screen coordinates in one cardinal direction through the shared native UI provider.",
+      "schema": {
+        "type": "object",
+        "properties": {
+          "target": {
+            "type": ["string", "object"],
+            "properties": {
+              "x": { "type": "number" },
+              "y": { "type": "number" }
+            },
+            "required": ["x", "y"],
+            "additionalProperties": false
+          },
+          "direction": {
+            "type": "string",
+            "enum": ["up", "down", "left", "right"]
+          },
+          "distance": { "type": "number", "minimum": 1 },
+          "duration_ms": { "type": "integer", "minimum": 1 },
+          "timeout_ms": { "type": "integer", "minimum": 1 },
+          "settle": { "type": "boolean" }
+        },
+        "required": ["target", "direction", "distance", "duration_ms"],
+        "additionalProperties": false
+      },
+      "examples": [
+        {
+          "action": "ui.swipe",
+          "target": "example-scrollable",
+          "direction": "up",
+          "distance": 240,
+          "duration_ms": 300,
+          "intent": "Swipe the requested visible surface.",
+          "next": "done"
+        }
+      ]
+    },
+    "ui.pan": {
+      "description": "Pan from a test-id target or screen coordinates through the shared native UI provider.",
+      "schema": {
+        "type": "object",
+        "properties": {
+          "target": {
+            "type": ["string", "object"],
+            "properties": {
+              "x": { "type": "number" },
+              "y": { "type": "number" }
+            },
+            "required": ["x", "y"],
+            "additionalProperties": false
+          },
+          "path": {
+            "type": "array",
+            "minItems": 1,
+            "maxItems": 1,
+            "items": {
+              "type": "object",
+              "properties": {
+                "x": { "type": "number" },
+                "y": { "type": "number" }
+              },
+              "required": ["x", "y"],
+              "additionalProperties": false
+            }
+          },
+          "delta": {
+            "type": "object",
+            "properties": {
+              "x": { "type": "number" },
+              "y": { "type": "number" }
+            },
+            "required": ["x", "y"],
+            "additionalProperties": false
+          },
+          "duration_ms": { "type": "integer", "minimum": 1 },
+          "timeout_ms": { "type": "integer", "minimum": 1 },
+          "settle": { "type": "boolean" }
+        },
+        "required": ["target", "duration_ms"],
+        "additionalProperties": false
+      },
+      "examples": [
+        {
+          "action": "ui.pan",
+          "target": "example-surface",
+          "path": [{ "x": 80, "y": 20 }],
+          "duration_ms": 400,
+          "intent": "Pan through the requested pointer path.",
+          "next": "done"
+        }
+      ]
+    },
+    "ui.drag": {
+      "description": "Drag a test-id target or screen coordinate through the shared native UI provider.",
+      "schema": {
+        "type": "object",
+        "properties": {
+          "target": {
+            "type": ["string", "object"],
+            "properties": {
+              "x": { "type": "number" },
+              "y": { "type": "number" }
+            },
+            "required": ["x", "y"],
+            "additionalProperties": false
+          },
+          "path": {
+            "type": "array",
+            "minItems": 1,
+            "maxItems": 1,
+            "items": {
+              "type": "object",
+              "properties": {
+                "x": { "type": "number" },
+                "y": { "type": "number" }
+              },
+              "required": ["x", "y"],
+              "additionalProperties": false
+            }
+          },
+          "delta": {
+            "type": "object",
+            "properties": {
+              "x": { "type": "number" },
+              "y": { "type": "number" }
+            },
+            "required": ["x", "y"],
+            "additionalProperties": false
+          },
+          "duration_ms": { "type": "integer", "minimum": 1 },
+          "timeout_ms": { "type": "integer", "minimum": 1 },
+          "settle": { "type": "boolean" }
+        },
+        "required": ["target", "duration_ms"],
+        "additionalProperties": false
+      },
+      "examples": [
+        {
+          "action": "ui.drag",
+          "target": "example-slider",
+          "delta": { "x": 120, "y": 0 },
+          "duration_ms": 500,
+          "intent": "Drag the requested control to a new value.",
+          "next": "done"
+        }
+      ]
+    },
+    "ui.long_press": {
+      "description": "Hold a test-id target or screen coordinate through the shared native UI provider.",
+      "schema": {
+        "type": "object",
+        "properties": {
+          "target": {
+            "type": ["string", "object"],
+            "properties": {
+              "x": { "type": "number" },
+              "y": { "type": "number" }
+            },
+            "required": ["x", "y"],
+            "additionalProperties": false
+          },
+          "hold_ms": { "type": "integer", "minimum": 1 },
+          "timeout_ms": { "type": "integer", "minimum": 1 },
+          "settle": { "type": "boolean" }
+        },
+        "required": ["target", "hold_ms"],
+        "additionalProperties": false
+      },
+      "examples": [
+        {
+          "action": "ui.long_press",
+          "target": "example-control",
+          "hold_ms": 700,
+          "intent": "Hold the requested visible control.",
+          "next": "done"
+        }
+      ]
+    },
     "ui.wait_for": {
       "description": "Wait for a mounted React Native testID/test_id. Optional text validates text returned by that testID; text alone is not supported on Mobile.",
       "schema": {
@@ -781,6 +960,31 @@
         "additionalProperties": false
       }
     },
+    "ui.capture_surface": {
+      "description": "Capture the complete current native scroll surface as one registered PNG while leaving ui.screenshot viewport-only.",
+      "schema": {
+        "type": "object",
+        "properties": {
+          "path": { "type": "string" },
+          "label": { "type": "string" },
+          "surface_test_id": { "type": "string" },
+          "until_test_id": { "type": "string" },
+          "max_scrolls": { "type": "integer", "minimum": 1 },
+          "restore_position": { "type": "boolean" }
+        },
+        "additionalProperties": false
+      },
+      "examples": [
+        {
+          "action": "ui.capture_surface",
+          "path": "screenshots/full-screen.png",
+          "surface_test_id": "screen-scroll-surface",
+          "until_test_id": "screen-bottom",
+          "intent": "Preserve the complete scrollable context for human review.",
+          "next": "done"
+        }
+      ]
+    },
     "app.lifecycle": {
       "description": "Drive outer mobile app lifecycle for deterministic performance start states without rebuilding: foreground/launch, background, terminate, or restart the installed app.",
       "examples": [
@@ -3101,7 +3305,12 @@
         "metamask.deeplink.open",
         "ui.navigate",
         "ui.press",
+        "ui.swipe",
+        "ui.pan",
+        "ui.drag",
+        "ui.long_press",
         "ui.screenshot",
+        "ui.capture_surface",
         "ui.scroll",
         "ui.set_input",
         "ui.wait_for"
@@ -3113,7 +3322,12 @@
         "metamask.deeplink.open",
         "ui.navigate",
         "ui.press",
+        "ui.swipe",
+        "ui.pan",
+        "ui.drag",
+        "ui.long_press",
         "ui.screenshot",
+        "ui.capture_surface",
         "ui.scroll",
         "ui.set_input",
         "ui.wait_for"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@deeeed/metamask-harness",
-  "version": "0.32.0",
+  "version": "0.33.0",
   "type": "module",
   "bin": {
     "mm-harness": "bin/mm-harness"
@@ -20,10 +20,12 @@
     "check:syntax": "find . -name '*.mjs' -print0 | xargs -0 -n1 node --check"
   },
   "dependencies": {
-    "@farmslot/agent-runtime": "0.5.1",
+    "@farmslot/agent-runtime": "0.7.0",
+    "@farmslot/expo-recipe": "0.7.1",
     "@farmslot/handoff": "^0.3.1",
-    "@farmslot/protocol": "0.16.0",
-    "@farmslot/recipe-harness": "0.13.0",
+    "@farmslot/protocol": "0.18.0",
+    "@farmslot/recipe-harness": "0.14.0",
+    "agent-device": "0.19.3",
     "commander": "^12.0.0",
     "es-module-lexer": "2.3.1",
     "esbuild": "0.28.1",