@deeeed/metamask-harness 0.24.0 → 0.25.0

package/CHANGELOG.md

@@ -2,6 +2,31 @@
 
 ## Unreleased
 
+## 0.25.0 - 2026-07-30
+
+### Added
+
+- Extension and Mobile recipes can start a local analytics collector, set MetaMetrics consent through each app's native state boundary, and assert captured event cardinality and properties.
+- Extension UI actions record default screen and visibility observations in recipe traces.
+
+### Changed
+
+- Updated to Recipe Protocol 0.14 and `@farmslot/recipe-harness` 0.10.3 for trustworthy observation, artifact, and hardened-page CDP behavior.
+
+### Fixed
+
+- Analytics capture rejects foreign collectors, replaces its own previous listener safely, and reverses the collector port to selected Android devices.
+- Analytics captures honor the selected runtime directory and store telemetry in private, symlink-safe files.
+- Analytics collector startup executes only plan-bound bundled bytes; opt-out consent defaults, typed property assertions, and long settle windows retain their declared semantics.
+- Extension analytics consent setup calls the background controller over a temporary Extension port without navigating through unrelated Settings UI.
+- Recipe evidence screenshots and recordings publish atomically with private permissions and reject symlink destinations.
+- Explicit Extension video recording waits for a captured frame and fails the run when the recorder cannot publish a valid MP4 instead of reporting a proof without its requested evidence.
+- Extension fixture seeding evaluates through raw CDP so LavaMoat scuttling cannot break live launch setup.
+- Extension launch output prints a runnable `runtime-launch` recovery command, including a selected Chrome profile, instead of suggesting unsupported or incomplete `launch` flags.
+- `runtime-launch` distinguishes launch failures from post-launch verification failures and points at the failing evidence.
+- Source-checkout dependency bootstrap verifies every runtime dependency and its declared version before declaring the harness ready, while preserving explicit local protocol package links.
+- Source checkouts run newer TypeScript sources instead of silently executing an older compiled `dist/`.
+
 ## 0.24.0 - 2026-07-30
 
 ### Added

package/adapters/extension/live.sh

@@ -279,10 +279,10 @@ if [ -n "$REMOTE_FLAGS" ] && [ "$REMOTE_FLAGS_APPLIED" != "true" ]; then
 fi
 
 echo "Extension live validation command:"
-display_args=(mm-harness launch --verify --target "$TARGET")
-$LAUNCH_EXISTING_DIST && display_args+=(--launch-existing-dist)
+display_args=(mm-harness runtime-launch --adapter extension --target "$TARGET" --cdp-port "$CDP_PORT")
 $START_WATCH && display_args+=(--start-watch)
 [ -n "$REMOTE_FLAGS" ] && display_args+=(--remote-flag "$REMOTE_FLAGS")
+[ -n "$CHROME_USER_DATA_DIR" ] && display_args+=(--chrome-user-data-dir "$CHROME_USER_DATA_DIR")
 printf '  '
 printf '%q ' "${display_args[@]}"
 printf '\n'

package/adapters/extension/wallet-fixture-state.cjs

@@ -708,11 +708,10 @@ async function waitForWalletScreen(page) {
   const unlockSelector = '[data-testid="unlock-password"]';
   const deadline = Date.now() + 45000;
   while (Date.now() < deadline) {
-    const fatalStartup = await page
-      .evaluate(() => {
-        const text = document.body?.innerText || '';
-        return text.includes('MetaMask had trouble starting') ? text.slice(0, 500) : null;
-      })
+    const fatalStartup = await evaluateViaCdp(page, () => {
+      const text = document.body?.innerText || '';
+      return text.includes('MetaMask had trouble starting') ? text.slice(0, 500) : null;
+    })
       .catch(() => null);
     if (fatalStartup) {
       return { state: 'fatal', selector: null, detail: fatalStartup };
@@ -733,24 +732,51 @@ async function waitForWalletScreen(page) {
   return { state: 'unknown', selector: null };
 }
 
+async function evaluateViaCdp(page, callback, argument) {
+  const session = await page.context().newCDPSession(page);
+  const invocation = argument === undefined
+    ? `(${callback.toString()})()`
+    : `(${callback.toString()})(${JSON.stringify(argument)})`;
+  try {
+    const response = await session.send('Runtime.evaluate', {
+      expression: invocation,
+      awaitPromise: true,
+      returnByValue: true,
+    });
+    if (response.exceptionDetails) {
+      const detail =
+        response.exceptionDetails.exception?.description ??
+        response.exceptionDetails.text ??
+        'unknown evaluation failure';
+      throw new Error(`CDP evaluation failed: ${detail}`);
+    }
+    return response.result?.value;
+  } finally {
+    await session.detach().catch(() => {});
+  }
+}
+
 async function attemptUnlock(page, password) {
   // Selector ladder: current testids first, then the generic selectors the CDP
   // unlock action uses (proven against the same build). A freshly seeded vault
   // can also still be initializing, so a single attempt is not conclusive —
   // the caller retries.
-  const filled = await page.evaluate((pw) => {
+  const filled = await evaluateViaCdp(page, (pw) => {
     const input =
       document.querySelector('[data-testid="unlock-password"]') ??
       document.querySelector('input[type="password"]');
     if (!input) return false;
-    const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value').set;
+    const setter = Object.getOwnPropertyDescriptor(Object.getPrototypeOf(input), 'value').set;
     setter.call(input, pw);
-    input.dispatchEvent(new Event('input', { bubbles: true }));
-    input.dispatchEvent(new Event('change', { bubbles: true }));
+    for (const type of ['input', 'change']) {
+      const event = input.ownerDocument.createEvent('Event');
+      event.initEvent(type, true, true);
+      input.dispatchEvent(event);
+    }
     return true;
   }, password);
   if (!filled) return false;
-  return page.evaluate(() => {
+  return evaluateViaCdp(page, () => {
     const button =
       document.querySelector('[data-testid="unlock-submit"]') ??
       document.querySelector('button[type="submit"]') ??
@@ -781,8 +807,8 @@ async function unlockIfNeeded(page, password) {
 }
 
 async function readLiveAccounts(page) {
-  const raw = await page.evaluate(() => {
-    const metamask = (window.stateHooks?.store?.getState?.() || {}).metamask || {};
+  const raw = await evaluateViaCdp(page, async () => {
+    const metamask = (await window.stateHooks?.getCleanAppState?.())?.metamask || {};
     const accts = metamask.internalAccounts || {};
     const byId = accts.accounts || {};
     const selectedId = accts.selectedAccount || null;
@@ -905,9 +931,10 @@ async function applyAccountNames(page, expectedAccounts) {
     if (!account.name || !account.address.startsWith('0x')) {
       continue;
     }
-    await page.evaluate(
+    await evaluateViaCdp(
+      page,
       async ({ address, name }) => {
-        const metamask = window.stateHooks?.store?.getState?.()?.metamask || {};
+        const metamask = (await window.stateHooks?.getCleanAppState?.())?.metamask || {};
         const normalizedAddress = String(address || '').toLowerCase();
         const accountsById = metamask.internalAccounts?.accounts || {};
         const accountId = Object.keys(accountsById).find(
@@ -937,9 +964,11 @@ async function applySelectedAccount(page, expectedSelected) {
   if (!expectedSelected?.address) {
     return;
   }
-  await page.evaluate(
+  await evaluateViaCdp(
+    page,
     async ({ address }) => {
-      const accounts = window.stateHooks?.store?.getState?.()?.metamask?.internalAccounts?.accounts || {};
+      const accounts =
+        (await window.stateHooks?.getCleanAppState?.())?.metamask?.internalAccounts?.accounts || {};
       const accountId = Object.keys(accounts).find(
         (id) => String(accounts[id]?.address || '').toLowerCase() === String(address || '').toLowerCase(),
       );
@@ -1009,9 +1038,18 @@ async function seedCdp(args) {
     fs.writeFileSync(extensionIdFile, `${extensionId}\n`);
   }
 
-  await page.evaluate(async (state) => {
-    await chrome.storage.local.set(state);
-  }, versionedState);
+  try {
+    await evaluateViaCdp(page, async (state) => {
+      await chrome.storage.local.set(state);
+    }, versionedState);
+  } catch (error) {
+    if (!String(error?.message || '').includes('property "chrome" of globalThis is inaccessible under scuttling mode')) {
+      throw error;
+    }
+    console.error(
+      '[fixture] chrome.storage is scuttled in the page realm; validating the prefilled profile state instead.',
+    );
+  }
   await page.goto(`chrome-extension://${extensionId}/home.html`, {
     waitUntil: 'load',
     timeout: 30000,

package/adapters/shared/ensure-runner-deps.sh

@@ -17,8 +17,65 @@ if [ -z "$RUNNER_DIR" ]; then
 fi
 RUNNER_DIR="$(cd "$RUNNER_DIR" && pwd -P)"
 
+PROTOCOL_ROOT="${METAMASK_RUNNER_PROTOCOL_ROOT:-${METAMASK_RUNNER_FARMSLOT_ROOT:-${FARMSLOT_ROOT:-}}}"
+if [ -z "$PROTOCOL_ROOT" ] && [ -f "$RUNNER_DIR/.farmslot-root" ]; then
+  PROTOCOL_ROOT="$(cat "$RUNNER_DIR/.farmslot-root")"
+fi
+
 deps_ready() {
-  [ -f "$RUNNER_DIR/node_modules/@farmslot/recipe-harness/package.json" ]
+  node --input-type=module - "$RUNNER_DIR" "$PROTOCOL_ROOT" <<'NODE'
+import fs from 'node:fs';
+import { createRequire } from 'node:module';
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+const runnerDir = process.argv[2];
+const overrideRootInput = process.argv[3];
+const pkg = JSON.parse(fs.readFileSync(path.join(runnerDir, 'package.json'), 'utf8'));
+let dependencyVersionSatisfies;
+try {
+  const requireFromRunner = createRequire(path.join(runnerDir, 'package.json'));
+  const helperPath = requireFromRunner.resolve(
+    '@farmslot/recipe-harness/runtime/deps-readiness',
+  );
+  ({ dependencyVersionSatisfies } = await import(pathToFileURL(helperPath).href));
+  if (typeof dependencyVersionSatisfies !== 'function') process.exit(1);
+} catch {
+  process.exit(1);
+}
+let overrideRoot = null;
+if (overrideRootInput) {
+  try {
+    overrideRoot = fs.realpathSync(overrideRootInput);
+  } catch {
+    overrideRoot = null;
+  }
+}
+function isOverrideLink(packageDir) {
+  if (!overrideRoot) return false;
+  try {
+    if (!fs.lstatSync(packageDir).isSymbolicLink()) return false;
+    const resolved = fs.realpathSync(packageDir);
+    const relative = path.relative(overrideRoot, resolved);
+    return relative === '' || (
+      relative !== '..' &&
+      !relative.startsWith(`..${path.sep}`) &&
+      !path.isAbsolute(relative)
+    );
+  } catch {
+    return false;
+  }
+}
+const invalid = Object.entries(pkg.dependencies ?? {}).filter(([name, request]) => {
+  const packageDir = path.join(runnerDir, 'node_modules', ...name.split('/'));
+  const installedPath = path.join(packageDir, 'package.json');
+  if (!fs.existsSync(installedPath)) return true;
+  if (isOverrideLink(packageDir)) return false;
+  const installed = JSON.parse(fs.readFileSync(installedPath, 'utf8'));
+  return !dependencyVersionSatisfies(installed.version, request);
+});
+process.exit(invalid.length === 0 ? 0 : 1);
+NODE
 }
 
 # Library actions import @deeeed/metamask-harness by package name; a source checkout
@@ -55,10 +112,6 @@ if [ ! -f "$RUNNER_DIR/package.json" ]; then
   exit 1
 fi
 
-PROTOCOL_ROOT="${METAMASK_RUNNER_PROTOCOL_ROOT:-${METAMASK_RUNNER_FARMSLOT_ROOT:-${FARMSLOT_ROOT:-}}}"
-if [ -z "$PROTOCOL_ROOT" ] && [ -f "$RUNNER_DIR/.farmslot-root" ]; then
-  PROTOCOL_ROOT="$(cat "$RUNNER_DIR/.farmslot-root")"
-fi
 if [ -n "$PROTOCOL_ROOT" ] && [ -f "$RUNNER_DIR/scripts/link-local-farmslot.mjs" ]; then
   if [ -f "$PROTOCOL_ROOT/packages/recipe-harness/package.json" ]; then
     FARMSLOT_ROOT="$(cd "$PROTOCOL_ROOT" && pwd -P)" \
@@ -77,6 +130,12 @@ echo "mm-harness: installing runner dependencies via npm in $RUNNER_DIR" >&2
   npm install --ignore-scripts >&2
 )
 
+# npm may replace the local protocol links with registry packages. Restore the
+# explicit co-development override after the complete dependency tree exists.
+if [ -n "$PROTOCOL_ROOT" ] && [ -f "$RUNNER_DIR/scripts/link-local-farmslot.mjs" ]; then
+  FARMSLOT_ROOT="$PROTOCOL_ROOT" node "$RUNNER_DIR/scripts/link-local-farmslot.mjs" >&2
+fi
+
 if ! deps_ready; then
   echo "mm-harness: runner dependencies still missing after npm install: $RUNNER_DIR" >&2
   exit 1

package/bin/mm-harness

@@ -128,20 +128,38 @@ node_can_run_source_typescript() {
   rm -rf "$probe_dir"
 }
 
-# In a source checkout dist wins when present — make that visible, and warn when
-# the compiled entry is older than the sources it shadows (a stale dist silently
-# runs old code while you edit src/).
+# Published installs use dist. A source checkout uses dist only while it is at
+# least as new as every TypeScript source it would shadow.
+USE_DIST=0
 if [ -f "$ENTRY_DIST" ] && [ -f "$ENTRY_TS" ]; then
+  case "$RUNNER_DIR/" in
+    */node_modules/*)
+      USE_DIST=1
+      export MM_HARNESS_RUN_MODE="dist"
+      ;;
+    *)
+      newer_source=""
+      while IFS= read -r newer_source; do
+        break
+      done < <(find "$RUNNER_DIR/src" -name '*.ts' -newer "$ENTRY_DIST" -print 2>/dev/null)
+      if [ -n "$newer_source" ]; then
+        export MM_HARNESS_RUN_MODE="src"
+        echo "mm-harness: dist/ is older than src/ — running the current source checkout." >&2
+        echo "  Next: npm run build to refresh dist/" >&2
+      else
+        USE_DIST=1
+        export MM_HARNESS_RUN_MODE="dist"
+      fi
+      ;;
+  esac
+elif [ -f "$ENTRY_DIST" ]; then
+  USE_DIST=1
   export MM_HARNESS_RUN_MODE="dist"
-  if find "$RUNNER_DIR/src" -name '*.ts' -newer "$ENTRY_DIST" 2>/dev/null | head -1 | grep -q .; then
-    echo "mm-harness: dist/ is OLDER than src/ — this run uses the stale compiled code." >&2
-    echo "  Next: npm run build   (or: trash dist/ to run straight from src via tsx)" >&2
-  fi
 elif [ -f "$ENTRY_TS" ]; then
   export MM_HARNESS_RUN_MODE="src"
 fi
 
-if [ -f "$ENTRY_DIST" ]; then
+if [ "$USE_DIST" -eq 1 ]; then
   exec node "$ENTRY_DIST" "$@"
 fi
 

package/dist/adapters/extension/runtime.js

@@ -146,16 +146,7 @@ async function checkExtensionRuntimeHealth(projectRoot, cdpPort, options = {}) {
     if (runtime.backgroundUnresponsive === true) {
       findings.push("Extension UI reports background connection unresponsive.");
     }
-    if (runtime.hasSubmitRequest !== true) {
-      findings.push("stateHooks.submitRequestToBackground is unavailable.");
-    }
-    if (runtime.hasStore !== true) {
-      findings.push("stateHooks.store is unavailable.");
-    }
-    if (runtime.hasPerpsStreamManager !== true) {
-      findings.push("stateHooks.getPerpsStreamManager is unavailable.");
-    }
-    if (runtime.backgroundProbeOk !== true) {
+    if (runtime.hasSubmitRequest === true && runtime.backgroundProbeOk !== true) {
       warnings.push(`Perps background read probe failed: ${runtime.backgroundProbeError ?? "unknown error"}.`);
     }
     const extensionId = safeExtensionId(target);

package/dist/adapters.js

@@ -496,7 +496,7 @@ function createMetaMaskUiTransport(platform, harness, preparedLiveAdapters) {
       );
     }
   });
-  return {
+  const transport = {
     async execute(action, node, context) {
       if (action === "ui.navigate") {
         const live = await runLiveFirst(
@@ -513,6 +513,10 @@ function createMetaMaskUiTransport(platform, harness, preparedLiveAdapters) {
       return base.execute(action, action === "ui.wait_for" ? normalizeUiWaitNode(node) : node, context);
     }
   };
+  if (base.observe) {
+    transport.observe = (refs, node, context) => base.observe(refs, node, context);
+  }
+  return transport;
 }
 function normalizeUiWaitNode(node) {
   if (node.expected !== void 0 || typeof node.visible !== "boolean") return node;

package/dist/commands/run-engine.js

@@ -64,24 +64,29 @@ async function runRecipe(adapter, recipe, artifactsDir, projectRoot, actionManif
       record: true,
       cdpPort: runtimeOptions.cdpPort
     }) : void 0;
+    let result;
     try {
-      let result;
+      result = await runner.run(runRequest);
+    } catch (error) {
       try {
-        result = await runner.run(runRequest);
-      } finally {
-        if (adapter === "mobile") {
-          const { hideMobileHudOnTeardown } = await import("../adapters.js");
-          await hideMobileHudOnTeardown(
-            projectRoot,
-            recipeRunEnv(adapter, runtimeOptions)
-          );
-        }
+        await stopRecipeRecording(recording);
+      } catch (recordingError) {
+        console.error(
+          `WARN: recipe and video recording both failed; preserving recipe failure: ${recordingError instanceof Error ? recordingError.message : String(recordingError)}`
+        );
       }
-      await stopRecipeRecording(recording, result);
-      return finishRunDiagnostics(diagnosticBaseline, result);
+      throw error;
     } finally {
-      await stopRecipeRecording(recording);
+      if (adapter === "mobile") {
+        const { hideMobileHudOnTeardown } = await import("../adapters.js");
+        await hideMobileHudOnTeardown(
+          projectRoot,
+          recipeRunEnv(adapter, runtimeOptions)
+        );
+      }
     }
+    await stopRecipeRecording(recording, result);
+    return finishRunDiagnostics(diagnosticBaseline, result);
   } finally {
     restoreRuntimeEnvironment();
   }

package/dist/commands/runtime-launch.js

@@ -89,6 +89,10 @@ async function handleRuntimeLaunch({ options }) {
     return 0;
   }
   const launchLogPath = path.join(artifactsDir, "launch", "logs", "launch.log");
+  const verifySummaryPath = path.join(artifactsDir, "verify", "summary.json");
+  const launchSucceeded = isRecord(summary) && isRecord(summary.launch) && summary.launch.exitCode === 0;
+  const failurePath = launchSucceeded && fs.existsSync(verifySummaryPath) ? verifySummaryPath : fs.existsSync(launchLogPath) ? launchLogPath : summaryPath;
+  const reason = launchSucceeded ? "runtime_verify_failed" : "runtime_launch_failed";
   const report = runtimeLaunchReport("fail", {
     adapter,
     target,
@@ -96,8 +100,9 @@ async function handleRuntimeLaunch({ options }) {
     artifactsDir,
     summaryPath: fs.existsSync(summaryPath) ? summaryPath : void 0,
     launchLogPath: fs.existsSync(launchLogPath) ? launchLogPath : void 0,
-    reason: "runtime_launch_failed",
-    fix: `Read ${fs.existsSync(launchLogPath) ? launchLogPath : summaryPath}, fix the first error, then rerun: ${runtimeLaunchCommand(target, cdpPort, chromeUserDataDir, startWatch)}`,
+    verifySummaryPath: fs.existsSync(verifySummaryPath) ? verifySummaryPath : void 0,
+    reason,
+    fix: `Read ${failurePath}, fix the first error, then rerun: ${runtimeLaunchCommand(target, cdpPort, chromeUserDataDir, startWatch)}`,
     command,
     exitCode: result.status ?? 1
   });

package/dist/live-adapter-contract.js

@@ -350,11 +350,13 @@ function liveAdapterProcessTimeoutMs(node) {
     const timeoutMs = Number(node.live_adapter_timeout_ms);
     return Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 6e4;
   }
+  const settleMs = Number(node.settle_ms);
+  const settleAllowance = Number.isFinite(settleMs) && settleMs > 0 ? settleMs : 0;
   if (node.timeout_ms != null) {
     const timeoutMs = Number(node.timeout_ms);
-    return Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs + 5e3 : 6e4;
+    return Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs + settleAllowance + 5e3 : 6e4;
   }
-  return 6e4;
+  return 6e4 + settleAllowance;
 }
 function commandForPrepared(projectRoot, platform) {
   if (platform !== "core") {

package/dist/recipe-security.js

@@ -10,11 +10,8 @@ const READ_ONLY_CUSTOM_ACTIONS = /* @__PURE__ */ new Set([
   "metamask.perps.read_account",
   "metamask.perps.assert_positions",
   "metamask.perps.assert_orders",
-  // Both only read the collector's JSONL and export/assert over it.
-  // `metamask.analytics.start_capture` is deliberately NOT listed: it spawns a
-  // detached HTTP listener, so it falls through to 'arbitrary-code' and needs
-  // explicit approval. Classifying a process-spawning action as read-only to
-  // save an approval prompt would defeat the point of this allowlist.
+  // Collector reads are host-only; starting capture remains arbitrary-code
+  // because it spawns an HTTP listener.
   "metamask.analytics.read_events",
   "metamask.analytics.assert_events"
 ]);
@@ -31,7 +28,7 @@ const APP_MUTATION_CUSTOM_ACTIONS = /* @__PURE__ */ new Set([
   "metamask.perps.ensure_orders",
   "metamask.perps.start_state",
   "metamask.perps.teardown_state",
-  // Flips MetaMetrics consent in the wallet, so app-mutation, not read-only.
+  // Consent changes wallet state and is therefore an app mutation.
   "metamask.analytics.set_consent"
 ]);
 const EXTERNAL_MUTATION_CUSTOM_ACTIONS = /* @__PURE__ */ new Set([