@deeeed/metamask-harness 0.17.5 → 0.19.0

package/library/recipes/runner/smoke.core.recipe.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "https://farmslot.io/schemas/recipe-v1.schema.json",
+  "title": "MetaMask Core runner smoke",
+  "description": "Proves the headless Core runner resolves the checkout, executes generic actions, and writes a valid artifact package.",
+  "workflow": {
+    "entry": "status",
+    "nodes": {
+      "status": {
+        "action": "app.status",
+        "next": "package",
+        "intent": "Resolve the Core checkout and report headless readiness"
+      },
+      "package": {
+        "action": "assert_file",
+        "path": "package.json",
+        "next": "done",
+        "intent": "Prove generic project-file assertions work in Core"
+      },
+      "done": {
+        "action": "end",
+        "status": "pass"
+      }
+    }
+  }
+}

package/library/recipes/runner/smoke.extension.recipe.json

@@ -1,30 +1,29 @@
 {
-  "schema_version": 1,
+  "$schema": "https://farmslot.io/schemas/recipe-v1.schema.json",
   "title": "MetaMask Extension v1 runner smoke",
   "description": "Proves the MetaMask Extension runner executes manifest-declared actions and writes a valid artifact package.",
-  "validate": {
-    "workflow": {
-      "entry": "status",
-      "nodes": {
-        "status": {
-          "action": "app.status",
-          "next": "fixture",
-          "intent": "Perform status for the recipe"
-        },
-        "fixture": {
-          "action": "metamask.wallet.fixture_status",
-          "next": "cdp",
-          "intent": "Perform fixture for the recipe"
-        },
-        "cdp": {
-          "action": "cdp.target",
-          "next": "done",
-          "intent": "Perform cdp for the recipe"
-        },
-        "done": {
-          "action": "end",
-          "status": "pass"
-        }
+  "workflow": {
+    "entry": "status",
+    "nodes": {
+      "status": {
+        "action": "app.status",
+        "next": "fixture",
+        "intent": "Confirm the Extension checkout is ready"
+      },
+      "fixture": {
+        "action": "metamask.wallet.fixture_status",
+        "next": "cdp",
+        "intent": "Confirm the wallet fixture is available"
+      },
+      "cdp": {
+        "action": "cdp.target",
+        "required": true,
+        "next": "done",
+        "intent": "Confirm the Extension runtime is reachable"
+      },
+      "done": {
+        "action": "end",
+        "status": "pass"
       }
     }
   }

package/library/recipes/runner/smoke.mobile.recipe.json

@@ -1,30 +1,29 @@
 {
-  "schema_version": 1,
+  "$schema": "https://farmslot.io/schemas/recipe-v1.schema.json",
   "title": "MetaMask Mobile v1 runner smoke",
   "description": "Proves the MetaMask Mobile runner executes manifest-declared actions and writes a valid artifact package.",
-  "validate": {
-    "workflow": {
-      "entry": "status",
-      "nodes": {
-        "status": {
-          "action": "app.status",
-          "next": "fixture",
-          "intent": "Perform status for the recipe"
-        },
-        "fixture": {
-          "action": "metamask.wallet.fixture_status",
-          "next": "cdp",
-          "intent": "Perform fixture for the recipe"
-        },
-        "cdp": {
-          "action": "cdp.target",
-          "next": "done",
-          "intent": "Perform cdp for the recipe"
-        },
-        "done": {
-          "action": "end",
-          "status": "pass"
-        }
+  "workflow": {
+    "entry": "status",
+    "nodes": {
+      "status": {
+        "action": "app.status",
+        "next": "fixture",
+        "intent": "Confirm the Mobile checkout is ready"
+      },
+      "fixture": {
+        "action": "metamask.wallet.fixture_status",
+        "next": "cdp",
+        "intent": "Confirm the wallet fixture is available"
+      },
+      "cdp": {
+        "action": "cdp.target",
+        "require_reachable": true,
+        "next": "done",
+        "intent": "Confirm the Mobile runtime is reachable"
+      },
+      "done": {
+        "action": "end",
+        "status": "pass"
       }
     }
   }

package/library/recipes/wallet/smoke.extension.recipe.json

@@ -0,0 +1,40 @@
+{
+  "$schema": "https://farmslot.io/schemas/recipe-v1.schema.json",
+  "title": "MetaMask Extension wallet smoke",
+  "description": "Fixture-backed proof that the Extension wallet is reachable, unlocked, readable, and visually capturable without depending on a product feature domain.",
+  "workflow": {
+    "entry": "status",
+    "nodes": {
+      "status": {
+        "action": "app.status",
+        "next": "fixture",
+        "intent": "Confirm the Extension runtime is reachable"
+      },
+      "fixture": {
+        "action": "metamask.wallet.fixture_status",
+        "next": "unlock",
+        "intent": "Confirm the checkout wallet fixture is ready"
+      },
+      "unlock": {
+        "action": "metamask.wallet.ensure_unlocked",
+        "next": "state",
+        "intent": "Ensure the fixture-backed wallet is unlocked"
+      },
+      "state": {
+        "action": "metamask.wallet.read_state",
+        "next": "screenshot",
+        "intent": "Read the current wallet state through the public adapter"
+      },
+      "screenshot": {
+        "action": "ui.screenshot",
+        "path": "screenshots/wallet-smoke.png",
+        "next": "done",
+        "intent": "Record the visible wallet state as evidence"
+      },
+      "done": {
+        "action": "end",
+        "status": "pass"
+      }
+    }
+  }
+}

package/library/recipes/wallet/smoke.mobile.recipe.json

@@ -0,0 +1,40 @@
+{
+  "$schema": "https://farmslot.io/schemas/recipe-v1.schema.json",
+  "title": "MetaMask Mobile wallet smoke",
+  "description": "Fixture-backed proof that the Mobile wallet is reachable, unlocked, readable, and visually capturable without depending on a product feature domain.",
+  "workflow": {
+    "entry": "status",
+    "nodes": {
+      "status": {
+        "action": "app.status",
+        "next": "fixture",
+        "intent": "Confirm the Mobile runtime is reachable"
+      },
+      "fixture": {
+        "action": "metamask.wallet.fixture_status",
+        "next": "unlock",
+        "intent": "Confirm the checkout wallet fixture is ready"
+      },
+      "unlock": {
+        "action": "metamask.wallet.ensure_unlocked",
+        "next": "state",
+        "intent": "Ensure the fixture-backed wallet is unlocked"
+      },
+      "state": {
+        "action": "metamask.wallet.read_state",
+        "next": "screenshot",
+        "intent": "Read the current wallet state through the public adapter"
+      },
+      "screenshot": {
+        "action": "ui.screenshot",
+        "path": "screenshots/wallet-smoke.png",
+        "next": "done",
+        "intent": "Record the visible wallet state as evidence"
+      },
+      "done": {
+        "action": "end",
+        "status": "pass"
+      }
+    }
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@deeeed/metamask-harness",
-  "version": "0.17.5",
+  "version": "0.19.0",
   "type": "module",
   "bin": {
     "mm-harness": "bin/mm-harness"
@@ -10,6 +10,9 @@
     "prepack": "npm run build",
     "dev:link-farmslot": "node scripts/link-local-farmslot.mjs",
     "check": "node scripts/check.mjs",
+    "test:unit": "vitest run --config scripts/vitest.config.mjs",
+    "test:coverage": "vitest run --coverage --config scripts/vitest.config.mjs",
+    "qa:human": "node scripts/validate-human-outcomes.mjs",
     "self-test": "bin/mm-harness self-test",
     "manifest:mobile": "bin/mm-harness actions --raw --adapter mobile --json",
     "manifest:extension": "bin/mm-harness actions --raw --adapter extension --json",
@@ -17,9 +20,11 @@
   },
   "dependencies": {
     "@farmslot/agent-runtime": "^0.2.0",
-    "@farmslot/protocol": "^0.8.0",
-    "@farmslot/recipe-harness": "^0.5.0",
+    "@farmslot/protocol": "^0.11.0",
+    "@farmslot/recipe-harness": "^0.9.0",
     "commander": "^12.0.0",
+    "es-module-lexer": "2.3.1",
+    "esbuild": "0.28.1",
     "viem": "^2.54.3",
     "ws": "8.21.0"
   },
@@ -31,12 +36,13 @@
   "devDependencies": {
     "@eslint/js": "^9",
     "@types/node": "^22.0.0",
-    "esbuild": "0.28.1",
+    "@vitest/coverage-v8": "^4.1.10",
     "eslint": "^9",
     "globals": "^17.7.0",
     "tsx": "^4.23.0",
     "typescript": "^5.6.0",
-    "typescript-eslint": "^8"
+    "typescript-eslint": "^8",
+    "vitest": "^4.1.10"
   },
   "main": "./dist/index.js",
   "exports": {
@@ -56,6 +62,7 @@
     "library",
     "scripts/completions.sh",
     "scripts/install-completions.sh",
+    "scripts/validate-human-outcomes.mjs",
     "!scripts/README.md",
     "docs",
     "README.md",

package/scripts/completions.sh

@@ -3,9 +3,9 @@
 #
 # Static parts (commands, positional targets, per-command flags) mirror the
 # mm-harness surface. Dynamic parts are sourced live from the hidden
-# `mm-harness completion-candidates <actions|flows>` command (per-checkout cached):
+# `mm-harness completion-candidates <actions|recipes>` command (per-checkout cached):
 #   call <TAB>   → action names from the adapter manifest
-#   run  <TAB>   → recipe files + flow refs from the recipe library
+#   run  <TAB>   → recipe files + named recipes from the recipe library
 # so completion reflects the installed overlay's real vocabulary. A candidate read
 # that fails is silently empty — completion never blocks the shell.
 #
@@ -29,7 +29,7 @@ _mmh_bin() {
   return 1
 }
 
-_mmh_commands="launch stop logs debug fixtures actions call flows run doctor check checklist recipe-quality install verify cleanup completions"
+_mmh_commands="launch stop logs debug fixtures actions call run doctor check checklist recipe-quality install verify cleanup completions"
 
 # Per-command flags (static, from the mm-harness surface).
 _mmh_flags_for() {
@@ -41,9 +41,8 @@ _mmh_flags_for() {
     fixtures) printf '%s' "--from --dev --force --fixture --adapter --target --json" ;;
     checklist) printf '%s' "" ;;
     actions)  printf '%s' "--adapter --action --kind --raw --action-manifest --target --json" ;;
-    call)     printf '%s' "--arg --adapter --target --artifacts-dir --action-manifest --heal --json" ;;
-    flows)    printf '%s' "--library --json" ;;
-    run)      printf '%s' "--plan --adapter --artifacts-dir --target --library --cdp-port --heal --record-video --json" ;;
+    call)     printf '%s' "--list --arg --adapter --target --artifacts-dir --action-manifest --heal --json" ;;
+    run)      printf '%s' "--list --describe --plan --adapter --artifacts-dir --target --library --cdp-port --heal --record-video --json" ;;
     doctor)   printf '%s' "--adapter --target --fix --json" ;;
     install|verify|cleanup) printf '%s' "--adapter --target --json" ;;
     *)        printf '%s' "" ;;
@@ -52,7 +51,7 @@ _mmh_flags_for() {
 
 # Dynamic candidates from the DISCOVER layer (silent empty on any failure).
 _mmh_actions() { local b; b="$(_mmh_bin)" || return 0; "$b" completion-candidates actions 2>/dev/null; }
-_mmh_flows()   { local b; b="$(_mmh_bin)" || return 0; "$b" completion-candidates flows 2>/dev/null; }
+_mmh_recipes() { local b; b="$(_mmh_bin)" || return 0; "$b" completion-candidates recipes 2>/dev/null; }
 
 # ── Bash ────────────────────────────────────────────────────────────────
 if [ -n "${BASH_VERSION:-}" ]; then
@@ -80,9 +79,8 @@ if [ -n "${BASH_VERSION:-}" ]; then
       fixtures)    [ "$cword" -eq 2 ] && COMPREPLY=($(compgen -W "init sync set generate finalize" -- "$cur")) ;;
       checklist)   [ "$cword" -eq 2 ] && COMPREPLY=($(compgen -W "mark" -- "$cur")) ;;
       completions) [ "$cword" -eq 2 ] && COMPREPLY=($(compgen -W "install bash zsh status" -- "$cur")) ;;
-      flows)       [ "$cword" -eq 2 ] && COMPREPLY=($(compgen -W "list promote" -- "$cur")) ;;
       call)        [ "$cword" -eq 2 ] && COMPREPLY=($(compgen -W "$(_mmh_actions)" -- "$cur")) ;;
-      run)         COMPREPLY=($(compgen -f -- "$cur"; compgen -W "$(_mmh_flows)" -- "$cur")) ;;
+      run)         COMPREPLY=($(compgen -f -- "$cur"; compgen -W "$(_mmh_recipes)" -- "$cur")) ;;
       *)           COMPREPLY=($(compgen -W "$(_mmh_flags_for "$cmd")" -- "$cur")) ;;
     esac
   }
@@ -115,9 +113,8 @@ if [ -n "${ZSH_VERSION:-}" ]; then
       fixtures)    (( CURRENT == 3 )) && { candidates=(init sync set generate finalize); compadd -a candidates; } ;;
       checklist)   (( CURRENT == 3 )) && { candidates=(mark); compadd -a candidates; } ;;
       completions) (( CURRENT == 3 )) && { candidates=(install bash zsh status); compadd -a candidates; } ;;
-      flows)       (( CURRENT == 3 )) && { candidates=(list promote); compadd -a candidates; } ;;
       call)        (( CURRENT == 3 )) && { candidates=("${(@f)$(_mmh_actions)}"); compadd -a candidates; } ;;
-      run)         candidates=("${(@f)$(_mmh_flows)}"); compadd -a candidates; _files ;;
+      run)         candidates=("${(@f)$(_mmh_recipes)}"); compadd -a candidates; _files ;;
       *)           candidates=(${=$(_mmh_flags_for "$cmd")}); compadd -a candidates ;;
     esac
   }

package/scripts/validate-human-outcomes.mjs

@@ -0,0 +1,169 @@
+#!/usr/bin/env node
+import { spawnSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const options = parseArgs(process.argv.slice(2));
+const harness = path.resolve(options.bin ?? path.join(repoRoot, 'bin', 'mm-harness'));
+const artifactsRoot = path.resolve(options.artifactsDir ?? path.join(repoRoot, 'temp', 'human-qa'));
+const targets = ['extension', 'mobile', 'core']
+  .filter((adapter) => options[adapter])
+  .map((adapter) => ({ adapter, target: path.resolve(options[adapter]) }));
+
+if (targets.length === 0) usage('pass at least one of --extension, --mobile, or --core');
+if (!fs.existsSync(harness)) usage(`mm-harness executable not found: ${harness}`);
+fs.mkdirSync(artifactsRoot, { recursive: true });
+
+const platforms = targets.map(runPlatform);
+const report = {
+  schemaVersion: 1,
+  command: 'validate-human-outcomes',
+  status: platforms.every((platform) => platform.status === 'pass') ? 'pass' : 'fail',
+  harness,
+  artifactsRoot,
+  platforms,
+  createdAt: new Date().toISOString(),
+};
+const reportPath = path.join(artifactsRoot, 'human-qa.json');
+const markdownPath = path.join(artifactsRoot, 'human-qa.md');
+fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`);
+fs.writeFileSync(markdownPath, renderMarkdown(report));
+
+if (options.json) process.stdout.write(`${JSON.stringify({ ...report, reportPath, markdownPath }, null, 2)}\n`);
+else {
+  console.log(`${report.status.toUpperCase()} human outcome QA`);
+  for (const platform of platforms) {
+    console.log(`  ${platform.status.toUpperCase()} ${platform.adapter}: ${platform.steps.filter((step) => step.status === 'pass').length}/${platform.steps.length}`);
+    for (const step of platform.steps.filter((candidate) => candidate.status !== 'pass')) {
+      console.log(`    ${step.name}: ${step.detail}`);
+    }
+  }
+  console.log(`report: ${reportPath}`);
+}
+process.exit(report.status === 'pass' ? 0 : 1);
+
+function runPlatform({ adapter, target }) {
+  const platformArtifacts = path.join(artifactsRoot, adapter);
+  fs.mkdirSync(platformArtifacts, { recursive: true });
+  const before = gitState(target);
+  const steps = [];
+
+  steps.push(runJsonStep('doctor-live', ['doctor', '--adapter', adapter, '--target', target, '--expect-live', '--json']));
+  steps.push(runJsonStep('verify', ['verify', '--adapter', adapter, '--target', target, '--json']));
+  if (adapter !== 'core') {
+    steps.push(runJsonStep('debug-target', ['debug', '--adapter', adapter, '--target', target, '--no-open', '--json'], (result) => {
+      if (adapter === 'extension' && !String(result.target?.url ?? '').startsWith('chrome-extension://')) {
+        throw new Error('debug did not resolve an Extension page');
+      }
+      if (adapter === 'mobile' && !result.endpoint) throw new Error('debug did not resolve a Mobile endpoint');
+    }));
+  }
+
+  const smokeArtifacts = path.join(platformArtifacts, 'runner-smoke');
+  steps.push(runJsonStep(
+    'runner-smoke',
+    ['run', 'runner.smoke', '--adapter', adapter, '--target', target, '--artifacts-dir', smokeArtifacts, '--json'],
+    (result) => assertRunEvidence(result, smokeArtifacts),
+  ));
+
+  if (adapter !== 'core') {
+    const walletArtifacts = path.join(platformArtifacts, 'wallet-smoke');
+    steps.push(runJsonStep(
+      'wallet-smoke',
+      ['run', 'wallet.smoke', '--adapter', adapter, '--target', target, '--artifacts-dir', walletArtifacts, '--json'],
+      (result) => assertRunEvidence(result, walletArtifacts),
+    ));
+  }
+
+  const after = gitState(target);
+  steps.push({
+    name: 'product-source-unchanged',
+    status: before === after ? 'pass' : 'fail',
+    detail: before === after ? 'tracked checkout state unchanged' : 'tracked checkout state changed during QA',
+  });
+
+  return {
+    adapter,
+    target,
+    status: steps.every((step) => step.status === 'pass') ? 'pass' : 'fail',
+    steps,
+  };
+}
+
+function runJsonStep(name, args, assertResult) {
+  const result = spawnSync(harness, args, {
+    encoding: 'utf8',
+    timeout: 10 * 60 * 1000,
+    // Installed platform delegates must exercise this exact candidate. Package
+    // managers prepend their own bin shims to PATH during scripts, which can
+    // otherwise send nested verify calls through a different/global version.
+    env: { ...process.env, MM_HARNESS_BIN: harness, MM_HARNESS_NO_UPDATE_CHECK: '1' },
+  });
+  let payload;
+  try {
+    payload = JSON.parse(result.stdout);
+  } catch {
+    return { name, status: 'fail', detail: `stdout was not one JSON document: ${compact(result.stdout || result.stderr)}` };
+  }
+  if (result.status !== 0 || payload.status === 'fail' || payload.exitCode > 0) {
+    return { name, status: 'fail', detail: compact(payload.error?.message || result.stderr || result.stdout) };
+  }
+  try {
+    assertResult?.(payload);
+  } catch (error) {
+    return { name, status: 'fail', detail: error instanceof Error ? error.message : String(error) };
+  }
+  return { name, status: 'pass', detail: 'human outcome verified' };
+}
+
+function assertRunEvidence(result, artifactsDir) {
+  if (result.status !== 'pass') throw new Error('recipe did not pass');
+  const required = ['report.md', 'summary.json', 'trace.json', 'recipe.json', 'artifact-manifest.json'];
+  for (const relative of required) {
+    if (!fs.existsSync(path.join(artifactsDir, relative))) throw new Error(`missing run evidence: ${relative}`);
+  }
+  const manifest = JSON.parse(fs.readFileSync(path.join(artifactsDir, 'artifact-manifest.json'), 'utf8'));
+  if (!Array.isArray(manifest.artifacts) || manifest.artifacts.length === 0) throw new Error('artifact manifest is empty');
+}
+
+function gitState(target) {
+  const result = spawnSync('git', ['status', '--porcelain', '--untracked-files=no'], { cwd: target, encoding: 'utf8' });
+  return result.status === 0 ? result.stdout : '';
+}
+
+function compact(value) {
+  return String(value ?? '').trim().replace(/\s+/gu, ' ').slice(0, 500) || 'command failed without detail';
+}
+
+function renderMarkdown(report) {
+  const lines = ['# mm-harness human outcome QA', '', `Status: **${report.status.toUpperCase()}**`, ''];
+  for (const platform of report.platforms) {
+    lines.push(`## ${platform.adapter}`, '', `Target: \`${platform.target}\``, '');
+    for (const step of platform.steps) lines.push(`- [${step.status === 'pass' ? 'x' : ' '}] ${step.name} — ${step.detail}`);
+    lines.push('');
+  }
+  return `${lines.join('\n')}\n`;
+}
+
+function parseArgs(argv) {
+  const parsed = {};
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === '--json') parsed.json = true;
+    else if (['--extension', '--mobile', '--core', '--bin', '--artifacts-dir'].includes(arg)) {
+      const value = argv[index + 1];
+      if (!value || value.startsWith('--')) usage(`missing value for ${arg}`);
+      parsed[arg.slice(2).replace(/-([a-z])/gu, (_, letter) => letter.toUpperCase())] = value;
+      index += 1;
+    } else usage(`unknown option: ${arg}`);
+  }
+  return parsed;
+}
+
+function usage(message) {
+  if (message) console.error(`validate-human-outcomes: ${message}`);
+  console.error('Usage: yarn qa:human [--extension <path>] [--mobile <path>] [--core <path>] [--artifacts-dir <path>] [--json]');
+  process.exit(2);
+}