@deeeed/metamask-harness 0.37.0 → 0.38.1

package/library/recipes/swap-bridge/release-quote.recipe.json

@@ -0,0 +1,72 @@
+{
+  "$schema": "https://farmslot.io/schemas/recipe-v1.schema.json",
+  "title": "Swap release quote",
+  "description": "Selects an exact live Swap pair, enters a capped amount, and proves a destination quote through visible product UI without submitting a transaction.",
+  "paramsSchema": {
+    "type": "object",
+    "additionalProperties": false,
+    "properties": {
+      "source_symbol": { "type": "string", "default": "ETH" },
+      "destination_symbol": { "type": "string", "default": "WETH" },
+      "source_chain": { "type": "string", "default": "eip155:1" },
+      "destination_chain": { "type": "string", "default": "eip155:1" },
+      "source_query": { "type": "string", "default": "ETH" },
+      "destination_query": { "type": "string", "default": "WETH" },
+      "source_asset_id": { "type": "string", "default": "" },
+      "destination_asset_id": { "type": "string", "default": "" },
+      "amount": { "type": "number", "minimum": 0.00000001, "default": 0.0001 },
+      "timeout_ms": { "type": "integer", "minimum": 1000, "default": 60000 }
+    }
+  },
+  "workflow": {
+    "entry": "open-swap",
+    "nodes": {
+      "open-swap": {
+        "action": "ui.navigate",
+        "page": "swap",
+        "timeout_ms": "{{params.timeout_ms}}",
+        "intent": "Open Swap through visible product navigation",
+        "next": "select-assets"
+      },
+      "select-assets": {
+        "action": "metamask.swap_bridge.select_assets",
+        "source_symbol": "{{params.source_symbol}}",
+        "destination_symbol": "{{params.destination_symbol}}",
+        "source_chain": "{{params.source_chain}}",
+        "destination_chain": "{{params.destination_chain}}",
+        "source_query": "{{params.source_query}}",
+        "destination_query": "{{params.destination_query}}",
+        "source_asset_id": "{{params.source_asset_id}}",
+        "destination_asset_id": "{{params.destination_asset_id}}",
+        "timeout_ms": "{{params.timeout_ms}}",
+        "intent": "Select the exact source and destination assets",
+        "next": "enter-amount"
+      },
+      "enter-amount": {
+        "action": "metamask.swap_bridge.set_amount",
+        "amount": "{{params.amount}}",
+        "wait_for_quote": true,
+        "timeout_ms": "{{params.timeout_ms}}",
+        "intent": "Enter the capped amount and wait for a live quote",
+        "next": "assert-quote"
+      },
+      "assert-quote": {
+        "action": "metamask.swap_bridge.read_visible_state",
+        "source_symbol": "{{params.source_symbol}}",
+        "destination_symbol": "{{params.destination_symbol}}",
+        "require_amount_inputs": true,
+        "require_settings": true,
+        "intent": "Prove the selected pair and quoted destination amount remain visible",
+        "next": "capture-quote"
+      },
+      "capture-quote": {
+        "action": "ui.screenshot",
+        "path": "screenshots/swap-quote.png",
+        "label": "Swap release quote",
+        "intent": "Preserve reviewer-visible proof of the live quote",
+        "next": "done"
+      },
+      "done": { "action": "end", "status": "pass" }
+    }
+  }
+}

package/library/recipes/swap-bridge/release-surface.recipe.json

@@ -0,0 +1,43 @@
+{
+  "$schema": "https://farmslot.io/schemas/recipe-v1.schema.json",
+  "title": "Swap release surface",
+  "description": "Opens Swap through visible product navigation and proves both token selectors, amount inputs, and settings are available without development-only bridges.",
+  "paramsSchema": {
+    "type": "object",
+    "additionalProperties": false,
+    "properties": {
+      "source_symbol": { "type": "string", "default": "" },
+      "destination_symbol": { "type": "string", "default": "" },
+      "timeout_ms": { "type": "integer", "minimum": 1000, "default": 45000 }
+    }
+  },
+  "workflow": {
+    "entry": "open-swap",
+    "nodes": {
+      "open-swap": {
+        "action": "ui.navigate",
+        "page": "swap",
+        "timeout_ms": "{{params.timeout_ms}}",
+        "intent": "Open Swap through visible product navigation",
+        "next": "assert-swap"
+      },
+      "assert-swap": {
+        "action": "metamask.swap_bridge.read_visible_state",
+        "source_symbol": "{{params.source_symbol}}",
+        "destination_symbol": "{{params.destination_symbol}}",
+        "require_amount_inputs": true,
+        "require_settings": true,
+        "intent": "Prove the visible Swap inputs, token selectors, and settings control",
+        "next": "capture-swap"
+      },
+      "capture-swap": {
+        "action": "ui.screenshot",
+        "path": "screenshots/swap-surface.png",
+        "label": "Swap release surface",
+        "intent": "Preserve reviewer-visible proof of the Swap surface",
+        "next": "done"
+      },
+      "done": { "action": "end", "status": "pass" }
+    }
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@deeeed/metamask-harness",
-  "version": "0.37.0",
+  "version": "0.38.1",
   "type": "module",
   "bin": {
     "mm-harness": "bin/mm-harness"
@@ -20,11 +20,11 @@
     "check:syntax": "find . -name '*.mjs' -print0 | xargs -0 -n1 node --check"
   },
   "dependencies": {
-    "@farmslot/agent-runtime": "0.7.0",
-    "@farmslot/expo-recipe": "0.7.1",
+    "@farmslot/agent-runtime": "0.8.1",
+    "@farmslot/expo-recipe": "0.8.1",
     "@farmslot/handoff": "^0.3.1",
-    "@farmslot/protocol": "0.18.0",
-    "@farmslot/recipe-harness": "0.14.0",
+    "@farmslot/protocol": "0.21.0",
+    "@farmslot/recipe-harness": "0.15.0",
     "agent-device": "0.19.3",
     "commander": "^12.0.0",
     "es-module-lexer": "2.3.1",
@@ -69,6 +69,7 @@
     "scripts/install-completions.sh",
     "scripts/site-contrast.mjs",
     "scripts/validate-human-outcomes.mjs",
+    "scripts/validate-release-report.mjs",
     "!scripts/README.md",
     "docs",
     "README.md",

package/scripts/validate-release-report.mjs

@@ -0,0 +1,333 @@
+#!/usr/bin/env node
+import fs from 'node:fs';
+import path from 'node:path';
+import { validateRecipeArtifactPackage } from '@farmslot/protocol';
+
+const options = parseArgs(process.argv.slice(2));
+const inventoryPath = path.resolve(options.inventory);
+const verdictsPath = path.resolve(options.verdicts);
+const inventory = readJson(inventoryPath, 'inventory');
+const verdicts = readJson(verdictsPath, 'verdicts');
+const errors = [];
+const allowedStatuses = new Set([
+  'PASS',
+  'FAIL',
+  'BLOCKED_FUNDS',
+  'BLOCKED_COST',
+  'BLOCKED_ENVIRONMENT',
+]);
+
+const inventoryCases = Array.isArray(inventory.cases) ? inventory.cases : [];
+const verdictCases = Array.isArray(verdicts.cases) ? verdicts.cases : [];
+for (const entry of inventoryCases) {
+  if (!entry || typeof entry !== 'object' || typeof entry.id !== 'string' || entry.id.trim().length === 0) {
+    errors.push('every inventory case must include a non-empty string ID');
+  }
+}
+for (const entry of verdictCases) {
+  if (!entry || typeof entry !== 'object' || typeof entry.id !== 'string' || entry.id.trim().length === 0) {
+    errors.push('every verdict case must include a non-empty string ID');
+  }
+}
+const inventoryIds = inventoryCases.map((entry) => entry?.id).filter((id) => typeof id === 'string' && id.trim().length > 0);
+const verdictIds = verdictCases.map((entry) => entry?.id).filter((id) => typeof id === 'string' && id.trim().length > 0);
+const duplicateInventoryIds = duplicates(inventoryIds);
+const duplicateVerdictIds = duplicates(verdictIds);
+if (inventory.schemaVersion !== 1) errors.push('inventory.schemaVersion must be 1');
+if (verdicts.schemaVersion !== 1) errors.push('verdicts.schemaVersion must be 1');
+if (inventoryCases.length === 0) errors.push('inventory.cases must not be empty');
+if (duplicateInventoryIds.length > 0) errors.push(`duplicate inventory case IDs: ${duplicateInventoryIds.join(', ')}`);
+if (duplicateVerdictIds.length > 0) errors.push(`duplicate verdict case IDs: ${duplicateVerdictIds.join(', ')}`);
+
+const inventoryIdSet = new Set(inventoryIds);
+const verdictIdSet = new Set(verdictIds);
+const missing = inventoryIds.filter((id) => !verdictIdSet.has(id));
+const unknown = verdictIds.filter((id) => !inventoryIdSet.has(id));
+if (missing.length > 0) errors.push(`missing case verdicts: ${missing.join(', ')}`);
+if (unknown.length > 0) errors.push(`unknown case verdicts: ${unknown.join(', ')}`);
+if (!Array.isArray(verdicts.remainingCases) || verdicts.remainingCases.length !== 0) {
+  errors.push('verdicts.remainingCases must be an empty array after terminal case coverage');
+}
+
+const evidenceRoot = path.resolve(verdicts.evidenceRoot ?? path.dirname(verdictsPath));
+let canonicalEvidenceRoot = evidenceRoot;
+try {
+  canonicalEvidenceRoot = fs.realpathSync(evidenceRoot);
+} catch {
+  errors.push(`evidenceRoot does not exist: ${evidenceRoot}`);
+}
+for (const entry of verdictCases) {
+  if (!entry || typeof entry !== 'object') {
+    errors.push('every verdict case must be an object');
+    continue;
+  }
+  if (!allowedStatuses.has(entry.status)) {
+    errors.push(`${entry.id ?? '<missing-id>'} has unsupported status ${JSON.stringify(entry.status)}`);
+  }
+  if (typeof entry.summary !== 'string' || entry.summary.trim().length === 0) {
+    errors.push(`${entry.id ?? '<missing-id>'} must include a non-empty summary`);
+  }
+  if (!Array.isArray(entry.evidence) || entry.evidence.length === 0) {
+    errors.push(`${entry.id ?? '<missing-id>'} must include evidence`);
+    continue;
+  }
+  for (const evidence of entry.evidence) {
+    if (typeof evidence !== 'string' || evidence.length === 0) {
+      errors.push(`${entry.id ?? '<missing-id>'} contains invalid evidence`);
+      continue;
+    }
+    const resolved = path.isAbsolute(evidence) ? path.resolve(evidence) : path.resolve(evidenceRoot, evidence);
+    if (resolved !== evidenceRoot && !resolved.startsWith(`${evidenceRoot}${path.sep}`)) {
+      errors.push(`${entry.id ?? '<missing-id>'} evidence escapes evidenceRoot: ${evidence}`);
+      continue;
+    }
+    if (!fs.existsSync(resolved)) {
+      errors.push(`${entry.id ?? '<missing-id>'} evidence does not exist: ${resolved}`);
+      continue;
+    }
+    const stat = fs.lstatSync(resolved);
+    if (stat.isSymbolicLink()) {
+      errors.push(`${entry.id ?? '<missing-id>'} evidence must not be a symbolic link: ${resolved}`);
+      continue;
+    }
+    const canonicalResolved = fs.realpathSync(resolved);
+    if (canonicalResolved !== canonicalEvidenceRoot && !canonicalResolved.startsWith(`${canonicalEvidenceRoot}${path.sep}`)) {
+      errors.push(`${entry.id ?? '<missing-id>'} evidence escapes evidenceRoot through a linked path: ${evidence}`);
+      continue;
+    }
+    if (stat.isDirectory()) {
+      const documents = {};
+      const requiredDocuments = [
+        'summary.json',
+        'trace.json',
+        'artifact-manifest.json',
+        'recipe.json',
+        'recipe-resolution.json',
+      ];
+      for (const required of requiredDocuments) {
+        const requiredPath = path.join(resolved, required);
+        const requiredDocument = readRegularJsonFile(requiredPath);
+        if (requiredDocument.status === 'missing') {
+          errors.push(`${entry.id ?? '<missing-id>'} evidence directory is missing ${required}: ${resolved}`);
+          continue;
+        }
+        if (requiredDocument.status === 'unsafe') {
+          errors.push(`${entry.id ?? '<missing-id>'} evidence directory ${required} must be a regular file: ${resolved}`);
+          continue;
+        }
+        if (requiredDocument.status === 'invalid') {
+          errors.push(`${entry.id ?? '<missing-id>'} evidence directory ${required} must contain valid JSON: ${resolved}`);
+          continue;
+        }
+        documents[required] = requiredDocument.value;
+      }
+      if (documents['summary.json'] && entry.status === 'PASS' && documents['summary.json'].status !== 'pass') {
+        errors.push(`${entry.id ?? '<missing-id>'} PASS evidence summary.json must have status pass: ${resolved}`);
+      }
+      if (requiredDocuments.every((required) => required in documents)) {
+        const resolvedRecipes = readResolvedRecipes(
+          resolved,
+          documents['recipe-resolution.json'],
+          entry.id ?? '<missing-id>',
+          errors,
+        );
+        const validation = validateRecipeArtifactPackage({
+          summary: documents['summary.json'],
+          trace: documents['trace.json'],
+          manifest: documents['artifact-manifest.json'],
+          recipe: documents['recipe.json'],
+          recipeResolution: documents['recipe-resolution.json'],
+          resolvedRecipes,
+          artifactPaths: listEvidenceFiles(resolved),
+        });
+        if (validation.status !== 'valid') {
+          const codes = validation.findings
+            .filter((finding) => finding.severity === 'error')
+            .map((finding) => finding.code)
+            .join(', ');
+          errors.push(`${entry.id ?? '<missing-id>'} evidence directory is not a valid Recipe Protocol artifact package: ${resolved} (${codes})`);
+        }
+      }
+    } else if (!stat.isFile()) {
+      errors.push(`${entry.id ?? '<missing-id>'} evidence must be a file or run directory: ${resolved}`);
+    }
+  }
+}
+
+const counts = Object.fromEntries([...allowedStatuses].map((status) => [
+  status,
+  verdictCases.filter((entry) => entry?.status === status).length,
+]));
+const result = {
+  schemaVersion: 1,
+  command: 'validate-release-report',
+  status: errors.length === 0 ? 'pass' : 'fail',
+  releaseStatus: counts.FAIL > 0 ? 'fail' : Object.entries(counts).some(([status, count]) => status.startsWith('BLOCKED_') && count > 0) ? 'blocked' : 'pass',
+  inventoryPath,
+  verdictsPath,
+  evidenceRoot,
+  expectedCases: inventoryIds.length,
+  coveredCases: verdictIds.filter((id) => inventoryIdSet.has(id)).length,
+  counts,
+  missingCases: missing,
+  unknownCases: unknown,
+  errors,
+};
+if (errors.length === 0 && options.outputDir) {
+  const outputDir = path.resolve(options.outputDir);
+  fs.mkdirSync(outputDir, { recursive: true });
+  const metadata = {
+    schemaVersion: 1,
+    release: verdicts.release ?? null,
+    platform: verdicts.platform ?? null,
+    domain: verdicts.domain ?? null,
+    artifactSha256: verdicts.artifactSha256 ?? null,
+    releaseStatus: result.releaseStatus,
+    total: inventoryIds.length,
+    counts,
+    remaining_cases: [],
+  };
+  const quality = {
+    schemaVersion: 1,
+    status: 'pass',
+    canonicalCaseCoverage: true,
+    terminalVerdicts: true,
+    evidenceComplete: true,
+    summariesComplete: true,
+    remainingCasesComplete: true,
+  };
+  fs.writeFileSync(path.join(outputDir, 'metadata.json'), `${JSON.stringify(metadata, null, 2)}\n`);
+  fs.writeFileSync(path.join(outputDir, 'report-quality.json'), `${JSON.stringify(quality, null, 2)}\n`);
+  fs.writeFileSync(path.join(outputDir, 'report.md'), renderMarkdown(verdicts, inventoryCases, counts, result.releaseStatus));
+  result.outputDir = outputDir;
+}
+process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+process.exit(errors.length === 0 ? 0 : 1);
+
+function readJson(file, label) {
+  try {
+    return JSON.parse(fs.readFileSync(file, 'utf8'));
+  } catch (error) {
+    console.error(`validate-release-report: could not read ${label} ${file}: ${error instanceof Error ? error.message : String(error)}`);
+    process.exit(2);
+  }
+}
+
+function readRegularJsonFile(file) {
+  let descriptor;
+  try {
+    descriptor = fs.openSync(file, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0));
+    if (!fs.fstatSync(descriptor).isFile()) return { status: 'unsafe' };
+    const contents = fs.readFileSync(descriptor, 'utf8');
+    try {
+      return { status: 'ok', value: JSON.parse(contents) };
+    } catch {
+      return { status: 'invalid' };
+    }
+  } catch (error) {
+    return { status: error?.code === 'ENOENT' ? 'missing' : 'unsafe' };
+  } finally {
+    if (descriptor !== undefined) fs.closeSync(descriptor);
+  }
+}
+
+function duplicates(values) {
+  const seen = new Set();
+  const repeated = new Set();
+  for (const value of values) {
+    if (seen.has(value)) repeated.add(value);
+    seen.add(value);
+  }
+  return [...repeated];
+}
+
+function listEvidenceFiles(root) {
+  const files = [];
+  const pending = [{ directory: root, prefix: '' }];
+  while (pending.length > 0) {
+    const { directory, prefix } = pending.pop();
+    for (const name of fs.readdirSync(directory)) {
+      const absolute = path.join(directory, name);
+      const relative = prefix ? `${prefix}/${name}` : name;
+      const stat = fs.lstatSync(absolute);
+      if (stat.isSymbolicLink()) continue;
+      if (stat.isDirectory()) pending.push({ directory: absolute, prefix: relative });
+      else if (stat.isFile()) files.push(relative);
+    }
+  }
+  return files;
+}
+
+function readResolvedRecipes(root, resolution, caseId, validationErrors) {
+  const recipes = {};
+  if (!resolution || typeof resolution !== 'object' || !Array.isArray(resolution.dependencies)) return recipes;
+  const canonicalRoot = fs.realpathSync(root);
+  for (const dependency of resolution.dependencies) {
+    const digest = dependency?.digest;
+    const artifact = dependency?.artifact;
+    if (
+      typeof digest !== 'string' ||
+      typeof artifact !== 'string' ||
+      !/^resolved-recipes\/sha256-[0-9a-f]{64}\.json$/u.test(artifact)
+    ) continue;
+    const artifactPath = path.join(root, ...artifact.split('/'));
+    if (!fs.existsSync(artifactPath)) continue;
+    const canonicalArtifact = fs.realpathSync(artifactPath);
+    if (!canonicalArtifact.startsWith(`${canonicalRoot}${path.sep}`)) continue;
+    const recipe = readRegularJsonFile(artifactPath);
+    if (recipe.status === 'ok') {
+      recipes[digest] = recipe.value;
+    } else if (recipe.status === 'invalid') {
+      validationErrors.push(`${caseId} evidence directory resolved recipe must contain valid JSON: ${artifactPath}`);
+    }
+  }
+  return recipes;
+}
+
+function renderMarkdown(verdicts, inventoryCases, counts, releaseStatus) {
+  const byId = new Map(verdicts.cases.map((entry) => [entry.id, entry]));
+  const lines = [
+    `# ${verdicts.platform ?? 'Release'} ${verdicts.release ?? ''} ${verdicts.domain ?? ''} validation`,
+    '',
+    `Release status: **${releaseStatus.toUpperCase()}**`,
+    '',
+    `Artifact SHA-256: \`${verdicts.artifactSha256 ?? 'not-recorded'}\``,
+    '',
+    `Cases: ${inventoryCases.length} total; ${counts.PASS} pass; ${counts.FAIL} fail; ${counts.BLOCKED_ENVIRONMENT + counts.BLOCKED_FUNDS + counts.BLOCKED_COST} blocked.`,
+    '',
+    '| Case | Verdict | Result |',
+    '| --- | --- | --- |',
+  ];
+  for (const inventoryCase of inventoryCases) {
+    const verdict = byId.get(inventoryCase.id);
+    const evidence = verdict.evidence.map((entry) => `\`${entry}\``).join(', ');
+    lines.push(`| ${inventoryCase.id} | ${verdict.status} | ${escapeTable(verdict.summary)} Evidence: ${evidence} |`);
+  }
+  lines.push('');
+  return `${lines.join('\n')}\n`;
+}
+
+function escapeTable(value) {
+  return String(value).replace(/\\/gu, '\\\\').replace(/\|/gu, '\\|').replace(/\s+/gu, ' ').trim();
+}
+
+function parseArgs(argv) {
+  const parsed = {};
+  for (let index = 0; index < argv.length; index += 1) {
+    const argument = argv[index];
+    if (argument === '--inventory' || argument === '--verdicts' || argument === '--output-dir') {
+      const value = argv[index + 1];
+      if (!value || value.startsWith('--')) usage(`missing value for ${argument}`);
+      parsed[argument.slice(2).replace(/-([a-z])/gu, (_, letter) => letter.toUpperCase())] = value;
+      index += 1;
+    } else usage(`unknown option: ${argument}`);
+  }
+  if (!parsed.inventory || !parsed.verdicts) usage('both --inventory and --verdicts are required');
+  return parsed;
+}
+
+function usage(message) {
+  if (message) console.error(`validate-release-report: ${message}`);
+  console.error('Usage: node scripts/validate-release-report.mjs --inventory <file> --verdicts <file> [--output-dir <dir>]');
+  process.exit(2);
+}