@deeeed/metamask-harness 0.17.2 → 0.17.4

package/scripts/validate-human-outcomes.mjs

@@ -0,0 +1,169 @@
+#!/usr/bin/env node
+import { spawnSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const options = parseArgs(process.argv.slice(2));
+const harness = path.resolve(options.bin ?? path.join(repoRoot, 'bin', 'mm-harness'));
+const artifactsRoot = path.resolve(options.artifactsDir ?? path.join(repoRoot, 'temp', 'human-qa'));
+const targets = ['extension', 'mobile', 'core']
+  .filter((adapter) => options[adapter])
+  .map((adapter) => ({ adapter, target: path.resolve(options[adapter]) }));
+
+if (targets.length === 0) usage('pass at least one of --extension, --mobile, or --core');
+if (!fs.existsSync(harness)) usage(`mm-harness executable not found: ${harness}`);
+fs.mkdirSync(artifactsRoot, { recursive: true });
+
+const platforms = targets.map(runPlatform);
+const report = {
+  schemaVersion: 1,
+  command: 'validate-human-outcomes',
+  status: platforms.every((platform) => platform.status === 'pass') ? 'pass' : 'fail',
+  harness,
+  artifactsRoot,
+  platforms,
+  createdAt: new Date().toISOString(),
+};
+const reportPath = path.join(artifactsRoot, 'human-qa.json');
+const markdownPath = path.join(artifactsRoot, 'human-qa.md');
+fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`);
+fs.writeFileSync(markdownPath, renderMarkdown(report));
+
+if (options.json) process.stdout.write(`${JSON.stringify({ ...report, reportPath, markdownPath }, null, 2)}\n`);
+else {
+  console.log(`${report.status.toUpperCase()} human outcome QA`);
+  for (const platform of platforms) {
+    console.log(`  ${platform.status.toUpperCase()} ${platform.adapter}: ${platform.steps.filter((step) => step.status === 'pass').length}/${platform.steps.length}`);
+    for (const step of platform.steps.filter((candidate) => candidate.status !== 'pass')) {
+      console.log(`    ${step.name}: ${step.detail}`);
+    }
+  }
+  console.log(`report: ${reportPath}`);
+}
+process.exit(report.status === 'pass' ? 0 : 1);
+
+function runPlatform({ adapter, target }) {
+  const platformArtifacts = path.join(artifactsRoot, adapter);
+  fs.mkdirSync(platformArtifacts, { recursive: true });
+  const before = gitState(target);
+  const steps = [];
+
+  steps.push(runJsonStep('doctor-live', ['doctor', '--adapter', adapter, '--target', target, '--expect-live', '--json']));
+  steps.push(runJsonStep('verify', ['verify', '--adapter', adapter, '--target', target, '--json']));
+  if (adapter !== 'core') {
+    steps.push(runJsonStep('debug-target', ['debug', '--adapter', adapter, '--target', target, '--no-open', '--json'], (result) => {
+      if (adapter === 'extension' && !String(result.target?.url ?? '').startsWith('chrome-extension://')) {
+        throw new Error('debug did not resolve an Extension page');
+      }
+      if (adapter === 'mobile' && !result.endpoint) throw new Error('debug did not resolve a Mobile endpoint');
+    }));
+  }
+
+  const smokeArtifacts = path.join(platformArtifacts, 'runner-smoke');
+  steps.push(runJsonStep(
+    'runner-smoke',
+    ['run', 'runner.smoke', '--adapter', adapter, '--target', target, '--artifacts-dir', smokeArtifacts, '--json'],
+    (result) => assertRunEvidence(result, smokeArtifacts),
+  ));
+
+  if (adapter !== 'core') {
+    const walletArtifacts = path.join(platformArtifacts, 'wallet-smoke');
+    steps.push(runJsonStep(
+      'wallet-smoke',
+      ['run', 'wallet.smoke', '--adapter', adapter, '--target', target, '--artifacts-dir', walletArtifacts, '--json'],
+      (result) => assertRunEvidence(result, walletArtifacts),
+    ));
+  }
+
+  const after = gitState(target);
+  steps.push({
+    name: 'product-source-unchanged',
+    status: before === after ? 'pass' : 'fail',
+    detail: before === after ? 'tracked checkout state unchanged' : 'tracked checkout state changed during QA',
+  });
+
+  return {
+    adapter,
+    target,
+    status: steps.every((step) => step.status === 'pass') ? 'pass' : 'fail',
+    steps,
+  };
+}
+
+function runJsonStep(name, args, assertResult) {
+  const result = spawnSync(harness, args, {
+    encoding: 'utf8',
+    timeout: 10 * 60 * 1000,
+    // Installed platform delegates must exercise this exact candidate. Package
+    // managers prepend their own bin shims to PATH during scripts, which can
+    // otherwise send nested verify calls through a different/global version.
+    env: { ...process.env, MM_HARNESS_BIN: harness, MM_HARNESS_NO_UPDATE_CHECK: '1' },
+  });
+  let payload;
+  try {
+    payload = JSON.parse(result.stdout);
+  } catch {
+    return { name, status: 'fail', detail: `stdout was not one JSON document: ${compact(result.stdout || result.stderr)}` };
+  }
+  if (result.status !== 0 || payload.status === 'fail' || payload.exitCode > 0) {
+    return { name, status: 'fail', detail: compact(payload.error?.message || result.stderr || result.stdout) };
+  }
+  try {
+    assertResult?.(payload);
+  } catch (error) {
+    return { name, status: 'fail', detail: error instanceof Error ? error.message : String(error) };
+  }
+  return { name, status: 'pass', detail: 'human outcome verified' };
+}
+
+function assertRunEvidence(result, artifactsDir) {
+  if (result.status !== 'pass') throw new Error('recipe did not pass');
+  const required = ['report.md', 'summary.json', 'trace.json', 'recipe.json', 'artifact-manifest.json'];
+  for (const relative of required) {
+    if (!fs.existsSync(path.join(artifactsDir, relative))) throw new Error(`missing run evidence: ${relative}`);
+  }
+  const manifest = JSON.parse(fs.readFileSync(path.join(artifactsDir, 'artifact-manifest.json'), 'utf8'));
+  if (!Array.isArray(manifest.artifacts) || manifest.artifacts.length === 0) throw new Error('artifact manifest is empty');
+}
+
+function gitState(target) {
+  const result = spawnSync('git', ['status', '--porcelain', '--untracked-files=no'], { cwd: target, encoding: 'utf8' });
+  return result.status === 0 ? result.stdout : '';
+}
+
+function compact(value) {
+  return String(value ?? '').trim().replace(/\s+/gu, ' ').slice(0, 500) || 'command failed without detail';
+}
+
+function renderMarkdown(report) {
+  const lines = ['# mm-harness human outcome QA', '', `Status: **${report.status.toUpperCase()}**`, ''];
+  for (const platform of report.platforms) {
+    lines.push(`## ${platform.adapter}`, '', `Target: \`${platform.target}\``, '');
+    for (const step of platform.steps) lines.push(`- [${step.status === 'pass' ? 'x' : ' '}] ${step.name} — ${step.detail}`);
+    lines.push('');
+  }
+  return `${lines.join('\n')}\n`;
+}
+
+function parseArgs(argv) {
+  const parsed = {};
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === '--json') parsed.json = true;
+    else if (['--extension', '--mobile', '--core', '--bin', '--artifacts-dir'].includes(arg)) {
+      const value = argv[index + 1];
+      if (!value || value.startsWith('--')) usage(`missing value for ${arg}`);
+      parsed[arg.slice(2).replace(/-([a-z])/gu, (_, letter) => letter.toUpperCase())] = value;
+      index += 1;
+    } else usage(`unknown option: ${arg}`);
+  }
+  return parsed;
+}
+
+function usage(message) {
+  if (message) console.error(`validate-human-outcomes: ${message}`);
+  console.error('Usage: yarn qa:human [--extension <path>] [--mobile <path>] [--core <path>] [--artifacts-dir <path>] [--json]');
+  process.exit(2);
+}