@getarbiter/cli 0.5.0

package/dist/templates/scripts/lib/e2e-reliability.mjs.ejs

@@ -0,0 +1,207 @@
+// SPDX-License-Identifier: Apache-2.0
+// arbiter — stack-agnostic E2E reliability library (#1445, INV-130).
+//
+// A small, framework-neutral toolkit for telling a FLAKE apart from a real REGRESSION
+// — the #1 source of fake-green for a prompt-only operator, where a flaky test that
+// goes green on re-run is indistinguishable from a regression that rides the retry
+// habit into production. It operates on a NORMALISED failure shape, so the same logic
+// serves any runner (Playwright / Vitest / pytest / JUnit / go test …); per-framework
+// adapters that map a native report onto this shape are thin and live in the target.
+//
+// Exports (all pure / fail-closed — an unknown failure is treated as a REGRESSION,
+// never silently dismissed):
+//   fingerprint(text)                  stable identity after volatile-token normalisation
+//   classify(failure, {quarantined})   INFRA | FLAKE | REGRESSION
+//   retryLadder(runAttempt, opts)      PASS | FLAKE | INFRA | REGRESSION over a retry ladder
+//   riskTier(summary)                  R0 (clean) … R4 (block / error) — fail-closes to R4
+//   appendLedger(path, entry)          append-only JSONL outcome ledger
+//   validateQuarantine(registry, now)  quarantine-registry schema + expiry validation
+//   QUARANTINE_REQUIRED_FIELDS         the registry entry contract
+import { createHash } from 'node:crypto'
+import { appendFileSync, mkdirSync } from 'node:fs'
+import { dirname } from 'node:path'
+
+// ─── Deterministic fingerprint ────────────────────────────────────────────────
+// Strip the volatile tokens that make two runs of the SAME failure look different,
+// then hash. Keeps failure identity stable across machines, runs, and clocks.
+const NORMALISERS = [
+  // ISO-ish timestamps (with or without millis / timezone)
+  [/\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?/g, '<TS>'],
+  // UUIDs
+  [/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi, '<UUID>'],
+  // hex pointers / addresses
+  [/0x[0-9a-fA-F]+/g, '<ADDR>'],
+  // file position suffixes (:line:col) — must run before the port rule
+  [/:\d+:\d+/g, ':<LC>'],
+  // ports (host:NNNN)
+  [/:\d{2,5}\b/g, ':<PORT>'],
+  // durations
+  [/\b\d+(?:\.\d+)?\s?(?:ms|s)\b/g, '<DUR>'],
+  // OS temp paths
+  [/(?:\/[^\s:'"]*)?\/(?:tmp|temp)\/[^\s:'")]*/gi, '<TMP>'],
+]
+
+export function fingerprint(input) {
+  let s = String(input == null ? '' : input)
+  for (const [re, rep] of NORMALISERS) s = s.replace(re, rep)
+  s = s.replace(/\s+/g, ' ').trim()
+  return 'fp_' + createHash('sha256').update(s).digest('hex').slice(0, 16)
+}
+
+// ─── Failure classification ───────────────────────────────────────────────────
+// Environmental breakage (network / DNS / OOM / container) is INFRA; a failure whose
+// fingerprint is in the known-flaky (quarantined) set is FLAKE; everything else is a
+// REGRESSION. INFRA is checked first because an environment fault is not the test's
+// fault regardless of quarantine status. Fail-closed: a malformed failure → REGRESSION.
+const INFRA_SIGNALS = [
+  /econnrefused/i,
+  /etimedout/i,
+  /enotfound/i,
+  /eai_again/i,
+  /econnreset/i,
+  /epipe/i,
+  /socket hang ?up/i,
+  /connection refused/i,
+  /could not connect/i,
+  /\bnetwork\b/i,
+  /\bdns\b/i,
+  /\b50[234]\b/, // 502 / 503 / 504
+  /no space left/i,
+  /out of memory|\boom\b/i,
+  /\b(docker|container)\b/i,
+]
+
+export function classify(failure, opts = {}) {
+  try {
+    if (!failure || typeof failure !== 'object') return 'REGRESSION'
+    const quarantined =
+      opts.quarantined instanceof Set ? opts.quarantined : new Set(opts.quarantined || [])
+    const text = [failure.message, failure.name, failure.error, failure.stack]
+      .filter(Boolean)
+      .join('\n')
+    if (INFRA_SIGNALS.some((re) => re.test(text))) return 'INFRA'
+    const fp = failure.fingerprint || fingerprint(text)
+    if (quarantined.has(fp)) return 'FLAKE'
+    return 'REGRESSION'
+  } catch {
+    return 'REGRESSION'
+  }
+}
+
+// ─── Retry ladder ─────────────────────────────────────────────────────────────
+// `runAttempt(scope)` is injected by the caller (this is what keeps the library
+// stack-agnostic): it runs the suite at the given scope and returns
+// `{ passed: boolean, failures?: NormalisedFailure[] }`. The ladder escalates
+// initial → single-test → spec. Passing on the first attempt is a PASS; passing only
+// after a retry is a FLAKE; an all-INFRA failure short-circuits to INFRA (retrying a
+// dead environment is pointless); an exhausted ladder is a REGRESSION. A runner that
+// throws is fail-closed to REGRESSION — never an implicit pass.
+//
+// A3 (#1817): `opts.tier === 'smoke'` forces the ladder down to a single ['initial']
+// attempt — @smoke tier gets zero retries, full stop. This overrides any caller-supplied
+// `opts.scopes`: retries hide races, and the smoke tier exists specifically to catch
+// them, so the zero-retry rule is not something a caller can opt out of.
+export function retryLadder(runAttempt, opts = {}) {
+  const quarantined =
+    opts.quarantined instanceof Set ? opts.quarantined : new Set(opts.quarantined || [])
+  const ladder =
+    opts.tier === 'smoke'
+      ? ['initial']
+      : Array.isArray(opts.scopes) && opts.scopes.length
+        ? opts.scopes
+        : ['initial', 'single-test', 'spec']
+  const attempts = []
+  let residual = []
+  for (let i = 0; i < ladder.length; i++) {
+    const scope = ladder[i]
+    let res
+    try {
+      res = runAttempt(scope)
+    } catch (err) {
+      attempts.push({ scope, error: String((err && err.message) || err) })
+      return { verdict: 'REGRESSION', attempts }
+    }
+    const passed = !!(res && res.passed)
+    const failures = res && Array.isArray(res.failures) ? res.failures : []
+    attempts.push({ scope, passed, failureCount: failures.length })
+    if (passed) return { verdict: i === 0 ? 'PASS' : 'FLAKE', attempts }
+    residual = failures
+    const labels = failures.map((f) => classify(f, { quarantined }))
+    if (labels.length && labels.every((l) => l === 'INFRA')) return { verdict: 'INFRA', attempts }
+  }
+  const labels = residual.map((f) => classify(f, { quarantined }))
+  if (labels.length && labels.every((l) => l === 'INFRA')) return { verdict: 'INFRA', attempts }
+  if (labels.length && labels.every((l) => l === 'FLAKE')) return { verdict: 'FLAKE', attempts }
+  return { verdict: 'REGRESSION', attempts }
+}
+
+// ─── Risk tier ────────────────────────────────────────────────────────────────
+// R0 clean · R1 known-flaky only · R2 infra present · R3 governance rot (expired
+// quarantine) · R4 regression present OR malformed input. Fail-closes to R4.
+export function riskTier(summary) {
+  try {
+    if (!summary || typeof summary !== 'object') return 'R4'
+    const n = (v) => (Number.isFinite(Number(v)) ? Number(v) : 0)
+    if (n(summary.regressions) > 0) return 'R4'
+    if (n(summary.expiredQuarantine) > 0) return 'R3'
+    if (n(summary.infra) > 0) return 'R2'
+    if (n(summary.flakes) > 0) return 'R1'
+    return 'R0'
+  } catch {
+    return 'R4'
+  }
+}
+
+// ─── Append-only ledger ───────────────────────────────────────────────────────
+// One JSON object per line; never rewrites prior lines. The historical record of
+// every reliability verdict, for auditing whether re-runs are masking regressions.
+export function appendLedger(path, entry) {
+  const record = { recorded_at: new Date().toISOString(), ...entry }
+  mkdirSync(dirname(path), { recursive: true })
+  appendFileSync(path, JSON.stringify(record) + '\n')
+  return record
+}
+
+// ─── Quarantine registry contract ─────────────────────────────────────────────
+// A quarantine entry annotates a known-unstable test; it does NOT suppress it. Every
+// entry must carry the full field set AND a FUTURE `expires` date so a quarantine
+// cannot rot into a permanent silent mute. validateQuarantine is the executable schema.
+export const QUARANTINE_REQUIRED_FIELDS = [
+  'id',
+  'fingerprint',
+  'reason',
+  'owner',
+  'added',
+  'expires',
+  'issue',
+]
+
+export function validateQuarantine(registry, now = new Date()) {
+  const errors = []
+  let entries
+  if (Array.isArray(registry)) entries = registry
+  else if (registry && Array.isArray(registry.entries)) entries = registry.entries
+  else return { ok: false, errors: ['quarantine registry must be an array or { entries: [...] }'] }
+
+  const nowMs = now instanceof Date ? now.getTime() : new Date(now).getTime()
+  entries.forEach((e, idx) => {
+    const label = (e && (e.id || e.fingerprint)) || `entry[${idx}]`
+    if (!e || typeof e !== 'object') {
+      errors.push(`${label}: not an object`)
+      return
+    }
+    for (const f of QUARANTINE_REQUIRED_FIELDS) {
+      if (e[f] === undefined || e[f] === null || e[f] === '') {
+        errors.push(`${label}: missing required field '${f}'`)
+      }
+    }
+    if (e.expires !== undefined && e.expires !== null && e.expires !== '') {
+      const expMs = new Date(e.expires).getTime()
+      if (Number.isNaN(expMs)) errors.push(`${label}: 'expires' is not a valid date (${e.expires})`)
+      else if (expMs < nowMs) {
+        errors.push(`${label}: quarantine expired on ${String(e.expires)} — re-triage or remove`)
+      }
+    }
+  })
+  return { ok: errors.length === 0, errors }
+}

package/dist/templates/scripts/lib/ephemeral-server.mjs.ejs

@@ -0,0 +1,148 @@
+#!/usr/bin/env node
+// Ephemeral-server runner (#358, CANON-02, CANON-15) — Phase 7F.
+//
+// Starts a server in the background, polls a TCP port for readiness, runs a
+// downstream test command, and tears the server down before exiting. The
+// runner's own exit code mirrors the test command's exit code so it composes
+// cleanly with the gate trinity (runCheck / runToolCheck).
+//
+// Usage:
+//   node scripts/lib/ephemeral-server.mjs \
+//     --start "<server command>" \
+//     --test  "<test command>" \
+//     --port  <port> \
+//     [--ready-timeout <ms>] \
+//     [--host <host>]
+//
+// Example (TS frontend-spa Playwright):
+//   node scripts/lib/ephemeral-server.mjs \
+//     --start "npm run preview -- --port 4173" \
+//     --test  "npx playwright test" \
+//     --port  4173
+//
+// Direct child_process use here is the documented exception for gate scripts
+// (mirrors scripts/check-all.mjs's carve-out); INV-12 applies to TS sources.
+import { spawn, spawnSync } from 'node:child_process';
+import net from 'node:net';
+
+const DEFAULT_READY_TIMEOUT_MS = 60 * 1000;
+const POLL_INTERVAL_MS = 250;
+
+function parseArgs(argv) {
+  const out = {
+    start: null,
+    test: null,
+    port: null,
+    host: '127.0.0.1',
+    readyTimeoutMs: DEFAULT_READY_TIMEOUT_MS,
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === '--start') out.start = argv[++i];
+    else if (a === '--test') out.test = argv[++i];
+    else if (a === '--port') out.port = Number(argv[++i]);
+    else if (a === '--host') out.host = argv[++i];
+    else if (a === '--ready-timeout') out.readyTimeoutMs = Number(argv[++i]);
+  }
+  return out;
+}
+
+function usage(msg) {
+  if (msg) process.stderr.write(`ephemeral-server: ${msg}\n`);
+  process.stderr.write(
+    'Usage: node scripts/lib/ephemeral-server.mjs --start "<cmd>" --test "<cmd>" --port <n> [--host <h>] [--ready-timeout <ms>]\n',
+  );
+  process.exit(2);
+}
+
+function probePort(host, port) {
+  return new Promise((resolve) => {
+    const socket = net.createConnection({ host, port });
+    socket.once('connect', () => {
+      socket.end();
+      resolve(true);
+    });
+    socket.once('error', () => {
+      socket.destroy();
+      resolve(false);
+    });
+  });
+}
+
+async function waitForPort(host, port, timeoutMs) {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (await probePort(host, port)) return true;
+    await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
+  }
+  return false;
+}
+
+let serverProcess = null;
+
+function teardown() {
+  if (serverProcess && serverProcess.pid && serverProcess.exitCode === null) {
+    try {
+      serverProcess.kill('SIGTERM');
+    } catch {
+      /* already exited */
+    }
+  }
+}
+
+process.on('SIGINT', () => {
+  teardown();
+  // Exit-code contract: 0=PASS / 1=FAIL / 2=ERROR. Interrupted run is FAIL.
+  process.exit(1);
+});
+process.on('SIGTERM', () => {
+  teardown();
+  process.exit(1);
+});
+
+async function main() {
+  const opts = parseArgs(process.argv.slice(2));
+  if (!opts.start) usage('missing --start');
+  if (!opts.test) usage('missing --test');
+  if (!opts.port || Number.isNaN(opts.port)) usage('missing or invalid --port');
+
+  process.stderr.write(`[ephemeral-server] starting: ${opts.start}\n`);
+  serverProcess = spawn(opts.start, {
+    shell: true,
+    stdio: 'inherit',
+    detached: false,
+  });
+  serverProcess.on('exit', (code, signal) => {
+    if (code !== null && code !== 0) {
+      process.stderr.write(`[ephemeral-server] server exited early code=${code}\n`);
+    } else if (signal) {
+      process.stderr.write(`[ephemeral-server] server signaled ${signal}\n`);
+    }
+  });
+
+  try {
+    process.stderr.write(
+      `[ephemeral-server] waiting for ${opts.host}:${opts.port} (timeout ${opts.readyTimeoutMs}ms)\n`,
+    );
+    const ready = await waitForPort(opts.host, opts.port, opts.readyTimeoutMs);
+    if (!ready) {
+      process.stderr.write('[ephemeral-server] readiness timeout\n');
+      process.exit(1);
+    }
+    process.stderr.write(`[ephemeral-server] ready — running: ${opts.test}\n`);
+    const testResult = spawnSync(opts.test, {
+      shell: true,
+      stdio: 'inherit',
+    });
+    const testStatus = typeof testResult.status === 'number' ? testResult.status : 1;
+    process.exit(testStatus);
+  } finally {
+    teardown();
+  }
+}
+
+main().catch((err) => {
+  process.stderr.write(`[ephemeral-server] error: ${err && err.message ? err.message : err}\n`);
+  teardown();
+  process.exit(1);
+});

package/dist/templates/scripts/lib/evidence-binding.mjs.ejs

@@ -0,0 +1,140 @@
+#!/usr/bin/env node
+// SPDX-License-Identifier: Apache-2.0
+//
+// #2399 — content binding for review/dispatch evidence.
+//
+// Binding review evidence to `sha === HEAD` makes every evidence-refresh commit
+// invalidate the evidence it just recorded: writing the artifact moves HEAD, so
+// the artifact must be rewritten and re-committed, forever (one PR carried nine
+// pure sha-bump commits). The property the evidence actually claims is "an agent
+// reviewed THIS SOURCE", so the binding is to source content:
+//
+//   valid iff  sha resolves
+//         AND  sha is an ancestor of HEAD          (same history, not a fork)
+//         AND  no path outside the evidence dirs changed between sha and HEAD
+//         AND  (when required) the checkout is on the branch the evidence names.
+//
+// Scope is deliberately the COMMITTED range only. Working-tree, index and
+// untracked dirt are NOT inspected here — every consumer keeps its own
+// dirty-checkout guard, so an uncommitted `src/` edit still invalidates the
+// evidence. Folding dirt in here would let a consumer drop its guard and end up
+// weaker than the exact-HEAD rule this replaces.
+//
+// The gate-pass marker (`.arbiter/gate-pass.json`, scripts/lib/gate-evidence.mjs)
+// deliberately stays EXACT-HEAD: the gate regenerates it on every run and it is
+// TTL-bound, so it has no refresh-loop problem to solve.
+//
+// Consumed by scripts/check-cross-model-review.mjs, scripts/check-review-completion.mjs,
+// scripts/check-agent-return.mjs and .claude/hooks/stop-evidence-guard.mjs.
+import { spawnSync } from 'node:child_process'
+
+/** Paths that hold evidence itself — a commit touching only these is not a source change. */
+const DEFAULT_EXCLUDES = Object.freeze(['.arbiter', '.agents'])
+
+function short(sha) {
+  return String(sha).slice(0, 7)
+}
+
+function git(root, args) {
+  return spawnSync('git', args, {
+    cwd: root,
+    encoding: 'utf-8',
+    stdio: ['ignore', 'pipe', 'ignore'],
+  })
+}
+
+/**
+ * Why `sha` no longer describes the current checkout, or `null` when it still does.
+ *
+ * FAIL-CLOSED: every unresolvable fact returns a REASON (a rejection), never `null` —
+ * an unreadable git is refused, not waved through.
+ *
+ * @param {string} root repository root
+ * @param {unknown} sha commit the evidence was recorded at
+ * @param {{ branch?: string, excludes?: readonly string[] }} [opts]
+ * @returns {string | null} a specific reason, or null when the evidence still binds
+ */
+export function evidenceStaleness(root, sha, opts = {}) {
+  const { branch, excludes = DEFAULT_EXCLUDES } = opts
+  if (typeof sha !== 'string' || sha.trim() === '') return 'evidence records no sha'
+
+  if (git(root, ['rev-parse', '--verify', '--quiet', `${sha}^{commit}`]).status !== 0) {
+    return `sha ${short(sha)} does not resolve in this repository`
+  }
+
+  const mismatch = branchMismatch(root, branch)
+  if (mismatch !== null) return mismatch
+
+  if (git(root, ['merge-base', '--is-ancestor', sha, 'HEAD']).status !== 0) {
+    return `sha ${short(sha)} is not an ancestor of HEAD`
+  }
+
+  return sourceChange(root, sha, excludes)
+}
+
+/**
+ * Why the checkout is not on the branch the evidence names, or null when it is (or when the
+ * evidence names none). An unreadable branch is a REASON, never a pass.
+ *
+ * @param {string} root repository root
+ * @param {string | undefined} branch branch the evidence was recorded on
+ * @returns {string | null}
+ */
+function branchMismatch(root, branch) {
+  if (branch === undefined) return null
+  const current = git(root, ['rev-parse', '--abbrev-ref', 'HEAD'])
+  if (current.status !== 0) return 'cannot read the current branch'
+  const currentBranch = String(current.stdout).trim()
+  if (currentBranch === branch) return null
+  return `branch mismatch: evidence is for ${branch}, checkout is on ${currentBranch}`
+}
+
+/**
+ * Why the committed tree outside `excludes` differs between `sha` and HEAD, or null when it
+ * does not. An unreadable diff is a REASON, never a pass.
+ *
+ * @param {string} root repository root
+ * @param {string} sha commit the evidence was recorded at
+ * @param {readonly string[]} excludes paths that hold evidence itself
+ * @returns {string | null}
+ */
+function sourceChange(root, sha, excludes) {
+  // `git diff --quiet` is a trichotomy: 0 unchanged, 1 changed, >=2 error.
+  const diff = git(root, [
+    'diff',
+    '--quiet',
+    sha,
+    'HEAD',
+    '--',
+    '.',
+    ...excludes.map((path) => `:(exclude)${path}`),
+  ])
+  if (diff.status === 1) return `source changed since ${short(sha)}`
+  if (diff.status !== 0) return `cannot compare the tree against ${short(sha)}`
+  return null
+}
+
+/**
+ * True when a dispatch sidecar was recorded for a DIFFERENT task than the active one.
+ *
+ * `.arbiter/agents-dispatched.json` is tracked and shared by every branch, so a
+ * sidecar left behind by task X otherwise fails every other branch's gate. A
+ * foreign sidecar is treated as ABSENT by the readers, not as a mismatch error.
+ *
+ * Conservative on both sides: a sidecar that declares no task, or an unknown
+ * active task, is NOT provably foreign — voiding those would turn every legacy
+ * sidecar into a silent hard failure.
+ *
+ * @param {unknown} sidecar parsed sidecar object
+ * @param {unknown} activeTaskId task id of the current task, when known
+ * @returns {boolean}
+ */
+export function isForeignSidecar(sidecar, activeTaskId) {
+  if (typeof sidecar !== 'object' || sidecar === null) return false
+  if (typeof activeTaskId !== 'string' || activeTaskId === '') return false
+  const declared = [sidecar.taskId, sidecar.task].filter(
+    (value) => typeof value === 'string' && value !== '',
+  )
+  if (declared.length === 0) return false
+  return declared.every((value) => value !== activeTaskId)
+}