@ticket-gate/engine 0.0.0-stage → 0.1.0-rc.1

package/run.mjs

@@ -0,0 +1,291 @@
+import { spawnSync } from 'node:child_process';
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { join } from 'node:path';
+import { GATE } from './project.mjs';
+import { pick } from './config.mjs';
+import { stopsOf } from './plan.mjs';
+import { receiptOk } from './receipt.mjs';
+import { oneLine } from './summary.mjs';
+import { ROOT } from './root.mjs';
+
+export const STEP_IDS = GATE.steps.map((s) => s.id);
+
+// Seconds a step may run before the watchdog kills its process group. A step
+// that needs longer gets `--timeout=<id>:<sec>`; the limit is never removed.
+export const STEP_LIMITS = Object.fromEntries(
+  GATE.steps.filter((s) => !s.human).map((s) => [s.id, s.limitS]),
+);
+
+const READY_LIMIT_S = 120;
+
+const STACK_STEPS = GATE.steps.filter((s) => s.stack).map((s) => s.id);
+
+// Steps before any GATE.stopOnRedBefore id in step order; a red one stops the stack steps.
+const preStack = (id) =>
+  stopsOf(GATE).some((s) => STEP_IDS.indexOf(id) < STEP_IDS.indexOf(s));
+
+const digestOn = (step) => (step.digest ? { digest: step.digest } : {});
+
+/**
+ * Turns a plan into receipt results. `runStep(step, { limitS })` executes one
+ * `run` step and returns `{ status, out, timedOut?, streamed? }`. Steps the
+ * diff did not select (`notSelected`) produce no result. Each run result gets
+ * `ms`, and `limitS` when `--timeout` overrode that step's limit. A red
+ * `preflight` stops the gate (D1): nothing after it runs, the receipt is red.
+ */
+export function buildResults(
+  plan,
+  runStep,
+  {
+    limits = STEP_LIMITS,
+    overridden = new Set(),
+    lock = null,
+    keepGoing = false,
+  } = {},
+) {
+  const results = [];
+  const failures = [];
+  // The id of the first red step before `seed`; once set, no stack
+  // step (and no preflight) that has not started is run.
+  let stoppedBy = null;
+  try {
+    for (const step of plan.steps) {
+      if (step.mode === 'human') {
+        console.log(`\n=== HUMAN STEP (not run): ${step.cmd}\n`);
+        results.push({
+          id: step.id,
+          cmd: step.cmd,
+          status: 'human',
+          summary: step.summary(''),
+        });
+        continue;
+      }
+      if (step.mode === 'reuse') {
+        console.log(`\n=== REUSED (${step.reusedFrom}): ${step.cmd}\n`);
+        results.push({
+          id: step.id,
+          cmd: step.cmd,
+          status: 'reused',
+          summary: `${step.reusedSummary} (reused from ${step.reusedFrom})`,
+          digest: step.digest,
+          reusedFrom: step.reusedFrom,
+        });
+        continue;
+      }
+      if (step.mode === 'skip') {
+        console.log(`\n=== SKIPPED (--skip=${step.id}): ${step.cmd}\n`);
+        results.push({
+          id: step.id,
+          cmd: step.cmd,
+          status: 'skipped',
+          summary: 'skipped by request: NOT verified',
+          ...digestOn(step),
+        });
+        continue;
+      }
+      if (
+        stoppedBy !== null &&
+        (STACK_STEPS.includes(step.id) ||
+          step.id === 'preflight' ||
+          step.health)
+      ) {
+        console.log(`\n=== STOPPED (${stoppedBy} red): ${step.cmd}\n`);
+        results.push({
+          id: step.id,
+          cmd: step.cmd,
+          status: 'stopped',
+          summary: `not run (stopped: ${stoppedBy} red)`,
+          ...digestOn(step),
+        });
+        continue;
+      }
+      console.log(`\n=== ${step.cmd}   (${step.why})\n`);
+      const started = Date.now();
+      if (step.id === 'preflight') {
+        const got = lock ? lock.acquire() : { ok: true };
+        if (!got.ok) {
+          results.push({
+            id: 'preflight',
+            cmd: step.cmd,
+            status: 'fail',
+            summary: got.summary,
+            ms: Date.now() - started,
+          });
+          failures.push({
+            id: 'preflight',
+            cmd: step.cmd,
+            why: step.why,
+            out: `${got.summary}\n`,
+          });
+          break;
+        }
+      }
+      if (typeof step.prepare === 'function') {
+        try {
+          step.prepare();
+        } catch (e) {
+          const summary = `e2e list: ${e.message}`;
+          results.push({
+            id: step.id,
+            cmd: step.cmd,
+            status: 'fail',
+            summary,
+            ms: Date.now() - started,
+            ...digestOn(step),
+          });
+          failures.push({
+            id: step.id,
+            cmd: step.cmd,
+            why: step.why,
+            out: `${summary}\n`,
+          });
+          continue;
+        }
+      }
+      const limitS = limits[step.id] ?? step.limitS;
+      const first = runStep(step, { limitS });
+      if (first.streamed !== true) process.stdout.write(first.out);
+      let ok = first.status === 0 && first.timedOut !== true;
+      let { out } = first;
+      let timedOut = first.timedOut === true;
+      let timedOutLimit = limitS;
+      let readyFailed = false;
+      if (ok && step.ready) {
+        console.log(
+          `\n=== ${step.ready}   (${step.id} is up; waiting for the services later steps need)\n`,
+        );
+        const r = runStep(
+          { id: step.id, cmd: step.ready, append: true },
+          { limitS: READY_LIMIT_S },
+        );
+        if (r.streamed !== true) process.stdout.write(r.out);
+        if (r.status !== 0 || r.timedOut === true) {
+          ok = false;
+          readyFailed = true;
+          out = r.out;
+          timedOut = r.timedOut === true;
+          timedOutLimit = READY_LIMIT_S;
+        }
+      }
+      if (!ok) {
+        const hint = STACK_STEPS.includes(step.id)
+          ? ` Check the stack with: ${GATE.stackHint(step.id)}`
+          : ` Rerun this step alone: ${step.cmd}`;
+        failures.push({
+          id: step.id,
+          cmd: step.cmd,
+          why: step.why,
+          out: timedOut
+            ? `${out}\nTIMEOUT after ${timedOutLimit} s (limit ${step.id}: ${timedOutLimit} s).${hint}\n`
+            : out,
+        });
+      }
+      let summary;
+      if (timedOut)
+        summary = `TIMEOUT after ${timedOutLimit} s (limit ${step.id}: ${timedOutLimit} s)`;
+      else if (readyFailed) summary = oneLine(out) || 'stack not ready';
+      else if (ok) summary = step.summary(out) ?? 'passed';
+      else
+        summary =
+          step.failSummary?.(out) ??
+          pick(out, [/Code style issues/, /error/i, /failed/i]) ??
+          'failed';
+      results.push({
+        id: step.id,
+        cmd: step.cmd,
+        status: ok ? 'pass' : 'fail',
+        summary,
+        ms: Date.now() - started,
+        ...digestOn(step),
+        ...(overridden.has(step.id) ? { limitS } : {}),
+      });
+      // prettier-ignore
+      if (!ok && !keepGoing && stoppedBy === null && preStack(step.id)) stoppedBy = step.id;
+      if (step.id === 'preflight' && !ok) break;
+    }
+  } finally {
+    lock?.release();
+  }
+  // `ok` means every required step actually ran and passed. A skipped step is
+  // not ok: the gate reads `results` too, but `ok` is what a human reads first.
+  return { results, ok: receiptOk(results), failures, stoppedBy };
+}
+
+// Each step runs in its own process group under a watchdog that kills the
+// group once this process is gone. Without it, a gate stopped by a signal to
+// its own pid (pkill, a closed agent shell) left `supabase db reset` running
+// at full CPU for over an hour, and the next gate timed out on load.
+// The same watchdog enforces the step's time limit: past it the group gets
+// TERM, then KILL 5 s later, and the marker file says the limit (not the step)
+// ended it. $1 cmd, $2 limit in seconds (0 = none), $3 log (appended), $4 marker.
+// Output reaches the terminal live through tee and is kept in the log.
+const STEP_WATCHDOG = String.raw`
+set -m
+( set -o pipefail; bash -lc "$1" 2>&1 | tee -a "$3" ) &
+child=$!
+set +m
+( while kill -0 "$VERIFY_TICKET_PID" 2>/dev/null; do sleep 2; done
+  kill -TERM -"$child" 2>/dev/null; sleep 5; kill -KILL -"$child" 2>/dev/null ) >/dev/null 2>&1 &
+watch=$!
+timer=
+if [ "$2" -gt 0 ]; then
+  ( sleep "$2"; echo timeout > "$4"; kill -TERM -"$child" 2>/dev/null; sleep 5; kill -KILL -"$child" 2>/dev/null ) >/dev/null 2>&1 &
+  timer=$!
+fi
+wait "$child"; status=$?
+# The timer arm's sleep is its child and outlives a kill of the subshell: stop the
+# subshell (it can fork no more), find its sleep, then kill both.
+kids=
+if [ -n "$timer" ]; then kill -STOP "$timer" 2>/dev/null; kids=$(pgrep -P "$timer" 2>/dev/null); fi
+kill "$watch" 2>/dev/null
+[ -n "$timer" ] && kill -KILL "$timer" $kids 2>/dev/null
+exit "$status"`;
+
+/**
+ * Runs one step under the watchdog. Logs under `logDir/<id>.log` (only .turbo/
+ * is ever written); `step.append` keeps the existing log and returns just the
+ * new part. `timedOut` is true only when the watchdog's timer fired: a step
+ * that itself exits 124 is an ordinary failure.
+ */
+export const runInShell = (
+  step,
+  { limitS, logDir = join(ROOT, '.turbo', 'verify-steps') } = {},
+) => {
+  mkdirSync(logDir, { recursive: true });
+  const log = join(logDir, `${step.id ?? 'step'}.log`);
+  const marker = `${log}.timeout`;
+  rmSync(marker, { force: true });
+  if (!step.append) writeFileSync(log, '');
+  const before = readFileSync(log, 'utf8').length;
+  const r = spawnSync(
+    'bash',
+    [
+      '-c',
+      STEP_WATCHDOG,
+      'verify-step',
+      step.cmd,
+      String(limitS ?? 0),
+      log,
+      marker,
+    ],
+    {
+      cwd: ROOT,
+      stdio: ['ignore', 'inherit', 'inherit'],
+      env: { ...process.env, VERIFY_TICKET_PID: String(process.pid) },
+    },
+  );
+  const timedOut = existsSync(marker);
+  rmSync(marker, { force: true });
+  return {
+    status: r.status ?? (timedOut ? 124 : 1),
+    out: readFileSync(log, 'utf8').slice(before),
+    timedOut,
+    streamed: true,
+  };
+};

package/runners/playwright.mjs

@@ -0,0 +1,132 @@
+import { execFileSync } from 'node:child_process';
+
+import { projectSelection } from '../selection/project.mjs';
+
+const ERROR_CAP = 2000;
+
+/** Walks Playwright's nested suites, calling visit(spec, test) for each test. */
+export function walkReport(suites, visit) {
+  for (const s of suites ?? []) {
+    for (const spec of s.specs ?? [])
+      for (const t of spec.tests ?? []) visit(spec, t, s);
+    walkReport(s.suites, visit);
+  }
+}
+
+/** Playwright JSON -> { stats, failedSpecs, flakySpecs, redTests }; null stats = interrupted. */
+export function summarizeReport(playwrightJson) {
+  const report = playwrightJson ?? {};
+  const stats = { expected: 0, unexpected: 0, flaky: 0, skipped: 0 };
+  const failed = new Set();
+  const flaky = new Set();
+  const redTests = [];
+  const navigation = {};
+  walkReport(report.suites, (spec, t) => {
+    if (t.status in stats) stats[t.status] += 1;
+    const file = spec.file ?? '';
+    const notes = [
+      ...(t.annotations ?? []),
+      ...(t.results ?? []).flatMap((r) => r.annotations ?? []),
+    ];
+    for (const a of notes) {
+      if (a?.type !== 'e2e-nav') continue;
+      const seen = (navigation[file] ??= []);
+      for (const p of String(a.description ?? '').split(' '))
+        if (p !== '' && !seen.includes(p)) seen.push(p);
+    }
+    if (t.status === 'unexpected') {
+      failed.add(file);
+      const last = (t.results ?? []).at(-1);
+      const msg = last?.error?.message ?? last?.errors?.[0]?.message ?? '';
+      redTests.push({
+        file,
+        title: spec.title,
+        project: t.projectName,
+        error: String(msg).slice(0, ERROR_CAP),
+      });
+    }
+    if (t.status === 'flaky') flaky.add(file);
+  });
+  return {
+    stats: report.stats ? stats : null,
+    failedSpecs: [...failed].sort(),
+    flakySpecs: [...flaky].sort(),
+    redTests,
+    navigation,
+  };
+}
+
+// ---------------------------------------------------------------- shadow and test list
+
+const isSmoke = (spec, t) =>
+  [...(spec.tags ?? []), ...(t.tags ?? [])].some(
+    (x) => String(x).replace(/^@/, '') === 'smoke',
+  ) || /@smoke\b/.test(spec.title ?? '');
+
+/** What a partial run would have missed (R6). Never changes the gate's result. */
+export function shadowOf(selection, report, cfg = projectSelection()) {
+  const selected = new Set(selection.specs);
+  const base = {
+    mode: selection.mode,
+    why: selection.why,
+    selected: [...selection.specs],
+  };
+  if (selection.mode === 'full') return { ...base, missed: [] };
+  if (report == null) return { ...base, missed: null, error: 'no-report' };
+  const failed = new Set();
+  walkReport(report.suites, (spec, t) => {
+    if (t.status !== 'unexpected') return;
+    const file = spec.file ?? '';
+    if (cfg.specs.setup.has(file) || isSmoke(spec, t)) return;
+    failed.add(file);
+  });
+  const failedFiles = [...failed];
+  const missed = failedFiles.filter((f) => !selected.has(f));
+  return { ...base, missed: missed.sort() };
+}
+
+/** Lines for `playwright test --test-list`: every test of a selected file plus smoke tests. */
+export function testListLines(listJson, selection) {
+  if (selection.mode === 'full') return null;
+  const selected = new Set(selection.specs);
+  const lines = new Set();
+  const visit = (suites, titles) => {
+    for (const s of suites ?? []) {
+      const own = titles.length === 0 && s.title === s.file ? [] : [s.title];
+      const path = [...titles, ...own.filter(Boolean)];
+      for (const spec of s.specs ?? []) {
+        const file = spec.file ?? s.file ?? '';
+        for (const t of spec.tests ?? [])
+          if (selected.has(file) || isSmoke(spec, t))
+            lines.add(`${file} › ${[...path, spec.title].join(' › ')}`);
+      }
+      visit(s.suites, path);
+    }
+  };
+  visit(listJson?.suites, []);
+  return [...lines];
+}
+
+// ---------------------------------------------------------------- runner
+
+/**
+ * The Playwright runner adapter. `cmd` is the command that runs `playwright test` from the repo
+ * root; `specPrefix` is the spec path prefix that command takes.
+ */
+export function playwright({ cmd, specPrefix }) {
+  return {
+    name: 'playwright',
+    command: cmd.join(' '),
+    specPrefix,
+    list: (cwd) =>
+      JSON.parse(
+        execFileSync(cmd[0], [...cmd.slice(1), '--list', '--reporter=json'], {
+          cwd,
+          encoding: 'utf8',
+          maxBuffer: 1 << 28,
+        }),
+      ),
+    summarize: summarizeReport,
+    invertFlag: '--test-list-invert',
+  };
+}

package/score/adapters/stryker.mjs

@@ -0,0 +1,111 @@
+// The Stryker adapter of the mutation score gate: how to run the mutation
+// tool on the branch and on a base worktree, and which lines disable it.
+import { spawnSync } from 'node:child_process';
+import { readFileSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+
+import { fail2, git } from '../base.mjs';
+
+export const DISABLE_WITH_REASON =
+  /Stryker\s+disable(?:\s+next-line)?\s+[\w,]+\s*:\s*\S/;
+
+/** Lines of `source` that disable Stryker without a stated reason. */
+export function disablesWithoutReason(source) {
+  const bad = [];
+  source.split('\n').forEach((text, i) => {
+    if (!/Stryker\s+disable/.test(text)) return;
+    const line = text;
+    if (!DISABLE_WITH_REASON.test(line)) bad.push({ line: i + 1 });
+  });
+  return bad;
+}
+
+function runStryker(cmd, args, cwd, label) {
+  const r = spawnSync(cmd, args, { cwd, stdio: 'inherit' });
+  if (r.status !== 0)
+    fail2(
+      `${label} Stryker run exited ${r.status ?? r.signal}. Run it by hand: cd ${cwd} && ${cmd} ${args.join(' ')}`,
+    );
+}
+
+function addedDisables(root, base, path, isNew) {
+  const found = [];
+  if (isNew) {
+    readFileSync(join(root, path), 'utf8')
+      .split('\n')
+      .forEach((text, i) => {
+        if (/Stryker\s+disable/.test(text))
+          found.push({ path, line: i + 1, text });
+      });
+    return found;
+  }
+  const diff = git(root, [
+    'diff',
+    '-U0',
+    '--no-renames',
+    '--no-color',
+    base,
+    '--',
+    path,
+  ]);
+  let line = 0;
+  for (const l of diff.split('\n')) {
+    const h = /^@@ -\d+(?:,\d+)? \+(\d+)/.exec(l);
+    if (h) line = Number(h[1]);
+    else if (l.startsWith('+') && !l.startsWith('+++')) {
+      if (/Stryker\s+disable/.test(l))
+        found.push({ path, line, text: l.slice(1) });
+      line++;
+    }
+  }
+  return found;
+}
+
+/**
+ * `branchCmd`: the command (argv) that runs Stryker for the branch, given
+ * `--mutate <files>` appended. `packageDir`: the package the files live in.
+ * `config`: Stryker's config file inside it. `reportPath`: the JSON report,
+ * relative to the repo root (and to a base worktree).
+ */
+export function stryker({ branchCmd, packageDir, config, reportPath }) {
+  const rel = (t) =>
+    packageDir === '.' ? t : t.slice(`${packageDir}/`.length);
+  const forceArg = (force) => (force ? ['--force'] : []);
+  return {
+    name: 'stryker',
+    reportPath,
+    runBranch({ root, files, force }) {
+      rmSync(join(root, reportPath), { force: true });
+      runStryker(
+        branchCmd[0],
+        [
+          ...branchCmd.slice(1),
+          '--mutate',
+          files.map(rel).join(','),
+          ...forceArg(force),
+        ],
+        root,
+        'branch',
+      );
+    },
+    runBase({ root, dir, files, force }) {
+      const out = join(dir, reportPath);
+      rmSync(out, { force: true });
+      runStryker(
+        join(root, packageDir, 'node_modules', '.bin', 'stryker'),
+        [
+          'run',
+          join(root, packageDir, config),
+          '--mutate',
+          files.map(rel).join(','),
+          ...forceArg(force),
+        ],
+        join(dir, packageDir),
+        'base',
+      );
+      return out;
+    },
+    disablesWithoutReason,
+    addedDisables,
+  };
+}

package/score/base.mjs

@@ -0,0 +1,82 @@
+// Git, report and base-worktree helpers of the mutation score gate. Every
+// project value (install command, marker, worktree folder) arrives as `cfg`.
+import { execFileSync, spawnSync } from 'node:child_process';
+import { existsSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import { schemaProblem } from './schema.mjs';
+
+export class GateError extends Error {}
+
+export const fail2 = (msg) => {
+  throw new GateError(`mutation: FAIL ${msg}`);
+};
+
+export function git(cwd, args) {
+  return execFileSync('git', args, {
+    cwd,
+    encoding: 'utf8',
+    maxBuffer: 256 * 1024 * 1024,
+    stdio: ['ignore', 'pipe', 'pipe'],
+  });
+}
+
+export const gitOk = (cwd, args) => {
+  try {
+    git(cwd, args);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+export function readReport(path, label) {
+  if (!existsSync(path)) fail2(`${label} report missing: ${path}`);
+  let report;
+  try {
+    report = JSON.parse(readFileSync(path, 'utf8'));
+  } catch (e) {
+    return fail2(`${label} report unreadable: ${path} (${e.message})`);
+  }
+  const problem = schemaProblem(report);
+  if (problem) fail2(`${label} report ${path}: ${problem}`);
+  return report;
+}
+
+/** The install runner for a `baseInstall` config: `(dir) => spawnResult`. */
+export const installWith = (baseInstall) => (dir) =>
+  spawnSync(baseInstall.cmd, baseInstall.args, { cwd: dir, stdio: 'inherit' });
+
+export function ensureBaseInstall(dir, sha, install, cfg) {
+  const { baseInstall } = cfg;
+  if (!existsSync(join(dir, baseInstall.marker))) {
+    const r = install(dir);
+    if (r.status !== 0)
+      fail2(
+        `${baseInstall.label} in the base worktree ${dir} exited ${r.status}. Run it by hand: cd ${dir} && ${baseInstall.hint}`,
+      );
+    writeFileSync(join(dir, baseInstall.marker), sha);
+  }
+}
+
+export function baseWorktree(root, sha, cfg) {
+  const dir = join(tmpdir(), cfg.worktreeDir, sha.slice(0, 12));
+  try {
+    git(root, ['worktree', 'prune']);
+    const head = existsSync(dir)
+      ? git(dir, ['rev-parse', 'HEAD']).trim()
+      : null;
+    if (head !== sha) {
+      if (existsSync(dir)) {
+        gitOk(root, ['worktree', 'remove', '--force', dir]);
+        rmSync(dir, { recursive: true, force: true });
+      }
+      git(root, ['worktree', 'add', '--detach', dir, sha]);
+    }
+  } catch (e) {
+    fail2(`base worktree ${dir} failed: ${e.stderr ?? e.message}`);
+  }
+  ensureBaseInstall(dir, sha, installWith(cfg.baseInstall), cfg);
+  return dir;
+}