@1aboveio/skills 0.10.0

package/skills/engineering/resolve-issues/scripts/sandbox-selftest.mjs

@@ -0,0 +1,395 @@
+#!/usr/bin/env node
+// sandbox-selftest — MECHANIZES references/sandboxed-testing.md. Mutation testing
+// (flip a predicate, delete a bound, revert a guard, then confirm the suite
+// notices) must never run against this skill's live working tree — a corruption
+// you forget to revert ships. (Named incident: during the #387 audit, a mutation
+// test in place left `PRUNE_AFTER_DAYS = 6000` behind in an uncommitted tree; had
+// it survived, catalog pruning would have shipped disabled for ~16 years.)
+//
+// The safe method is a scratch copy — but a bare copy of resolve-issues does not
+// run clean: some tests import `../../review-pr/scripts/...` (a bare copy has no
+// sibling `review-pr` beside it — the flat distribution model installs
+// this skill AS a sibling of review-pr and engineering-runtime, not standalone), and some tests walk up
+// to the repo root that a scratch copy has no parent for. So the recipe is: copy
+// the skill to a temp dir, symlink the sibling skills beside it, run the suites,
+// and confirm the ONLY failures are the KNOWN repo-root-dependent tests — the ones
+// that read files above the copy (a repo-root fixture, or repo-root SKILL.md /
+// CLAUDE.md) and therefore cannot pass in any isolated sandbox.
+//
+// The baseline is NOT a magic count ("total minus one"): the set of repo-root
+// tests grows and shrinks as the suite evolves, so the check is set-based, matched
+// by test name + file. Every non-baseline verdict NAMES which test moved:
+//   - a failure OUTSIDE the known set → a real regression;
+//   - a known-set test that unexpectedly PASSES → it no longer depends on repo
+//     root; the environment changed, and the fix is to remove it from the set.
+//
+//   node sandbox-selftest.mjs                      # sandbox resolve-issues, run, verify
+//   node sandbox-selftest.mjs --skill review-pr    # a different skill under test
+//   node sandbox-selftest.mjs --keep               # leave the sandbox on disk for debugging
+//   node sandbox-selftest.mjs --help               # usage + exit 0
+//
+// NEVER mutates the real skill tree: every file operation lands in an OS temp dir
+// (`mkdtempSync`), the siblings are symlinked read-only-in-intent, and the sandbox
+// is removed on exit unless `--keep`.
+//
+// EXIT CODES:
+//   0  baseline held — exactly the known repo-root-dependent tests failed, nothing else
+//   1  anomaly — a real regression, or a known-set test that moved (named either way)
+//   2  usage error, or setup failure (skill not found, no tests ran)
+
+import fs from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { execFileSync } from 'node:child_process'
+import { isMainModule } from '../../engineering-runtime/scripts/main-module.mjs'
+
+// The sibling skills symlinked beside the copy. `review-pr` is the one actually
+// imported by the suite (`scripts/fix-rounds.mjs` → `../../review-pr/scripts/...`);
+// the rest round out parity with the skills resolve-issues orchestrates
+// (SKILL.md frontmatter `dependencies:`), in case a broader sandbox session
+// touches them too.
+export const SIBLINGS = ['review-pr', 'implement-and-pr', 'ensure-coverage', 'e2e-test', 'smoke']
+
+// `engineering-runtime` is a support package and is NOT optional: every CLI script in this family
+// imports `../../engineering-runtime/scripts/main-module.mjs` for its main-guard (#718/#446), so a
+// sandbox without it fails at module load in every suite at once. Listed separately from SIBLINGS
+// because it supplies runtime support rather than an orchestrated user workflow.
+export const SHARED_RUNTIME = 'engineering-runtime'
+
+// The tests that CANNOT pass in an isolated sandbox because they resolve paths
+// ABOVE the copy — a repo-root fixture, or repo-root SKILL.md/CLAUDE.md/run-state.md.
+// The baseline is "total minus this set", verified by FILE + a distinctive
+// name fragment (`nameIncludes`), robust to reordering AND to a suffix tweak.
+// Keyed by skill; a skill with no repo-root-dependent tests has an empty set (any
+// failure is then a real regression). Adding/removing a repo-root test here is a
+// one-line diff, and this script flags a member that stops failing so the set
+// can't silently drift out of date (the drift that made "total minus one" wrong —
+// see references/sandboxed-testing.md).
+//
+// The fragments deliberately omit the obsolete `units[].`-prefixed field token that
+// some of these artifact-audit test names carry: this file is a real gate script,
+// and artifact-audit's own delete-safety test asserts NO gate script references
+// that deleted field. A fragment without that token is still distinctive enough to
+// identify the test uniquely without re-introducing a false reader.
+//
+// resolve-issues, as of #406:
+//   - run-state-review6: cause 3 — reads ../../../../docs/evidence/809-runtime-smoke.json.
+//   - 5 × artifact-audit (#379/#397): cause 4 — walk ../../../.. to read repo-root
+//     skills/SKILL.md, CLAUDE.md, run-state.md, and to scan the whole skills/ tree.
+export const KNOWN_FAILING_TESTS = {
+  'resolve-issues': [
+    {
+      file: 'run-state-review6.test.mjs',
+      nameIncludes: 'one public table-driven executor runs every literal 42-cell case',
+    },
+    { file: 'artifact-audit.test.mjs', nameIncludes: 'has no machine reader anywhere in the real gate scripts' },
+    { file: 'artifact-audit.test.mjs', nameIncludes: 'none of the three SKILL.md files mandate' },
+    { file: 'artifact-audit.test.mjs', nameIncludes: 'run-state.md no longer documents' },
+    { file: 'artifact-audit.test.mjs', nameIncludes: 'resolve-issues SKILL.md instructs only unit statuses the code accepts' },
+    { file: 'artifact-audit.test.mjs', nameIncludes: 'CLAUDE.md records which CI machinery is unwired in this repo' },
+  ],
+}
+
+export function knownFailingFor(skill) {
+  return KNOWN_FAILING_TESTS[skill] ?? []
+}
+
+// ---- setupSandbox: the copy + symlink layout (pure fs, unit-testable) ----
+
+// Copy <repoRoot>/skills/engineering/<skill> into <parentDir>/<skill> as a REAL
+// copy, then symlink each sibling skill beside it (skipping the skill under test
+// itself, so a `--skill review-pr` run does not shadow its own copy) plus the shared
+// runtime package. Returns the copy dir and the symlinks created. Throws (→ exit 2) if the
+// skill is not found, or if engineering-runtime is missing — see SHARED_RUNTIME.
+export function setupSandbox({ repoRoot, skill = 'resolve-issues', parentDir, siblings = SIBLINGS }) {
+  const engineeringDir = path.join(repoRoot, 'skills', 'engineering')
+  const src = path.join(engineeringDir, skill)
+  if (!fs.existsSync(src)) {
+    throw new Error(`sandbox-selftest: skill "${skill}" not found at ${src}`)
+  }
+
+  const skillCopyDir = path.join(parentDir, skill)
+  // cpSync recursive makes an independent copy — a mutation in the copy never
+  // reaches the source. (Symlinks inside the source, if any, are copied as links.)
+  fs.cpSync(src, skillCopyDir, { recursive: true })
+
+  const symlinks = []
+  for (const name of siblings) {
+    if (name === skill) continue // never symlink a sibling over the copy under test
+    const target = path.join(engineeringDir, name)
+    if (!fs.existsSync(target)) continue // a missing optional sibling is not fatal
+    const link = path.join(parentDir, name)
+    fs.symlinkSync(target, link)
+    symlinks.push({ name, target, link })
+  }
+
+  // Hard-failed rather than skipped like an optional sibling: without it EVERY suite dies at import
+  // and the run reports a meaningless zero.
+  const runtimeTarget = path.join(engineeringDir, SHARED_RUNTIME)
+  if (!fs.existsSync(runtimeTarget)) {
+    throw new Error(`sandbox-selftest: shared ${SHARED_RUNTIME} not found at ${runtimeTarget} — every main-guard imports it`)
+  }
+  const runtimeLink = path.join(parentDir, SHARED_RUNTIME)
+  fs.symlinkSync(runtimeTarget, runtimeLink)
+  symlinks.push({ name: SHARED_RUNTIME, target: runtimeTarget, link: runtimeLink })
+
+  return { skillCopyDir, symlinks }
+}
+
+// ---- parseTapSummary: pass/total/failing-test extraction (pure, unit-testable) ----
+
+// Parse the tail of `node --test --test-reporter=tap` output into
+// { tests, pass, fail, failingTests }. The summary block is the authoritative
+// count (`# tests N`, `# pass N`, `# fail N`); each failing test is captured as
+// { name, file } — the name from its `not ok N - <name>` line (TAP directive
+// suffixes stripped), the file basename from the `location:` field of its YAML
+// diagnostic block (null if absent). Throws if no summary block is present — a run
+// that produced no counts is a setup failure, not a zero-pass result.
+export function parseTapSummary(output) {
+  const num = (label) => {
+    const m = output.match(new RegExp(`^# ${label} (\\d+)$`, 'm'))
+    return m ? Number(m[1]) : null
+  }
+  const tests = num('tests')
+  const pass = num('pass')
+  const fail = num('fail')
+  if (tests === null || pass === null || fail === null) {
+    throw new Error('sandbox-selftest: no TAP summary (# tests/# pass/# fail) found in `node --test` output')
+  }
+
+  const failingTests = []
+  let pending = null
+  const flush = () => {
+    if (pending) failingTests.push(pending)
+    pending = null
+  }
+  for (const line of output.split('\n')) {
+    const nok = /^not ok \d+ - (.+)$/.exec(line)
+    if (nok) {
+      flush()
+      pending = { name: nok[1].replace(/\s+#\s+(TODO|SKIP).*$/i, '').trim(), file: null }
+      continue
+    }
+    if (pending) {
+      // location: '/abs/path/to/file.test.mjs:LINE:COL'  → take the basename.
+      const loc = /^\s*location:\s*'(.+):\d+:\d+'\s*$/.exec(line)
+      if (loc) {
+        pending.file = path.basename(loc[1])
+        flush()
+        continue
+      }
+      if (/^\s*\.\.\.\s*$/.test(line)) flush() // YAML block ended with no location
+    }
+  }
+  flush()
+
+  return { tests, pass, fail, failingTests }
+}
+
+// ---- evaluateBaseline: the set-based verdict (pure, unit-testable) ----
+
+// A known member matches a failing test when the failing test's name contains the
+// member's distinctive fragment and the files agree (or the failing test carried
+// no parseable file).
+function matches(known, failing) {
+  return failing.name.includes(known.nameIncludes) && (failing.file == null || known.file === failing.file)
+}
+
+// Given combined { tests, pass, failingTests } and the skill's knownSet of
+// repo-root-dependent tests, decide whether the sandbox baseline held: the ONLY
+// failures are exactly the known set. Every non-baseline verdict names the
+// offending test. kind ∈ 'baseline' | 'regression' | 'moved' | 'empty'.
+export function evaluateBaseline({ tests, pass, failingTests = [], knownSet = [] }) {
+  if (tests === 0) {
+    return {
+      ok: false,
+      kind: 'empty',
+      message:
+        'no tests ran in the sandbox — the copy or the suites did not execute. This is a setup failure, not a baseline result.',
+    }
+  }
+
+  const unexpected = failingTests.filter((f) => !knownSet.some((k) => matches(k, f)))
+  const missing = knownSet.filter((k) => !failingTests.some((f) => matches(k, f)))
+
+  if (unexpected.length > 0) {
+    let message =
+      `REGRESSION: ${pass}/${tests} passing, and ${unexpected.length} failure(s) are NOT in the known ` +
+      `repo-root-dependent set — a real regression, not a sandbox artifact:\n  - ` +
+      unexpected.map((f) => `${f.name}${f.file ? ` (${f.file})` : ''}`).join('\n  - ')
+    if (missing.length > 0) {
+      message +=
+        `\nAlso, ${missing.length} known-set test(s) unexpectedly PASSED — the known set is also out of date ` +
+        `(they no longer depend on repo root; remove them from KNOWN_FAILING_TESTS):\n  - ` +
+        missing.map((k) => `${k.nameIncludes} (${k.file})`).join('\n  - ')
+    }
+    return { ok: false, kind: 'regression', message }
+  }
+
+  if (missing.length > 0) {
+    return {
+      ok: false,
+      kind: 'moved',
+      message:
+        `A KNOWN TEST MOVED: ${pass}/${tests} passing, no unexpected failures, but ${missing.length} test(s) ` +
+        `that were expected to fail in a sandbox PASSED:\n  - ` +
+        missing.map((k) => `${k.nameIncludes} (${k.file})`).join('\n  - ') +
+        `\nThis test no longer depends on repo root — remove it from the known set ` +
+        `(KNOWN_FAILING_TESTS in sandbox-selftest.mjs). Until then the baseline can't be trusted.`,
+    }
+  }
+
+  return {
+    ok: true,
+    kind: 'baseline',
+    message:
+      `baseline held: ${pass}/${tests} passing. Exactly the ${knownSet.length} known repo-root-dependent ` +
+      `test(s) failed, nothing else.`,
+  }
+}
+
+// ---- suite execution (the multi-minute part; not unit-tested) ----
+
+// Enumerate the *.test.mjs files under scripts/ and workflows/ of the copy.
+function discoverTestFiles(skillCopyDir) {
+  const files = []
+  for (const sub of ['scripts', 'workflows']) {
+    const dir = path.join(skillCopyDir, sub)
+    if (!fs.existsSync(dir)) continue
+    for (const f of fs.readdirSync(dir)) {
+      if (f.endsWith('.test.mjs')) files.push(path.join(dir, f))
+    }
+  }
+  return files.sort()
+}
+
+// Run `node --test --test-reporter=tap <files>` from the copy dir and return the
+// captured stdout (the suite is EXPECTED to exit non-zero: the known repo-root
+// tests fail). cwd is the copy so repo-root-relative walks resolve — and fail —
+// exactly as in a real sandbox.
+function runSuites(skillCopyDir, testFiles) {
+  let stdout = ''
+  try {
+    stdout = execFileSync('node', ['--test', '--test-reporter=tap', ...testFiles], {
+      cwd: skillCopyDir,
+      encoding: 'utf8',
+      maxBuffer: 64 * 1024 * 1024,
+    })
+  } catch (err) {
+    // node --test exits non-zero when any test fails — that is the expected path.
+    stdout = err.stdout ?? ''
+  }
+  return stdout
+}
+
+// ---- CLI ----
+
+const USAGE = `sandbox-selftest — mechanizes references/sandboxed-testing.md.
+
+Copies a skill (default resolve-issues) into an OS temp dir, symlinks the sibling
+skills beside it, runs its scripts/*.test.mjs + workflows/*.test.mjs suites, and
+verifies the ONLY failures are the known repo-root-dependent tests — the ones that
+read files above the copy (a repo-root fixture, or repo-root SKILL.md/CLAUDE.md)
+and so cannot pass in any isolated sandbox. The baseline is the suite TOTAL MINUS
+that named set (not a magic count): a failure outside the set is a real regression,
+and a known-set test that starts passing is flagged so the set stays honest. On
+any mismatch it names which test moved. The real skill tree is never mutated — all
+work happens in the temp dir.
+
+Usage:
+  node sandbox-selftest.mjs [--skill <name>] [--keep]
+  node sandbox-selftest.mjs --help
+
+Options:
+  --skill <name>   skill under test (default: resolve-issues)
+  --keep           leave the temp sandbox on disk (prints its path) for debugging
+  --help, -h       print this usage and exit 0
+
+Exit codes:
+  0  baseline held: exactly the known repo-root-dependent tests failed, nothing else
+  1  anomaly: a real regression, or a known-set test that moved (named either way)
+  2  usage error, or setup failure (skill not found / no tests ran)`
+
+export function parseArgs(argv) {
+  const f = { skill: 'resolve-issues' }
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i]
+    if (a === '--help' || a === '-h' || a === 'help') f.help = true
+    else if (a === '--keep') f.keep = true
+    else if (a === '--skill') {
+      f.skill = argv[++i]
+      if (!f.skill) f.unknown = '--skill (missing value)'
+    } else if (a.startsWith('--skill=')) f.skill = a.slice('--skill='.length)
+    else f.unknown = a
+  }
+  return f
+}
+
+// repoRoot is four levels up from this script: <root>/skills/engineering/resolve-issues/scripts.
+export function repoRootFromScript(scriptPath) {
+  return path.resolve(path.dirname(scriptPath), '..', '..', '..', '..')
+}
+
+export function runCli(argv, { scriptPath } = {}) {
+  const f = parseArgs(argv)
+  if (f.help) {
+    console.log(USAGE)
+    return 0
+  }
+  if (f.unknown) {
+    console.error(`sandbox-selftest: unknown argument "${f.unknown}"\n\n${USAGE}`)
+    return 2
+  }
+
+  const repoRoot = repoRootFromScript(scriptPath ?? fileURLToPath(import.meta.url))
+  const parentDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sandbox-selftest-'))
+  const cleanup = () => {
+    if (f.keep) console.error(`sandbox-selftest: --keep set, sandbox retained at ${parentDir}`)
+    else fs.rmSync(parentDir, { recursive: true, force: true })
+  }
+
+  let setup
+  try {
+    setup = setupSandbox({ repoRoot, skill: f.skill, parentDir })
+  } catch (err) {
+    console.error(err.message)
+    cleanup()
+    return 2
+  }
+
+  console.error(`sandbox-selftest: sandbox at ${setup.skillCopyDir}`)
+  console.error(`sandbox-selftest: symlinked siblings — ${setup.symlinks.map((s) => s.name).join(', ')}`)
+
+  const testFiles = discoverTestFiles(setup.skillCopyDir)
+  if (testFiles.length === 0) {
+    console.error(`sandbox-selftest: no *.test.mjs files found under ${f.skill}/{scripts,workflows}`)
+    cleanup()
+    return 2
+  }
+
+  const output = runSuites(setup.skillCopyDir, testFiles)
+
+  let summary
+  try {
+    summary = parseTapSummary(output)
+  } catch (err) {
+    console.error(err.message)
+    cleanup()
+    return 2
+  }
+
+  const verdict = evaluateBaseline({ ...summary, knownSet: knownFailingFor(f.skill) })
+  cleanup()
+
+  if (verdict.ok) {
+    console.error(`sandbox-selftest: ✓ ${verdict.message}`)
+    return 0
+  }
+  console.error(`sandbox-selftest: ✗ ${verdict.message}`)
+  return verdict.kind === 'empty' ? 2 : 1
+}
+
+if (isMainModule(import.meta.url)) {
+  process.exit(runCli(process.argv.slice(2), { scriptPath: fileURLToPath(import.meta.url) }))
+}