@rungs/cli 0.1.3 → 0.3.0

package/src/selftest.ts

@@ -0,0 +1,237 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { ENGINES, type Finding } from './engines.ts';
+
+/**
+ * Execute a gate's `[[self_test]]` fixtures instead of only checking they exist.
+ *
+ * F-006 / WI-045. `gateMeta` confirmed that a `pass` block and a `fail` block
+ * were declared and never ran either, so every fixture in the repo was
+ * documentation shaped like a test. `gates-links-resolve`'s
+ * `See [the plan](./does-not-exist.md).` had never been executed — and had it
+ * been, it would have caught F-005 months before a person found it by hand.
+ *
+ * That is `gate-self-test`'s own argument turned on itself: *a gate whose rules
+ * are all currently satisfied is indistinguishable from a gate that matches
+ * nothing*, and a self-test that never runs is indistinguishable from one that
+ * would fail.
+ *
+ * **What this does not do is as important as what it does.** A fixture whose
+ * shape has no builder is reported **unrun, by name**. It is never counted as a
+ * pass, because a suite that reports green while three quarters of it never
+ * executed is this finding again, one level up.
+ */
+
+export interface SelfTestResult {
+  gate: string;
+  expect: 'pass' | 'fail';
+  outcome: 'ok' | 'mismatch' | 'unrun';
+  detail?: string;
+}
+
+/**
+ * A concrete path the table's `scan` would match, for writing a fixture into.
+ *
+ * The table may be an **array** — `[[frontmatter_schema]]` declares several — in
+ * which case `t.file`/`t.scan` are undefined and the first "scan pattern" was
+ * an entire schema object. That silently produced a nonsense path, the engine
+ * saw no file, and the fixture was reported as a gate failure (F-018).
+ */
+function targetPath(table: any): string {
+  const t = Array.isArray(table) ? table[0] ?? {} : table ?? {};
+  const pattern: string = t.file ?? [t.scan ?? []].flat()[0] ?? '**/*.md';
+  if (!pattern.includes('*')) return pattern;
+  return pattern
+    .replace(/\*\*\//g, 'probe/')
+    .replace(/\/\*\*/g, '/probe')
+    .replace(/\*/g, 'probe')
+    .replace(/probe\.probe$/, 'probe.md');
+}
+
+/**
+ * Build the fixture's repo state, or return null when its shape has no builder.
+ *
+ * The declarative keys are deliberately handled generically — `frontmatter`,
+ * `sections`, `opening` and `body` are all "a markdown file with this in it",
+ * and writing one builder per key would be the same per-shape sprawl that left
+ * 85 of 114 fixtures unrunnable in the first place.
+ */
+function build(root: string, table: any, fx: any, input?: string): string[] | null {
+  const write = (rel: string, body: string) => {
+    const full = join(root, rel);
+    mkdirSync(dirname(full), { recursive: true });
+    writeFileSync(full, body);
+    return rel;
+  };
+
+  if (typeof input === 'string') return [write(targetPath(table), `${input}\n`)];
+  if (!fx || typeof fx !== 'object') return null;
+
+  // A set of manifests and the version each states — the computed-claim shapes.
+  // `{ "package.json": "1.2.0", "site/package.json": "1.1.0" }`.
+  if (fx.packages && typeof fx.packages === 'object') {
+    return Object.entries(fx.packages).map(([rel, version]) =>
+      write(rel, JSON.stringify({ name: rel.replace(/\W/g, '-'), version })),
+    );
+  }
+
+  // Named files in a parameterised directory, plus the version they are judged
+  // against — the changelog shapes. `dir` is stated by the fixture rather than
+  // assumed here, because the self-test sees the module's *raw* table and a
+  // `{{changelog_dir}}` glob cannot match anything on disk; see `deparam`.
+  if (Array.isArray(fx.fragments) && typeof fx.version === 'string') {
+    const dir = fx.dir ?? 'changelog.d';
+    // Forward slashes, not `join`: the returned paths are matched against the
+    // spec's globs, and on Windows `join` yields `changelog.d\0.1.1.md`, which
+    // `changelog.d/*.md` does not match. The gate then reports "did not fire"
+    // about the harness rather than the fixture.
+    const written = fx.fragments.map((n: string) => write(`${dir}/${n}`, `# ${n}\n`));
+    written.push(write('package.json', JSON.stringify({ version: fx.version })));
+    return written;
+  }
+
+  // N files matching the table's scan — the population shapes.
+  if (typeof fx.matching_files === 'number') {
+    const base = fx.location ?? dirname(targetPath(table));
+    const marker = fx.exempt ? `<!-- ${fx.exempt} -->\n` : '';
+    return Array.from({ length: fx.matching_files }, (_, i) =>
+      write(join(base === '.' ? '' : base, `probe-${i}.md`), `${marker}# probe ${i}\n`),
+    );
+  }
+
+  // A single markdown file described declaratively.
+  const contentKeys = ['frontmatter', 'sections', 'opening', 'body', 'row', 'table'];
+  if (contentKeys.some((k) => k in fx)) {
+    const parts: string[] = [];
+    // `table = "Closed"` names the heading the row belongs under. A register
+    // engine finds rows by section, so a row written without its heading is in
+    // no table at all — which read as "the gate did not fire" (F-018).
+    if (fx.table) parts.push(`## ${fx.table}`, '');
+    if (fx.frontmatter && typeof fx.frontmatter === 'object') {
+      parts.push('---');
+      for (const [k, v] of Object.entries(fx.frontmatter)) parts.push(`${k}: ${v}`);
+      parts.push('---', '');
+    }
+    if (fx.opening) parts.push(String(fx.opening), '');
+    for (const s of fx.sections ?? []) parts.push(`## ${s}`, '', 'text', '');
+    if (fx.row && typeof fx.row === 'object') {
+      const cols = Object.keys(fx.row);
+      parts.push(`| ${cols.join(' | ')} |`, `| ${cols.map(() => '---').join(' | ')} |`,
+        `| ${cols.map((c) => (fx.row as any)[c]).join(' | ')} |`, '');
+    }
+    if (fx.body) parts.push(String(fx.body), '');
+    return [write(fx.file ?? targetPath(table), `${parts.join('\n')}\n`)];
+  }
+
+  return null;
+}
+
+/**
+ * Engines whose verdict depends **only on the content of the file the fixture
+ * describes**, so a fixture can be executed faithfully in an empty directory.
+ *
+ * The rest need context the fixture does not carry, and running them anyway
+ * produces confident nonsense. `gates-links-resolve`'s `pass` fixture is
+ * `See [this table](./structural.toml).` — it asserts that a link *which
+ * resolves* passes, and in a temp directory it does not resolve, so the runner
+ * would report the gate broken. Creating the target to make it pass would be
+ * assuming the answer, and only for `expect = "pass"` blocks, which is worse.
+ *
+ * Measured 2026-08-16: without this restriction the runner reported 17
+ * "failures", and the ones inspected were all its own artifacts. A test harness
+ * that cries wolf gets deleted faster than the gate it was checking.
+ *
+ * This is the same shape as `applicability` in ADR-0007, one level down: ask
+ * whether the check can legitimately run before running it.
+ */
+const CONTEXT_FREE: ReadonlySet<string> = new Set([
+  'frontmatter-schema',
+  'sections',
+  'file-budget',
+  'register-schema',
+  'file-population',
+  'changelog-freshness',
+  'computed-claim',
+]);
+
+/**
+ * Replace `{{param}}` segments in a spec's globs with the literal the fixture
+ * stands in for.
+ *
+ * The runner reads the **module's** gate table, where paths are still written as
+ * parameters — `{{changelog_dir}}/*.md`. Nothing substitutes them here, because
+ * there is no installed repo to take values from. So a fixture that exercises a
+ * parameterised path has to say what it is standing in for, and the spec has to
+ * be told the same thing, or the glob matches a file the builder just wrote and
+ * the gate reports "did not fire" about its own harness.
+ */
+function deparam<T>(spec: T, dir: string): T {
+  const walk = (v: any): any =>
+    typeof v === 'string' ? v.replace(/\{\{[^}]+\}\}/g, dir)
+      : Array.isArray(v) ? v.map(walk)
+      : v && typeof v === 'object' ? Object.fromEntries(Object.entries(v).map(([k, x]) => [k, walk(x)]))
+      : v;
+  return walk(spec);
+}
+
+export function runSelfTests(
+  gateId: string,
+  engine: string,
+  table: any,
+  blocks: { expect: string; input?: string; fixture?: any }[],
+): SelfTestResult[] {
+  const out: SelfTestResult[] = [];
+  for (const b of blocks) {
+    const expect = b.expect === 'fail' ? 'fail' : 'pass';
+    if (!CONTEXT_FREE.has(engine)) {
+      out.push({ gate: gateId, expect, outcome: 'unrun', detail: `${engine} fixtures need context the fixture does not carry` });
+      continue;
+    }
+    if (!(engine in ENGINES)) {
+      out.push({ gate: gateId, expect, outcome: 'unrun', detail: `engine '${engine}' not implemented` });
+      continue;
+    }
+    const root = mkdtempSync(join(tmpdir(), 'rungs-selftest-'));
+    try {
+      const files = build(root, table, b.fixture, b.input);
+      // A fixture states an opt-in with `opted_in`; the engine reads it from the
+      // spec as `extensions_opted_in`. Without the bridge, the `pass` fixture for
+      // an opted-in extension fired `non-spec key` — the harness asserting the
+      // opposite of what the fixture said (F-018).
+      let spec = b.fixture?.opted_in
+        ? (Array.isArray(table) ? table.map((s: any) => ({ ...s, extensions_opted_in: b.fixture.opted_in })) : { ...table, extensions_opted_in: b.fixture.opted_in })
+        : table;
+      // Same bridge, for paths: a fixture that names a parameterised directory
+      // has to hand the spec the same literal it wrote the files into.
+      if (Array.isArray(b.fixture?.fragments)) spec = deparam(spec, b.fixture.dir ?? 'changelog.d');
+      // And for `exclude`, which is the thing under test in half these fixtures:
+      // the table ships it empty by default, so a fixture proving exclusion works
+      // has to set it, exactly as a repo would.
+      if (Array.isArray(b.fixture?.exclude)) {
+        const ex = b.fixture.exclude;
+        spec = Array.isArray(spec) ? spec.map((s: any) => ({ ...s, exclude: ex })) : { ...spec, exclude: ex };
+      }
+      if (!files) {
+        out.push({ gate: gateId, expect, outcome: 'unrun', detail: `no builder for fixture ${JSON.stringify(b.fixture).slice(0, 60)}` });
+        continue;
+      }
+      let findings: Finding[];
+      try {
+        findings = ENGINES[engine](spec, root, files).findings;
+      } catch (e: any) {
+        out.push({ gate: gateId, expect, outcome: 'unrun', detail: `engine threw: ${e.message}`.slice(0, 90) });
+        continue;
+      }
+      const fired = findings.length > 0;
+      out.push(
+        fired === (expect === 'fail')
+          ? { gate: gateId, expect, outcome: 'ok' }
+          : { gate: gateId, expect, outcome: 'mismatch', detail: `expected ${expect}, ${fired ? `fired: ${findings[0].message}`.slice(0, 80) : 'did not fire'}` },
+      );
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  }
+  return out;
+}

package/src/types.ts

@@ -15,10 +15,35 @@ export interface ParamSpec {
   consumed_by?: string;
 }
 
+/**
+ * Whether a detector can legitimately interpret a repo that did not adopt our
+ * conventions — asked *before* "did the condition fire" (WI-052).
+ *
+ * This lived as two hard-coded sets of engine names inside `explain.ts`, which
+ * meant a new gate inherited an applicability nobody chose for it and could not
+ * see. The three cases are the ones measurement produced, not a taxonomy:
+ *
+ * - `repo-content`  measures the repo's own content. A broken link is a broken
+ *   link in anybody's methodology, and so is a 1,358-line file.
+ * - `our-artifacts` checks something rungs wrote. On a repo that never installed
+ *   it the artifact cannot exist, so the finding is guaranteed by the repo's
+ *   *state* rather than its *content* — `adr-index-current` reporting a missing
+ *   `adr-index` block against hexguard's perfectly healthy decision index.
+ * - `our-schema`    reads their file against a shape we defined.
+ *   `specs-status-evidence` produced 70 findings on hexguard-templates whose
+ *   spec register is fine and simply has its own columns.
+ *
+ * Only `repo-content` runs on a repo that is not ours. There is deliberately no
+ * default: an undeclared gate is reported, never quietly assumed safe — the same
+ * rule `enforcement-declaration` applies to `gated` / `review-only`.
+ */
+export type Applicability = 'repo-content' | 'our-artifacts' | 'our-schema';
+
 export interface GateSpec {
   id: string;
   kind: 'declared' | 'command';
   engine?: string;
+  applicability?: Applicability;
   table?: string;
   command?: string;
   tier?: string;
@@ -79,6 +104,15 @@ export interface Manifest {
   params: Record<string, ParamSpec>;
   gates: GateSpec[];
   detect: DetectSpec;
+  /**
+   * Per skill, the harness extensions this module opted into — `[skills.<name>]`
+   * in `module.toml`.
+   *
+   * Was declared in manifests and documented in `modules/README.md` while being
+   * parsed by nothing (F-019), so the opt-in that is meant to attach a
+   * portability cost to a decision attached to nothing at all.
+   */
+  skills?: Record<string, { extensions?: Record<string, unknown>; extension_note?: string }>;
   provenance: Provenance;
   threshold?: { metric: string; minimum: number; confirm?: boolean };
   /** Absolute path to the module directory. */