control-arm 0.1.0

package/.github/workflows/ci.yml

@@ -0,0 +1,82 @@
+# The tool that asks whether your tests can fail, running its own.
+#
+# It shipped for a day without this. That is the defect it exists to find, one level up:
+# a suite that is never executed by anything but its author's terminal is not a gate, and
+# "46/46 green" meant "green on one laptop, when I remembered".
+#
+# GitHub-hosted runners on purpose: control-arm has no dependencies and must work on a
+# stock box. Pinning it to a self-hosted fleet would hide exactly the assumptions —
+# a preinstalled binary, a warm cache, a particular git version — that break for the
+# first stranger who clones it.
+name: CI
+
+on:
+  push:
+    branches: [master, main]
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: node ${{ matrix.node }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # 20 is the oldest LTS with a stable node:test reporter API, which the TAP parser
+        # depends on. 24 is what it is developed against. A break in either is worth knowing.
+        node: ['20', '22', '24']
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # FULL HISTORY, not the default shallow clone. The dogfood step verifies the tool
+          # against one of its OWN past commits, and `git show 1845c1d3` on a depth-1
+          # checkout fails with "unknown revision". Caught by this workflow's first run,
+          # which is the argument for having it.
+          fetch-depth: 0
+      - uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node }}
+
+      - name: Unit + fixtures
+        run: node --test --test-concurrency=1 test/*.test.mjs
+
+      # DOGFOOD. The fixtures prove the verdicts on repos built to have known answers.
+      # This proves the whole two-arm machinery works on a REAL history with real commits,
+      # worktrees and module resolution — which is where every bug so far has come from.
+      - name: Judge its own history
+        run: |
+          set -euo pipefail
+          git config --global user.email ci@control-arm
+          git config --global user.name ci
+
+          OUT=$(node bin/ca.mjs verify 1845c1d3 --repo "$GITHUB_WORKSPACE" --timeout 120000)
+          echo "$OUT"
+
+          # That commit added the rule "a SKIPPED case blocks BLIND", with a test written
+          # for it. If the tool cannot still see that test discriminate, the tool is broken
+          # — regardless of what its unit suite says.
+          # Match the VERDICT WORD, not the whole line. This grep was 'VERDICT  CAUGHT'
+          # and broke the moment a glyph was added between them — a cosmetic change that
+          # failed a correctness gate, which trains people to edit the gate rather than
+          # believe it. Anchor on what the check is actually about.
+          echo "$OUT" | grep -qE '^ *VERDICT .*\bCAUGHT\b' \
+            || { echo "::error::control-arm no longer judges its own fix correctly"; exit 1; }
+          # And it must be the STRONG claim: 1845c1d3 is a repair, so a "weak evidence"
+          # qualifier here would mean the new-code heuristic has started misfiring on fixes.
+          echo "$OUT" | grep -q 'weak evidence' \
+            && { echo "::error::a repair was labelled new code — the kind heuristic is wrong"; exit 1; }
+          echo "$OUT" | grep -q 'module identity verified' \
+            || { echo "::error::module identity was not proven — a verdict here is not trustworthy"; exit 1; }
+
+      - name: Determinism
+        run: |
+          set -euo pipefail
+          A=$(node bin/ca.mjs verify 1845c1d3 --repo "$GITHUB_WORKSPACE" --work /tmp/d1 --timeout 120000 | tail -14)
+          B=$(node bin/ca.mjs verify 1845c1d3 --repo "$GITHUB_WORKSPACE" --work /tmp/d2 --timeout 120000 | tail -14)
+          [ "$A" = "$B" ] || { echo "::error::two runs disagreed — the tool is not deterministic"; exit 1; }
+          echo "two independent runs are byte-identical"

package/DESIGN.md

@@ -0,0 +1,161 @@
+# How it works, and why you should believe it
+
+[README](README.md) covers what `ca` is and how to run it. This is the part a skeptic
+needs: the mechanism, the traps that separate an instrument from a random number
+generator, and an honest account of how often it is wrong.
+
+## The mechanism
+
+You do **not** run the parent commit's tests — the parent does not have this test.
+You take the new test and ask it about the old code.
+
+```
+  commit C ("fix: …")
+      │
+      ├── [worktree @ C]   run the new test on the FIXED code ──► must be GREEN
+      │                        └ not green? INCONCLUSIVE: no before/after to compare
+      │
+      ├── [worktree @ C~1] transplant ONLY the test file onto the BROKEN code
+      │                    └ prove module identity, then run
+      │        ┌───────────────┬──────────────┬──────────────────┐
+      │        ▼               ▼              ▼                  ▼
+      │   AssertionError   passed        SyntaxError/        timeout/crash
+      │        │               │         import failure           │
+      │        ▼               ▼              ▼                  ▼
+      │     CAUGHT      NON-DISCRIM.     INCONCLUSIVE       INCONCLUSIVE
+      │
+      └── N runs disagree? ──► FLAKY
+```
+
+## Three things that decide whether this is an instrument or a random number generator
+
+### 1. Exit code is not the signal
+
+A test that "fails" at the parent with `SyntaxError: does not provide an export named
+'TITLE_SEPARATOR'` — because the fix *added* that export — exits non-zero and has told you
+nothing. It never ran. Read the exit code only and your headline number is inflated
+garbage.
+
+The discriminator is **assertion-failure vs error-failure**, which every test runner
+reports and almost nothing reads.
+
+### 2. Prove the control arm loaded the control arm's code
+
+On its first real run this tool reported `4 pass / 0 fail` on the parent — a `BLIND`
+verdict on one of the most carefully tested commits in the target repo. The harness had
+symlinked the repo's whole `node_modules` into the worktree, and workspace packages inside
+it are symlinks *relative to the real repo*:
+
+```
+node_modules/@acme/core -> ../../packages/core
+resolved: file:///…/the-repo/packages/core/src/parser.js   ← today's code
+```
+
+Arm B never loaded the code under test. Every language has this trap somewhere — venvs,
+module caches, classpaths. So `ca` asks the runtime where each of the test's imports
+actually resolved, and **refuses to answer** if any landed outside the worktree.
+
+### 3. A false `BLIND` is the worst output this tool can produce
+
+`CAUGHT` is good news. `INCONCLUSIVE` is an honest shrug. **`BLIND` accuses an engineer of
+writing a test that cannot fail.** Get it wrong once and nobody trusts the tool again.
+
+So **every ambiguity resolves away from `BLIND`**, and the verdict is withheld rather than
+guessed.
+
+That is also why the case level and the commit level use different words. The first real
+run printed this:
+
+```
+✗ BLIND   TITLE-022 CONTROL ARM: the whitespace-only patterns really do split one phrase
+✗ BLIND   TITLE-022: the prefix test survives the wider separators
+✗ BLIND   TITLE-022: the brands the separator change must not touch
+```
+
+All three are **regression guards**. Staying green on both arms is their entire job.
+Red/green cannot distinguish "meant to catch this bug and failed" from "meant to stay
+green" — that is *intent*, and no tool can read it. So a case states the fact
+(`NON-DISCRIMINATING`) and only a **commit** gets judged `BLIND`, when not one of its cases
+discriminated.
+
+## Why you should believe a verdict
+
+`npm test` runs eight fixture repos where the right answer is known by construction —
+including three different shapes of blind test and two different shapes of unrunnable test.
+**If `ca` cannot tell `02-blind-direction` from `01-caught-value`, it does not ship.**
+
+The verdict engine is pure and has no I/O, so it is tested exhaustively. It also keeps a
+control arm executing the naive exit-code implementation and asserting it still gets the
+answers wrong.
+
+That is the standard this tool asks of other people's tests, so it is the standard its own
+suite is held to.
+
+## Measured results
+
+Three repositories, random draws, seeds recorded so the samples are reproducible.
+
+| | a private monorepo | nodejs/undici | iamkun/dayjs |
+|---|---|---|---|
+| fix commits sampled | 300 | 25 | 40 |
+| answerable | 214 | 15 | 25 |
+| **CAUGHT** | **201 — 93.9%** | **12 — 80.0%** | **23 — 92.0%** |
+| BLIND | 13 — 6.1% | 3 — 20.0% | 2 — 8.0% |
+| INCONCLUSIVE | 86 | 10 | 9 |
+| runtime | 7.1 s/commit | 24.9 s/commit | 1.4 s/commit |
+
+The dayjs run (`--n 40 --since '6 years' --grep '^fix' --seed 11`) is the one worth
+weighting: a codebase the author did not write, did not pick for a flattering result, and
+could not tune against. 190 commits matched the subject filter; 99 ship both a test and a
+source change; 40 were drawn from those.
+
+**One usability trap found while running it.** `audit` defaults to a six-month window. On
+a mature repo that silently shrinks the sample — the first dayjs run judged **5 commits
+instead of 40** and printed a confident, meaningless 100%. The window *is* echoed in the
+header line, so it is visible rather than hidden, but a `--n 40` that quietly returns 5
+deserves a louder signal. Pass `--since` explicitly on any repo older than six months.
+
+`INCONCLUSIVE` is 29% and 40% respectively. That is the honest denominator, not a rounding
+error: a fix that adds an export its test imports cannot be replayed against the parent,
+because the test will not load there. Those are excluded from the ratio rather than assumed
+either way.
+
+## How often is this tool wrong?
+
+Ask any measuring instrument this. Here is the answer for this one, on the 300-commit run.
+
+The raw headline was **13 `BLIND` commits**. After running arm C on each and checking them
+by hand, **5 were real**:
+
+| | |
+|---|---|
+| 13 | raw `BLIND` |
+| −3 | `↻ REPAIRED SINCE` — the gap was closed after that commit |
+| −2 | `? cannot tell` — the test file no longer exists at HEAD |
+| −3 | category errors: two `fix(test):` (the bug WAS the test) and one build failure no unit test can catch |
+| **5** | genuinely open, **2.3% of answerable commits** |
+
+**A 62% false-positive rate on the raw number.** Arm C and the corpus filter exist because
+of it. Publish the raw count and you hand someone thirteen tickets, eight of which waste
+their afternoon.
+
+## Every bug found in this tool so far
+
+None were found by its own test suite. All were found by pointing it at real work.
+
+| bug | found by |
+|---|---|
+| a slash in a test NAME silently dropped the case | random hand-audit of `BLIND` verdicts |
+| skipped cases counted toward `BLIND` | hand-audit of the `BLIND` list |
+| config and CI YAML not treated as source | a live PR it was asked to check |
+| commit-vs-parent is wrong for a PR branch | a reviewer's suggested command |
+| `.env` absent from worktrees → 955 phantom skips | getting a real database running |
+| a historical `BLIND` is not an open defect | shipping a redundant ticket to a colleague |
+| test detection required BOTH a `test/` dir AND a `.test.` suffix | first run on a repo the author did not write |
+
+That last one returned **an entirely empty audit** on undici — 284 commits matched, zero
+judged — because undici names its tests `test/client-request.js`. Not a wrong answer: no
+answer.
+
+**It is the argument for running this on somebody else's code before believing any number
+it prints.**

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mekan Hanov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,164 @@
+# control-arm (`ca`)
+
+**Does a test actually fail on the code it was written to catch?**
+
+When you fix a bug you usually add a test so it can't come back. Everyone does this.
+**Nobody checks whether that test would actually have caught the bug.**
+
+This checks. It takes the test you wrote, rewinds your code to just before the fix, and
+runs the test there. Red means it works. Green means it was never capable of catching
+anything — it just sits in your suite looking like protection.
+
+It does not write tests, and there is no AI in it. It runs your tests and reports what
+happened.
+
+## Two tests for the same bug
+
+```js
+// A
+assert.equal(priority('HIGH-PRIORITY'), 1);
+
+// B
+const p = priority('HIGH-PRIORITY');
+assert.ok(p);                 // any number but 0 is truthy
+assert.ok(SLA_HOURS[p] > 0);  // 24 > 0 is true
+```
+
+The bug: `priority()` matched `HIGH PRIORITY` with a space but not `HIGH-PRIORITY` with a
+hyphen, so an urgent ticket came back as normal.
+
+On the fixed code both pass. On the broken code **A fails and B still passes** — the wrong
+answer, `2`, is also truthy and also has a positive SLA. In CI they are identical: two
+green checkmarks.
+
+**That is the gap.** Nothing in a normal pipeline can tell those two apart. `ca` can,
+because it runs the new test against the old code.
+
+## Install
+
+```bash
+npm i -g control-arm     # or run it from a clone: node bin/ca.mjs
+```
+
+Node 18+. No other dependencies.
+
+## Use it
+
+```bash
+# Can this repo be measured at all?
+ca probe
+
+# Judge one commit — a fix, with the test that shipped alongside it
+ca verify <sha>
+
+# Judge a branch against where it will merge
+ca verify HEAD --against origin/main
+```
+
+### As a PR check
+
+```yaml
+# .github/workflows/control-arm.yml
+name: control-arm
+on: pull_request
+
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with: { fetch-depth: 0 }        # the merge base needs real history
+      - uses: actions/setup-node@v4
+        with: { node-version: 22 }
+      - run: npm ci
+      - uses: mekanhan/control-arm@v1
+        with:
+          base: main
+          comment: 'true'
+```
+
+It comments on the PR with a line per test. It does **not** fail the build by default: a
+PR can legitimately ship only regression guards, and a gate that fires on those gets
+switched off within a week.
+
+## What you get
+
+> ### `control-arm` — this branch is proven
+>
+> **1 of 5 tests here genuinely catch this change.** Run against the code as it was before
+> this PR, they fail; with the change, they pass.
+>
+> | | test | what it did on the code WITHOUT this change |
+> |---|---|---|
+> | ✅ | `RESET-002: does not render the next UTC midnight` | **failed** — `expect(received).toBeNull()` |
+> | ⚪️ | `RESET-003: the pre-fix expression disagrees` | _passed too — so it is not what catches this bug_ |
+> | ⚠️ | `RESET-001: renders the time the server sent` | _could not run there — crashed rather than disagreeing_ |
+
+**One `CAUGHT` and a pile of `NON-DISCRIMINATING` is the healthy result**, not a problem.
+One test was aimed at the bug; the rest guard other things. The only worrying outcome is a
+commit where *nothing* caught it.
+
+## Verdicts
+
+| level | verdict | meaning |
+|---|---|---|
+| case | `CAUGHT` | ran on the broken code and disagreed with it |
+| case | `NON-DISCRIMINATING` | ran and was fine with it — a fact, not a fault |
+| case | `INCONCLUSIVE` | never ran, or its identity could not be proven |
+| case | `FLAKY` | repeated runs disagreed |
+| commit | `CAUGHT` | at least one case discriminates |
+| commit | `BLIND` | every case ran, **not one** discriminated |
+| commit | `INCONCLUSIVE` | excluded from the ratio rather than assumed |
+
+`INCONCLUSIVE` is reported in its own column and never folded into the others. A
+measurement that cannot tell zero from failure is not a measurement.
+
+## Which of your tests it looks at
+
+It does not care what *kind* of test it is, only whether the runner can execute the file.
+
+| kind | covered | why |
+|---|---|---|
+| unit tests | yes | the easy case |
+| backend / server logic | yes | same runner, same rewind |
+| database-backed tests | yes | needs a live DB, else they report `SKIPPED` |
+| component tests (React / RN) | yes | via vitest and jest |
+| browser e2e (Playwright) | **no** | no Playwright runner yet |
+| performance / load | **no** | they measure speed, not correctness |
+| manual QA | **no** | nothing to execute |
+
+## Does it work?
+
+Three repositories, random draws, seeds recorded so the samples are reproducible.
+
+| | a private monorepo | nodejs/undici | iamkun/dayjs |
+|---|---|---|---|
+| fix commits sampled | 300 | 25 | 40 |
+| answerable | 214 | 15 | 25 |
+| **CAUGHT** | **93.9%** | **80.0%** | **92.0%** |
+| BLIND | 6.1% | 20.0% | 8.0% |
+| runtime | 7.1 s/commit | 24.9 s/commit | 1.4 s/commit |
+
+**The dayjs column is the one to weigh.** It is a codebase the author did not write, did
+not choose for a flattering result, and could not tune the tool against — 190 fix commits
+matched the filter, 99 ship both a test and a source change, 40 drawn at random with the
+seed recorded. It lands within a point of the private monorepo it was built on.
+
+Of the 13 raw `BLIND` verdicts in that 300-commit run, **5 were genuinely open** — a 62%
+false-positive rate on the raw number, which is why the filters exist.
+
+**[DESIGN.md](DESIGN.md) has the rest**: how the rewind works, the three traps that
+separate an instrument from a random number generator, the full accuracy audit, and every
+bug found in this tool so far — none of which were found by its own test suite.
+
+## Scope, stated plainly
+
+- **A test can only be judged against the bug it was written for.** No fix commit, no
+  broken code, no verdict. The measurable universe is fix commits that shipped a test. For
+  tests with no such pairing, the instrument is mutation testing, not this.
+- One runner today (`node:test`, plus vitest and jest via adapters). The runner seam is
+  isolated in `src/runner.mjs`; the decision logic never sees raw runner output.
+- `ca` never touches your working tree — no `stash`, no `checkout` in your clone.
+  Everything happens in detached worktrees under `.ca-work/`.
+
+MIT.

package/action.yml

@@ -0,0 +1,95 @@
+# A composite action, not a Docker one: the tool is plain Node with no dependencies, so a
+# container would add a minute of build time to a check that otherwise takes seconds.
+name: 'control-arm'
+description: 'Prove the tests in this PR fail without it'
+inputs:
+  base:
+    description: >-
+      Branch this PR targets. The comparison uses the MERGE BASE with it, never its tip.
+      Defaults to `main`; set it explicitly if your project integrates somewhere else
+      (`develop`, `trunk`, a release branch).
+    required: false
+    default: 'main'
+  comment:
+    description: 'Post the result as a PR comment. Needs pull-requests:write and GH_TOKEN.'
+    required: false
+    default: 'true'
+  fail-on-blind:
+    description: >-
+      Fail the check when NO case in the PR discriminates. Default false: a PR can
+      legitimately ship only regression guards, and a gate that fires on those gets
+      switched off within a week.
+    required: false
+    default: 'false'
+  timeout-ms:
+    required: false
+    default: '180000'
+outputs:
+  verdict:
+    description: 'CAUGHT | BLIND | INCONCLUSIVE | FLAKY | SKIPPED'
+    value: ${{ steps.run.outputs.verdict }}
+runs:
+  using: composite
+  steps:
+    - id: run
+      shell: bash
+      env:
+        CA_BASE: ${{ inputs.base }}
+        CA_TIMEOUT: ${{ inputs.timeout-ms }}
+      run: |
+        set -uo pipefail
+        # The merge base needs both histories. A shallow checkout has neither.
+        git -C "$GITHUB_WORKSPACE" fetch --no-tags --depth=200 origin "$CA_BASE" 2>/dev/null || true
+        TIP="$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)"
+
+        OUT=$(node "${{ github.action_path }}/bin/ca.mjs" verify "$TIP" \
+                --against "origin/$CA_BASE" \
+                --repo "$GITHUB_WORKSPACE" \
+                --timeout "$CA_TIMEOUT" 2>&1) || true
+        echo "$OUT"
+
+        # `|| true` is load-bearing. GitHub runs bash steps with `set -e` by default, and
+        # `set -uo pipefail` above does not unset it. A grep that finds nothing exits 1,
+        # pipefail propagates it, and the step dies BEFORE the :-INCONCLUSIVE fallback can
+        # run. That is exactly what happens on a PR the tool declines — a workflow-only
+        # change with no test — so the one case designed to be a non-event failed the check.
+        VERDICT=$(printf '%s' "$OUT" | grep -oE 'VERDICT +[A-Z]+' | awk '{print $2}' | head -1 || true)
+        VERDICT="${VERDICT:-INCONCLUSIVE}"
+        echo "verdict=$VERDICT" >> "$GITHUB_OUTPUT"
+
+        # `--pr-comment` returns EMPTY for a commit the tool declined to judge, and the
+        # comment step skips on an empty file. The decision is the tool's, not a shell's:
+        # scraping stdout for a VERDICT line a declined run never prints is what put
+        # "SKIPPED · no case fails without the change" onto a PR that simply has no tests.
+        MD=$(node "${{ github.action_path }}/bin/ca.mjs" verify "$TIP" \
+               --against "origin/$CA_BASE" --pr-comment \
+               --repo "$GITHUB_WORKSPACE" --timeout "$CA_TIMEOUT" 2>/dev/null) || true
+        printf '%s' "$MD" > "$RUNNER_TEMP/ca-comment.md"
+        [ -s "$RUNNER_TEMP/ca-comment.md" ] || echo "declined — nothing to post: $(printf '%s' "$OUT" | tail -1)"
+
+    - shell: bash
+      if: inputs.comment == 'true' && github.event_name == 'pull_request'
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: |
+        # node, not `gh`. The CLI is not installed on every runner — a self-hosted
+        # bare-metal box failed here with `gh: command not found` after every other step
+        # had passed, so the tool ran, reached the right verdict, and could not say so.
+        node "${{ github.action_path }}/scripts/gh-api.mjs" upsert-pr-comment \
+          "${{ github.repository }}" \
+          "${{ github.event.pull_request.number }}" \
+          "$RUNNER_TEMP/ca-comment.md" \
+          '### `control-arm`'
+
+    - shell: bash
+      if: inputs.fail-on-blind == 'true' && steps.run.outputs.verdict == 'BLIND'
+      run: |
+        echo "::error::No test in this PR fails without the change."
+        exit 1
+
+# Required by the GitHub Marketplace listing. `rewind` is not decoration: it is
+# literally what this does — wind the code back to before the fix, then run the
+# new test against it.
+branding:
+  icon: 'rewind'
+  color: 'orange'