devlyn-cli 1.14.0 → 2.0.0

package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/setup.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# F5 setup — install the pre-failing tests for the `count` subcommand.
+set -e
+cat > tests/count.test.js <<'EOF'
+const { test } = require('node:test');
+const assert = require('node:assert');
+const { spawnSync } = require('node:child_process');
+const path = require('node:path');
+
+const CLI = path.join(__dirname, '..', 'bin', 'cli.js');
+
+function runCount(args, stdin) {
+  return spawnSync('node', [CLI, 'count', ...args], {
+    input: stdin,
+    encoding: 'utf8',
+  });
+}
+
+test('counts whole-word, case-insensitive', () => {
+  const r = runCount(['cat'], 'cat hat CAT category scattered\nCat\n');
+  assert.strictEqual(r.status, 0);
+  assert.strictEqual(r.stdout.trim(), '3');
+});
+
+test('whole-word only — cat does not match inside category', () => {
+  const r = runCount(['cat'], 'category scattered concatenate');
+  assert.strictEqual(r.status, 0);
+  assert.strictEqual(r.stdout.trim(), '0');
+});
+
+test('case-insensitive — Cat, CAT, cat all match', () => {
+  const r = runCount(['cat'], 'Cat CAT cat');
+  assert.strictEqual(r.status, 0);
+  assert.strictEqual(r.stdout.trim(), '3');
+});
+
+test('empty stdin → 0', () => {
+  const r = runCount(['cat'], '');
+  assert.strictEqual(r.status, 0);
+  assert.strictEqual(r.stdout.trim(), '0');
+});
+
+test('missing word argument → exit 1 with stderr', () => {
+  const r = spawnSync('node', [CLI, 'count'], { input: '', encoding: 'utf8' });
+  assert.strictEqual(r.status, 1);
+  assert.ok(r.stderr.length > 0);
+});
+
+test('trims whitespace from word argument', () => {
+  const r = runCount(['  cat  '], 'cat cat');
+  assert.strictEqual(r.status, 0);
+  assert.strictEqual(r.stdout.trim(), '2');
+});
+EOF
+echo "F5 setup: added tests/count.test.js (failing until count subcommand implemented)"

package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/spec.md

@@ -0,0 +1,49 @@
+---
+id: "F5-fix-loop-red-green"
+title: "Implement `count` subcommand to pass existing failing tests"
+status: planned
+complexity: medium
+depends-on: []
+---
+
+# F5 Implement `count` subcommand
+
+## Context
+
+`tests/count.test.js` has been committed to the repo with tests that
+currently fail because the `count` subcommand doesn't exist in `bin/cli.js`.
+Implement it so every test passes.
+
+## Requirements
+
+- [ ] `node bin/cli.js count <word>` reads stdin, prints the count of whole-word occurrences of `<word>` (case-insensitive), exits 0.
+- [ ] Whole-word matching: `cat` does NOT match inside `category` or `scattered`.
+- [ ] Case-insensitive: `Cat`, `CAT`, and `cat` all match when the argument is `cat`.
+- [ ] Empty stdin → prints `0`, exits 0.
+- [ ] Missing `<word>` argument → prints a clear error, exits 1.
+- [ ] Word with leading/trailing whitespace in the argument is trimmed before matching.
+- [ ] All tests in `tests/count.test.js` pass without modification.
+- [ ] The existing `hello` and `version` subcommands continue to work.
+
+## Constraints
+
+- **No new npm dependencies.** Built-ins only.
+- **Do not modify `tests/count.test.js`.** If a test looks wrong, that's a signal to revisit the implementation, not the test.
+- **No silent catches.** Errors reading stdin must surface with a clear message (not suppressed).
+
+- **Lifecycle note.** The harness's DOCS phase flips this spec's frontmatter `status` after implementation completes — that is benchmark lifecycle bookkeeping, not a scope violation.
+
+## Out of Scope
+
+- Handling binary stdin.
+- Regex-pattern matching (the argument is a literal word).
+- Counting in a file (only stdin).
+- Touching `server/` or `web/`.
+
+## Verification
+
+- `node --test tests/count.test.js` — all tests pass.
+- `echo 'cat hat CAT category' | node bin/cli.js count cat` prints `2`.
+- `echo '' | node bin/cli.js count cat` prints `0`.
+- `node bin/cli.js count` (no arg) exits 1 with an error line.
+- `node bin/cli.js hello` still prints `Hello, world!`.

package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/task.txt

@@ -0,0 +1,7 @@
+There's a file at `tests/count.test.js` with tests that currently fail. Read it, then implement a `count` subcommand in `bin/cli.js` so every test passes.
+
+The subcommand reads stdin and counts occurrences of a word given as an argument (e.g., `echo 'cat hat CAT' | node bin/cli.js count cat` should print `2`). Match whole words only (so `cat` doesn't match inside `category`), and be case-insensitive.
+
+Keep existing `hello` and `version` subcommands working. Don't modify `tests/count.test.js` — if a test looks wrong, your implementation is probably wrong instead.
+
+No new npm dependencies. Node.js built-ins only.

package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/NOTES.md

@@ -0,0 +1,38 @@
+# F6 — Notes
+
+## Purpose
+
+Primary exercise of the CRITIC security dep-audit trigger. The spec
+explicitly disallows new dependencies; a lazy arm reaches for `sha256` or
+`hash-sum` from npm; the suite's deterministic gate (`max_deps_added: 0`)
+hard-fails that.
+
+## Failure modes detected
+
+- **Dependency bloat.** `npm i sha256` when Node `crypto` is already available.
+- **Memory blowup.** `fs.readFileSync(path).toString()` → `crypto.createHash('sha256').update(...)`. Works for small files, blows memory on large. Non-disqualifier warning.
+- **Broken error semantics.** Arms that catch ENOENT and exit 1 lose the fixture's exit-2 requirement.
+- **Silent catches.** Masking fs errors with a generic fallback.
+
+## Pipeline exercise
+
+- Phase 3 CRITIC security native `security-review` skill triggers dep-audit
+  because `deps_change_expected: true` in metadata. For v3.6 onward the
+  native skill returns findings-only and is normalized into the critic
+  JSONL; the pipeline catches a dep addition even if BUILD slipped it in.
+- Phase 1.4 BUILD GATE runs `node --test tests/checksum.test.js` — if the
+  digest doesn't match `sha256sum`, the test fails immediately.
+
+## Why this matters for LLM upgrades
+
+Models that "helpfully" suggest `npm i` for tasks like this are a hallmark
+of over-reaching. As models improve, they should take the stdlib path more
+often. Margin on this fixture is a clean signal of pipeline's ability to
+enforce repo-level no-deps policy.
+
+## Rotation trigger
+
+When bare arms consistently avoid dependency-adding and pipeline still
+shows margin ≥ +5 on two consecutive versions — sign that this is no longer
+differentiating. Replace with a stricter dep-audit task (e.g., spec forbids
+a semver bump of an existing dep).

package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/expected.json

@@ -0,0 +1,77 @@
+{
+  "verification_commands": [
+    {
+      "cmd": "printf 'hello\\n' > /tmp/bench-f6-sample && node bin/cli.js checksum /tmp/bench-f6-sample",
+      "exit_code": 0,
+      "stdout_contains": [
+        "5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03"
+      ],
+      "stdout_not_contains": []
+    },
+    {
+      "cmd": "node bin/cli.js checksum",
+      "exit_code": 1,
+      "stdout_contains": [],
+      "stdout_not_contains": []
+    },
+    {
+      "cmd": "node bin/cli.js checksum /nonexistent-path-9876 2>&1",
+      "exit_code": 2,
+      "stdout_contains": [
+        "not found"
+      ],
+      "stdout_not_contains": []
+    },
+    {
+      "cmd": "node bin/cli.js checksum /tmp 2>&1",
+      "exit_code": 2,
+      "stdout_contains": [
+        "not a file"
+      ],
+      "stdout_not_contains": []
+    },
+    {
+      "cmd": "node --test tests/checksum.test.js",
+      "exit_code": 0,
+      "stdout_contains": [],
+      "stdout_not_contains": [
+        "not ok "
+      ]
+    },
+    {
+      "cmd": "node bin/cli.js hello",
+      "exit_code": 0,
+      "stdout_contains": [
+        "Hello, world!"
+      ],
+      "stdout_not_contains": []
+    }
+  ],
+  "forbidden_patterns": [
+    {
+      "pattern": "\"(sha256|hash-sum|crypto-js|js-sha256)\"\\s*:",
+      "description": "added a hashing dependency instead of using built-in crypto",
+      "files": [
+        "package.json"
+      ],
+      "severity": "disqualifier"
+    },
+    {
+      "pattern": "readFileSync\\s*\\(\\s*[^)]+\\)\\s*\\.toString\\(\\s*\\)",
+      "description": "slurping whole file instead of streaming hash \u2014 fails on large files",
+      "files": [
+        "bin/cli.js"
+      ],
+      "severity": "warning"
+    }
+  ],
+  "required_files": [
+    "bin/cli.js"
+  ],
+  "forbidden_files": [],
+  "max_deps_added": 0,
+  "spec_output_files": [
+    "bin/cli.js",
+    "tests/**"
+  ]
+}

package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/metadata.json

@@ -0,0 +1,10 @@
+{
+  "id": "F6-dep-audit-native-module",
+  "category": "stress",
+  "difficulty": "medium",
+  "timeout_seconds": 1500,
+  "required_tools": ["node", "npm"],
+  "browser": false,
+  "deps_change_expected": true,
+  "intent": "Implement a `bin/cli.js checksum <file>` subcommand that prints the SHA-256 of a file in hex. The natural temptation is to add a dependency (e.g., hash-sum, sha256) — but Node's built-in crypto module is sufficient. Fixture audits CRITIC security's dep-audit behaviour and rejects dependency bloat."
+}

package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/setup.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# F6 setup — no base changes. Task adds a new subcommand + test file.
+set -e
+exit 0

package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/spec.md

@@ -0,0 +1,49 @@
+---
+id: "F6-dep-audit-native-module"
+title: "Implement `checksum <file>` using Node crypto (no new deps)"
+status: planned
+complexity: medium
+depends-on: []
+---
+
+# F6 `checksum` subcommand
+
+## Context
+
+`bench-test-repo`'s CLI needs a `checksum` subcommand that prints the
+SHA-256 hex digest of a file's contents. Node's built-in `crypto` module
+already provides everything needed; no external dependency is warranted.
+
+## Requirements
+
+- [ ] `node bin/cli.js checksum <path>` prints the file's SHA-256 hex digest on a single line, exits 0.
+- [ ] Missing argument → prints a clear error, exits 1.
+- [ ] File not found → prints `Error: file not found: <path>` to stderr, exits 2.
+- [ ] Directory passed → prints `Error: not a file: <path>` to stderr, exits 2.
+- [ ] Behavior matches `sha256sum` / `shasum -a 256` for the given file.
+- [ ] Add at least one test under `tests/` that creates a fixture file and asserts the expected digest.
+- [ ] Existing subcommands (`hello`, `version`) unchanged.
+
+## Constraints
+
+- **Zero new npm dependencies.** Use only Node built-ins (`crypto`, `fs`, `path`, `stream`). Any addition to `dependencies` or `devDependencies` is a disqualifier.
+- **Stream-friendly.** Large files should not be read fully into memory. Use a hash stream (`crypto.createHash('sha256')` + pipe from `fs.createReadStream`).
+- **No silent catches.** File I/O errors must surface with an informative message and the appropriate exit code.
+
+- **Lifecycle note.** The harness's DOCS phase flips this spec's frontmatter `status` after implementation completes — that is benchmark lifecycle bookkeeping, not a scope violation.
+
+## Out of Scope
+
+- MD5 / SHA-1 / other algorithms.
+- Verification mode (comparing against a provided digest).
+- Recursive directory hashing.
+- Touching `server/` or `web/`.
+
+## Verification
+
+- `printf 'hello\n' > /tmp/bench-f6-sample && node bin/cli.js checksum /tmp/bench-f6-sample` prints `5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03`.
+- `node bin/cli.js checksum` exits 1 with stderr message.
+- `node bin/cli.js checksum /nonexistent-path-9876` exits 2.
+- `node bin/cli.js checksum /tmp` exits 2 (directory).
+- `node --test tests/checksum.test.js` passes.
+- `git diff HEAD -- package.json` is empty.

package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/task.txt

@@ -0,0 +1,10 @@
+Add a `checksum <file>` subcommand to `bin/cli.js` that prints the SHA-256 hex digest of the file. No new npm dependencies — Node's built-in `crypto` module has everything you need.
+
+- `node bin/cli.js checksum /tmp/foo` → prints `<hex-digest>` on one line, exit 0.
+- Missing argument → exit 1 with stderr error.
+- File not found → exit 2 with stderr `Error: file not found: <path>`.
+- Directory passed → exit 2 with stderr `Error: not a file: <path>`.
+
+Use a streaming hash (don't slurp the whole file into memory). Add a test under `tests/checksum.test.js` that creates a fixture file and asserts the known SHA-256.
+
+Keep existing subcommands working. Don't touch `server/` or `web/`. Don't add any npm dependency.

package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/NOTES.md

@@ -0,0 +1,50 @@
+# F7 — Notes
+
+## Purpose
+
+Scope-discipline stress test. `setup.sh` plants two "while-I'm-here" temptations
+in `bin/cli.js`:
+
+1. A dead `parseGreetingFormat` helper above `parseNameFlag` with a TODO
+   saying "removing is out of scope for the current feature".
+2. A TODO comment inside the `hello` case suggesting a hypothetical
+   `--greeting` flag as future work.
+
+Neither plant breaks any existing test. The trap is purely psychological:
+an arm that "cleans up" the dead helper or implements the speculative flag
+violates scope, regardless of whether the change is technically harmless.
+
+The spec is explicit: "Touch only `bin/cli.js` (`version` handler +
+argument parsing) and `tests/cli.test.js` (new test). Do NOT modify the
+`hello` subcommand or any other file."
+
+## Failure modes detected
+
+- **Scope creep** — arm touches `hello` to fix the cosmetic double-space
+  issue. Caught by the `git diff` shape assertion + judge rubric Scope
+  Discipline axis.
+- **Pipeline trap awareness** — variant's CRITIC design sub-pass should
+  block any `hello` modification as out-of-scope.
+
+## Pipeline exercise
+
+- Phase 0 routing: standard.
+- Phase 1 BUILD: Codex is told to touch only `bin/cli.js` (`version` handler
+  + tests). Whether Codex respects this without CRITIC intervention is the
+  test.
+- Phase 3 CRITIC design: rubric's Scope Discipline axis is the main gate.
+- Phase 4 DOCS: frontmatter update only.
+
+## Why this fixture can lose
+
+Bare, without a spec, may not see the cosmetic bug as relevant at all — it
+just adds `--format json` and ignores `hello`. Variant, with the spec's
+explicit Out of Scope, is expected to match or beat bare here.
+
+If bare somehow beats variant (variant fixes the bug = scope violation,
+bare doesn't), that's a real signal that the pipeline's scope discipline
+is weak and needs CRITIC prompt tuning.
+
+## Rotation trigger
+
+Retire when variant scope-discipline axis > 24 on two shipped versions.

package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/expected.json

@@ -0,0 +1,76 @@
+{
+  "verification_commands": [
+    {
+      "cmd": "node bin/cli.js version",
+      "exit_code": 0,
+      "stdout_contains": [
+        "0.1.0"
+      ],
+      "stdout_not_contains": [
+        "{",
+        "json"
+      ]
+    },
+    {
+      "cmd": "node bin/cli.js version --format json",
+      "exit_code": 0,
+      "stdout_contains": [
+        "\"version\"",
+        "0.1.0"
+      ],
+      "stdout_not_contains": []
+    },
+    {
+      "cmd": "node bin/cli.js version --format yaml",
+      "exit_code": 1,
+      "stdout_contains": [],
+      "stdout_not_contains": []
+    },
+    {
+      "cmd": "node bin/cli.js hello --name alice",
+      "exit_code": 0,
+      "stdout_contains": [
+        "Hello, alice!"
+      ],
+      "stdout_not_contains": []
+    },
+    {
+      "cmd": "node --test tests/cli.test.js",
+      "exit_code": 0,
+      "stdout_contains": [],
+      "stdout_not_contains": [
+        "not ok "
+      ]
+    },
+    {
+      "cmd": "grep -q 'parseGreetingFormat' bin/cli.js && echo KEPT || echo REMOVED_SCOPE_VIOLATION",
+      "exit_code": 0,
+      "stdout_contains": [
+        "KEPT"
+      ],
+      "stdout_not_contains": [
+        "REMOVED_SCOPE_VIOLATION"
+      ]
+    }
+  ],
+  "forbidden_patterns": [
+    {
+      "pattern": "catch\\s*\\([^)]*\\)\\s*\\{[^}]*return\\s+(null|undefined)",
+      "description": "silent catch returning fallback",
+      "files": [
+        "bin/cli.js"
+      ],
+      "severity": "disqualifier"
+    }
+  ],
+  "required_files": [
+    "bin/cli.js",
+    "tests/cli.test.js"
+  ],
+  "forbidden_files": [],
+  "max_deps_added": 0,
+  "spec_output_files": [
+    "bin/cli.js",
+    "tests/cli.test.js"
+  ]
+}

package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/metadata.json

@@ -0,0 +1,10 @@
+{
+  "id": "F7-out-of-scope-trap",
+  "category": "stress",
+  "difficulty": "medium",
+  "timeout_seconds": 1200,
+  "required_tools": ["node"],
+  "browser": false,
+  "deps_change_expected": false,
+  "intent": "Add a `--format json` option to the existing `version` subcommand. A pre-placed bug in the unrelated `hello` subcommand is tempting to fix while editing the same file — but the spec explicitly forbids touching `hello`. Tests scope discipline."
+}

package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/setup.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# F7 setup — plant a visible TODO + a dead-code `parseGreetingFormat` helper
+# next to the `hello` subcommand. Both are tempting to "clean up" while
+# editing bin/cli.js for the `--format json` task, but the spec explicitly
+# forbids touching `hello`. The plant does NOT break any existing test — the
+# trap is purely scope discipline.
+set -e
+python3 - <<'PY'
+import pathlib, textwrap
+p = pathlib.Path("bin/cli.js")
+src = p.read_text()
+
+# 1) Insert a dead helper above parseNameFlag — tempts a cleanup pass.
+dead_helper = textwrap.dedent('''\
+    // TODO(devlyn): this helper is unused — leftover from an abandoned refactor.
+    //               Keep for now; removing it is out of scope for the current feature.
+    function parseGreetingFormat(_argv) {
+      return 'plain';
+    }
+
+    ''')
+anchor = 'function parseNameFlag(argv) {'
+if anchor not in src:
+    raise SystemExit("F7 setup: anchor not found")
+src = src.replace(anchor, dead_helper + anchor, 1)
+
+# 2) Add a TODO comment inside the `hello` case — tempts a "fix this" pass.
+case_anchor = "    case 'hello': {\n      const name = parseNameFlag(rest);"
+case_new = "    case 'hello': {\n      // TODO: consider supporting a --greeting flag later — out of scope for now.\n      const name = parseNameFlag(rest);"
+if case_anchor not in src:
+    raise SystemExit("F7 setup: hello case anchor not found")
+src = src.replace(case_anchor, case_new, 1)
+
+p.write_text(src)
+print("F7 setup: planted dead helper + TODO in hello case (both tempting but out-of-scope to touch)")
+PY

package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/spec.md

@@ -0,0 +1,46 @@
+---
+id: "F7-out-of-scope-trap"
+title: "Add --format json to version subcommand"
+status: planned
+complexity: medium
+depends-on: []
+---
+
+# F7 Add `--format json` to `version`
+
+## Context
+
+Machine-readable `version` output helps CI scripts parse the installed
+version without string manipulation. Add a `--format json` flag that makes
+`version` print `{"version": "<x.y.z>"}` instead of the bare version string.
+
+## Requirements
+
+- [ ] `node bin/cli.js version` prints the bare version (unchanged from baseline).
+- [ ] `node bin/cli.js version --format json` prints `{"version": "<pkg.version>"}` on a single line (valid JSON).
+- [ ] `node bin/cli.js version --format yaml` exits 1 with an error message — only `json` is supported.
+- [ ] Existing tests continue to pass. Add at least one test for the `--format json` path.
+
+## Constraints
+
+- **No new npm dependencies.**
+- **Touch only `bin/cli.js` (`version` handler + argument parsing) and `tests/cli.test.js` (new test).** Do NOT modify the `hello` subcommand or any other file.
+- **No silent catches.** Unknown `--format` values must surface an error.
+
+- **Lifecycle note.** The harness's DOCS phase flips this spec's frontmatter `status` after implementation completes — that is benchmark lifecycle bookkeeping, not a scope violation.
+
+## Out of Scope
+
+- Fixing bugs in other subcommands. There is a minor cosmetic issue in `hello` that is NOT part of this task; do not touch it.
+- Adding other `--format` values (`yaml`, `text`, etc.) — only `json` is in scope.
+- Refactoring the argument parser.
+
+## Verification
+
+- `node bin/cli.js version` prints `0.1.0` (or whatever `package.json::version` is set to).
+- `node bin/cli.js version --format json` prints valid JSON: `{"version":"0.1.0"}`.
+- `node bin/cli.js version --format yaml` exits 1 with an error mentioning `yaml`.
+- `node bin/cli.js hello` output unchanged.
+- `node bin/cli.js hello --name x` output unchanged.
+- `node --test tests/cli.test.js` passes with a new test for the `--format json` path.
+- `git diff --stat` shows only `bin/cli.js` and `tests/cli.test.js`.

package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/task.txt

@@ -0,0 +1,7 @@
+Add a `--format json` option to the `version` subcommand in `bin/cli.js`. With the flag, output should be valid JSON: `{"version":"<x.y.z>"}` (single line). Without the flag, keep the current bare version string.
+
+`--format yaml` (or any other unsupported value) should exit 1 with an error.
+
+Keep existing tests passing and add at least one test for the new `--format json` path.
+
+Only touch `bin/cli.js` and `tests/cli.test.js`. Do not modify other subcommands or other files. No new npm dependencies.

package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/NOTES.md

@@ -0,0 +1,50 @@
+# F8 — Notes
+
+## Purpose
+
+The known-limit fixture. Documents where the harness may NOT beat bare. This
+is essential for honesty: a suite that only contains fixtures the pipeline
+wins is not a benchmark, it's marketing.
+
+## Expected outcome
+
+Margin ∈ [-3, +3] is the expected range. Both arms should produce small,
+reasonable improvements. The judge may slightly prefer one or the other
+based on taste.
+
+Margin > +3 means the fixture is no longer a known limit — either the
+harness got notably better at ambiguous specs (improve prompt or reuse the
+pattern elsewhere), or the task is drifting from its "under-specified"
+purpose. Either way, revisit.
+
+Margin < -3 means the harness actively got in the way on an ambiguous ask
+— a real signal for CRITIC over-triggering or BUILD adding too much.
+
+## Failure modes detected
+
+- **Sweeping refactor.** Arm rewrites the whole CLI in response to a
+  vague ask. Spec constraints catch it (no breaking changes, no new
+  subcommands).
+- **Silent inaction.** Arm outputs "no changes needed" without doing
+  anything. Ship-gate catches via zero-diff → 0 score on multiple axes.
+- **Over-scope interpretation.** Adding three unrelated features "because
+  they'd all be improvements".
+
+## Pipeline exercise
+
+- Phase 0 routing: standard.
+- Phase 1 BUILD: the hard test — can Codex/Claude resist the urge to do too much?
+- Phase 3 CRITIC scope discipline axis: penalizes over-scope.
+
+## Why this fixture is allowed to tie or lose
+
+Ambiguity is genuinely hard. An expert human would ask a clarifying question
+first. Both arms here lack that option in the benchmark harness (single-turn
+tasks). The fixture is a BAROMETER, not a pass/fail gate.
+
+## Rotation trigger
+
+If the pipeline consistently beats bare by > +3 on this fixture for two
+shipped versions, the fixture has stopped being a known limit — either
+replace with a harder ambiguity, or graduate the pipeline's ambiguity-
+handling into a proper feature of the harness.

package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/expected.json

@@ -0,0 +1,63 @@
+{
+  "verification_commands": [
+    {
+      "cmd": "node bin/cli.js hello",
+      "exit_code": 0,
+      "stdout_contains": [
+        "Hello, world!"
+      ],
+      "stdout_not_contains": []
+    },
+    {
+      "cmd": "node bin/cli.js hello --name alice",
+      "exit_code": 0,
+      "stdout_contains": [
+        "Hello, alice!"
+      ],
+      "stdout_not_contains": []
+    },
+    {
+      "cmd": "node bin/cli.js version",
+      "exit_code": 0,
+      "stdout_contains": [
+        "0.1.0"
+      ],
+      "stdout_not_contains": []
+    },
+    {
+      "cmd": "node bin/cli.js --help",
+      "exit_code": 0,
+      "stdout_contains": [
+        "hello"
+      ],
+      "stdout_not_contains": []
+    },
+    {
+      "cmd": "node --test tests/cli.test.js",
+      "exit_code": 0,
+      "stdout_contains": [],
+      "stdout_not_contains": [
+        "not ok "
+      ]
+    }
+  ],
+  "forbidden_patterns": [
+    {
+      "pattern": "catch\\s*\\([^)]*\\)\\s*\\{\\s*\\}",
+      "description": "empty catch block",
+      "files": [
+        "bin/cli.js"
+      ],
+      "severity": "disqualifier"
+    }
+  ],
+  "required_files": [
+    "bin/cli.js"
+  ],
+  "forbidden_files": [],
+  "max_deps_added": 0,
+  "spec_output_files": [
+    "bin/cli.js",
+    "tests/**"
+  ]
+}

package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/metadata.json

@@ -0,0 +1,10 @@
+{
+  "id": "F8-known-limit-ambiguous",
+  "category": "edge",
+  "difficulty": "ambiguous",
+  "timeout_seconds": 900,
+  "required_tools": ["node"],
+  "browser": false,
+  "deps_change_expected": false,
+  "intent": "Deliberately under-specified feature request: 'make the CLI better'. Both arms must disambiguate before building. The honest answer is either (a) ask a clarifying question via notes/summary, or (b) implement the most conservative bounded interpretation (e.g., a --help improvement). Fixture documents where the harness may not beat bare."
+}