devlyn-cli 2.0.0 → 2.2.0

package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/setup.sh

@@ -0,0 +1,251 @@
+#!/usr/bin/env bash
+# F12 setup — seed the shared secret and stage verifier scripts.
+set -e
+
+mkdir -p data
+
+# Single-line secret, no trailing newline (printf, not echo).
+printf 'wh_test_secret_a3f9e1c2_d4b6e7' > data/webhook-secret.txt
+
+# Sample event payload for human inspection (not used by verifiers).
+cat > data/_sample-event.json <<'JSON'
+{"id":"evt_001","type":"order.created","timestamp":1735689600,"data":{"order_id":"o_42","amount_cents":1500}}
+JSON
+
+# Verifier 1: happy path — exact bytes the provider signed → 200 accepted.
+cat > data/_verify-happy.js <<'JS'
+'use strict';
+const http = require('http');
+const fs = require('fs');
+const crypto = require('crypto');
+const { app } = require('../server');
+
+const SECRET = fs.readFileSync('data/webhook-secret.txt');
+
+function hmacHex(bytes) {
+  return crypto.createHmac('sha256', SECRET).update(bytes).digest('hex');
+}
+
+function postRaw(port, bytes, sig) {
+  return new Promise((resolve) => {
+    const req = http.request(
+      { host: '127.0.0.1', port, method: 'POST', path: '/webhook',
+        headers: {
+          'Content-Type': 'application/json',
+          'Content-Length': Buffer.byteLength(bytes),
+          'X-Signature': sig,
+        } },
+      (r) => {
+        let b = ''; r.on('data', (c) => (b += c));
+        r.on('end', () => {
+          let d = null; try { d = JSON.parse(b); } catch {}
+          resolve({ status: r.statusCode, body: d });
+        });
+      }
+    );
+    req.on('error', () => resolve({ status: 0, body: null }));
+    req.write(bytes);
+    req.end();
+  });
+}
+
+const s = http.createServer(app).listen(0, async () => {
+  const { port } = s.address();
+  const body = Buffer.from('{"id":"evt_h1","type":"order.created","timestamp":1,"data":{}}');
+  const sig = hmacHex(body);
+  const r = await postRaw(port, body, sig);
+  const ok = r.status === 200 && r.body && r.body.accepted === true && r.body.id === 'evt_h1';
+  console.log(JSON.stringify({ status: r.status, body: r.body, ok }));
+  s.close();
+  process.exit(ok ? 0 : 1);
+});
+JS
+
+# Verifier 2: replay (same id) — first 200, second 409 even with valid sig.
+cat > data/_verify-replay.js <<'JS'
+'use strict';
+const http = require('http');
+const fs = require('fs');
+const crypto = require('crypto');
+const { app } = require('../server');
+
+const SECRET = fs.readFileSync('data/webhook-secret.txt');
+function hmacHex(bytes) {
+  return crypto.createHmac('sha256', SECRET).update(bytes).digest('hex');
+}
+function postRaw(port, bytes, sig) {
+  return new Promise((resolve) => {
+    const req = http.request(
+      { host: '127.0.0.1', port, method: 'POST', path: '/webhook',
+        headers: {
+          'Content-Type': 'application/json',
+          'Content-Length': Buffer.byteLength(bytes),
+          'X-Signature': sig,
+        } },
+      (r) => {
+        let b = ''; r.on('data', (c) => (b += c));
+        r.on('end', () => {
+          let d = null; try { d = JSON.parse(b); } catch {}
+          resolve({ status: r.statusCode, body: d });
+        });
+      }
+    );
+    req.on('error', () => resolve({ status: 0, body: null }));
+    req.write(bytes);
+    req.end();
+  });
+}
+
+const s = http.createServer(app).listen(0, async () => {
+  const { port } = s.address();
+  const body = Buffer.from('{"id":"evt_r1","type":"x","timestamp":1,"data":{}}');
+  const sig = hmacHex(body);
+  const first  = await postRaw(port, body, sig);
+  const second = await postRaw(port, body, sig);
+  const ok = first.status === 200 &&
+             second.status === 409 &&
+             second.body && second.body.error === 'duplicate_event' && second.body.id === 'evt_r1';
+  console.log(JSON.stringify({ first: first.status, second: second.status, second_body: second.body, ok }));
+  s.close();
+  process.exit(ok ? 0 : 1);
+});
+JS
+
+# Verifier 3: tampered body → 401. Body changed AFTER signing; the original
+# sig now corresponds to bytes that are no longer in the request.
+cat > data/_verify-tampered.js <<'JS'
+'use strict';
+const http = require('http');
+const fs = require('fs');
+const crypto = require('crypto');
+const { app } = require('../server');
+
+const SECRET = fs.readFileSync('data/webhook-secret.txt');
+function hmacHex(bytes) {
+  return crypto.createHmac('sha256', SECRET).update(bytes).digest('hex');
+}
+function postRaw(port, bytes, sig) {
+  return new Promise((resolve) => {
+    const req = http.request(
+      { host: '127.0.0.1', port, method: 'POST', path: '/webhook',
+        headers: {
+          'Content-Type': 'application/json',
+          'Content-Length': Buffer.byteLength(bytes),
+          'X-Signature': sig,
+        } },
+      (r) => {
+        let b = ''; r.on('data', (c) => (b += c));
+        r.on('end', () => {
+          let d = null; try { d = JSON.parse(b); } catch {}
+          resolve({ status: r.statusCode, body: d });
+        });
+      }
+    );
+    req.on('error', () => resolve({ status: 0, body: null }));
+    req.write(bytes);
+    req.end();
+  });
+}
+
+const s = http.createServer(app).listen(0, async () => {
+  const { port } = s.address();
+  const original = Buffer.from('{"id":"evt_t1","type":"x","timestamp":1,"data":{"amount":100}}');
+  const sig = hmacHex(original); // sig over the original
+  const tampered = Buffer.from('{"id":"evt_t1","type":"x","timestamp":1,"data":{"amount":9999}}');
+  const r = await postRaw(port, tampered, sig);
+  const ok = r.status === 401 && r.body && r.body.error === 'invalid_signature';
+  console.log(JSON.stringify({ status: r.status, body: r.body, ok }));
+  s.close();
+  process.exit(ok ? 0 : 1);
+});
+JS
+
+# Verifier 4: missing/malformed X-Signature → 401.
+cat > data/_verify-missing-sig.js <<'JS'
+'use strict';
+const http = require('http');
+const { app } = require('../server');
+
+const s = http.createServer(app).listen(0, async () => {
+  const { port } = s.address();
+  const body = Buffer.from('{"id":"evt_m1","type":"x","timestamp":1,"data":{}}');
+  const r = await new Promise((resolve) => {
+    const req = http.request(
+      { host: '127.0.0.1', port, method: 'POST', path: '/webhook',
+        headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) } },
+      (rs) => {
+        let b = ''; rs.on('data', (c) => (b += c));
+        rs.on('end', () => {
+          let d = null; try { d = JSON.parse(b); } catch {}
+          resolve({ status: rs.statusCode, body: d });
+        });
+      }
+    );
+    req.on('error', () => resolve({ status: 0, body: null }));
+    req.write(body);
+    req.end();
+  });
+  const ok = r.status === 401 && r.body && r.body.error === 'invalid_signature';
+  console.log(JSON.stringify({ status: r.status, body: r.body, ok }));
+  s.close();
+  process.exit(ok ? 0 : 1);
+});
+JS
+
+# Verifier 5: raw-body discrimination — same parsed object, different on-wire
+# bytes, signature of an alternate-bytes form. A naive impl that re-serializes
+# req.body and HMACs the result will accept this; a correct impl that HMACs
+# the actual received bytes will reject. The "alternate" bytes here are the
+# canonical (compact) form of the same JSON, so naive's re-stringify produces
+# bytes whose HMAC matches the provided signature.
+cat > data/_verify-raw-body.js <<'JS'
+'use strict';
+const http = require('http');
+const fs = require('fs');
+const crypto = require('crypto');
+const { app } = require('../server');
+
+const SECRET = fs.readFileSync('data/webhook-secret.txt');
+function hmacHex(bytes) {
+  return crypto.createHmac('sha256', SECRET).update(bytes).digest('hex');
+}
+
+const s = http.createServer(app).listen(0, async () => {
+  const { port } = s.address();
+  // Canonical bytes (compact form) — what JSON.stringify(parsed_obj) emits.
+  const canonical = Buffer.from('{"id":"evt_rb1","type":"x","timestamp":1,"data":{}}');
+  // Sig over canonical bytes — what a naive impl will produce when it
+  // re-stringifies req.body and HMACs the result.
+  const sig = hmacHex(canonical);
+  // Wire bytes: same parsed JSON, but with extra whitespace. Different bytes
+  // than canonical; an HMAC over THESE bytes is NOT `sig`.
+  const wire = Buffer.from('{ "id":"evt_rb1","type":"x","timestamp":1,"data":{} }');
+  const r = await new Promise((resolve) => {
+    const req = http.request(
+      { host: '127.0.0.1', port, method: 'POST', path: '/webhook',
+        headers: {
+          'Content-Type': 'application/json',
+          'Content-Length': Buffer.byteLength(wire),
+          'X-Signature': sig,
+        } },
+      (rs) => {
+        let b = ''; rs.on('data', (c) => (b += c));
+        rs.on('end', () => {
+          let d = null; try { d = JSON.parse(b); } catch {}
+          resolve({ status: rs.statusCode, body: d });
+        });
+      }
+    );
+    req.on('error', () => resolve({ status: 0, body: null }));
+    req.write(wire);
+    req.end();
+  });
+  // Correct: 401 (HMAC over wire bytes ≠ sig). Naive (JSON.stringify path): 200.
+  const ok = r.status === 401 && r.body && r.body.error === 'invalid_signature';
+  console.log(JSON.stringify({ status: r.status, body: r.body, ok }));
+  s.close();
+  process.exit(ok ? 0 : 1);
+});
+JS
+
+exit 0

package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/spec.md

@@ -0,0 +1,58 @@
+---
+id: "F12-webhook-raw-body-signature"
+title: "Webhook signature verification endpoint"
+status: planned
+complexity: high
+depends-on: []
+---
+
+# F12 Webhook signature verification endpoint
+
+## Context
+
+A third-party provider sends event notifications via webhook. Each
+delivery is signed: the provider computes HMAC-SHA256 over the body
+bytes using a shared secret and includes the lowercase hex-encoded
+result in the `X-Signature` header.
+
+The provider may retry a delivery (network blip, ack lost). When it
+does, it sends the same payload with the same `id` field — the server
+must accept the event at most once and respond `409` on a re-delivery.
+
+The shared secret is stored at `data/webhook-secret.txt` (single line,
+no trailing newline).
+
+## Requirements
+
+- [ ] `POST /webhook` accepts JSON-shaped bodies of the form `{ id, type, timestamp, data }` where `id` and `type` are non-empty strings, `timestamp` is a number, and `data` is an object.
+- [ ] On valid signature + first delivery of a given `id`: respond `200` with `{ accepted: true, id }`.
+- [ ] On valid signature + re-delivery of an already-seen `id`: respond `409` with `{ error: 'duplicate_event', id }`. The body of the duplicate is irrelevant — once an `id` is accepted, the same `id` is permanently rejected for the lifetime of the server process.
+- [ ] On missing or invalid `X-Signature`, or signature does not match the body: respond `401` with `{ error: 'invalid_signature' }`. Verification compares the signature the provider sent against an HMAC-SHA256 the server computes the same way the provider did.
+- [ ] On a body that fails the shape check (missing fields, wrong types, empty `id`/`type`): respond `400` with `{ error: 'invalid_body' }`. Sig check still runs first — a bad body with a valid sig is still 400, not 401.
+- [ ] `tests/server.test.js` is updated. Existing assertions still hold AND at least three new tests cover: happy path, replay (same id) → 409, tampered body with stale signature → 401.
+
+## Constraints
+
+- **No new npm dependencies.** Express + Node `crypto` + Node built-ins only.
+- **No silent catches.** Errors in the verification path surface as `500` with a clear body.
+- **Use `crypto.timingSafeEqual` for the signature comparison.** A non-constant-time `===` between hex strings leaks information about the true MAC byte-by-byte.
+- **No breaking change** to existing `/items`, `/items/:id`, `/health`.
+- **Lifecycle note.** The harness's DOCS phase flips this spec's frontmatter `status` after implementation completes — that is benchmark lifecycle bookkeeping, not a scope violation.
+
+## Out of Scope
+
+- Authentication beyond signature verification.
+- Rate limiting, replay window TTL (the seen-id set is process-lifetime only).
+- Persistence of seen-ids across restarts.
+- Touching `bin/cli.js`, `web/`, or `tests/cli.test.js`.
+
+## Verification
+
+- `node --test tests/server.test.js` exits 0.
+- A POST with the provider's sig over the exact body bytes returns 200.
+- A second POST with the same `id` returns 409 even if the signature is valid.
+- A second POST with the same accepted `id` returns 409 even if the duplicate
+  body would otherwise fail shape validation; duplicate id wins after acceptance.
+- A POST whose body has been modified after signing (or whose `X-Signature` was computed against different bytes than the body now contains) returns 401.
+- A POST with a missing or malformed `X-Signature` header returns 401.
+- `git diff --stat` shows only `server/index.js` and `tests/server.test.js` touched.

package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/task.txt

@@ -0,0 +1,13 @@
+Add `POST /webhook` to `server/index.js` to receive signed event notifications from a third-party provider.
+
+Each request carries an `X-Signature` header with a lowercase hex HMAC-SHA256 over the body, computed using the shared secret stored at `data/webhook-secret.txt`. Verify the signature; if it doesn't match what you compute the same way, respond 401 with `{ error: 'invalid_signature' }`.
+
+The body shape is `{ id, type, timestamp, data }` where `id` and `type` must be non-empty strings, `timestamp` is a number, and `data` is an object. A body that fails this shape returns 400 with `{ error: 'invalid_body' }` (sig check still runs first — a bad body with a valid sig is 400, not 401).
+
+The provider retries on network failure, so the same `id` may arrive twice. After accepting an `id`, any subsequent delivery of that same `id` returns 409 with `{ error: 'duplicate_event', id }` — the second body is irrelevant. First delivery returns 200 with `{ accepted: true, id }`.
+
+Use `crypto.timingSafeEqual` for the signature comparison.
+
+Update `tests/server.test.js` so existing tests still pass AND add at least three new tests covering: happy path, replay of the same id returns 409, tampered body with the original signature returns 401.
+
+No new npm dependencies. Only touch `server/index.js` and `tests/server.test.js`.

package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/verifiers/replay-malformed-body.js

@@ -0,0 +1,64 @@
+'use strict';
+
+const http = require('node:http');
+const fs = require('node:fs');
+const crypto = require('node:crypto');
+const path = require('node:path');
+
+const workdir = process.env.BENCH_WORKDIR;
+const { app } = require(path.join(workdir, 'server'));
+const secret = fs.readFileSync(path.join(workdir, 'data', 'webhook-secret.txt'));
+
+function hmacHex(bytes) {
+  return crypto.createHmac('sha256', secret).update(bytes).digest('hex');
+}
+
+function postRaw(port, bytes) {
+  return new Promise((resolve) => {
+    const req = http.request(
+      {
+        host: '127.0.0.1',
+        port,
+        method: 'POST',
+        path: '/webhook',
+        headers: {
+          'Content-Type': 'application/json',
+          'Content-Length': Buffer.byteLength(bytes),
+          'X-Signature': hmacHex(bytes)
+        }
+      },
+      (res) => {
+        let body = '';
+        res.on('data', (chunk) => { body += chunk; });
+        res.on('end', () => {
+          let parsed = null;
+          try {
+            parsed = JSON.parse(body);
+          } catch {
+            parsed = null;
+          }
+          resolve({ status: res.statusCode, body: parsed });
+        });
+      }
+    );
+    req.on('error', () => resolve({ status: 0, body: null }));
+    req.write(bytes);
+    req.end();
+  });
+}
+
+const server = http.createServer(app).listen(0, async () => {
+  const { port } = server.address();
+  const accepted = Buffer.from('{"id":"evt_hidden_replay","type":"x","timestamp":1,"data":{}}');
+  const malformedDuplicate = Buffer.from('{"id":"evt_hidden_replay","type":"x","timestamp":"bad","data":{}}');
+  const first = await postRaw(port, accepted);
+  const second = await postRaw(port, malformedDuplicate);
+  const ok = first.status === 200
+    && second.status === 409
+    && second.body
+    && second.body.error === 'duplicate_event'
+    && second.body.id === 'evt_hidden_replay';
+  console.log(JSON.stringify({ ok, first: first.status, second: second.status, second_body: second.body }));
+  server.close();
+  process.exit(ok ? 0 : 1);
+});

package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/NOTES.md

@@ -0,0 +1,98 @@
+# F15 — Notes
+
+## Purpose
+
+**Frozen-diff review fixture.** The 5th option from Codex R3 (2026-05-05):
+"Seed a plausible, green-but-wrong implementation, then ask arms to
+review/fix against the spec. This isolates external review value from
+implementation variance."
+
+Where F10/F11/F12 ask the arm to BUILD an implementation, F15 asks the
+arm to REVIEW pre-existing code and identify a latent correctness issue
+that single-request testing does not surface.
+
+This directly tests pair-mode's empirically-validated strength: in iter-
+0033c the dual-engine VERIFY pair found a unique EACCES/EPERM finding on
+a frozen diff that single-engine review missed (autoresearch/iterations/
+0033c-l2-new-vs-new-l1.md:216). Generation tasks did not show pair lift;
+review tasks did.
+
+## The seeded bug
+
+`server/index.js` ships with a POST `/items` handler that does an awaited
+read-modify-write on `data/items.json`:
+
+```js
+const data = await readStore();        // T1 reads, T2 reads (same view)
+const newId = data.items.length + 1;   // both compute the same id
+data.items.push(newItem);              // both mutate (separate copies)
+await writeStore(data);                // last writer wins
+```
+
+Two concurrent POSTs interleave during the await gaps. The visible failure
+modes:
+
+- Duplicate ids: both T1 and T2 compute `length + 1`, return 201 to both
+  callers, but final state contains only one of the two new items (still
+  with same id).
+- Lost writes: state has length+1 instead of length+2.
+
+Single-request testing never triggers this — the race window only opens
+when at least two POSTs are in flight. Pre-staged tests cover happy path
+and validation but NOT concurrency.
+
+## Failure modes detected
+
+- **Race ignored**: arm reviews the code, doesn't notice the await gap,
+  responds "the implementation looks correct" or makes cosmetic changes.
+  Verifier 1 (concurrent POSTs) catches.
+- **Race noticed, fix breaks single-POST**: e.g., adds a global mutex but
+  forgets to release it on error. Verifier 2 catches.
+- **Race noticed, fix violates scope**: e.g., replaces JSON file with
+  SQLite. Out-of-scope per spec; surfaces in oracle-scope-tier-a / -b.
+- **Silent catch added** to "fix" by hiding the error. Caught by
+  forbidden_pattern.
+
+## Pipeline exercise
+
+- Phase 0 routing: standard.
+- Phase 1 BUILD: this fixture is review-shaped, not build-shaped.
+  Implementer reads the existing `server/index.js`, must derive the race
+  from the await pattern alone — no spec wording to anchor on.
+- Phase 2 EVAL: verifies the fix actually addresses concurrency, not just
+  cosmetic.
+- Phase 3 CRITIC: scope discipline (no out-of-scope refactor) + production-
+  ready (no silent catches in the fix).
+
+## Discrimination expectation
+
+Calibration target:
+
+- bare arm: 30-65 (passes baseline tests + single-POST verifier; may or
+  may not derive the race from cold-read of the code without an explicit
+  spec hint).
+- solo arm: 55-80 (skill review pass increases the chance of catching it,
+  but the same model is doing both implementation and review).
+- pair arm: 75-90 (cross-perspective review; the second engine reads the
+  same code with different priors and is more likely to surface the race).
+
+The `intent` and `spec.md` deliberately use "load test" and "correctness
+under load" rather than "race", "concurrency", or "atomic" — those would
+be leading keywords. "Load" alone is not specific enough for the model
+to anchor on a single failure mode without reading the code.
+
+## Public-spec wording — load-bearing
+
+The spec says "issues a load test would surface that single-request
+testing would not" — natural product framing, no implementation hint.
+The Requirements bullet lists "lost updates, duplicated identifiers, and
+inconsistent response/state pairs" as common load-related issues — this
+is education for a generic reviewer, not a leak. A naive review-only pass
+that doesn't read the await sequence carefully will gloss over these.
+
+## Rotation trigger
+
+Retire when both bare and solo arms consistently land > 85 across two
+shipped versions. If 2026 baseline reliably catches the awaited RMW race
+on cold read of someone else's code, the frozen-diff review thesis also
+needs updating — not just the seeded bug.

package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/expected.json

@@ -0,0 +1,46 @@
+{
+  "verification_commands": [
+    {
+      "cmd": "node --test tests/server.test.js",
+      "exit_code": 0,
+      "stdout_contains": [],
+      "stdout_not_contains": ["not ok "]
+    },
+    {
+      "cmd": "node data/_verify-concurrent.js",
+      "exit_code": 0,
+      "stdout_contains": ["\"ok\":true"],
+      "stdout_not_contains": []
+    },
+    {
+      "cmd": "node data/_verify-single.js",
+      "exit_code": 0,
+      "stdout_contains": ["\"ok\":true"],
+      "stdout_not_contains": []
+    }
+  ],
+  "forbidden_patterns": [
+    {
+      "pattern": "catch\\s*\\([^)]*\\)\\s*\\{[^}]*return\\s+(null|undefined|''|\\{\\})",
+      "description": "silent catch returning fallback",
+      "files": ["server/index.js"],
+      "severity": "disqualifier"
+    },
+    {
+      "pattern": "catch\\s*\\([^)]*\\)\\s*\\{\\s*\\}",
+      "description": "empty catch block",
+      "files": ["server/index.js"],
+      "severity": "disqualifier"
+    },
+    {
+      "pattern": "/\\*\\s*eslint-disable",
+      "description": "eslint-disable without scoped justification",
+      "files": ["server/index.js"],
+      "severity": "disqualifier"
+    }
+  ],
+  "required_files": ["server/index.js", "tests/server.test.js", "data/items.json"],
+  "forbidden_files": [],
+  "max_deps_added": 0,
+  "spec_output_files": ["server/index.js", "tests/server.test.js"]
+}

package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/metadata.json

@@ -0,0 +1,10 @@
+{
+  "id": "F15-frozen-diff-race-review",
+  "category": "high-risk",
+  "difficulty": "high",
+  "timeout_seconds": 1500,
+  "required_tools": ["node"],
+  "browser": false,
+  "deps_change_expected": false,
+  "intent": "Review the existing /items API in server/index.js for correctness issues a load test would surface. Fix what you find and add tests covering the fix. The current implementation has been deployed for a few weeks; existing tests pass."
+}