@dsalazardev/ai-hub-cli 0.0.0-stage → 0.1.0

package/skills/shared/research-ops/scripts/verify.sh

@@ -0,0 +1,181 @@
+#!/usr/bin/env bash
+#
+# verify.sh — provenance gate for the `research-ops` skill.
+#
+# WHAT IT DOES (read-only; never edits a file)
+#   Static checks over a produced research memo (Markdown). It validates that the
+#   memo is SOURCED, DATED, and REVIEW-READY — never whether the answer is correct.
+#   For each memo found:
+#     1. Structure: an Answer/Summary section AND an Open-questions/Unverified
+#        section both exist. A memo with no stated boundary of the evidence fails.
+#     2. Findings carry citations: under the Findings section, lines that state a
+#        claim carry a citation token (a Markdown link or a bare URL).
+#     3. Dates: every cited finding line carries a date — YYYY-MM-DD, "pub", or
+#        "accessed". An undated citation is a fail.
+#     4. Confidence: a confidence tier token (high|med|medium|low) appears among
+#        the findings. A memo that never tiers its claims fails.
+#
+#   A missing/empty target is a SKIP, never a failure. grep/awk only; no network.
+#
+# HOW TO RUN (inside YOUR project, not the skills repo)
+#   ./verify.sh                      # scan ./ for *memo*/*research* .md
+#   ./verify.sh --path memo.md       # check one memo
+#   ./verify.sh --path research/     # scan a directory of memos
+#   ./verify.sh --strict             # treat warnings as failures (CI gate)
+#
+# EXIT CODES
+#   0  clean, empty target, or warnings only without --strict
+#   1  a real failure (no answer section, undated citation, no confidence tier)
+#   2  bad usage
+#
+# Runs on stock macOS bash 3.2: no mapfile, no associative arrays.
+
+set -euo pipefail
+
+if [ -t 1 ]; then
+  RED=$'\033[31m'; GREEN=$'\033[32m'; YELLOW=$'\033[33m'; NC=$'\033[0m'
+else
+  RED=''; GREEN=''; YELLOW=''; NC=''
+fi
+
+usage() { sed -n '2,34p' "$0" | sed 's/^# \{0,1\}//'; }
+
+SCAN_PATH="."
+STRICT=0
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --path)    SCAN_PATH="${2:?--path needs a value}"; shift 2 ;;
+    --strict)  STRICT=1; shift ;;
+    -h|--help) usage; exit 0 ;;
+    *) printf '%sUnknown argument: %s%s\n\n' "$RED" "$1" "$NC"; usage; exit 2 ;;
+  esac
+done
+
+ok_count=0; skip_count=0; warn_count=0; fail_count=0
+summary() { printf '\nok=%d skip=%d warn=%d fail=%d\n' "$ok_count" "$skip_count" "$warn_count" "$fail_count"; }
+
+if [ ! -e "$SCAN_PATH" ]; then
+  printf '%s[skip]%s path not found: %s — nothing to check\n' "$YELLOW" "$NC" "$SCAN_PATH"
+  skip_count=1; summary; exit 0
+fi
+
+have() { command -v "$1" >/dev/null 2>&1; }
+if ! have awk; then
+  printf '%s[skip]%s awk not found — cannot parse memo\n' "$YELLOW" "$NC"
+  skip_count=1; summary; exit 0
+fi
+
+# Collect candidate memo files: prefer named memos/research files, else any .md.
+FILES=""
+if [ -f "$SCAN_PATH" ]; then
+  FILES="$SCAN_PATH"
+else
+  FILES="$(find "$SCAN_PATH" -type f -name '*.md' \( -iname '*memo*' -o -iname '*research*' -o -iname '*findings*' \) 2>/dev/null || true)"
+  if [ -z "$FILES" ]; then
+    FILES="$(find "$SCAN_PATH" -type f -name '*.md' 2>/dev/null || true)"
+  fi
+fi
+
+if [ -z "$FILES" ]; then
+  printf '%s[skip]%s no research memo found under %s — nothing to check (clean)\n' "$YELLOW" "$NC" "$SCAN_PATH"
+  skip_count=1; summary; exit 0
+fi
+
+# check_one <file>: one awk pass emits "LEVEL message" lines; the shell tallies.
+check_one() {
+  f="$1"
+  [ -s "$f" ] || { printf '%s[skip]%s empty file: %s\n' "$YELLOW" "$NC" "$f"; return; }
+  printf '%s—— %s%s\n' "$YELLOW" "$f" "$NC"
+
+  awk '
+    # BSD awk: no IGNORECASE, no \b. Lowercase with tolower(); explicit boundaries.
+    {
+      lc=tolower($0)
+
+      # --- section detection (headings) ---
+      if (lc ~ /^#+/) {
+        infind=0
+        if (lc ~ /(answer|summary|bottom line|tl;dr|tldr)/) hasanswer=1
+        if (lc ~ /(open question|unverified|could not verify|couldnt verify|not verified|limitations)/) hasopen=1
+        if (lc ~ /(finding|claim|evidence)/) infind=1
+      }
+
+      # --- a finding line: a bullet/numbered claim, inside findings or anywhere ---
+      isbullet = ($0 ~ /^[ \t]*([-*]|[0-9]+\.)[ \t]+/)
+      if (isbullet && (infind || lc ~ /confidence:/)) {
+        findlines++
+        haslink = ($0 ~ /\]\(http/ || $0 ~ /https?:\/\//)
+        hasdate = ($0 ~ /(19|20)[0-9][0-9]-[0-1][0-9]-[0-3][0-9]/ || lc ~ /pub /|| lc ~ /accessed/ || lc ~ /pub n\/a/)
+        if (haslink) {
+          citedlines++
+          if (!hasdate) {
+            undated++
+            if (undated <= 3) print "DETAIL cited finding with no date: " substr($0,1,80)
+          }
+        } else {
+          uncited++
+          if (uncited <= 3) print "DETAIL finding with no citation token: " substr($0,1,80)
+        }
+      }
+
+      if (lc ~ /confidence: *(high|med|medium|low)/) tiers++
+    }
+    END {
+      if (hasanswer) print "OK answer/summary section present"
+      else           print "FAIL no Answer/Summary section found"
+
+      if (hasopen) print "OK open-questions/unverified section present"
+      else         print "FAIL no Open-questions/Unverified section found"
+
+      if (findlines == 0) {
+        print "WARN no finding/claim bullet lines detected — nothing to check provenance on"
+      } else {
+        if (uncited > 0) print "FAIL "uncited" finding line(s) carry no citation token"
+        else             print "OK every finding line carries a citation token ("citedlines" cited)"
+
+        if (undated > 0) print "FAIL "undated" cited finding(s) carry no date"
+        else if (citedlines > 0) print "OK "citedlines" cited finding(s); all dated"
+
+        if (tiers > 0) print "OK "tiers" confidence tier token(s) present"
+        else           print "FAIL no confidence tier (high|med|low) found on any finding"
+      }
+    }
+  ' "$f" 2>/dev/null
+}
+
+printf 'Checking research memos under: %s\n\n' "$SCAN_PATH"
+printf '%s\n' "$FILES" | while IFS= read -r f; do
+  [ -z "$f" ] && continue
+  check_one "$f"
+done > "/tmp/ro_verify.$$" 2>&1 || true
+
+while IFS= read -r line; do
+  case "$line" in
+    FAIL*)   printf '%s[fail]%s %s\n' "$RED"    "$NC" "${line#FAIL }" ;;
+    WARN*)   printf '%s[warn]%s %s\n' "$YELLOW" "$NC" "${line#WARN }" ;;
+    OK*)     printf '%s[ ok ]%s %s\n' "$GREEN"  "$NC" "${line#OK }" ;;
+    DETAIL*) printf '        %s\n' "${line#DETAIL }" ;;
+    *)       printf '%s\n' "$line" ;;
+  esac
+done < "/tmp/ro_verify.$$"
+
+tally() { { grep -c "$1" "/tmp/ro_verify.$$" 2>/dev/null || true; } | tr -dc '0-9'; }
+fail_count=$(tally '^FAIL'); fail_count=${fail_count:-0}
+warn_count=$(tally '^WARN'); warn_count=${warn_count:-0}
+ok_count=$(tally '^OK');     ok_count=${ok_count:-0}
+sk=$(tally '\[skip\]');      skip_count=$((skip_count + ${sk:-0}))
+rm -f "/tmp/ro_verify.$$" 2>/dev/null || true
+
+summary
+
+cat <<'EOF'
+
+Note: this gate proves the memo is SOURCED, DATED, and REVIEW-READY (answer section,
+cited + dated findings, confidence tiers, an open-questions section). It does NOT judge
+whether the answer is correct — that is the capability eval's and the researcher's job.
+Re-run with --strict to gate CI on a clean pass.
+EOF
+
+if [ "$fail_count" -gt 0 ]; then exit 1; fi
+if [ "$STRICT" -eq 1 ] && [ "$warn_count" -gt 0 ]; then exit 1; fi
+exit 0

package/skills/shared/review/SKILL.md

@@ -0,0 +1,314 @@
+---
+name: review
+description: "Use when a green diff needs adversarial judgment in the ai-hub Spec-Kit chain, between verify and ship — give a review keyed to the spec/plan/constitution in 02-DOCS/wiki/sdd/, or receive one and verify each comment before agreeing. NOT running lint/type/tests (that is `verify`), NOT the spec-less standalone pass (that is `code-review`), NOT merging (that is `ship`)."
+tags: [sdd, review, code-review]
+recommends: [ship]
+profiles: [core, full]
+origin: ai-hub
+---
+
+# Review — adversarial code review, both directions
+
+Review is the **penultimate gate** of the ai-hub Spec-Kit chain: `constitution → specify → clarify → plan → tasks → analyze → implement → verify → **review** → ship`. By the time a diff reaches review, the `verify` phase has already proven that lint, types, tests and the stack `verify.sh` are green. Review answers a different question: **is the green diff actually correct, well-scoped, and faithful to the spec — or did it pass for the wrong reasons?**
+
+This skill owns **two roles that share one discipline**:
+
+- **Giving** a review — read a diff adversarially, find the real defects, rank them by impact, ship the verdict as evidence not opinion.
+- **Receiving** a review — take incoming feedback (human or machine), verify each point against the code *before* acting, fix the real findings, and push back — with proof — on the wrong ones.
+
+The discipline is the same in both directions: **every finding and every rebuttal carries evidence — no agreement without verification, no objection without a defect.** A reviewer who waves through a bug to be agreeable, and an author who edits working code because a comment *sounded* authoritative, are making the same mistake. Everything below is in service of that one rule.
+
+**Route out** when the ask isn't this gate: a deep OWASP / threat-model pass belongs to `../secure-coding/SKILL.md` (review folds its findings in, but the dedicated pass is owned there); a "is this idiomatic React/FastAPI/Go/SQL" question belongs to the stack skill (`../nextjs/SKILL.md`, `../fastapi/SKILL.md`, `../go/SKILL.md`, `../postgresdb/SKILL.md`, `../flutter/SKILL.md`) — pull the idiom there, then judge against it here.
+
+## Read these first
+
+1. `02-DOCS/wiki/sdd/specs/<slug>.md` and `02-DOCS/wiki/sdd/plans/<slug>.md` — what the diff was *supposed* to do. A review with no spec is a review of vibes.
+2. `02-DOCS/wiki/sdd/constitution.md` — the project's non-negotiables (stack canon, quality bars, conventions). Constitution violations are findings even when the code "works".
+3. `02-DOCS/wiki/harness/user-profile.md` — `technical_level`, the register (see "How a review reads" below).
+
+If there is no spec/plan (someone jumped straight to code), say so and review against the constitution + the diff's own stated intent. Don't pretend a spec exists.
+
+## Confidence filtering
+
+A review's value is its signal-to-noise. Every finding you report costs the author triage time, so report only what you can stand behind:
+
+- **>80% sure, or it doesn't ship.** If you're not >80% confident a finding is a real defect, either trace it until you are, or downgrade it to `[question]` and ask. Plausible-looking ≠ verified.
+- **Zero findings is an acceptable verdict.** A clean diff gets `APPROVE`, not a manufactured nit. Padding a report to look thorough is the opposite of thorough.
+- **Common false positives to skip:** patterns guarded two functions up; "unsafe" calls on values that are provably constant/internal; missing checks the framework already enforces; style the linter owns; defects behind a flag that's off everywhere (note as `nit`, not blocker); test-only or generated code held to prod standards.
+- **No severity inflation.** A `should-fix` dressed as a `blocker` burns the same trust as a missed bug. Rank by actual blast radius and reachability — if everything is a blocker, nothing is.
+
+This skill is the discipline; the **`ai-hub-review` bundle is it executed** — `/code-review [pr]` fans a diff out to the per-language reviewer fleet and aggregates one ranked verdict, `/security-scan` merges automated scanners with the `security-reviewer` agent into an exploitability-ranked report. Both enforce the same evidence bar and confidence filtering described here.
+
+---
+
+## GIVING a review
+
+A good review is not a list of everything you'd have done differently. It is a ranked set of **defects that matter**, each one reproducible from the diff itself.
+
+### The pass order
+
+Run these passes in order; each is cheap and catches a different class of defect. Stop padding the report once a pass is clean — a clean pass is a finding too ("auth path: checked ownership scoping, correct").
+
+| Pass | The question | Typical defects |
+| --- | --- | --- |
+| **1. Spec fidelity** | Does the diff do what the spec/plan said — no more, no less? | Missing acceptance criteria; scope creep; a TODO masquerading as done |
+| **2. Correctness** | Is it right on the boundaries, not just the happy path? | Off-by-one, null/empty/zero, error paths swallowed, race, wrong operator |
+| **3. Contracts & data** | Do the interfaces and data shapes hold? | Breaking API change, nullable mismatch, migration that loses data, N+1 |
+| **4. Security boundary** | Any untrusted input reaching a sink? | Injection, authz gap (authenticated ≠ authorized), secret in diff, SSRF |
+| **5. Tests as evidence** | Do the tests actually exercise the change, or just pass? | Asserts on mocks, no failing case, happy-path-only, deleted assertions |
+| **6. Constitution & fit** | Does it honor the project's canon and read like the codebase? | Banned pattern, wrong layer, duplicated logic, dead code left behind |
+
+Passes 1–5 are correctness/safety — those produce **blocking** findings. Pass 6 is fit — usually **non-blocking** unless it violates a stated constitution rule.
+
+### Severity — rank or it's noise
+
+Tag every finding. An unranked review forces the author to triage your opinions; a ranked one tells them exactly what blocks the merge.
+
+- **`blocker`** — ships a bug, a vuln, data loss, or breaks the spec. Merge does not happen until resolved.
+- **`should-fix`** — real defect, narrow blast radius; fix now or file a tracked follow-up the author agrees to.
+- **`nit`** — style/preference with no correctness impact. Explicitly labelled, never blocking, the author may decline freely.
+
+If everything is a blocker, nothing is. If everything is a nit, you didn't review.
+
+### Finding format
+
+Each finding is a quoted location, the defect, the evidence, and a concrete fix. No "consider maybe looking at error handling here."
+
+```text
+[blocker] auth: authenticated user can read any document
+  where:  api/documents.py:42  return db.get(Document, doc_id)
+  why:    doc_id comes from the path; no owner check. Any logged-in
+          user enumerates every document by id.
+  repro:  GET /documents/<other_users_id> with a valid session → 200 + body.
+  fix:    scope the query — WHERE id == doc_id AND owner_id == current_user.id;
+          return 404 (not 403) on miss to avoid leaking existence.
+```
+
+A finding without a `repro` or a mechanism is a *suspicion*. Label it `[question]` and ask, don't assert it as a defect.
+
+### Verify before you flag
+
+The reviewer is held to the same evidence bar as the author, because a reviewer who cries blocker on a non-bug burns the same trust as an author who ships one. Before writing a `blocker`:
+
+- **Read the surrounding code**, not just the diff hunk. The check you think is missing may live two functions up.
+- **Trace the value**, don't pattern-match. "This looks like SQL injection" is not a finding; "this f-string interpolates `request.args['q']` straight into `execute()`" is.
+- **Confirm the path is reachable.** A defect behind a flag that's off in every environment is a `nit`, not a `blocker` — say so.
+
+### The verdict
+
+End every review you give with one of three, and nothing mushy in between:
+
+- **`APPROVE`** — no blockers, no unresolved should-fix. Say it plainly and **point to ship**.
+- **`APPROVE WITH NITS`** — mergeable; the nits are the author's call.
+- **`CHANGES REQUESTED`** — one or more blockers/should-fix. List exactly what unblocks it.
+
+### The sello — when this project opted in
+
+If the sello is on — `.ai-hub/sello-config.json` in the project, or `~/.ai-hub/sello-config.json` for
+every project (`sello on --global`; the project switch always wins) — the review's verdict is
+**sealed to the exact bytes reviewed**, and the ship gate refuses commit/push/PR on anything else.
+
+**What the sello does and does not prove.** It binds bytes, not intent: it guarantees *what ships
+is what was reviewed*, never *the review was good*. You are the one calling `sello approve`, so it
+is self-attested — drift protection between review and delivery, not tamper-evidence. Sealing
+without actually running the lenses produces a valid sello and a worthless one.
+
+Every state transition is deterministic CLI, never tokens:
+
+```text
+1. npx ai-hub sello freeze      → hashes the candidate, prints risk tier + lens count
+2. Run the lenses (below), filter findings, decide
+3a. approved → npx ai-hub sello approve --lenses correctness,security,tests
+3b. blocked  → npx ai-hub sello block --reason "<the blocking finding>"
+```
+
+`approve` refuses to seal with fewer lenses than the tier requires — pass them all, or accept the
+gap deliberately with `--accept-partial-lenses` (it is recorded). Every approval also appends to
+`.ai-hub/sello-log.jsonl`, so what shipped under which verdict survives the next freeze.
+
+**Lenses by risk tier** — tier 0 never reaches you (the gate passes docs/copy silently);
+tier 1 → run the single most relevant pass from the table above yourself; tier 2 → dispatch
+**three parallel fresh-context subagents** (`refuter-correctness` · `refuter-security` ·
+`refuter-tests`), each told
+to *refute* readiness, not confirm it. Fresh context is the point: a reviewer who inherits the
+implementer's context inherits its blind spots.
+
+**Give each refuter exactly four inputs — no more, no less.** The diff alone is not enough, and
+that is not a detail: with only the diff, the one failure class no test suite can structurally
+catch — *the change is correct and is not what the approved spec asked for* — is invisible to the
+whole panel.
+
+1. **The task contract** — the original request **plus every scope change a human explicitly
+   approved since.** Not just the first message: without the approved changes a legitimate scope
+   revision reads as a spec gap and you get a confident false positive.
+2. **The approved spec** (`02-DOCS/wiki/sdd/specs/<slug>.md`).
+3. **The exact source state** — commit SHA, or a tree hash when git is absent. A verdict belongs to
+   the state that was reviewed, not to the project.
+4. **The entry point** — the one command that reruns the checks.
+
+**And say what they do not get:** your conversation, your reasoning, your defences, and your draft
+verdict. If a claim needs your justification to stand, it is not proven.
+
+**Blind first, compare second.** Each refuter records what it attacked and what it found *before*
+being shown your conclusions. Only then may it compare and add findings — the blind record is
+append-only after that, never rewritten. Skip this and the fresh context is spent confirming your
+framing, which is the one thing it was bought to avoid.
+
+**The attack list is the deliverable, not just the findings.** "Nothing found" without saying where
+you looked is indistinguishable from not having looked — the same rule that governs a dismissal.
+
+Two extra finding classes the four inputs unlock, neither reachable from a diff:
+
+- **Contract vs spec** — what would a caller reasonably expect, given the stated deployment, that
+  no clause covers? An exclusion the spec deliberately approved is **not** a finding; an exclusion
+  the spec describes *inaccurately* is.
+- **Mapping, both directions** — an acceptance criterion with no test that can be made to fail, and
+  a test pinning behaviour no criterion asked for.
+
+**Consolidate, then verify — before anything reaches the author.** Three lenses on one diff often
+report one defect three ways, and a lens hunting for something can overstate what it found. Two
+steps, in this order:
+
+1. **Group (deterministic).** Each lens ends its report with a ` ```json findings ` block. Save the
+   three reports and run `npx ai-hub review consolidate <reports…>`. It merges findings on
+   the same file within 3 lines, or quoting the same evidence; keeps the highest severity and every
+   lens that saw it; and downgrades a blocker or should-fix with no `file:line` or failure scenario to
+   a question. Several lenses on one finding is a reason to look first, never a reason to raise its
+   severity.
+2. **Verify (judgement).** For each finding it marks `verify`, dispatch the **`finding-verifier`**
+   agent with that one finding plus the same four inputs, in parallel. It starts from "this is false"
+   and returns **CONFIRMED** (reproduced or traced, with the proof), **REFUTED** (cites the guard that
+   makes it impossible), or **UNPROVEN** (becomes a question). It never gets the lens's reasoning.
+
+Report the consolidated count honestly ("11 reported → 4 unique → 2 confirmed"), and list what was
+refuted and why: a refuted finding hidden from the author is a review nobody can audit. At tier 1
+(one pass, run by you) there is nothing to group, but a blocker still goes to `finding-verifier`
+before it blocks.
+
+**A finding blocks only if it survives all three filters** — no exceptions, and eagerness to
+find something is not evidence:
+
+1. **Causal** — introduced by *this* change. Pre-existing defects → note with
+   `--note "<finding>"` (they land in `.ai-hub/sello-findings.md`, surfaced by `doctor`) and
+   suggest an issue; they never block this delivery.
+2. **Severity** — only `blocker` blocks. `should-fix`/`nit` → `--note`.
+3. **Evidence** — `finding-verifier` returned **CONFIRMED** with its proof. REFUTED is dropped (with
+   its reason in the report); UNPROVEN is a `[question]`, and questions don't block.
+
+**Fixing a blocker is budgeted, one attempt.** Before touching anything: estimate the fix and
+declare it — `npx ai-hub sello budget --lines <N>`. After the fix:
+`sello budget-check` (over budget → justify with `--justify "…"` or shrink; an unexplained
+overrun is how over-engineering enters disguised as a fix). Then `sello freeze` + re-review
+**only the divergence**, and approve. Still broken after one attempt → stop, hand it to the human.
+
+---
+
+## RECEIVING a review
+
+This is the half everyone skips. Feedback arrives and the reflex is to either comply with all of it (looks cooperative, ships bugs) or dismiss the annoying parts (looks confident, ships bugs). Both skip the only step that matters: **checking whether the finding is true.**
+
+### Process every comment through this gate
+
+For each incoming finding, before touching a line of code:
+
+```text
+1. RESTATE   — what is the reviewer actually claiming is wrong? (in one sentence)
+2. VERIFY    — go to the code. Is the claim true? Trace it. Can you reproduce it?
+3. CLASSIFY  — true defect / partially-true / false / unclear
+4. ACT       —
+     true        → fix it, and add the test that would have caught it
+     partial     → fix the real part, reply on the rest with evidence
+     false       → DON'T change the code. Reply with the proof it's already handled
+     unclear     → ask one specific question; don't guess and don't pre-emptively edit
+```
+
+The expensive failure is step 2 skipped. Editing working code because a comment *sounded* right introduces bugs the original author (you) already prevented.
+
+### Pushback is a feature, not rudeness
+
+Disagreeing with a wrong finding — **with evidence** — is doing the job, not being difficult. The reply is not "you're wrong"; it's the trace:
+
+```text
+> Reviewer: this can NPE if `user` is null.
+Verified: `user` is non-null here — it's the return of `require_auth()` on
+line 12, which raises 401 before this line on a null session. Added a test
+(test_get_doc_unauthenticated) that asserts the 401 so this stays true.
+Leaving the code as-is.
+```
+
+That reply *strengthens* the diff (it added a regression test) while declining the change. That's the bar.
+
+### Performative agreement is the failure mode
+
+The tells, and what to do instead:
+
+| Tell | Reality | Do instead |
+| --- | --- | --- |
+| "Good catch!" then editing without checking | You don't yet know if it's a catch | Verify first; *then* "good catch" or "checked — already handled" |
+| Rewriting a whole function to satisfy a nit | Scope creep dressed as responsiveness | Make the minimal change the nit asks for, or decline it as a nit |
+| Silently making the change the bot suggested | Bots flag plausible-looking non-bugs constantly | Trace it like any other finding; reply with the verdict |
+| "You're probably right" with no trace | Probably-right is not verified | There is no probably. Reproduce it or refute it |
+
+### Author's verdict
+
+When you've processed the review, summarize for the reviewer (and the decisions log): findings accepted + fixed (with the commit/test), findings declined + why (with evidence), findings deferred + tracked. Then the diff is ready to re-review or, if clean, to ship.
+
+---
+
+## Anti-patterns → STOP
+
+| Rationalization | Reality |
+| --- | --- |
+| "It passed verify, so it's correct" | Green gates prove it runs and tests pass, not that the logic is right. Review is the logic gate. |
+| "I'll list everything I'd do differently" | A review is ranked defects, not your preferences. Tag nits as nits or drop them. |
+| "This pattern usually means a bug" | Usually isn't a finding. Trace this instance or label it `[question]`. |
+| "The reviewer is senior, they're probably right" | Seniority isn't evidence. Verify the claim against the code like any other. |
+| "Just apply all the comments, it's faster" | Applying a false finding ships a regression. Each comment goes through the gate. |
+| "Pushing back will look defensive" | Pushing back *with a trace* strengthens the diff. Silent compliance hides bugs. |
+| "I'll approve it, the issues are minor" | If they're truly minor, label them nits and approve. If they block, don't approve. No mushy middle. |
+| "No spec, so I'll just eyeball it" | Say there's no spec and review against the constitution + stated intent. Don't fake a baseline. |
+
+## How a review reads
+
+Read `technical_level` from `02-DOCS/wiki/harness/user-profile.md`. It changes the words, never the rigor — every reader gets the same passes and the same evidence bar. No profile → use analogies and proceed; don't stall a review to ask.
+
+In the `orient` voice: the verdict first, then the blocker list — e.g. `CHANGES REQUESTED: 1 blocker (auth, documents.py:42), 1 nit. Fix the auth scope and re-run verify.` Each finding keeps its full format (where/why/repro/fix) and names its defect class (IDOR, N+1, TOCTOU). For a `non-technical` author, translate the impact ("any logged-in person could read everyone else's documents").
+
+## Model tier — `heavy` (opt-in routing)
+
+This phase's default model tier is **`heavy`** — adversarial diff reading is where the strongest model pays off most. Routing is **off** unless `models.enabled: true` in `02-DOCS/wiki/sdd/config.yaml`. When on: resolve this phase's tier (`models.overrides` wins over `models.phases`), map it to a model via `models.tiers`, and apply per `../sdd/references/model-routing.md` — announce the switch in one line when it differs from the session model, and dispatch any `Task`/`parallel` subagents on that model. Routing off or no profile → honor the session model silently. Never fake a switch a tool can't make; skip routing on a one-line change.
+
+## Where this writes
+
+Review is mostly a conversation, but two artifacts persist into the harness wiki so the knowledge model grows:
+
+- **Accepted/declined findings** of consequence → append to `02-DOCS/wiki/sdd/decisions.md` (e.g. "declined NPE finding on documents.py:42 — guarded by require_auth; added regression test"). This is the same append-only log `implement` writes to.
+- If a finding reveals a **missing constitution rule** (a defect class the project keeps hitting), propose adding it to `02-DOCS/wiki/sdd/constitution.md` so the next review catches it earlier.
+
+Index both in `02-DOCS/wiki/index.md` (the Knowledge map; root `CLAUDE.md` keeps only a short pointer) under the `sdd/` topic — the harness owns that map; this skill just keeps its rows honest.
+
+## Result envelope
+
+End with the parseable block every SDD phase shares, so the dispatcher can chain without
+interpreting prose (contract: `../sdd/SKILL.md`):
+
+```json result-envelope
+{
+  "status": "complete|blocked|failed",
+  "executive_summary": "Adversarial review against spec/plan/constitution; verdict and blocking findings.",
+  "artifact": "02-DOCS/wiki/sdd/reviews/<slug>.md",
+  "next_recommended": "ship",
+  "risk": "low|medium|high",
+  "skill_resolution": {
+    "used": ["review"],
+    "missing": [],
+    "fallback": [],
+    "compact_rules": ["Rank or it is noise: blocker / should-fix / nit.", "A finding without a repro is a question, not a defect."]
+  },
+  "evidence": ["verdict stated (APPROVE | APPROVE WITH NITS | CHANGES REQUESTED)", "each blocker carries location + repro + fix"]
+}
+```
+
+## Next in the chain
+
+When the diff carries an **APPROVE** / **APPROVE WITH NITS** verdict and every blocker is resolved, hand off to **ship** — close the branch via PR / merge / cleanup, with **git authorship as Daner Alejandro Salazar Colorado, never Claude**. If the review came back **CHANGES REQUESTED**, the loop goes back to **implement** (fix), then **verify** (re-prove green), then back here for re-review. Don't ship a diff that hasn't earned its verdict.

package/skills/shared/review/evals/README.md

@@ -0,0 +1,84 @@
+# Eval harness — review
+
+These evals run through an **agent harness** (an agent that loads skills on
+demand), not a plain shell script. `cases.yaml` is the fixture; this file is the
+procedure. Two things are measured: **triggering** (does `review` fire when it
+should and stay quiet when it shouldn't, vs its siblings) and **capability**
+(does loading `review` make the answer materially better — sharper findings when
+giving a review, correct verify-before-acting when receiving one).
+
+## Setup
+
+- Use the same agent/model for every trial; vary only which skills are loaded.
+- Triggering trials: load the **full skill catalog** so the agent can route to a
+  sibling, then observe which skill it selects.
+- Capability trials: compare **only `review` loaded** vs **no skill loaded**.
+
+## 1. Triggering
+
+For each item in `should_trigger` and `should_not_trigger`:
+
+1. Start a fresh agent session with the full catalog available.
+2. Feed the `prompt` verbatim as the user message.
+3. Record which skill (if any) the agent invokes.
+4. Run **3–5 trials** per prompt — selection is stochastic.
+
+Pass conditions:
+
+- `should_trigger`: **review** is invoked in the majority of trials. This set
+  deliberately mixes the two roles — *giving* a review ("review this PR", "tear
+  this apart", "critique against the spec") and *receiving* one ("do I have to
+  change it?", "which bot comments should I act on?"). Both must fire `review`.
+- `should_not_trigger`: review is **not** invoked; ideally the agent routes to
+  the listed `route_to` sibling (or declines when appropriate).
+
+**Pass bar: >= 90% trigger accuracy** across all trials. A prompt that flaps
+below majority counts as a fail for that prompt; >= 90% of prompts must pass
+clean.
+
+## 2. Capability
+
+For each `capability` scenario, run two arms:
+
+- **WITH**: only `review` loaded.
+- **WITHOUT**: no skill loaded (baseline model behavior).
+
+Run each arm 3 times. Grade every response against the scenario's `must_include`
+rubric — one point per item that is genuinely present (correct, traced to the
+code, not hand-waved). The two scenarios test opposite halves of the skill: one
+**receiving** a *false* finding (the win is declining the change with evidence
+and adding a regression test, not editing working code), one **giving** a review
+of a real IDOR (the win is a ranked, repro-backed blocker with a concrete fix and
+an explicit verdict).
+
+Pass conditions:
+
+- **WITH** the skill covers **>= 80%** of `must_include` items on average.
+- The skill **measurably improves** the output: WITH coverage must beat WITHOUT
+  by a clear margin (target **>= 25 percentage points**). The receiving scenario
+  is the discriminating one — baselines tend to performatively "fix" the false
+  NPE finding; the skill's verify-before-acting gate should flip that to a
+  correct, evidenced decline.
+
+## Scoring summary
+
+| Dimension | Metric | Pass bar |
+|---|---|---|
+| Triggering | trigger accuracy across all prompts/trials | >= 90% |
+| Capability | rubric coverage WITH skill | >= 80% |
+| Capability | WITH minus WITHOUT (lift) | >= 25 pts |
+
+## Notes / honesty
+
+- These are LLM-graded, stochastic evals — re-run on skill edits and treat small
+  score deltas as noise, not signal.
+- `route_to` targets name the sibling skill the prompt should land on instead of
+  `review`. The adjacent SDD phases this skill hands off to — **verify** before it
+  and **ship** after it — are present as their own skills in this repo, so the two
+  phase-boundary near-misses route straight to them (`verify` for the run-the-gates
+  prompt, `ship` for the open-the-PR-and-merge prompt). Every `route_to` target
+  must resolve to a real sibling skill; a mis-route to the wrong sibling counts
+  against `review`'s triggering accuracy.
+- The receiving-side capability scenario is intentionally a *false* finding. A
+  pass means the skill declined the edit with a trace; "fixed the NPE" is a
+  failure even though it looks responsive.

package/skills/shared/review/evals/cases.yaml

@@ -0,0 +1,55 @@
+skill: review
+
+should_trigger:
+  - prompt: "Can you review this PR before I merge it? It refactors the order-total calculation and adds a discount field."
+    why: "Explicit 'review this PR before merge' — the flagship giving-a-review trigger, on a correctness-sensitive money path. Squarely the review phase."
+  - prompt: "Revisa este diff y dime qué está mal antes de hacer ship."
+    why: "Spanish 'review this diff, tell me what's wrong, before ship' — same trigger in the user's other language, and explicitly the gate that sits before ship."
+  - prompt: "The reviewer left a comment saying my function can NPE if the user is null. Do I actually have to change it?"
+    why: "Non-obvious RECEIVING-side trigger: the user is processing incoming feedback and is unsure whether to comply. The skill's verify-before-acting half owns exactly this."
+  - prompt: "A bot left like twelve comments on my PR. Most look like nitpicks. Which ones should I actually act on?"
+    why: "Receiving a machine review and needing to triage real findings from noise — the gate (restate/verify/classify/act) and the nit-vs-blocker ranking are the core capability."
+  - prompt: "Tear this change apart for me — I want the harshest review you can give before it goes out."
+    why: "Adversarial framing with no literal 'review the PR' phrasing ('tear this apart', 'harshest'). This is the giving-a-review role at full rigor."
+  - prompt: "My teammate said I should rewrite this whole module, but I think the current code already handles the case they're worried about. What do I do?"
+    why: "Receiving a review that may be a false finding; the user wants to push back with evidence rather than blindly comply. The skill's pushback-with-a-trace discipline applies."
+  - prompt: "Critique my implementation against the spec — did I actually build what we specified or did I drift?"
+    why: "Spec-fidelity pass (pass 1) explicitly invoked: checking the diff against .specify specs for scope drift. A giving-a-review trigger keyed to the SDD artifacts."
+
+should_not_trigger:
+  - prompt: "Run lint, type-check and the test suite on my branch and tell me if it's green."
+    route_to: "verify"
+    why: "Running the gates to green is the verify phase, not review — the verify skill owns exactly 'lint/type/test before merge', 'run the checks', 'are we green'. Review judges an already-green diff; this routes out to verify."
+  - prompt: "The code's been reviewed and approved. Open the PR and merge it into main."
+    route_to: "ship"
+    why: "Closing the branch / opening the PR / merge with human-only authorship is the ship phase — the ship skill owns 'open the PR', 'merge into main', 'close the branch'. It is post-review, not review, so it routes out to ship."
+  - prompt: "Do a security threat-model of this endpoint that fetches a user-supplied URL — what's the SSRF risk?"
+    route_to: "secure-coding"
+    why: "A deep OWASP/SSRF threat-model is owned by secure-coding. Review folds its findings in but defers the dedicated security pass; this is explicitly routed out in 'When NOT to use'."
+  - prompt: "Is this the idiomatic way to write a Server Component data fetch in Next.js 15?"
+    route_to: "nextjs"
+    why: "A pure 'is this framework-idiomatic' question with no diff to judge belongs to the stack skill that owns the idiom, not the adversarial review gate."
+  - prompt: "Write me a Postgres query for the top 5 customers by monthly order value."
+    route_to: "postgresdb"
+    why: "Authoring new SQL from scratch — there is no existing change to review. Query design is owned by the postgresdb skill."
+  - prompt: "Rename this Go helper and split the formatting logic out — no behavior change, just tidy it up."
+    route_to: "go"
+    why: "A trivial mechanical refactor the author is performing themselves, not a review of someone's change. Routes to the Go skill for the idiom; nothing to review adversarially."
+
+capability:
+  - scenario: "Receiving review: a reviewer commented 'this can throw NPE if `user` is null' on the line `return user.documents.filter(owner=user.id)`. Two lines up, `user = require_auth(request)`. require_auth raises 401 on a null/absent session. Decide what to do."
+    must_include:
+      - "Runs the receiving gate: restates the claim, then VERIFIES it against the code instead of immediately editing."
+      - "Traces that require_auth raises 401 before the flagged line, so `user` cannot be null there — classifies the finding as FALSE."
+      - "Declines to change the working code (no performative 'good catch' edit)."
+      - "Replies to the reviewer with the evidence/trace (require_auth raises 401), not just an assertion of disagreement."
+      - "Adds or proposes a regression test (e.g. unauthenticated request asserts 401) so the guarantee stays true, strengthening the diff while declining the change."
+      - "Logs the declined finding + reasoning to .planning/history.md."
+  - scenario: "Giving review: review this diff before ship — `@router.get('/invoices/{invoice_id}') def get(invoice_id: int, db=Depends(get_db)): return db.get(Invoice, invoice_id)`, behind login. Spec said 'a user can view their own invoices'."
+    must_include:
+      - "Flags a [blocker]: authenticated != authorized — any logged-in user reads any invoice by id (IDOR / broken access control), and ties it to the spec ('their OWN invoices') as a fidelity failure."
+      - "Provides evidence/repro (GET another user's invoice_id with a valid session returns 200 + body), not a vague 'consider authorizing'."
+      - "Gives a concrete fix: scope the query by owner (WHERE id == invoice_id AND owner_id == current_user.id) and return 404 (not 403) on miss to avoid leaking existence."
+      - "Ranks findings by severity (blocker vs nit) rather than an unranked list of opinions."
+      - "Ends with an explicit verdict — CHANGES REQUESTED — and states exactly what unblocks the merge."
+      - "Points to the next phase (ship) only once the blocker is resolved; routes the deep security angle to secure-coding if a fuller pass is wanted."