@dzhechkov/p-replicator 1.5.14 → 1.5.15

package/templates/.claude/skills/goap-research-ed25519/scripts/check_report_evidence.py

@@ -0,0 +1,363 @@
+#!/usr/bin/env python3
+"""
+Report evidence gate (FR-4) — the rule that makes the evidence axis load-bearing.
+
+    ASSERTED facts do not reach a report at all.
+    LISTING_ONLY facts reach it only with an explicit marker next to the claim.
+
+This is an exit code, not advice. The lesson it encodes: a checklist item is
+satisfied by writing it; only an executable check refuses. The whole point of the
+evidence axis is lost if "don't include unread claims" stays a sentence in
+SKILL.md that a tired model skips at 2am.
+
+Usage:
+    python3 check_report_evidence.py --report report.md --facts facts.json [--json]
+
+Exit codes:
+    0  clean — no ASSERTED used, every used LISTING_ONLY carries a marker
+    1  violation — named facts printed
+    2  usage/IO error (a gate that cannot read its inputs must not report "clean")
+
+HONEST SCOPE: this gate proves the report does not LEAN ON unread sources. It
+does not prove the cited sources support the claims, and it cannot judge legacy
+facts that predate the evidence axis — those are counted and NAMED separately,
+never silently folded into "clean".
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+
+EVIDENCE_FETCH_VERIFIED = "FETCH_VERIFIED"
+EVIDENCE_LISTING_ONLY = "LISTING_ONLY"
+EVIDENCE_ASSERTED = "ASSERTED"
+
+# A LISTING_ONLY claim is admissible only if the report says so near the claim.
+# Both spellings are accepted because the report may be written in either
+# language; the marker must be VISIBLE to a reader, not a hidden attribute.
+LISTING_MARKERS = (
+    "LISTING_ONLY",
+    "listing-only",
+    "не открывалась",
+    "карточка не открывалась",
+    "из поисковой выдачи",
+    "not opened directly",
+    "from a search listing",
+)
+
+# How far from the claim's mention the marker may sit. A marker in the appendix
+# does not warn the reader of a sentence on page 2.
+MARKER_WINDOW_CHARS = 400
+
+
+@dataclass
+class Finding:
+    kind: str
+    claim: str
+    source_url: str
+    detail: str
+
+
+def _load_facts(path: str) -> List[Dict[str, Any]]:
+    """Load the ledger. Malformed entries RAISE rather than being skipped: a
+    silently-dropped record is a record the gate did not judge, and this gate must
+    never clear a report it could not fully read (Codex QE #9)."""
+    with open(path, "r", encoding="utf-8") as handle:
+        data = json.load(handle)
+    if isinstance(data, dict):
+        for key in ("facts", "claims", "items"):
+            if isinstance(data.get(key), list):
+                data = data[key]
+                break
+        else:
+            raise ValueError("facts JSON object has no 'facts'/'claims'/'items' array")
+    if not isinstance(data, list):
+        raise ValueError("facts JSON must be a list or an object containing one")
+    bad = [i for i, item in enumerate(data) if not isinstance(item, dict)]
+    if bad:
+        raise ValueError(f"facts ledger has non-object entries at index/indices {bad} — refusing to judge a partial ledger")
+    if not data:
+        # An EMPTY ledger against a non-empty report is the total bypass: pass `[]`
+        # and any medical report exits 0 (Codex QE #2). Emptiness is unevaluable,
+        # not clean.
+        raise ValueError("facts ledger is empty — a report with no recorded facts cannot be cleared, only unevaluated")
+    return data
+
+
+# Tokens that carry a claim's identity. NUMBERS COUNT (Codex QE #4): "LDL rose
+# 40%" vs "LDL increased 40%" shares almost no long words but shares the number,
+# and dosage/threshold claims are precisely where a silent miss is dangerous.
+_TOKEN_RE = re.compile(r"[\wЀ-ӿ]{4,}|\d+(?:[.,]\d+)?", re.UNICODE)
+_STOPWORDS = {
+    "that", "this", "with", "from", "these", "those", "have", "been", "were", "which", "their",
+    "there", "about", "would", "could", "should", "than", "then", "when", "what", "into",
+    "что", "этот", "этого", "было", "были", "который", "которая", "если", "также", "более",
+}
+
+
+def _significant_words(text: str, limit: int = 12) -> List[str]:
+    """Identity tokens of a claim: numerals plus content words, stopwords dropped.
+
+    The first draft required 5+ letters, ignored digits and kept only 8 tokens, so
+    ordinary paraphrases walked through. Detection here is deliberately GENEROUS —
+    an over-detection is a visible, arguable false alarm; a miss lets an unread
+    claim into a medical document silently.
+    """
+    tokens = [t for t in _TOKEN_RE.findall((text or "").lower()) if t not in _STOPWORDS]
+    seen: List[str] = []
+    for token in tokens:
+        if token not in seen:
+            seen.append(token)
+        if len(seen) >= limit:
+            break
+    return seen
+
+
+def claim_positions(report_text: str, claim: str) -> List[int]:
+    """Offsets where the claim appears to be used.
+
+    Exact substring first; otherwise a word-overlap probe, because a report
+    paraphrases claims rather than pasting them. Deliberately generous: a MISSED
+    usage would let an ASSERTED claim through, which is the failure this gate
+    exists to prevent, so we prefer over-detection (a false alarm is visible and
+    arguable; a miss is silent).
+    """
+    text_lower = report_text.lower()
+    claim_lower = (claim or "").strip().lower()
+    if not claim_lower:
+        return []
+    positions = [m.start() for m in re.finditer(re.escape(claim_lower), text_lower)]
+    if positions:
+        return positions
+    words = _significant_words(claim_lower)
+    if len(words) < 3:
+        return []
+    # Scan a SLIDING WINDOW over the whole text, not per physical line: markdown
+    # wrapping split a claim across lines and hid it from the line-based scan.
+    hits: List[int] = []
+    window, step = 360, 120
+    threshold = max(3, int(len(words) * 0.5))
+    position = 0
+    while position < max(len(text_lower), 1):
+        chunk = text_lower[position:position + window]
+        present = [w for w in words if w in chunk]
+        if len(present) >= threshold:
+            # Report where the claim ACTUALLY sits, not where the window started:
+            # marker proximity is measured from this offset, and using the window
+            # start put the measurement up to a full window away from the text it
+            # was supposed to be next to (Codex QE r2).
+            earliest = min(chunk.find(w) for w in present)
+            hits.append(position + max(earliest, 0))
+            position += window  # one hit per window; neighbours would double-count
+        else:
+            position += step
+    return hits
+
+
+def has_marker_near(report_text: str, position: int, window: int = MARKER_WINDOW_CHARS) -> bool:
+    start = max(0, position - window)
+    end = min(len(report_text), position + window)
+    neighbourhood = report_text[start:end].lower()
+    return any(marker.lower() in neighbourhood for marker in LISTING_MARKERS)
+
+
+def verify_ledger_signatures(facts: Sequence[Dict[str, Any]], pins: Optional[Dict[str, str]] = None) -> List[Finding]:
+    """Reject records whose signature does not cover their own contents.
+
+    Without this the gate reads `evidence_class` as a plain unsigned JSON string:
+    flipping a signed ASSERTED record to FETCH_VERIFIED in a text editor made the
+    gate pass while `verify_fact()` would have rejected it (Codex QE #3). The
+    signature protection and the report protection were not composed — each was
+    sound alone and the pair had a hole between them.
+
+    Verification is best-effort by design: if the verifier module or its crypto
+    backend is unavailable, that is reported as a NAMED limitation, never as a
+    pass — a gate that could not check must not imply it did.
+    """
+    try:
+        import ed25519_verifier as ev
+    except Exception as exc:
+        return [Finding(kind="SIGNATURES_UNCHECKED", claim="(whole ledger)", source_url="",
+                        detail=f"cannot import ed25519_verifier ({exc}) — evidence classes were read UNVERIFIED")]
+    if getattr(ev, "CRYPTO_BACKEND", None) is None:
+        return [Finding(kind="SIGNATURES_UNCHECKED", claim="(whole ledger)", source_url="",
+                        detail="no Ed25519 backend installed — evidence classes were read UNVERIFIED")]
+
+    verifier = ev.Ed25519Verifier()
+    for issuer, pubkey in (pins or {}).items():
+        try:
+            verifier.registry.add(issuer, pubkey)
+        except Exception:
+            pass
+    out: List[Finding] = []
+    for fact in facts:
+        if not fact.get("signature"):
+            continue  # unsigned ledgers predate signing; the class fields are still judged
+        trust = fact.get("trust_class")
+        if trust == "ISSUER_SIGNED" and fact.get("issuer") not in (pins or {}):
+            # REGRESSION GUARD (Codex QE r2, critical): the first version built an
+            # EMPTY registry, so every legitimate ISSUER_SIGNED fact failed as
+            # "unknown issuer" and was reported as TAMPERED. Accusing a sound record
+            # of forgery is worse than not checking it: it teaches the reader to
+            # ignore the gate. Without pins we say what we could not check.
+            out.append(Finding(kind="SIGNATURES_UNCHECKED", claim=str(fact.get("claim", ""))[:120],
+                               source_url=str(fact.get("source_url", "")),
+                               detail=("ISSUER_SIGNED fact but no pinned key was supplied (--pins) — "
+                                       "its signature was NOT checked; this is not an accusation")))
+            continue
+        try:
+            result = verifier.verify_fact(ev.SignedFact.from_dict(fact))
+        except Exception as exc:
+            out.append(Finding(kind="UNVERIFIABLE_FACT", claim=str(fact.get("claim", ""))[:120],
+                               source_url=str(fact.get("source_url", "")), detail=f"verification raised {exc}"))
+            continue
+        if not result.verified:
+            out.append(Finding(kind="TAMPERED_FACT", claim=str(fact.get("claim", ""))[:120],
+                               source_url=str(fact.get("source_url", "")),
+                               detail=f"signature does not verify ({result.error}) — its evidence_class cannot be trusted"))
+    return out
+
+
+def evaluate(report_text: str, facts: Sequence[Dict[str, Any]]) -> Tuple[List[Finding], Dict[str, int]]:
+    findings: List[Finding] = []
+    counts = {EVIDENCE_FETCH_VERIFIED: 0, EVIDENCE_LISTING_ONLY: 0, EVIDENCE_ASSERTED: 0, "UNKNOWN_LEGACY": 0}
+
+    for fact in facts:
+        claim = str(fact.get("claim", ""))
+        url = str(fact.get("source_url", ""))
+        evidence = fact.get("evidence_class")
+        bucket = evidence if evidence in counts else ("UNKNOWN_LEGACY" if evidence is None else "UNRECOGNISED")
+        counts[bucket] = counts.get(bucket, 0) + 1
+
+        positions = claim_positions(report_text, claim)
+        if not positions:
+            continue  # recorded but not used in this report — not this gate's business
+
+        if evidence == EVIDENCE_ASSERTED:
+            findings.append(
+                Finding(
+                    kind="ASSERTED_IN_REPORT",
+                    claim=claim,
+                    source_url=url,
+                    detail="stated from memory, source never opened — must not appear in a report at all",
+                )
+            )
+        elif evidence == EVIDENCE_LISTING_ONLY:
+            # EVERY occurrence must be marked, not just one (Codex QE #5): a claim
+            # marked on page 1 and repeated bare on page 4 warns nobody on page 4.
+            unmarked = [pos for pos in positions if not has_marker_near(report_text, pos)]
+            if unmarked:
+                findings.append(
+                    Finding(
+                        kind="UNMARKED_LISTING_ONLY",
+                        claim=claim,
+                        source_url=url,
+                        detail=(
+                            f"{len(unmarked)} of {len(positions)} occurrence(s) unmarked; the source was never "
+                            f"opened directly and the report must say so next to EACH mention "
+                            f"(within {MARKER_WINDOW_CHARS} chars)"
+                        ),
+                    )
+                )
+        elif evidence not in (EVIDENCE_FETCH_VERIFIED, None):
+            # An unrecognised or whitespace-damaged class ("ASSERTED ", "asserted",
+            # null) must NOT read as legacy-and-therefore-fine (Codex QE #9). A
+            # ledger we cannot interpret is a ledger we cannot clear.
+            findings.append(
+                Finding(
+                    kind="UNRECOGNISED_EVIDENCE_CLASS",
+                    claim=claim,
+                    source_url=url,
+                    detail=f"evidence_class {evidence!r} is not one of FETCH_VERIFIED/LISTING_ONLY/ASSERTED",
+                )
+            )
+    return findings, counts
+
+
+def render(findings: Sequence[Finding], counts: Dict[str, int]) -> str:
+    lines: List[str] = []
+    total = sum(counts.values())
+    lines.append("report evidence gate")
+    lines.append(
+        "  facts: {total} — FETCH_VERIFIED {f} / LISTING_ONLY {l} / ASSERTED {a} / legacy-unknown {u}".format(
+            total=total,
+            f=counts[EVIDENCE_FETCH_VERIFIED],
+            l=counts[EVIDENCE_LISTING_ONLY],
+            a=counts[EVIDENCE_ASSERTED],
+            u=counts["UNKNOWN_LEGACY"],
+        )
+    )
+    if counts["UNKNOWN_LEGACY"]:
+        lines.append(
+            "  NOTE: {n} fact(s) predate the evidence axis — this gate cannot judge them, "
+            "and does not pretend to.".format(n=counts["UNKNOWN_LEGACY"])
+        )
+    if not findings:
+        lines.append("  PASS — no ASSERTED claim used; every used LISTING_ONLY claim is marked.")
+    else:
+        lines.append("  FAIL — {n} violation(s):".format(n=len(findings)))
+        for finding in findings:
+            lines.append("    [{kind}] {claim}".format(kind=finding.kind, claim=finding.claim[:120]))
+            lines.append("       source: {url}".format(url=finding.source_url or "(none)"))
+            lines.append("       {detail}".format(detail=finding.detail))
+    lines.append(
+        "  scope: proves the report does not lean on unread sources AND that the ledger's evidence "
+        "classes carry intact signatures. It does NOT prove the cited sources support the claims, "
+        "and it can only judge claims it recognises as used — paraphrase detection is deliberately "
+        "over-eager, because a missed unread claim is silent while a false alarm is arguable."
+    )
+    return "\n".join(lines)
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    parser = argparse.ArgumentParser(description="Refuse reports that lean on unread sources.")
+    parser.add_argument("--report", required=True, help="path to the report markdown")
+    parser.add_argument("--facts", required=True, help="path to facts.json")
+    parser.add_argument("--json", action="store_true", help="emit machine-readable JSON")
+    parser.add_argument("--pins", help="JSON file of {issuer: pubkey_b64} so ISSUER_SIGNED facts can be checked")
+    args = parser.parse_args(argv)
+
+    try:
+        with open(args.report, "r", encoding="utf-8") as handle:
+            report_text = handle.read()
+        facts = _load_facts(args.facts)
+        pins = None
+        if args.pins:
+            with open(args.pins, "r", encoding="utf-8") as handle:
+                pins = json.load(handle)
+            if not isinstance(pins, dict):
+                raise ValueError("--pins must be a JSON object of {issuer: pubkey_b64}")
+    except (OSError, ValueError, json.JSONDecodeError) as exc:
+        # Exit 2, never 0: a gate that could not read its inputs has not cleared anything.
+        message = f"check_report_evidence: cannot evaluate — {exc}"
+        print(json.dumps({"error": str(exc), "exitCode": 2}) if args.json else message, file=sys.stderr)
+        return 2
+
+    findings, counts = evaluate(report_text, facts)
+    # Signature check FIRST in severity terms: a tampered record's evidence_class
+    # is not evidence of anything, so its verdict above cannot be trusted either.
+    findings = verify_ledger_signatures(facts, pins) + findings
+    if args.json:
+        print(
+            json.dumps(
+                {
+                    "ok": not findings,
+                    "counts": counts,
+                    "findings": [f.__dict__ for f in findings],
+                    "exitCode": 1 if findings else 0,
+                },
+                ensure_ascii=False,
+            )
+        )
+    else:
+        print(render(findings, counts))
+    return 1 if findings else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())