medsci-skills 4.1.0

package/skills/self-review/scripts/check_claim_artifact.py

@@ -0,0 +1,248 @@
+#!/usr/bin/env python3
+"""Claim-vs-artifact cross-check for self-review Phase 2.5f (narrowed 1st scope).
+
+The errors that survive a single-pass review are the ones where the manuscript
+text is internally consistent but disagrees with an external artifact — the
+pre-registration, the analysis code, the qc outputs. This gate checks the two
+highest-value, deterministic instances:
+
+  1. ESTIMAND PROVENANCE — does the manuscript's stated primary contrast match
+     the pre-registered / protocol primary, and is there language admitting the
+     primary was re-designated after results were known (outcome-dependent
+     primary selection)?
+  2. E-VALUE — is a reported E-value arithmetically consistent with its adjacent
+     effect estimate, and is it attached to the *primary* estimate rather than a
+     secondary/exploratory one quoted as if it bounded the headline claim?
+
+Figure/flow-count reconciliation, Methods-promised-analysis completeness, and
+imputation-input integrity are separate subchecks (see /make-figures and
+/write-paper); the JSON schema below reserves their `type` values so they can be
+added without a breaking change.
+
+INPUTS
+  --manuscript  manuscript markdown/text.
+  --prereg      pre-registration / protocol / project.yaml text (for estimand
+                provenance). Optional; without it only the post-hoc-reassignment
+                language scan and the E-value check run.
+
+OUTPUT  (--out path)
+  {"claims": [{claim_id, type, prose_value, artifact_source, verdict, detail}],
+   "summary": {...}}
+  Major verdicts: ESTIMAND_DRIFT, PRIMARY_REASSIGNED, EVALUE_ARITHMETIC.
+  Flags (review, not strict-fail): EVALUE_NON_PRIMARY, EVALUE_UNVERIFIABLE.
+
+Stdlib-only (re / json / math / argparse). Exit codes: 0 clean (or report-only),
+1 a Major verdict exists (with --strict), 2 input/usage error.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import re
+import sys
+from pathlib import Path
+
+EVALUE_TOL = 0.15  # relative tolerance for E-value recompute
+
+PRIMARY_RE = re.compile(
+    r"[^.]*\bprimary\b[^.]*\b(outcome|endpoint|analysis|objective|contrast|comparison|model|estimand)[^.]*\.",
+    re.I,
+)
+REASSIGN_RE = re.compile(
+    r"\bprimary\b[^.]*\b(re-?designat|re-?assign|re-?defin|switch|chang)\w*|"
+    r"\b(re-?designat|re-?assign)\w*[^.]*\bprimary\b|"
+    r"\bmanuscript[- ]stage\b[^.]*\b(decision|primary|analy)|"
+    r"\bpost[- ]?hoc\b[^.]*\bprimary\b|"
+    r"\bprimary\b[^.]*\bafter\b[^.]*\bresult",
+    re.I,
+)
+EFFECT_RE = re.compile(r"\b(s?HR|a?HR|a?OR|RR|hazard ratio|odds ratio|risk ratio)\b\D{0,8}(\d+\.\d+)", re.I)
+# The E-value figure follows a connective (was / of / = / :), so the non-greedy
+# scan does not grab the effect estimate's number from an adjacent "(HR 1.34)".
+EVALUE_RE = re.compile(
+    r"E[- ]?value\b[^\n]{0,90}?\b(?:was|were|of|is|=|:|reached|equals?(?:\s+to)?)\s*\(?(\d+\.\d+)",
+    re.I,
+)
+NONPRIMARY_KW = ("secondary", "exploratory", "subgroup", "sensitivity", "supporting",
+                 "cause-specific", "cancer-specific", "post-hoc", "post hoc", "non-primary")
+
+STOP = set("the a an of for in on to and or with by is was were are be been being this that "
+           "between association associated estimated using model analysis primary outcome "
+           "endpoint study patients group as at from".split())
+
+
+def _tokens(s: str) -> set[str]:
+    return {w for w in re.findall(r"[a-z0-9]+", s.lower()) if w not in STOP and len(w) > 2}
+
+
+def _jaccard(a: set[str], b: set[str]) -> float:
+    if not a or not b:
+        return 0.0
+    return len(a & b) / len(a | b)
+
+
+def evalue_point(rr: float) -> float:
+    """VanderWeele-Ding E-value for a point estimate (risk-ratio scale)."""
+    g = rr if rr >= 1 else 1.0 / rr
+    return g + math.sqrt(g * (g - 1.0))
+
+
+def check_estimand(manuscript: str, prereg: str | None) -> list[dict]:
+    claims = []
+    man_primary = [m.group(0).strip() for m in PRIMARY_RE.finditer(manuscript)]
+
+    # 1. Explicit post-hoc reassignment language (highest-confidence catch).
+    m = REASSIGN_RE.search(manuscript)
+    if m:
+        claims.append({
+            "claim_id": "EST-reassign",
+            "type": "estimand",
+            "prose_value": re.sub(r"\s+", " ", manuscript[max(0, m.start() - 40):m.end() + 40]).strip(),
+            "artifact_source": "manuscript (self-admission)",
+            "verdict": "PRIMARY_REASSIGNED",
+            "detail": "Language indicates the primary was re-designated after results were known; "
+                      "report the pre-specified and revised models coequally and disclose the change.",
+        })
+
+    # 2. Manuscript primary vs prereg primary (token overlap).
+    if prereg:
+        pre_primary = [m.group(0).strip() for m in PRIMARY_RE.finditer(prereg)]
+        if man_primary and pre_primary:
+            best = max(
+                ((_jaccard(_tokens(a), _tokens(b)), a, b) for a in man_primary for b in pre_primary),
+                key=lambda t: t[0],
+            )
+            score, a, b = best
+            claims.append({
+                "claim_id": "EST-primary",
+                "type": "estimand",
+                "prose_value": re.sub(r"\s+", " ", a)[:160],
+                "artifact_source": re.sub(r"\s+", " ", b)[:160],
+                "verdict": "ESTIMAND_DRIFT" if score < 0.30 else "OK",
+                "detail": f"manuscript↔prereg primary token overlap = {score:.2f} "
+                          f"(<0.30 → drift candidate; confirm the estimand matches the registration).",
+            })
+        elif man_primary and not pre_primary:
+            claims.append({
+                "claim_id": "EST-primary",
+                "type": "estimand",
+                "prose_value": re.sub(r"\s+", " ", man_primary[0])[:160],
+                "artifact_source": "prereg (no primary statement found)",
+                "verdict": "FLAG_NO_PREREG_PRIMARY",
+                "detail": "No primary-outcome statement located in the prereg/protocol; confirm one exists.",
+            })
+    return claims
+
+
+def check_evalue(manuscript: str) -> list[dict]:
+    claims = []
+    for i, m in enumerate(EVALUE_RE.finditer(manuscript), 1):
+        stated = float(m.group(1))
+        # sentence window around the E-value
+        start = manuscript.rfind(".", 0, m.start()) + 1
+        end = manuscript.find(".", m.end())
+        sent = manuscript[start:(end if end != -1 else len(manuscript))]
+        eff = EFFECT_RE.search(sent)
+        nonprimary = any(kw in sent.lower() for kw in NONPRIMARY_KW)
+
+        if not eff:
+            claims.append({
+                "claim_id": f"EVAL-{i}",
+                "type": "evalue",
+                "prose_value": f"E-value {stated}",
+                "artifact_source": "no adjacent effect estimate",
+                "verdict": "EVALUE_UNVERIFIABLE",
+                "detail": "No HR/OR/RR found in the same sentence; confirm the E-value is computed "
+                          "for the declared primary estimate.",
+            })
+            continue
+
+        rr = float(eff.group(2))
+        recomputed = evalue_point(rr)
+        rel = abs(stated - recomputed) / recomputed if recomputed else 1.0
+        if rel > EVALUE_TOL:
+            verdict = "EVALUE_ARITHMETIC"
+            detail = (f"stated E-value {stated} but {eff.group(1)} {rr} recomputes to "
+                      f"{recomputed:.2f} (rel. diff {rel:.0%} > {EVALUE_TOL:.0%}); the stated value "
+                      "likely belongs to a different (e.g. non-primary) estimate.")
+        elif nonprimary:
+            verdict = "EVALUE_NON_PRIMARY"
+            detail = (f"E-value {stated} matches {eff.group(1)} {rr} (recompute {recomputed:.2f}), "
+                      "but the sentence references a secondary/exploratory estimate; confirm the "
+                      "headline E-value bounds the PRIMARY contrast, not this one.")
+        else:
+            verdict = "OK"
+            detail = f"E-value {stated} consistent with {eff.group(1)} {rr} (recompute {recomputed:.2f})."
+        claims.append({
+            "claim_id": f"EVAL-{i}",
+            "type": "evalue",
+            "prose_value": f"E-value {stated} ({eff.group(1)} {rr})",
+            "artifact_source": "recomputed (VanderWeele-Ding)",
+            "verdict": verdict,
+            "detail": detail,
+        })
+    return claims
+
+
+MAJOR = {"ESTIMAND_DRIFT", "PRIMARY_REASSIGNED", "EVALUE_ARITHMETIC"}
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="Claim-vs-artifact cross-check (estimand + E-value).")
+    ap.add_argument("--manuscript", required=True, help="manuscript markdown/text")
+    ap.add_argument("--prereg", help="pre-registration / protocol / project.yaml text")
+    ap.add_argument("--out", help="write JSON artifact to this path")
+    ap.add_argument("--strict", action="store_true", help="exit 1 if any Major verdict")
+    args = ap.parse_args()
+
+    mp = Path(args.manuscript)
+    if not mp.is_file():
+        sys.stderr.write(f"ERROR: manuscript not found: {args.manuscript}\n")
+        return 2
+    # Collapse hard-wrap newlines to spaces so a sentence-level claim is not split
+    # across lines (the decimal points inside HR/E-value figures must survive).
+    def _unwrap(t: str) -> str:
+        return re.sub(r"\s*\n\s*", " ", t)
+
+    manuscript = _unwrap(mp.read_text(encoding="utf-8"))
+    prereg = None
+    if args.prereg:
+        pp = Path(args.prereg)
+        if pp.is_file():
+            prereg = _unwrap(pp.read_text(encoding="utf-8"))
+        else:
+            sys.stderr.write(f"WARN: prereg not found: {args.prereg} (estimand provenance limited)\n")
+
+    claims = check_estimand(manuscript, prereg) + check_evalue(manuscript)
+    n_major = sum(1 for c in claims if c["verdict"] in MAJOR)
+    n_flag = sum(1 for c in claims if c["verdict"] not in MAJOR and c["verdict"] != "OK")
+
+    result = {
+        "manuscript": str(mp),
+        "prereg": args.prereg,
+        "claims": claims,
+        "summary": {"n_claims": len(claims), "n_major": n_major, "n_flag": n_flag,
+                    "verdict": "MAJOR_CANDIDATE" if n_major else ("REVIEW" if n_flag else "OK")},
+    }
+
+    print("=" * 41)
+    print(" Claim-vs-Artifact Cross-Check (Phase 2.5f)")
+    print("=" * 41)
+    for c in claims:
+        mark = "✗" if c["verdict"] in MAJOR else ("△" if c["verdict"] != "OK" else "✓")
+        print(f"{mark} [{c['claim_id']}] {c['verdict']}")
+        print(f"    {c['detail']}")
+    print(f"\n{n_major} Major candidate(s), {n_flag} flag(s).")
+
+    if args.out:
+        Path(args.out).parent.mkdir(parents=True, exist_ok=True)
+        Path(args.out).write_text(json.dumps(result, indent=2), encoding="utf-8")
+        print(f"wrote {args.out}")
+
+    return 1 if (args.strict and n_major) else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/skills/self-review/scripts/check_classical_style.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+"""Classical-style body lint (self-review §J / write-paper Phase 7.1).
+
+A senior meta-analysis reviewer reads several surface signals as "AI wrote this"
+or as a policy violation. They are all deterministic greps, so they belong in a
+gate rather than a prose checklist (manuscript-style-classical.md §5/§6/§7/§8):
+
+  SECTION_SYMBOL        (Major) the § symbol anywhere in the body — the canonical
+                        AI tell; also catches "see Methods §2" self-references.
+  INBODY_AI_DISCLOSURE  (Major) an AI/LLM-use disclosure paragraph in the body
+                        ("Generative AI was not used", "During the preparation of
+                        this manuscript the authors used …"). For a classical /
+                        senior-MA target this belongs on the title page, not the body.
+  ELIGIBILITY_PROSE     (Minor) eligibility/inclusion criteria written as a prose
+                        sentence rather than a numbered list.
+  DECIMAL_INCONSISTENCY (Minor) OR/HR/RR reported with mixed decimal places (some
+                        2 dp, some 3 dp) in the same manuscript.
+  EM_DASH_OVERUSE       (Minor) more than 25 em-dashes — a generation tell.
+
+INPUTS
+  --manuscript   manuscript markdown/text (required).
+  --em-dash-max  em-dash threshold (default 25).
+
+OUTPUT
+  A reconciliation table (stdout) and, with --out, a JSON artifact:
+    {manuscript, claims[{verdict, severity, detail, where}], summary}
+  Exit 1 (with --strict) when any Major-severity claim exists (§ symbol or in-body
+  AI disclosure).
+
+Stdlib-only (json / re / argparse / pathlib). Exit codes: 0 clean (or report-only),
+1 Major claim(s) found (with --strict), 2 input/usage error.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+INBODY_AI_DISCLOSURE = re.compile(
+    r"generative ai was not used|artificial intelligence disclosure|"
+    r"during the preparation of (?:this|the) (?:manuscript|work|study)[^.]{0,120}?"
+    r"(?:used|use[d]?|employed)[^.]{0,60}?(?:ai|gpt|chatgpt|claude|copilot|gemini|language model)",
+    re.IGNORECASE)
+
+ELIGIBILITY_LEAD = re.compile(
+    r"(?:studies|articles|records|participants|patients|trials)\s+were\s+eligible\s+if|"
+    r"eligibility criteria were|inclusion criteria (?:were|included)|"
+    r"(?:studies|articles) were included if",
+    re.IGNORECASE)
+NUMBERED_MARKER = re.compile(r"\(\s*1\s*\)|\(\s*i\s*\)|(?:^|\s)1\.\s")
+
+EFFECT_DECIMAL = re.compile(
+    r"(?:\b(?:a?OR|a?HR|RR|sHR)\b|\bodds ratio\b|\bhazard ratio\b|\brisk ratio\b)"
+    r"\s*(?:of|was|were|=|:|,)?\s*(\d+\.(\d+))", re.IGNORECASE)
+
+
+def check(text: str, em_dash_max: int) -> list[dict]:
+    claims = []
+
+    # SECTION_SYMBOL (Major)
+    if "§" in text:
+        n = text.count("§")
+        first = text.find("§")
+        claims.append({
+            "verdict": "SECTION_SYMBOL",
+            "severity": "Major",
+            "detail": f"the § symbol appears {n} time(s) — a senior-reviewer AI tell; "
+                      f"replace with the section name (manuscript-style-classical §6)",
+            "where": text[max(0, first - 30):first + 20].replace("\n", " ").strip()[:120],
+        })
+
+    # INBODY_AI_DISCLOSURE (Major)
+    m = INBODY_AI_DISCLOSURE.search(text)
+    if m:
+        claims.append({
+            "verdict": "INBODY_AI_DISCLOSURE",
+            "severity": "Major",
+            "detail": "an AI/LLM-use disclosure paragraph is in the body; for a classical / "
+                      "senior-MA target it belongs on the title page (manuscript-style-classical §7)",
+            "where": m.group(0)[:120],
+        })
+
+    # ELIGIBILITY_PROSE (Minor)
+    em = ELIGIBILITY_LEAD.search(text)
+    if em and not NUMBERED_MARKER.search(text[em.start():em.start() + 320]):
+        claims.append({
+            "verdict": "ELIGIBILITY_PROSE",
+            "severity": "Minor",
+            "detail": "eligibility/inclusion criteria are written as prose; senior reviewers "
+                      "expect a numbered list (1)…(2)…(3) (manuscript-style-classical §5)",
+            "where": em.group(0)[:120],
+        })
+
+    # DECIMAL_INCONSISTENCY (Minor)
+    dps = {len(mm.group(2)) for mm in EFFECT_DECIMAL.finditer(text)}
+    if len(dps) > 1:
+        claims.append({
+            "verdict": "DECIMAL_INCONSISTENCY",
+            "severity": "Minor",
+            "detail": f"OR/HR/RR reported with mixed decimal places ({sorted(dps)}); "
+                      f"standardize (OR/HR to 2 dp)",
+            "where": "effect-size decimals",
+        })
+
+    # EM_DASH_OVERUSE (Minor)
+    n_dash = text.count("—")
+    if n_dash > em_dash_max:
+        claims.append({
+            "verdict": "EM_DASH_OVERUSE",
+            "severity": "Minor",
+            "detail": f"{n_dash} em-dashes (> {em_dash_max}); a generation tell — "
+                      f"replace some with commas/colons or split sentences",
+            "where": f"{n_dash} em-dashes",
+        })
+
+    return claims
+
+
+def analyze(manuscript: str, em_dash_max: int) -> dict:
+    p = Path(manuscript)
+    if not p.is_file():
+        sys.stderr.write(f"ERROR: manuscript not found: {manuscript}\n")
+        sys.exit(2)
+    claims = check(p.read_text(encoding="utf-8"), em_dash_max)
+    n_major = sum(1 for c in claims if c["severity"] == "Major")
+    return {
+        "manuscript": str(p),
+        "claims": claims,
+        "summary": {
+            "n_claims": len(claims),
+            "n_major": n_major,
+            "n_flag": len(claims) - n_major,
+            "verdict": "MAJOR_CANDIDATE" if n_major else ("FLAG" if claims else "OK"),
+        },
+    }
+
+
+def render(result: dict) -> str:
+    lines = ["| Check | Severity | Detail |", "|---|---|---|"]
+    for c in result["claims"]:
+        lines.append(f"| {c['verdict']} | {c['severity']} | {c['detail']} |")
+    if len(lines) == 2:
+        lines.append("| (none) | — | classical-style body conventions satisfied |")
+    return "\n".join(lines)
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="Classical-style body lint (§J).")
+    ap.add_argument("--manuscript", required=True, help="manuscript markdown/text")
+    ap.add_argument("--em-dash-max", type=int, default=25, help="em-dash threshold (default 25)")
+    ap.add_argument("--out", help="write JSON artifact to this path")
+    ap.add_argument("--strict", action="store_true", help="exit 1 if any Major claim exists")
+    ap.add_argument("--quiet", action="store_true", help="suppress stdout table")
+    args = ap.parse_args()
+
+    result = analyze(args.manuscript, args.em_dash_max)
+
+    if not args.quiet:
+        print("=" * 41)
+        print(" Classical-Style Body Lint (§J)")
+        print("=" * 41)
+        print(render(result))
+        print()
+        s = result["summary"]
+        if s["n_major"]:
+            print(f"MAJOR candidate: {s['n_major']} policy/AI-tell violation(s).")
+        elif s["n_flag"]:
+            print(f"FLAG: {s['n_flag']} style inconsistency(ies).")
+        else:
+            print("OK: classical-style body conventions satisfied.")
+
+    if args.out:
+        Path(args.out).parent.mkdir(parents=True, exist_ok=True)
+        Path(args.out).write_text(json.dumps(result, indent=2), encoding="utf-8")
+        if not args.quiet:
+            print(f"\nwrote {args.out}")
+
+    return 1 if (args.strict and result["summary"]["n_major"]) else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())