medsci-skills 5.11.0 → 5.13.0

package/metadata/distribution_files.json

@@ -153,8 +153,18 @@
     },
     {
       "path": "skills/analyze-stats/SKILL.md",
-      "size": 55302,
-      "sha256": "3809b6a55e8a79b7dc48f617e3be0a7faf2a96a6f0ba03d58977a39f267cce81"
+      "size": 56422,
+      "sha256": "ddb73598fe60601942e8b44a87771e3ba583a585304effac4ae206d53ab4f154"
+    },
+    {
+      "path": "skills/analyze-stats/references/analysis_guides/agreement_reliability.md",
+      "size": 6674,
+      "sha256": "599ad2551f547043b2bec79e3139d9c2c3ae33c5025784359b22022f882504c2"
+    },
+    {
+      "path": "skills/analyze-stats/references/analysis_guides/diagnostic_accuracy.md",
+      "size": 9303,
+      "sha256": "e788d612aa4227500e997ec405ab15a300d28c3b6fb2a1e1df33823764cb9537"
     },
     {
       "path": "skills/analyze-stats/references/analysis_guides/health_economic_evaluation.md",
@@ -216,6 +226,11 @@
       "size": 12803,
       "sha256": "1639f4810fb5946b24e782b3ce463c000222dcac75709e1e3dac78ad4a2ad72f"
     },
+    {
+      "path": "skills/analyze-stats/references/analysis_guides/survival.md",
+      "size": 7191,
+      "sha256": "6b7f220f2aa165fd16eda58b0b40ac743c122c03757bc00f61efc884b203f337"
+    },
     {
       "path": "skills/analyze-stats/references/analysis_guides/test_selection.md",
       "size": 3439,
@@ -2923,8 +2938,8 @@
     },
     {
       "path": "skills/peer-review/SKILL.md",
-      "size": 68302,
-      "sha256": "bf56fbc16bb3e43fb43f09982d36e68f53f8b0f666564e2828e692472ad0660f"
+      "size": 69384,
+      "sha256": "94033b84b41304d7dbd7919c84c7e9502bdaff1b7e2ef8f5dd7ec9cd7147c532"
     },
     {
       "path": "skills/peer-review/references/aczel_2021_reviewer2_patterns.md",
@@ -2948,8 +2963,8 @@
     },
     {
       "path": "skills/peer-review/references/domain-probes/diagnostic_accuracy.md",
-      "size": 11692,
-      "sha256": "afc69e56578babcc1cf7c7952d2df301fdc829329fb480a5cf426ff6ab65d9f5"
+      "size": 11967,
+      "sha256": "8c0a8bd30fe16507e2997ddba1bbcfb4ee7eade7df44f1981e1427cd3a28d233"
     },
     {
       "path": "skills/peer-review/references/domain-probes/equity_fairness.md",
@@ -2993,8 +3008,8 @@
     },
     {
       "path": "skills/peer-review/references/domain-probes/observational_confounding.md",
-      "size": 34617,
-      "sha256": "ec4a11caed6d604e510437827f2b979fcf43ab1ddea6b298eb95673472b2b459"
+      "size": 34795,
+      "sha256": "ff8cb910c7fb83ec08a04ba88866bdf1e0d2f43d972e2861b825595a85b035ee"
     },
     {
       "path": "skills/peer-review/references/domain-probes/polygenic_risk_score.md",
@@ -3038,8 +3053,8 @@
     },
     {
       "path": "skills/peer-review/references/domain-probes/survival_prognostic.md",
-      "size": 13765,
-      "sha256": "af0e0323876424eec8ba07be22fe99b3100356e8e4895b5b79635e9a2d76e98e"
+      "size": 14022,
+      "sha256": "96195cd1b575535bc12a133125b9d85e2ca7e756b96a0649e03d4c3696e5312b"
     },
     {
       "path": "skills/peer-review/references/exemplar_reviews/README.md",
@@ -3488,8 +3503,8 @@
     },
     {
       "path": "skills/self-review/SKILL.md",
-      "size": 104028,
-      "sha256": "22bc51c9adcb1fc9ff9120c2650faaa2f5f3a0482fe1fd781e231e2191e21081"
+      "size": 104731,
+      "sha256": "9c74f317d10c7b491e710debb619dcf536689092a59fe749050a0ad41f4aacd5"
     },
     {
       "path": "skills/self-review/references/domain-probes/ai_overclaiming.md",
@@ -3508,8 +3523,8 @@
     },
     {
       "path": "skills/self-review/references/domain-probes/diagnostic_accuracy.md",
-      "size": 11692,
-      "sha256": "afc69e56578babcc1cf7c7952d2df301fdc829329fb480a5cf426ff6ab65d9f5"
+      "size": 11967,
+      "sha256": "8c0a8bd30fe16507e2997ddba1bbcfb4ee7eade7df44f1981e1427cd3a28d233"
     },
     {
       "path": "skills/self-review/references/domain-probes/equity_fairness.md",
@@ -3553,8 +3568,8 @@
     },
     {
       "path": "skills/self-review/references/domain-probes/observational_confounding.md",
-      "size": 34617,
-      "sha256": "ec4a11caed6d604e510437827f2b979fcf43ab1ddea6b298eb95673472b2b459"
+      "size": 34795,
+      "sha256": "ff8cb910c7fb83ec08a04ba88866bdf1e0d2f43d972e2861b825595a85b035ee"
     },
     {
       "path": "skills/self-review/references/domain-probes/polygenic_risk_score.md",
@@ -3598,8 +3613,8 @@
     },
     {
       "path": "skills/self-review/references/domain-probes/survival_prognostic.md",
-      "size": 13765,
-      "sha256": "af0e0323876424eec8ba07be22fe99b3100356e8e4895b5b79635e9a2d76e98e"
+      "size": 14022,
+      "sha256": "96195cd1b575535bc12a133125b9d85e2ca7e756b96a0649e03d4c3696e5312b"
     },
     {
       "path": "skills/self-review/references/exemplar_findings/README.md",
@@ -3668,8 +3683,8 @@
     },
     {
       "path": "skills/self-review/scripts/check_claim_artifact.py",
-      "size": 15879,
-      "sha256": "1daa64eaabe493f05ae7e2129a2056b673ec588487d5b6c73f41f10104822935"
+      "size": 18475,
+      "sha256": "c99c8090205734b0eae981550ed09e6275d94add608e66df78f4533c35f20924"
     },
     {
       "path": "skills/self-review/scripts/check_classical_style.py",
@@ -3686,6 +3701,11 @@
       "size": 21506,
       "sha256": "7d3e67074d58a28ffee52ce64b486231f103a3ddcaf6b3b6ee83ba5f89c63bc2"
     },
+    {
+      "path": "skills/self-review/scripts/check_cv_leakage.py",
+      "size": 6349,
+      "sha256": "431c5f14ad2c4f59e85b2a5e86150ac708d476e8f2f1da859d81953f522b2117"
+    },
     {
       "path": "skills/self-review/scripts/check_editorial_impression.py",
       "size": 22032,
@@ -3743,8 +3763,8 @@
     },
     {
       "path": "skills/self-review/skill.yml",
-      "size": 2345,
-      "sha256": "91883d75f33ee873a6ebfbd4d5f2cf0c292d3b90fe4705d6a35e82a26ecf6523"
+      "size": 2381,
+      "sha256": "9499a87d985d981eac9418d6cd95ed79c1d9724aba8854b7d6713d89679d9f1d"
     },
     {
       "path": "skills/setup-medsci/SKILL.md",
@@ -3898,8 +3918,8 @@
     },
     {
       "path": "skills/write-paper/SKILL.md",
-      "size": 66426,
-      "sha256": "910808b73103a74c26a3a92acb9e7b99529de1b8d9af8efaf740e7ad737fdd09"
+      "size": 66771,
+      "sha256": "1a50c9ceb040a79feaa60e809c7196f9f04b85cab2f5f3402dcaed3f7aee552c"
     },
     {
       "path": "skills/write-paper/references/exemplar_abstract.md",
@@ -3918,8 +3938,8 @@
     },
     {
       "path": "skills/write-paper/references/exemplar_discussion/README.md",
-      "size": 2524,
-      "sha256": "46d1b9e874f669608cc2c2abc400ab5efa74e37958f59e0c3b8f842dd4934e7d"
+      "size": 2727,
+      "sha256": "3bfcd4eb4df4ca0309446a606550cb11adc4d88daa676f2217249691d2e6b042"
     },
     {
       "path": "skills/write-paper/references/exemplar_discussion/ai_validation_tripod_claim.md",
@@ -3931,6 +3951,11 @@
       "size": 2500,
       "sha256": "7d49d926eb9aa78c42ce0e6579ca299f528dec77dafc17ed9451ada0c2340b3c"
     },
+    {
+      "path": "skills/write-paper/references/exemplar_discussion/meta_analysis_prisma.md",
+      "size": 3357,
+      "sha256": "3cc8542c754372b19acbc25b3c6d6211479b4bbec361c7638c5d2550db3ecc5f"
+    },
     {
       "path": "skills/write-paper/references/exemplar_discussion/observational_cohort_strobe.md",
       "size": 3652,
@@ -3943,8 +3968,8 @@
     },
     {
       "path": "skills/write-paper/references/exemplar_methods/README.md",
-      "size": 2247,
-      "sha256": "158c4894b92c1b6c19bda8becd947e455c5050cbef4d7d3606c2c46c3123d006"
+      "size": 2342,
+      "sha256": "6966e8b723d0c4ccc012d54e2b8a22ef59b39c22619e63a8597f9badb380b4e7"
     },
     {
       "path": "skills/write-paper/references/exemplar_methods/ai_validation_tripod_claim.md",
@@ -3956,6 +3981,11 @@
       "size": 2739,
       "sha256": "f403613a0f3625c5c7dab8fa16a21635007821ede2b58d265440db51b55165a6"
     },
+    {
+      "path": "skills/write-paper/references/exemplar_methods/meta_analysis_prisma.md",
+      "size": 4160,
+      "sha256": "a0e696e2f581fa6ae8b1aeb86ff7b9a5aaa6bdb2184fee4352e74d2ee6559d90"
+    },
     {
       "path": "skills/write-paper/references/exemplar_methods/observational_cohort_strobe.md",
       "size": 2364,
@@ -3963,8 +3993,8 @@
     },
     {
       "path": "skills/write-paper/references/exemplar_results/README.md",
-      "size": 2588,
-      "sha256": "0ececbb937fa44a2d89d56124bc7d5d2fea540a459fd0328b0cf534c463baaa1"
+      "size": 2799,
+      "sha256": "14b588f1f4433be71c39f11627148fff797eb87b3b74be75e61514095d4c38ea"
     },
     {
       "path": "skills/write-paper/references/exemplar_results/ai_validation_tripod_claim.md",
@@ -3976,6 +4006,11 @@
       "size": 2772,
       "sha256": "8e45e0b30b4ab505b4eb8c37007f970bfb5aa81375fd011915b07d85c9419598"
     },
+    {
+      "path": "skills/write-paper/references/exemplar_results/meta_analysis_prisma.md",
+      "size": 3170,
+      "sha256": "8aed3ee5d36c15515843d77b5f69fe76f433c9972827a9e18303b38802fac782"
+    },
     {
       "path": "skills/write-paper/references/exemplar_results/observational_cohort_strobe.md",
       "size": 2452,
@@ -4288,8 +4323,8 @@
     },
     {
       "path": "skills/write-paper/references/paper_types/meta_analysis.md",
-      "size": 10163,
-      "sha256": "5384224227f54deb1829a400ec6994b45a1b7cb34c167c158804e9d361466b26"
+      "size": 10536,
+      "sha256": "2c237409005214baab63627966d8f2b31498e3c28bea34c571a6d5591a068ed8"
     },
     {
       "path": "skills/write-paper/references/paper_types/nhis_cohort.md",

package/metadata/distribution_manifest.json

@@ -1,6 +1,6 @@
 {
   "schema_version": 1,
-  "version": "5.11.0",
+  "version": "5.13.0",
   "owned_skills": [
     "academic-aio",
     "add-journal",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "medsci-skills",
-  "version": "5.11.0",
+  "version": "5.13.0",
   "description": "MedSci Skills — a medical/scientific research skill suite for AI coding agents (Claude Code, Codex, Cursor, Copilot). The npm package is a terminal-friendly installer shortcut; the canonical distribution remains the GitHub repository and the Claude Code plugin marketplace.",
   "license": "SEE LICENSE IN LICENSE",
   "homepage": "https://github.com/Aperivue/medsci-skills#readme",

package/skills/analyze-stats/SKILL.md

@@ -393,6 +393,7 @@ tbl %>% as_flex_table() %>% flextable::save_as_docx(path = "table.docx")
 
 ### Diagnostic Accuracy
 
+- **Methodology guide**: `references/analysis_guides/diagnostic_accuracy.md` (**load before generating code** — every metric with a CI on a stated analysis unit; the confidence-weighted trap [unweighted-baseline AUC + monotonic-encoding check, produce-side of probe D9]; paired DeLong vs MRMC for reader-generalising claims; per-stratum admissibility [D10]; one-scale-per-comparison [D11])
 - Template: `references/templates/diagnostic_accuracy.py`
 - Always report: sensitivity, specificity, PPV, NPV, accuracy, AUC
 - CIs: Wilson score for proportions, DeLong for AUC
@@ -411,6 +412,7 @@ tbl %>% as_flex_table() %>% flextable::save_as_docx(path = "table.docx")
 
 ### Inter-rater Agreement
 
+- **Methodology guide**: `references/analysis_guides/agreement_reliability.md` (**load before generating code** — the pseudoreplication trap for clustered/repeated measurements + the pseudoreplication-safe per-subject / mixed-effects code, ICC model/type selection, agreement-vs-reliability distinction; pairs with self-review probe O18)
 - Table type guide: `references/table-standards/table-types/agreement.md` (ICC with model/type + CI, weighted κ for ordinal, Bland–Altman bias + LoA, reliability-vs-agreement distinction, common errors)
 - Template: `references/templates/agreement_analysis.py`
 - 2 raters + categorical: Cohen's kappa
@@ -486,6 +488,7 @@ tbl %>% as_flex_table() %>% flextable::save_as_docx(path = "table.docx")
 
 ### Survival Analysis
 
+- **Methodology guide**: `references/analysis_guides/survival.md` (**load before generating code** — competing risks first [naive 1−KM overestimates → produce the Aalen–Johansen/Fine–Gray CIF; cause-specific vs subdistribution for which question, produce-side of probe S3]; PH check → RMST when violated; reverse-KM follow-up + C-index variant [S6]; estimand provenance [S8])
 - Table type guide: `references/table-standards/table-types/survival_results.md` (Cox results table: events/person-time, reverse-KM median follow-up, univariable + adjusted HR with CI, PH-assumption footnote, EPV/sparse-stratum and RMST-when-PH-violated rules)
 - Kaplan-Meier curves with number-at-risk table
 - Log-rank test for group comparison

package/skills/analyze-stats/references/analysis_guides/agreement_reliability.md

@@ -0,0 +1,130 @@
+# Inter-rater Agreement & Reliability Guide
+
+Quantifying how well two or more raters (or a rater and a reference, or repeated
+measurements) **agree**. The coefficient is easy to compute; the two ways these
+analyses fail review are (1) treating **clustered** measurements as independent
+(pseudoreplication) and (2) confusing **agreement** with **reliability**.
+
+---
+
+## When to Use
+
+- **Cohen's kappa** — 2 raters, categorical (nominal) labels.
+- **Weighted kappa** — 2 raters, **ordinal** labels (linear or quadratic weights; disagreement
+  by one category counts less than by three).
+- **Fleiss' kappa** — ≥3 raters, categorical.
+- **Krippendorff's alpha** — any number of raters, any measurement level, tolerates missing data.
+- **ICC (intraclass correlation)** — **continuous** measurements; report the model + type (below).
+- **Bland–Altman** — two continuous methods/raters: bias (mean difference) + 95% limits of agreement.
+- NOT for: a single 2×2 vs a reference standard (that is diagnostic accuracy — see
+  `table-types/diagnostic_accuracy.md`); not for a multi-reader AI-vs-human comparison with reader +
+  case variance (that is an MRMC reader study — see `table-types/reader_study.md`).
+
+---
+
+## Pseudoreplication comes first (this, not the coefficient, is the issue)
+
+If each **subject contributes more than one measurement** — several lesions, aneurysms, nodules,
+slices, or time-points per patient — the rows are **not independent**. Computing agreement on the
+**pooled** rows (or on all pairwise distances) uses an inflated *n*, narrows the CI, and gives an
+**anti-conservative** p-value. This is the single most common reliability-study error a reviewer
+catches (it is flagged by the self-review probe **O18** in `observational_confounding.md`).
+
+Two correct paths — pick one and state it:
+
+1. **Aggregate to the independent unit first**, then compute agreement per subject. This is the
+   simplest defensible analysis when a per-subject summary is meaningful (e.g. mean measurement,
+   majority label, or one index lesion per subject).
+2. **Model the clustering** — a mixed-effects / variance-components ICC with a **subject random
+   effect** (or a GEE with an exchangeable working correlation), so the within-subject correlation
+   is estimated rather than ignored.
+
+A pooled-pairwise test can *flip* on correction: e.g. Mann–Whitney p = 0.02 on 448 pooled
+pairwise distances became p = 0.59 at the per-aneurysm level (n = 112). **Report the unit of
+analysis explicitly**, and when subjects have multiple measurements report a per-subject
+sensitivity analysis.
+
+### Produce the pseudoreplication-safe version
+
+```python
+import pandas as pd
+import pingouin as pg   # ICC with model/type + CI
+
+# long format: one row per (subject, measurement); rater columns rater1..raterK
+df = pd.read_csv("ratings.csv")
+
+# 1) DETECT clustering: more rows than independent subjects
+n_rows, n_subjects = len(df), df["subject_id"].nunique()
+if n_rows > n_subjects:
+    print(f"CLUSTERED: {n_rows} measurements from {n_subjects} subjects "
+          f"({n_rows / n_subjects:.1f} per subject) — do NOT pool as independent.")
+
+# 2a) PER-SUBJECT AGGREGATION (continuous): mean per subject, then ICC on subject means
+per_subj = df.groupby("subject_id")[["rater1", "rater2"]].mean().reset_index()
+long = per_subj.melt(id_vars="subject_id", var_name="rater", value_name="score")
+icc = pg.intraclass_corr(data=long, targets="subject_id", raters="rater", nan_policy="omit")
+print(icc[["Type", "ICC", "CI95%"]])   # report Type (e.g. ICC2/ICC2k) + CI
+
+# 2b) OR MODEL THE CLUSTERING (keep every measurement, subject random effect)
+import statsmodels.formula.api as smf
+df_long = df.melt(id_vars="subject_id", value_vars=["rater1", "rater2"],
+                  var_name="rater", value_name="score")
+m = smf.mixedlm("score ~ 1", data=df_long, groups=df_long["subject_id"])
+res = m.fit()
+var_between = float(res.cov_re.iloc[0, 0]); var_resid = float(res.scale)
+icc_clustered = var_between / (var_between + var_resid)
+print(f"variance-components ICC (subject random effect) = {icc_clustered:.3f}")
+```
+
+---
+
+## ICC: state the model and the type (they are not interchangeable)
+
+- **Model**: one-way random (raters differ per subject), two-way random (same raters, generalise to
+  a rater population), two-way mixed (same raters, these raters only).
+- **Type**: **agreement** vs **consistency** (agreement penalises systematic rater bias; consistency
+  does not), and **single** vs **average** measurement (average-of-k is higher — only report it if
+  the clinical use averages k raters).
+- Report as e.g. **ICC(2,1) = 0.82 (95% CI 0.74–0.88), two-way random, absolute agreement, single
+  rater**. An ICC with no model/type is not interpretable.
+
+---
+
+## Agreement is not reliability
+
+- **Agreement** = do raters give the *same value* (absolute; Bland–Altman bias, absolute-agreement ICC).
+- **Reliability** = can raters *rank/discriminate subjects* consistently (relative; consistency ICC,
+  Pearson/Spearman). A method can be highly reliable yet have poor agreement (a constant offset).
+  State which one the clinical claim needs, and use the matching coefficient.
+
+---
+
+## Reporting
+
+- The coefficient **with a 95% CI** (bootstrap or analytic), the model/type (for ICC), and the
+  **unit of analysis** (per-subject vs per-lesion, and the clustering handling).
+- The interpretation band used (e.g. Landis–Koch), but do not over-interpret a point estimate whose
+  CI spans two bands.
+- For continuous methods: Bland–Altman **bias + 95% limits of agreement**, not just a correlation.
+
+---
+
+## Common failures (flag at review)
+
+- **Pooled/pairwise agreement on clustered data** (pseudoreplication) — the headline coefficient's
+  CI is too narrow; re-run per-subject or with a subject random effect (probe O18).
+- **ICC reported with no model/type** — uninterpretable; the same data yields different ICCs.
+- **Reliability coefficient used to claim agreement** (or vice versa) — a high consistency ICC does
+  not establish that the two methods are interchangeable.
+- **Correlation (r) reported as agreement** for two methods — r ignores a constant/proportional bias;
+  Bland–Altman is required.
+- **Kappa on ordinal labels unweighted** — treats a one-category disagreement as a full disagreement.
+
+---
+
+## Anti-Hallucination
+
+- Never hand-type a coefficient or CI — compute it from the ratings CSV with a seeded script.
+- Do not quote an ICC without the model/type actually estimated by the code.
+- If subjects have multiple measurements, the per-subject sensitivity analysis is **mandatory** —
+  do not report only the pooled number.

package/skills/analyze-stats/references/analysis_guides/diagnostic_accuracy.md

@@ -0,0 +1,184 @@
+# Diagnostic Accuracy & Reader-Study Guide
+
+Estimating how well an index test (or an AI model / reader) separates disease from
+no-disease against a reference standard. The point estimates are easy to compute; the
+ways these analyses fail review are (1) reporting AUC / sensitivity / specificity **without
+CIs or on the wrong analysis unit**, (2) a **confidence-weighted / rating** score whose
+novelty is never tested against the simpler **unweighted** baseline, and (3) comparing two
+AUCs with a **fixed-reader** test when the claim is meant to **generalise to readers**.
+
+---
+
+## When to Use
+
+- **Single index test** vs a reference standard → sensitivity, specificity, PPV, NPV, accuracy
+  (each with a 95% CI), and AUC (with a DeLong or bootstrap CI).
+- **Two tests / models on the same cases** → a **paired** AUC comparison (DeLong `roc.test`,
+  `paired = TRUE`) — never two independent CIs eyeballed for overlap.
+- **Multi-reader multi-case (MRMC)** reader study (AI-vs-reader, modality comparison) → an
+  MRMC method (Obuchowski–Rockette / DBM) that carries **reader + case** variance; see
+  `table-standards/table-types/reader_study.md` and `make-figures` `exemplar_plots/mrmc_roc.md`.
+- NOT for: pooling accuracy across studies (that is a DTA meta-analysis — bivariate / HSROC via
+  `mada`; see the `/meta-analysis` DTA path); not for rater **agreement** (that is
+  `analysis_guides/agreement_reliability.md`).
+
+---
+
+## Every metric with a CI, on a stated analysis unit (the floor)
+
+A bare AUC / sensitivity / specificity is not reportable. Compute the CI, and state the unit
+(**per-patient vs per-lesion** — multiple lesions per patient are clustered, exactly as in
+`agreement_reliability.md`; a per-lesion count inflates *n*).
+
+```python
+import numpy as np, pandas as pd
+from sklearn.metrics import roc_auc_score
+from statsmodels.stats.proportion import proportion_confint
+
+df = pd.read_csv("reader_calls.csv")   # truth (0/1), call (0/1), confidence (1..K), stratum
+
+# sensitivity / specificity / PPV / NPV at the operating point, each with a Wilson CI
+tp = int(((df.truth == 1) & (df.call == 1)).sum()); fn = int(((df.truth == 1) & (df.call == 0)).sum())
+tn = int(((df.truth == 0) & (df.call == 0)).sum()); fp = int(((df.truth == 0) & (df.call == 1)).sum())
+for name, num, den in [("sensitivity", tp, tp+fn), ("specificity", tn, tn+fp),
+                       ("PPV", tp, tp+fp), ("NPV", tn, tn+fn)]:
+    lo, hi = proportion_confint(num, den, method="wilson")
+    print(f"{name} = {num/den:.3f} (95% CI {lo:.3f}-{hi:.3f}), n={den}")
+```
+
+PPV and NPV are **prevalence-dependent** — do not transport them to a population with a
+different base rate; report sensitivity/specificity (prevalence-invariant) plus the study
+prevalence, and recompute predictive values at the target prevalence with Bayes' rule.
+
+---
+
+## The confidence-weighted trap comes first (this, not the AUC, is the issue)
+
+When the novelty is a **confidence-weighted / rating-collapsed** score used as the ROC
+predictor, you must (a) confirm the (call × confidence) encoding is **strictly monotone** — a
+folded encoding silently collides the most-confident-positive with a negative call — and
+(b) report the **unweighted binary-call AUC** beside the weighted one, so the weighting earns
+its place. This is the produce-side of self-review / peer-review probe **D9**.
+
+```python
+K = int(df.confidence.max())
+
+# intended strictly-monotone composite: negative calls rank below positive calls, and within
+# a call higher confidence pushes further from the decision boundary.
+def composite(call, conf, K):
+    return np.where(call == 1, K + conf, K + 1 - conf)   # neg: 1..K, pos: K+1..2K
+
+# (1) MONOTONIC-ENCODING CHECK — every distinct (call, confidence) must map to a distinct,
+#     correctly-ordered score. A collision is the folded-score bug (e.g. real/1 == ai/1).
+combos = df[["call", "confidence"]].drop_duplicates().copy()
+combos["score"] = composite(combos.call.values, combos.confidence.values, K)
+if combos.score.nunique() != len(combos):
+    raise ValueError("ENCODING COLLISION — the confidence weighting is not strictly monotone "
+                     "(folded-score bug); fix the encoding before computing AUC.")
+
+# (2) UNWEIGHTED BASELINE beside the weighted primary
+df["score"] = composite(df.call.values, df.confidence.values, K)
+print(f"AUC (confidence-weighted) = {roc_auc_score(df.truth, df.score):.3f}")
+print(f"AUC (unweighted binary call) = {roc_auc_score(df.truth, df.call):.3f}")
+```
+
+If the weighted AUC materially exceeds the unweighted one, the weighting must be justified;
+if it does not, report the simpler estimator as primary. A weighted score that changed a
+hypothesis's direction versus its unweighted baseline is a **post-lock change to disclose**,
+not a silent primary.
+
+---
+
+## AUC confidence intervals and comparing two AUCs
+
+DeLong is the field-standard analytic AUC CI and paired comparison; `pROC` (R) is canonical.
+
+```r
+library(pROC)
+d  <- read.csv("reader_calls.csv")
+rw <- roc(d$truth, d$score, quiet = TRUE)
+ci.auc(rw)                                   # DeLong 95% CI for the weighted AUC
+ru <- roc(d$truth, d$call,  quiet = TRUE)
+roc.test(rw, ru, method = "delong", paired = TRUE)   # paired, SAME cases (fixed-reader)
+```
+
+A portable Python bootstrap CI (use when a DeLong implementation is unavailable):
+
+```python
+def auc_ci_bootstrap(y, s, n_boot=2000, seed=42):
+    rng = np.random.default_rng(seed); y = np.asarray(y); s = np.asarray(s); n = len(y)
+    boots = [roc_auc_score(y[i], s[i]) for i in (rng.integers(0, n, n) for _ in range(n_boot))
+             if len(np.unique(y[i])) == 2]
+    return tuple(np.percentile(boots, [2.5, 97.5]))
+```
+
+**Fixed-reader vs MRMC.** A paired DeLong test treats the readers/cases as fixed. If the claim
+is that the result **generalises to readers** (a reader sample), a fixed-reader CI understates
+uncertainty — use an MRMC method (Obuchowski–Rockette / DBM) that carries reader + case
+variance, report per-reader and reader-averaged AUC, and state the unit (per-patient vs
+per-lesion) and the superiority/non-inferiority margin. This is probe D9's MRMC lead.
+
+---
+
+## Per-stratum admissibility: produce the table the claim is checked against (D10)
+
+A blanket "no subgroup met AUC ≥ 0.75 with lower bound ≥ 0.70" is falsified by a single tabled
+stratum that meets it. Produce the per-stratum AUC + CI and test each row against the rule
+rather than asserting a global negative.
+
+```python
+rule = lambda auc, lo: (auc >= 0.75) and (lo >= 0.70)
+for name, g in df.groupby("stratum"):
+    if g.truth.nunique() < 2:                       # single-class stratum: not estimable
+        print(f"{name}: not estimable (one class)"); continue
+    auc = roc_auc_score(g.truth, g.score); lo, hi = auc_ci_bootstrap(g.truth, g.score)
+    print(f"{name}: AUC {auc:.3f} (95% CI {lo:.3f}-{hi:.3f}) "
+          f"{'QUALIFIES' if rule(auc, lo) else 'below'}")
+```
+
+With *k* strata some crossings are expected — frame qualifying strata as hypothesis-generating
+(note the multiplicity), but do **not** deny a row the paper's own table satisfies.
+
+---
+
+## One scale per comparison (D11)
+
+Two values sharing a comparison column (or a row-wise "A vs B") must be computed under the
+**same normalisation / definition** — e.g. both volume errors as a standard relative error, not
+one relative and one range-normalised. If they are not identical, recompute on a common scale
+before any superiority/comparability claim, or footnote "not on the same scale".
+
+---
+
+## Reporting
+
+- Sensitivity, specificity, PPV, NPV, accuracy, and AUC each **with a 95% CI**; the operating
+  point and how it was chosen (Youden vs a prespecified clinical threshold — a threshold picked
+  on the same data is optimistic and needs a held-out or cross-validated estimate).
+- The **analysis unit** (per-patient vs per-lesion) and clustering handling; study prevalence.
+- For a weighted-score primary: the **unweighted-baseline AUC** beside it (D9).
+- AUC alone is insufficient for a clinical claim — pair discrimination with **calibration** and a
+  **decision-curve / net-benefit** pass at the relevant threshold (see the incremental-value
+  table type and the `make-figures` decision-curve exemplar).
+
+---
+
+## Common failures (flag at review)
+
+- **AUC / sensitivity / specificity with no CI**, or a per-lesion count reported as if per-patient.
+- **Confidence-weighted AUC with no unweighted baseline** and no monotonic-encoding check (D9) —
+  the weighting may have created the result or hidden a folded-score bug.
+- **Two AUCs compared by CI overlap** instead of a paired DeLong test; a **fixed-reader** CI used
+  for a claim that generalises to readers (needs MRMC).
+- **"No stratum met the rule" contradicted by the per-stratum table** (D10).
+- **Mixed-normalisation head-to-head** in one column (D11).
+- **PPV/NPV transported** across a prevalence change.
+
+---
+
+## Anti-Hallucination
+
+- Never hand-type an AUC, sensitivity/specificity, or CI — compute it from the calls CSV with a
+  seeded script; carry each estimate together with its CI (never a bare AUC).
+- Do not report a weighted-score AUC without the unweighted baseline the code produced.
+- Do not quote a DeLong comparison p-value that a paired test on the same cases did not produce.