medsci-skills 5.9.2 → 5.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/metadata/distribution_files.json +10 -10
- package/metadata/distribution_manifest.json +1 -1
- package/package.json +1 -1
- package/skills/make-figures/references/exemplar_plots/README.md +14 -5
- package/skills/make-figures/scripts/render_core_figures.py +290 -2
- package/skills/make-figures/scripts/render_core_figures_challenge/fixture/synthetic_inputs.json +49 -0
- package/skills/make-figures/scripts/render_core_figures_challenge/problem.md +28 -12
- package/skills/make-figures/scripts/render_core_figures_challenge/verify.sh +21 -12
|
@@ -2003,8 +2003,8 @@
|
|
|
2003
2003
|
},
|
|
2004
2004
|
{
|
|
2005
2005
|
"path": "skills/make-figures/references/exemplar_plots/README.md",
|
|
2006
|
-
"size":
|
|
2007
|
-
"sha256": "
|
|
2006
|
+
"size": 6524,
|
|
2007
|
+
"sha256": "6dded03ce2aa6cf81277d4a64fa15e0f6814aa568132fc817e2fc753517d5338"
|
|
2008
2008
|
},
|
|
2009
2009
|
{
|
|
2010
2010
|
"path": "skills/make-figures/references/exemplar_plots/bland_altman.md",
|
|
@@ -2168,23 +2168,23 @@
|
|
|
2168
2168
|
},
|
|
2169
2169
|
{
|
|
2170
2170
|
"path": "skills/make-figures/scripts/render_core_figures.py",
|
|
2171
|
-
"size":
|
|
2172
|
-
"sha256": "
|
|
2171
|
+
"size": 29667,
|
|
2172
|
+
"sha256": "8c5b8a3b564b74977f25931094e11b55e2d1bad7ec6c05436825bd1bf69dc0b0"
|
|
2173
2173
|
},
|
|
2174
2174
|
{
|
|
2175
2175
|
"path": "skills/make-figures/scripts/render_core_figures_challenge/fixture/synthetic_inputs.json",
|
|
2176
|
-
"size":
|
|
2177
|
-
"sha256": "
|
|
2176
|
+
"size": 3609,
|
|
2177
|
+
"sha256": "5ca97899b1f61fec1c933039fafdae13dbe20bec175561019a4c947109a9b88f"
|
|
2178
2178
|
},
|
|
2179
2179
|
{
|
|
2180
2180
|
"path": "skills/make-figures/scripts/render_core_figures_challenge/problem.md",
|
|
2181
|
-
"size":
|
|
2182
|
-
"sha256": "
|
|
2181
|
+
"size": 3486,
|
|
2182
|
+
"sha256": "86c57b26720a6ae1112a641f9ef1e1736c1bc3e413452de94d7e25ec298c26ba"
|
|
2183
2183
|
},
|
|
2184
2184
|
{
|
|
2185
2185
|
"path": "skills/make-figures/scripts/render_core_figures_challenge/verify.sh",
|
|
2186
|
-
"size":
|
|
2187
|
-
"sha256": "
|
|
2186
|
+
"size": 2378,
|
|
2187
|
+
"sha256": "2961607318f21052d1b1e607b024520ac6a53ebf4afeaf4287199ed252c1c034"
|
|
2188
2188
|
},
|
|
2189
2189
|
{
|
|
2190
2190
|
"path": "skills/make-figures/scripts/validate_pptx_mac_compat.py",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "medsci-skills",
|
|
3
|
-
"version": "5.
|
|
3
|
+
"version": "5.10.0",
|
|
4
4
|
"description": "MedSci Skills — a medical/scientific research skill suite for AI coding agents (Claude Code, Codex, Cursor, Copilot). The npm package is a terminal-friendly installer shortcut; the canonical distribution remains the GitHub repository and the Claude Code plugin marketplace.",
|
|
5
5
|
"license": "SEE LICENSE IN LICENSE",
|
|
6
6
|
"homepage": "https://github.com/Aperivue/medsci-skills#readme",
|
|
@@ -13,15 +13,24 @@ against `critic_rubrics/data_plot.md`; do not copy an image.
|
|
|
13
13
|
|
|
14
14
|
## Runnable render layer (tested)
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
decision-curve
|
|
18
|
-
|
|
16
|
+
Ten of the highest-yield clinical figures — **Kaplan–Meier, ROC, calibration,
|
|
17
|
+
decision-curve, forest, Bland–Altman, confusion matrix, multi-reader ROC (MRMC),
|
|
18
|
+
Manhattan, and clinical timeline** — have a matching **runnable, deterministic generator**
|
|
19
|
+
in `../../scripts/render_core_figures.py`. It renders already-computed inputs
|
|
19
20
|
(the statistical estimation stays in `/analyze-stats`; the render layer never recomputes a
|
|
20
21
|
number) into the canonical anatomy and asserts each figure's load-bearing elements
|
|
21
22
|
(number-at-risk table, chance diagonal, identity line, treat-all/treat-none references,
|
|
22
|
-
no extrapolation past follow-up
|
|
23
|
+
no extrapolation past follow-up; forest per-study CI rows + null line + pooled diamond;
|
|
24
|
+
Bland–Altman bias + 95% limits of agreement; confusion Predicted/Actual axes + annotated
|
|
25
|
+
cells; MRMC per-reader + averaged curves; Manhattan significance-threshold line; timeline
|
|
26
|
+
baseline + event markers). A network-free render-regression challenge
|
|
23
27
|
(`scripts/render_core_figures_challenge/`, wired into `skill.yml` validation) renders all
|
|
24
|
-
|
|
28
|
+
ten from a synthetic fixture and confirms the structural gate fails on a malformed figure.
|
|
29
|
+
|
|
30
|
+
`imaging_panel.md` stays a **prose-only** exemplar by design: it composes real medical
|
|
31
|
+
images with panel labels, scale bars, and arrows — an image-arrangement task, not a plot of
|
|
32
|
+
computed numbers — so a synthetic generator would only draw placeholder boxes. Use the
|
|
33
|
+
prose model to compose it against real images.
|
|
25
34
|
Use the generator to produce these four directly; use the prose models below to compose
|
|
26
35
|
the figure types that do not yet have a generator.
|
|
27
36
|
|
|
@@ -33,6 +33,7 @@ import matplotlib
|
|
|
33
33
|
matplotlib.use("Agg") # headless, deterministic
|
|
34
34
|
import matplotlib.pyplot as plt # noqa: E402
|
|
35
35
|
import numpy as np # noqa: E402
|
|
36
|
+
from matplotlib.patches import Polygon # noqa: E402
|
|
36
37
|
|
|
37
38
|
|
|
38
39
|
# --------------------------------------------------------------------------- KM
|
|
@@ -164,6 +165,188 @@ def decision_curve(thresholds, net_benefit_model, prevalence: float, *,
|
|
|
164
165
|
return fig
|
|
165
166
|
|
|
166
167
|
|
|
168
|
+
# ------------------------------------------------------------------- forest
|
|
169
|
+
def forest_plot(studies: list[dict], pooled: dict, *, null_value: float = 1.0,
|
|
170
|
+
effect_label: str = "Effect (95% CI)", title: str = "",
|
|
171
|
+
log_x: bool = True) -> plt.Figure:
|
|
172
|
+
"""Meta-analysis forest plot from already-computed per-study estimates.
|
|
173
|
+
|
|
174
|
+
studies: [{name, est, lo, hi, weight?}] — each study's point estimate + CI.
|
|
175
|
+
pooled: {est, lo, hi, label} — the pooled estimate + CI + model name.
|
|
176
|
+
Draws a weight-scaled marker + CI whisker per study, the null reference line, and a
|
|
177
|
+
pooled diamond; the pooled row is always last. For a ratio measure keep log_x=True and
|
|
178
|
+
null_value=1.0; for a mean difference pass log_x=False, null_value=0.0."""
|
|
179
|
+
n = len(studies)
|
|
180
|
+
fig, ax = plt.subplots(figsize=(6.6, 0.5 * n + 2.0))
|
|
181
|
+
ys = list(range(n, 0, -1)) # top study at the highest y
|
|
182
|
+
weights = np.asarray([s.get("weight", 1.0) for s in studies], float)
|
|
183
|
+
wnorm = weights / weights.max() if weights.max() > 0 else np.ones(n)
|
|
184
|
+
for y, s, w in zip(ys, studies, wnorm):
|
|
185
|
+
ax.plot([s["lo"], s["hi"]], [y, y], color="0.3") # CI whisker
|
|
186
|
+
ax.scatter([s["est"]], [y], s=30 + 120 * w, marker="s",
|
|
187
|
+
color="steelblue", zorder=4) # weight-scaled box
|
|
188
|
+
ax.axvline(null_value, linestyle="--", color="0.5") # null reference
|
|
189
|
+
# pooled diamond on a row below the studies
|
|
190
|
+
yd = 0
|
|
191
|
+
d = pooled
|
|
192
|
+
ax.add_patch(Polygon([[d["lo"], yd], [d["est"], yd + 0.35],
|
|
193
|
+
[d["hi"], yd], [d["est"], yd - 0.35]],
|
|
194
|
+
closed=True, facecolor="crimson", edgecolor="black", zorder=5))
|
|
195
|
+
labels = [s["name"] for s in studies] + [pooled.get("label", "Pooled")]
|
|
196
|
+
ax.set_yticks(ys + [yd])
|
|
197
|
+
ax.set_yticklabels(labels)
|
|
198
|
+
ax.set_ylim(-1, n + 1)
|
|
199
|
+
if log_x:
|
|
200
|
+
ax.set_xscale("log")
|
|
201
|
+
ax.set_xlabel(effect_label)
|
|
202
|
+
ax.set_title(title or "Meta-analysis forest plot")
|
|
203
|
+
fig.tight_layout()
|
|
204
|
+
fig._mf_kind = "forest"
|
|
205
|
+
fig._mf_n_studies = n
|
|
206
|
+
fig._mf_null = null_value
|
|
207
|
+
return fig
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
# -------------------------------------------------------------- Bland–Altman
|
|
211
|
+
def bland_altman(mean_vals, diff_vals, *, bias: float, sd_diff: float,
|
|
212
|
+
title: str = "") -> plt.Figure:
|
|
213
|
+
"""Bland–Altman agreement plot: difference vs mean, with the bias line and the
|
|
214
|
+
95% limits of agreement (bias ± 1.96·SD) — the load-bearing agreement elements."""
|
|
215
|
+
mean_vals = np.asarray(mean_vals, float)
|
|
216
|
+
diff_vals = np.asarray(diff_vals, float)
|
|
217
|
+
loa_hi, loa_lo = bias + 1.96 * sd_diff, bias - 1.96 * sd_diff
|
|
218
|
+
fig, ax = plt.subplots(figsize=(6.0, 5.0))
|
|
219
|
+
ax.scatter(mean_vals, diff_vals, s=25, color="steelblue", alpha=0.8)
|
|
220
|
+
ax.axhline(bias, color="crimson", label=f"Bias {bias:.2f}")
|
|
221
|
+
ax.axhline(loa_hi, linestyle="--", color="0.4", label=f"+1.96 SD {loa_hi:.2f}")
|
|
222
|
+
ax.axhline(loa_lo, linestyle="--", color="0.4", label=f"−1.96 SD {loa_lo:.2f}")
|
|
223
|
+
ax.set_xlabel("Mean of the two measurements")
|
|
224
|
+
ax.set_ylabel("Difference between measurements")
|
|
225
|
+
ax.set_title(title or "Bland–Altman agreement")
|
|
226
|
+
ax.legend(loc="upper right", frameon=False)
|
|
227
|
+
fig.tight_layout()
|
|
228
|
+
fig._mf_kind = "bland_altman"
|
|
229
|
+
fig._mf_loa = (loa_lo, loa_hi)
|
|
230
|
+
return fig
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
# ---------------------------------------------------------- confusion matrix
|
|
234
|
+
def confusion_matrix(matrix, labels, *, title: str = "") -> plt.Figure:
|
|
235
|
+
"""Confusion matrix from an already-computed count grid. Rows = actual, cols =
|
|
236
|
+
predicted; every cell is annotated with its count (for a 2×2, TN/FP/FN/TP)."""
|
|
237
|
+
m = np.asarray(matrix, float)
|
|
238
|
+
k = m.shape[0]
|
|
239
|
+
if m.shape[0] != m.shape[1] or k != len(labels):
|
|
240
|
+
raise AssertionError("confusion matrix must be square and match the label count")
|
|
241
|
+
fig, ax = plt.subplots(figsize=(1.4 * k + 2, 1.4 * k + 2))
|
|
242
|
+
ax.imshow(m, cmap="Blues")
|
|
243
|
+
thresh = m.max() / 2.0 if m.max() else 0.5
|
|
244
|
+
for i in range(k):
|
|
245
|
+
for j in range(k):
|
|
246
|
+
ax.text(j, i, str(int(m[i, j])), ha="center", va="center",
|
|
247
|
+
color="white" if m[i, j] > thresh else "black")
|
|
248
|
+
ax.set_xticks(range(k)); ax.set_xticklabels(labels)
|
|
249
|
+
ax.set_yticks(range(k)); ax.set_yticklabels(labels)
|
|
250
|
+
ax.set_xlabel("Predicted")
|
|
251
|
+
ax.set_ylabel("Actual")
|
|
252
|
+
ax.set_title(title or "Confusion matrix")
|
|
253
|
+
fig.tight_layout()
|
|
254
|
+
fig._mf_kind = "confusion"
|
|
255
|
+
fig._mf_k = k
|
|
256
|
+
return fig
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
# ------------------------------------------------------------------ MRMC ROC
|
|
260
|
+
def mrmc_roc(readers: list[dict], averaged: dict, *, delta_auc: dict | None = None,
|
|
261
|
+
title: str = "") -> plt.Figure:
|
|
262
|
+
"""Multi-reader multi-case ROC: each reader's ROC curve plus the reader-averaged
|
|
263
|
+
curve and the chance diagonal (the load-bearing MRMC-reader-study elements).
|
|
264
|
+
|
|
265
|
+
readers: [{name, fpr, tpr, auc}] — per-reader ROC coordinates.
|
|
266
|
+
averaged: {fpr, tpr, auc, label} — the reader-averaged curve.
|
|
267
|
+
delta_auc (optional): {value, margin} — a ΔAUC-vs-margin annotation."""
|
|
268
|
+
fig, ax = plt.subplots(figsize=(5.4, 5.2))
|
|
269
|
+
for r in readers:
|
|
270
|
+
ax.plot(np.asarray(r["fpr"], float), np.asarray(r["tpr"], float),
|
|
271
|
+
color="0.7", linewidth=1) # thin per-reader
|
|
272
|
+
a = averaged
|
|
273
|
+
ax.plot(np.asarray(a["fpr"], float), np.asarray(a["tpr"], float),
|
|
274
|
+
color="crimson", linewidth=2.4,
|
|
275
|
+
label=f"{a.get('label', 'Reader-averaged')} (AUC = {a['auc']:.3f})")
|
|
276
|
+
ax.plot([0, 1], [0, 1], linestyle="--", color="0.5", label="Chance")
|
|
277
|
+
ax.set_xlim(0, 1); ax.set_ylim(0, 1.02)
|
|
278
|
+
ax.set_xlabel("1 − specificity (false-positive rate)")
|
|
279
|
+
ax.set_ylabel("Sensitivity (true-positive rate)")
|
|
280
|
+
ax.set_title(title or "Multi-reader multi-case ROC")
|
|
281
|
+
txt = f"averaged AUC = {a['auc']:.3f}"
|
|
282
|
+
if delta_auc:
|
|
283
|
+
txt += f"\nΔAUC = {delta_auc['value']:.3f} (margin {delta_auc['margin']:.3f})"
|
|
284
|
+
ax.text(0.55, 0.08, txt, fontsize=10,
|
|
285
|
+
bbox=dict(boxstyle="round", fc="white", ec="0.7"))
|
|
286
|
+
ax.legend(loc="lower right", frameon=False)
|
|
287
|
+
fig.tight_layout()
|
|
288
|
+
fig._mf_kind = "mrmc_roc"
|
|
289
|
+
fig._mf_n_readers = len(readers)
|
|
290
|
+
return fig
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
# ------------------------------------------------------------------ Manhattan
|
|
294
|
+
def manhattan(x, neglogp, threshold: float, *, labels=None,
|
|
295
|
+
ylabel: str = "−log10(p)", xlabel: str = "Exposure / position",
|
|
296
|
+
title: str = "") -> plt.Figure:
|
|
297
|
+
"""Manhattan / *-wide-scan plot: −log10(p) vs position with the significance
|
|
298
|
+
threshold line (the two load-bearing elements of an agnostic many-test scan)."""
|
|
299
|
+
x = np.asarray(x, float)
|
|
300
|
+
y = np.asarray(neglogp, float)
|
|
301
|
+
fig, ax = plt.subplots(figsize=(7.0, 4.4))
|
|
302
|
+
ax.scatter(x, y, s=14, color="steelblue", alpha=0.8)
|
|
303
|
+
ax.axhline(threshold, color="crimson", linestyle="--",
|
|
304
|
+
label=f"significance threshold (−log10 = {threshold:.2f})")
|
|
305
|
+
if labels: # sparse labelling of hits above the threshold
|
|
306
|
+
for xi, yi, lab in zip(x, y, labels):
|
|
307
|
+
if lab and yi >= threshold:
|
|
308
|
+
ax.annotate(lab, (xi, yi), fontsize=8,
|
|
309
|
+
xytext=(0, 4), textcoords="offset points", ha="center")
|
|
310
|
+
ax.set_xlabel(xlabel)
|
|
311
|
+
ax.set_ylabel(ylabel)
|
|
312
|
+
ax.set_title(title or "Manhattan plot")
|
|
313
|
+
ax.legend(loc="upper right", frameon=False)
|
|
314
|
+
fig.tight_layout()
|
|
315
|
+
fig._mf_kind = "manhattan"
|
|
316
|
+
fig._mf_threshold = threshold
|
|
317
|
+
return fig
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
# --------------------------------------------------------- clinical timeline
|
|
321
|
+
def clinical_timeline(events: list[dict], *, time_unit: str = "days from admission",
|
|
322
|
+
title: str = "") -> plt.Figure:
|
|
323
|
+
"""Case-report clinical timeline: an event marker + label at each time on a single
|
|
324
|
+
time axis (the load-bearing elements of a longitudinal case figure)."""
|
|
325
|
+
times = [float(e["time"]) for e in events]
|
|
326
|
+
fig, ax = plt.subplots(figsize=(max(6.0, 0.9 * len(events)), 3.4))
|
|
327
|
+
lo, hi = (min(times), max(times)) if times else (0, 1)
|
|
328
|
+
pad = max(1.0, (hi - lo) * 0.08)
|
|
329
|
+
ax.axhline(0, color="0.4") # the timeline
|
|
330
|
+
for i, e in enumerate(events):
|
|
331
|
+
t = float(e["time"])
|
|
332
|
+
up = 1 if i % 2 == 0 else -1
|
|
333
|
+
ax.plot([t, t], [0, up * 0.6], color="0.6") # stem
|
|
334
|
+
ax.scatter([t], [0], s=40, color="crimson", zorder=5) # event marker
|
|
335
|
+
ax.annotate(str(e["label"]), (t, up * 0.65), ha="center",
|
|
336
|
+
va="bottom" if up > 0 else "top", fontsize=8)
|
|
337
|
+
ax.set_xlim(lo - pad, hi + pad)
|
|
338
|
+
ax.set_ylim(-1.3, 1.3)
|
|
339
|
+
ax.get_yaxis().set_visible(False)
|
|
340
|
+
for spine in ("left", "right", "top"):
|
|
341
|
+
ax.spines[spine].set_visible(False)
|
|
342
|
+
ax.set_xlabel(f"Time ({time_unit})")
|
|
343
|
+
ax.set_title(title or "Clinical timeline")
|
|
344
|
+
fig.tight_layout()
|
|
345
|
+
fig._mf_kind = "timeline"
|
|
346
|
+
fig._mf_n_events = len(events)
|
|
347
|
+
return fig
|
|
348
|
+
|
|
349
|
+
|
|
167
350
|
# ----------------------------------------------------- structural invariants
|
|
168
351
|
def assert_structure(fig: plt.Figure) -> list[str]:
|
|
169
352
|
"""Assert the load-bearing elements for the figure's kind. Returns the list of
|
|
@@ -241,6 +424,97 @@ def assert_structure(fig: plt.Figure) -> list[str]:
|
|
|
241
424
|
"DCA: treat-all / treat-none not labelled"
|
|
242
425
|
passed.append("DCA reference strategies labelled")
|
|
243
426
|
|
|
427
|
+
elif kind == "forest":
|
|
428
|
+
ax = fig.axes[0]
|
|
429
|
+
n = getattr(fig, "_mf_n_studies")
|
|
430
|
+
# one horizontal CI whisker (2-point line, equal y) per study
|
|
431
|
+
whiskers = [ln for ln in ax.lines
|
|
432
|
+
if len(ln.get_xdata()) == 2 and np.allclose(np.diff(ln.get_ydata()), 0.0)]
|
|
433
|
+
assert len(whiskers) >= n, "forest: missing per-study CI whiskers"
|
|
434
|
+
passed.append(f"forest per-study CI rows present ({n})")
|
|
435
|
+
null = getattr(fig, "_mf_null")
|
|
436
|
+
assert any(len(ln.get_xdata()) == 2 and np.allclose(ln.get_xdata(), [null, null])
|
|
437
|
+
for ln in ax.lines), "forest: null reference line missing"
|
|
438
|
+
passed.append("forest null reference line present")
|
|
439
|
+
assert any(isinstance(p, Polygon) and len(p.get_xy()) >= 4 for p in ax.patches), \
|
|
440
|
+
"forest: pooled diamond missing"
|
|
441
|
+
passed.append("forest pooled diamond present")
|
|
442
|
+
assert len(ax.get_yticklabels()) >= n + 1, "forest: study/pooled row labels missing"
|
|
443
|
+
passed.append("forest study + pooled row labels present")
|
|
444
|
+
|
|
445
|
+
elif kind == "bland_altman":
|
|
446
|
+
ax = fig.axes[0]
|
|
447
|
+
assert ax.collections, "Bland–Altman: scatter of differences missing"
|
|
448
|
+
passed.append("Bland–Altman difference scatter present")
|
|
449
|
+
hlines = [ln for ln in ax.lines if np.allclose(np.diff(ln.get_ydata()), 0.0)]
|
|
450
|
+
assert len(hlines) >= 3, "Bland–Altman: need bias + two limits-of-agreement lines"
|
|
451
|
+
passed.append("Bland–Altman bias + 2 LoA lines present")
|
|
452
|
+
loa_lo, loa_hi = getattr(fig, "_mf_loa")
|
|
453
|
+
yvals = [float(ln.get_ydata()[0]) for ln in hlines]
|
|
454
|
+
assert any(abs(y - loa_hi) < 1e-6 for y in yvals) and any(abs(y - loa_lo) < 1e-6 for y in yvals), \
|
|
455
|
+
"Bland–Altman: LoA lines not at bias ± 1.96·SD"
|
|
456
|
+
passed.append("Bland–Altman LoA at bias ± 1.96·SD")
|
|
457
|
+
assert "difference" in ax.get_ylabel().lower() and "mean" in ax.get_xlabel().lower(), \
|
|
458
|
+
"Bland–Altman: axes not difference-vs-mean"
|
|
459
|
+
passed.append("Bland–Altman difference-vs-mean axes")
|
|
460
|
+
|
|
461
|
+
elif kind == "confusion":
|
|
462
|
+
ax = fig.axes[0]
|
|
463
|
+
k = getattr(fig, "_mf_k")
|
|
464
|
+
assert ax.images, "confusion: matrix image missing"
|
|
465
|
+
passed.append("confusion matrix image present")
|
|
466
|
+
cells = [t for t in ax.texts if t.get_text().strip().lstrip("-").isdigit()]
|
|
467
|
+
assert len(cells) >= k * k, f"confusion: expected {k * k} annotated cells"
|
|
468
|
+
passed.append(f"confusion all {k}×{k} cells annotated")
|
|
469
|
+
assert "predicted" in ax.get_xlabel().lower() and "actual" in ax.get_ylabel().lower(), \
|
|
470
|
+
"confusion: axes not Predicted/Actual"
|
|
471
|
+
passed.append("confusion Predicted/Actual axes")
|
|
472
|
+
|
|
473
|
+
elif kind == "mrmc_roc":
|
|
474
|
+
ax = fig.axes[0]
|
|
475
|
+
n = getattr(fig, "_mf_n_readers")
|
|
476
|
+
curves = [ln for ln in ax.lines
|
|
477
|
+
if len(ln.get_xdata()) > 2] # multi-point ROC curves
|
|
478
|
+
assert len(curves) >= n + 1, "MRMC-ROC: fewer curves than readers + averaged"
|
|
479
|
+
passed.append(f"MRMC-ROC per-reader + averaged curves present ({n}+1)")
|
|
480
|
+
diag = [ln for ln in ax.lines
|
|
481
|
+
if len(ln.get_xdata()) == 2 and np.allclose(ln.get_xdata(), [0, 1])
|
|
482
|
+
and np.allclose(ln.get_ydata(), [0, 1])]
|
|
483
|
+
assert diag, "MRMC-ROC: chance diagonal missing"
|
|
484
|
+
passed.append("MRMC-ROC chance diagonal present")
|
|
485
|
+
assert has_text(ax, "auc"), "MRMC-ROC: averaged-AUC annotation missing"
|
|
486
|
+
passed.append("MRMC-ROC averaged-AUC annotation present")
|
|
487
|
+
assert "sensitiv" in ax.get_ylabel().lower(), "MRMC-ROC: y-label not sensitivity"
|
|
488
|
+
passed.append("MRMC-ROC sensitivity y-label")
|
|
489
|
+
|
|
490
|
+
elif kind == "manhattan":
|
|
491
|
+
ax = fig.axes[0]
|
|
492
|
+
assert ax.collections, "Manhattan: point scatter missing"
|
|
493
|
+
passed.append("Manhattan scatter present")
|
|
494
|
+
thr = getattr(fig, "_mf_threshold")
|
|
495
|
+
assert any(np.allclose(ln.get_ydata(), thr) for ln in ax.lines
|
|
496
|
+
if np.allclose(np.diff(ln.get_ydata()), 0.0)), \
|
|
497
|
+
"Manhattan: significance threshold line missing"
|
|
498
|
+
passed.append("Manhattan significance threshold line present")
|
|
499
|
+
yl = ax.get_ylabel().lower()
|
|
500
|
+
assert "log" in yl and ("10" in yl or "log10" in yl or "−log" in yl or "-log" in yl), \
|
|
501
|
+
"Manhattan: y-label not −log10(p)"
|
|
502
|
+
passed.append("Manhattan −log10(p) y-label")
|
|
503
|
+
|
|
504
|
+
elif kind == "timeline":
|
|
505
|
+
ax = fig.axes[0]
|
|
506
|
+
ne = getattr(fig, "_mf_n_events")
|
|
507
|
+
assert any(np.allclose(ln.get_ydata(), 0.0) for ln in ax.lines), \
|
|
508
|
+
"timeline: baseline axis missing"
|
|
509
|
+
passed.append("timeline baseline present")
|
|
510
|
+
assert ax.collections, "timeline: event markers missing"
|
|
511
|
+
passed.append("timeline event markers present")
|
|
512
|
+
assert len([t for t in ax.texts if t.get_text().strip()]) >= ne, \
|
|
513
|
+
"timeline: an event label is missing"
|
|
514
|
+
passed.append(f"timeline all {ne} event labels present")
|
|
515
|
+
assert "time" in ax.get_xlabel().lower(), "timeline: x-axis not a time axis"
|
|
516
|
+
passed.append("timeline time x-axis")
|
|
517
|
+
|
|
244
518
|
else:
|
|
245
519
|
raise AssertionError(f"unknown figure kind: {kind!r}")
|
|
246
520
|
return passed
|
|
@@ -248,7 +522,7 @@ def assert_structure(fig: plt.Figure) -> list[str]:
|
|
|
248
522
|
|
|
249
523
|
# ----------------------------------------------------------------- driver
|
|
250
524
|
def render_all(inputs: dict, out_dir: Path) -> dict:
|
|
251
|
-
"""Render
|
|
525
|
+
"""Render each figure kind present in ``inputs``, save PNGs, and assert structure.
|
|
252
526
|
Returns {kind: [passed checks]}. Raises on any structural violation."""
|
|
253
527
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
254
528
|
results: dict[str, list[str]] = {}
|
|
@@ -261,6 +535,20 @@ def render_all(inputs: dict, out_dir: Path) -> dict:
|
|
|
261
535
|
ci_low=d.get("ci_low"), ci_high=d.get("ci_high")),
|
|
262
536
|
"dca": lambda d: decision_curve(d["thresholds"], d["net_benefit_model"],
|
|
263
537
|
d["prevalence"]),
|
|
538
|
+
"forest": lambda d: forest_plot(d["studies"], d["pooled"],
|
|
539
|
+
null_value=d.get("null_value", 1.0),
|
|
540
|
+
effect_label=d.get("effect_label", "Effect (95% CI)"),
|
|
541
|
+
log_x=d.get("log_x", True)),
|
|
542
|
+
"bland_altman": lambda d: bland_altman(d["mean_vals"], d["diff_vals"],
|
|
543
|
+
bias=d["bias"], sd_diff=d["sd_diff"]),
|
|
544
|
+
"confusion": lambda d: confusion_matrix(d["matrix"], d["labels"]),
|
|
545
|
+
"mrmc_roc": lambda d: mrmc_roc(d["readers"], d["averaged"],
|
|
546
|
+
delta_auc=d.get("delta_auc")),
|
|
547
|
+
"manhattan": lambda d: manhattan(d["x"], d["neglogp"], d["threshold"],
|
|
548
|
+
labels=d.get("labels"),
|
|
549
|
+
xlabel=d.get("xlabel", "Exposure / position")),
|
|
550
|
+
"timeline": lambda d: clinical_timeline(d["events"],
|
|
551
|
+
time_unit=d.get("time_unit", "days from admission")),
|
|
264
552
|
}
|
|
265
553
|
for kind, build in builders.items():
|
|
266
554
|
if kind not in inputs:
|
|
@@ -287,7 +575,7 @@ def main(argv=None) -> int:
|
|
|
287
575
|
return 1
|
|
288
576
|
for kind, checks in results.items():
|
|
289
577
|
print(f"OK [{kind}] {len(checks)} structural invariants: {'; '.join(checks)}")
|
|
290
|
-
print(f"PASS: {len(results)}
|
|
578
|
+
print(f"PASS: {len(results)} figure(s) rendered + structurally verified.")
|
|
291
579
|
return 0
|
|
292
580
|
|
|
293
581
|
|
package/skills/make-figures/scripts/render_core_figures_challenge/fixture/synthetic_inputs.json
CHANGED
|
@@ -38,5 +38,54 @@
|
|
|
38
38
|
"thresholds": [0.05, 0.10, 0.20, 0.30, 0.40, 0.50],
|
|
39
39
|
"net_benefit_model": [0.34, 0.30, 0.22, 0.15, 0.09, 0.04],
|
|
40
40
|
"prevalence": 0.35
|
|
41
|
+
},
|
|
42
|
+
"forest": {
|
|
43
|
+
"effect_label": "Odds ratio (95% CI)",
|
|
44
|
+
"null_value": 1.0,
|
|
45
|
+
"log_x": true,
|
|
46
|
+
"studies": [
|
|
47
|
+
{"name": "Study A 2019", "est": 1.20, "lo": 0.90, "hi": 1.60, "weight": 25},
|
|
48
|
+
{"name": "Study B 2020", "est": 1.45, "lo": 1.10, "hi": 1.92, "weight": 30},
|
|
49
|
+
{"name": "Study C 2021", "est": 0.95, "lo": 0.70, "hi": 1.29, "weight": 20},
|
|
50
|
+
{"name": "Study D 2022", "est": 1.60, "lo": 1.15, "hi": 2.22, "weight": 25}
|
|
51
|
+
],
|
|
52
|
+
"pooled": {"est": 1.30, "lo": 1.10, "hi": 1.54, "label": "Random-effects pooled"}
|
|
53
|
+
},
|
|
54
|
+
"bland_altman": {
|
|
55
|
+
"mean_vals": [10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32],
|
|
56
|
+
"diff_vals": [0.2, -0.5, 0.8, 1.1, -0.3, 0.6, -1.2, 0.9, 0.1, 1.5, -0.7, 0.4],
|
|
57
|
+
"bias": 0.24,
|
|
58
|
+
"sd_diff": 0.80
|
|
59
|
+
},
|
|
60
|
+
"confusion": {
|
|
61
|
+
"matrix": [[85, 15], [10, 90]],
|
|
62
|
+
"labels": ["Negative", "Positive"]
|
|
63
|
+
},
|
|
64
|
+
"mrmc_roc": {
|
|
65
|
+
"readers": [
|
|
66
|
+
{"name": "Reader 1", "fpr": [0.0, 0.10, 0.25, 0.50, 1.0], "tpr": [0.0, 0.55, 0.72, 0.88, 1.0], "auc": 0.80},
|
|
67
|
+
{"name": "Reader 2", "fpr": [0.0, 0.08, 0.20, 0.45, 1.0], "tpr": [0.0, 0.62, 0.78, 0.90, 1.0], "auc": 0.84},
|
|
68
|
+
{"name": "Reader 3", "fpr": [0.0, 0.12, 0.30, 0.55, 1.0], "tpr": [0.0, 0.50, 0.68, 0.85, 1.0], "auc": 0.77}
|
|
69
|
+
],
|
|
70
|
+
"averaged": {"fpr": [0.0, 0.10, 0.25, 0.50, 1.0], "tpr": [0.0, 0.56, 0.73, 0.88, 1.0], "auc": 0.805, "label": "Reader-averaged"},
|
|
71
|
+
"delta_auc": {"value": 0.06, "margin": 0.05}
|
|
72
|
+
},
|
|
73
|
+
"manhattan": {
|
|
74
|
+
"xlabel": "Candidate exposure (index)",
|
|
75
|
+
"threshold": 2.6,
|
|
76
|
+
"x": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20],
|
|
77
|
+
"neglogp": [0.4, 1.2, 0.8, 3.5, 0.6, 1.1, 2.1, 0.9, 4.2, 0.5, 1.3, 0.7, 2.8, 1.0, 0.4, 5.1, 0.9, 1.4, 0.6, 2.2],
|
|
78
|
+
"labels": ["", "", "", "Exposure D", "", "", "", "", "Exposure I", "", "", "", "Exposure M", "", "", "Exposure P", "", "", "", ""]
|
|
79
|
+
},
|
|
80
|
+
"timeline": {
|
|
81
|
+
"time_unit": "days from admission",
|
|
82
|
+
"events": [
|
|
83
|
+
{"time": 0, "label": "Admission"},
|
|
84
|
+
{"time": 2, "label": "CT scan"},
|
|
85
|
+
{"time": 5, "label": "Biopsy"},
|
|
86
|
+
{"time": 9, "label": "Diagnosis"},
|
|
87
|
+
{"time": 14, "label": "Treatment start"},
|
|
88
|
+
{"time": 30, "label": "Follow-up"}
|
|
89
|
+
]
|
|
41
90
|
}
|
|
42
91
|
}
|
|
@@ -1,15 +1,17 @@
|
|
|
1
1
|
# Challenge card — core clinical-figure render regression (make-figures)
|
|
2
2
|
|
|
3
3
|
## Problem
|
|
4
|
-
The
|
|
5
|
-
|
|
4
|
+
The highest-yield clinical figures — Kaplan–Meier, ROC, calibration, decision-curve,
|
|
5
|
+
forest, Bland–Altman, and confusion matrix — were documented only as **prose anatomy** in
|
|
6
6
|
`references/exemplar_plots/`, and the actual matplotlib rendering had **no deterministic
|
|
7
7
|
test of any kind**. A regression in figure code (a dropped number-at-risk table, a
|
|
8
8
|
missing chance diagonal, a calibration plot without its identity line, a DCA without the
|
|
9
|
-
treat-all / treat-none references,
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
9
|
+
treat-all / treat-none references, a KM curve extrapolated past follow-up, a forest without
|
|
10
|
+
its pooled diamond or null line, a Bland–Altman without its limits of agreement, or a
|
|
11
|
+
confusion matrix without annotated cells) would pass every prose read and only be caught by
|
|
12
|
+
a reviewer — the gap that left the suite's self-identified weakest area with the same
|
|
13
|
+
defense/enablement asymmetry the rest of the repo has closed (integrity detectors have
|
|
14
|
+
challenge fixtures; the figure generators did not).
|
|
13
15
|
|
|
14
16
|
## What the generator does
|
|
15
17
|
`scripts/render_core_figures.py` is the **render** layer for the exemplar anatomies. It
|
|
@@ -26,17 +28,31 @@ elements are present:
|
|
|
26
28
|
predicted-vs-observed axes.
|
|
27
29
|
- **Decision curve** — model + treat-all + treat-none strategies, the treat-none
|
|
28
30
|
(net benefit = 0) reference, a net-benefit y-axis.
|
|
31
|
+
- **Forest** — a per-study CI whisker for every study, the null reference line, and the
|
|
32
|
+
pooled diamond; study + pooled row labels.
|
|
33
|
+
- **Bland–Altman** — the difference scatter, the bias line, and the 95% limits of
|
|
34
|
+
agreement (bias ± 1.96·SD); difference-vs-mean axes.
|
|
35
|
+
- **Confusion matrix** — a matrix image with every cell annotated and Predicted/Actual axes.
|
|
36
|
+
- **MRMC ROC** — a curve per reader + the reader-averaged curve, the chance diagonal, and
|
|
37
|
+
the averaged-AUC annotation.
|
|
38
|
+
- **Manhattan** — the point scatter, the named significance-threshold line, and a
|
|
39
|
+
−log10(p) y-axis.
|
|
40
|
+
- **Clinical timeline** — the time baseline, an event marker + label at each event, and a
|
|
41
|
+
time x-axis.
|
|
42
|
+
|
|
43
|
+
(`imaging_panel` stays a prose-only exemplar — it composes real images, not computed
|
|
44
|
+
numbers, so it has no synthetic generator.)
|
|
29
45
|
|
|
30
46
|
## Fixture (synthetic only — no real data)
|
|
31
|
-
- `fixture/synthetic_inputs.json` — hand-authored
|
|
32
|
-
|
|
47
|
+
- `fixture/synthetic_inputs.json` — hand-authored coordinates and summary statistics for
|
|
48
|
+
all ten figures.
|
|
33
49
|
|
|
34
50
|
## Expected (`verify.sh`, network-free)
|
|
35
|
-
- All
|
|
51
|
+
- All ten figures render to PNGs (each > 2 KB) **and** every structural invariant holds
|
|
36
52
|
→ exit 0.
|
|
37
|
-
-
|
|
38
|
-
|
|
39
|
-
|
|
53
|
+
- Mutated inputs that drop a load-bearing element (a non-monotonic KM curve; a non-square
|
|
54
|
+
confusion matrix) raise `AssertionError` → the negative cases in `verify.sh` confirm the
|
|
55
|
+
gate actually fails when it should.
|
|
40
56
|
|
|
41
57
|
Requires matplotlib + numpy (already make-figures runtime deps); the verifier skips with
|
|
42
58
|
a clear message if matplotlib is unavailable, so it never hard-fails a minimal host.
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env bash
|
|
2
2
|
# Deterministic verifier for the core-figure render challenge (make-figures).
|
|
3
|
-
# Network-free. Renders the
|
|
3
|
+
# Network-free. Renders the ten canonical clinical figures from a synthetic fixture and
|
|
4
4
|
# asserts each figure's load-bearing elements; then confirms the structural gate FAILS on
|
|
5
|
-
#
|
|
5
|
+
# mutated inputs (so the assertions are proven to bite). Exit 0 = all expectations hold.
|
|
6
6
|
set -euo pipefail
|
|
7
7
|
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
8
8
|
GEN="$HERE/../render_core_figures.py"
|
|
@@ -13,25 +13,34 @@ TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
|
|
|
13
13
|
python3 -c "import matplotlib, numpy" 2>/dev/null \
|
|
14
14
|
|| { echo "SKIP: matplotlib/numpy unavailable on this host"; exit 0; }
|
|
15
15
|
|
|
16
|
-
# (1) Positive: all
|
|
16
|
+
# (1) Positive: all ten figures render + every structural invariant holds.
|
|
17
17
|
python3 "$GEN" --inputs "$FIX" --out-dir "$TMP/out" >"$TMP/log" 2>&1 \
|
|
18
18
|
|| { echo "FAIL: valid fixture did not render/verify" >&2; cat "$TMP/log" >&2; exit 1; }
|
|
19
|
-
for k in km roc calibration dca; do
|
|
19
|
+
for k in km roc calibration dca forest bland_altman confusion mrmc_roc manhattan timeline; do
|
|
20
20
|
[ -s "$TMP/out/$k.png" ] || { echo "FAIL: $k.png not written" >&2; exit 1; }
|
|
21
21
|
done
|
|
22
|
-
grep -q "PASS:
|
|
22
|
+
grep -q "PASS: 10 figure" "$TMP/log" || { echo "FAIL: not all ten figures verified" >&2; cat "$TMP/log" >&2; exit 1; }
|
|
23
23
|
|
|
24
|
-
# (2) Negative
|
|
25
|
-
#
|
|
26
|
-
python3 - "$FIX" "$TMP/
|
|
24
|
+
# (2) Negative — the assertions must actually fire on malformed inputs.
|
|
25
|
+
# (2a) a KM survival curve mutated to be non-monotonic.
|
|
26
|
+
python3 - "$FIX" "$TMP/bad_km.json" <<'PY'
|
|
27
27
|
import json, sys
|
|
28
28
|
d = json.load(open(sys.argv[1]))
|
|
29
29
|
d["km"]["groups"][0]["surv"] = [1.0, 0.92, 0.95, 0.74, 0.68] # 0.81 -> 0.95 (increasing)
|
|
30
30
|
json.dump(d, open(sys.argv[2], "w"))
|
|
31
31
|
PY
|
|
32
|
-
if python3 "$GEN" --inputs "$TMP/
|
|
33
|
-
echo "FAIL: a non-monotonic KM survival curve was NOT caught
|
|
34
|
-
|
|
32
|
+
if python3 "$GEN" --inputs "$TMP/bad_km.json" --out-dir "$TMP/bk" >/dev/null 2>&1; then
|
|
33
|
+
echo "FAIL: a non-monotonic KM survival curve was NOT caught" >&2; exit 1
|
|
34
|
+
fi
|
|
35
|
+
# (2b) a non-square confusion matrix (label/shape mismatch).
|
|
36
|
+
python3 - "$FIX" "$TMP/bad_cm.json" <<'PY'
|
|
37
|
+
import json, sys
|
|
38
|
+
d = json.load(open(sys.argv[1]))
|
|
39
|
+
d["confusion"]["matrix"] = [[85, 15, 0], [10, 90, 0]] # 2x3, not square
|
|
40
|
+
json.dump(d, open(sys.argv[2], "w"))
|
|
41
|
+
PY
|
|
42
|
+
if python3 "$GEN" --inputs "$TMP/bad_cm.json" --out-dir "$TMP/bc" >/dev/null 2>&1; then
|
|
43
|
+
echo "FAIL: a non-square confusion matrix was NOT caught" >&2; exit 1
|
|
35
44
|
fi
|
|
36
45
|
|
|
37
|
-
echo "PASS:
|
|
46
|
+
echo "PASS: 10 core figures render + structurally verify; the gate fails on a non-monotonic KM curve and a non-square confusion matrix."
|