opencode-bioresearcher 1.6.0

package/skills/bioresearcher-plot-making/scripts/audit_pdf_text.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""Audit text font sizes used by PDF content-stream ``Tf`` operators.
+
+This dependency-free check catches reduced mathtext superscripts/subscripts and
+other glyph runs that can fall below a journal font-size floor even when the
+parent matplotlib ``fontsize`` is compliant. It supports plain and FlateDecode
+content streams, which covers normal matplotlib PDF output.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+import zlib
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+
+STREAM_START = re.compile(rb"stream\r?\n")
+TF_OPERATOR = re.compile(
+    rb"/([^\s/<>]+)\s+([-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[Ee][-+]?\d+)?)\s+Tf\b"
+)
+
+
+@dataclass(frozen=True)
+class TextRun:
+    stream: int
+    font: str
+    size_pt: float
+
+
+def decoded_streams(data: bytes) -> tuple[list[bytes], list[str]]:
+    streams: list[bytes] = []
+    warnings: list[str] = []
+    cursor = 0
+    stream_number = 0
+    while True:
+        match = STREAM_START.search(data, cursor)
+        if not match:
+            break
+        stream_number += 1
+        end = data.find(b"endstream", match.end())
+        if end < 0:
+            warnings.append(f"stream {stream_number} has no endstream marker")
+            break
+        # Keep the raw stream bytes. zlib accepts PDF's trailing line break,
+        # while stripping could accidentally remove a legitimate compressed
+        # byte that happens to equal CR or LF.
+        payload = data[match.end() : end]
+        header = data[max(0, match.start() - 2048) : match.start()]
+        dictionary_start = header.rfind(b"<<")
+        dictionary = header[dictionary_start:] if dictionary_start >= 0 else header
+        if b"/FlateDecode" in dictionary:
+            try:
+                payload = zlib.decompress(payload)
+            except zlib.error as exc:
+                warnings.append(f"stream {stream_number} FlateDecode failed: {exc}")
+                cursor = end + len(b"endstream")
+                continue
+        elif b"/Filter" in dictionary:
+            warnings.append(f"stream {stream_number} uses an unsupported PDF filter")
+            cursor = end + len(b"endstream")
+            continue
+        streams.append(payload)
+        cursor = end + len(b"endstream")
+    return streams, warnings
+
+
+def audit_pdf(data: bytes, minimum_pt: float = 5.0) -> dict[str, object]:
+    streams, warnings = decoded_streams(data)
+    runs: list[TextRun] = []
+    for stream_index, stream in enumerate(streams, 1):
+        for match in TF_OPERATOR.finditer(stream):
+            try:
+                font = match.group(1).decode("ascii", errors="replace")
+                size = float(match.group(2))
+            except ValueError:
+                continue
+            if size > 0:
+                runs.append(TextRun(stream=stream_index, font=font, size_pt=size))
+    below = [run for run in runs if run.size_pt < minimum_pt]
+    return {
+        "auditable": bool(runs),
+        "minimum_required_pt": minimum_pt,
+        "minimum_found_pt": min((run.size_pt for run in runs), default=None),
+        "text_run_count": len(runs),
+        "below_minimum_count": len(below),
+        "below_minimum": [asdict(run) for run in below],
+        "warnings": warnings,
+    }
+
+
+def render_text(path: Path, result: dict[str, object]) -> str:
+    lines = [
+        "Bioresearcher Plot PDF Text Audit",
+        f"pdf: {path}",
+        f"minimum required: {result['minimum_required_pt']:g} pt",
+    ]
+    if not result["auditable"]:
+        lines.append("verdict: NOT AUDITABLE — no supported Tf text operators were found")
+    else:
+        lines.extend(
+            [
+                f"minimum found: {result['minimum_found_pt']:g} pt",
+                f"text runs: {result['text_run_count']}",
+                f"below minimum: {result['below_minimum_count']}",
+                f"verdict: {'FAIL' if result['below_minimum_count'] else 'PASS'}",
+            ]
+        )
+    for run in result["below_minimum"]:
+        lines.append(f"  - stream {run['stream']}: /{run['font']} {run['size_pt']:g} Tf")
+    for warning in result["warnings"]:
+        lines.append(f"warning: {warning}")
+    lines.append("note: Tf scanning does not replace final-size visual inspection or account for every PDF transform")
+    return "\n".join(lines)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("pdf", type=Path, help="exported PDF figure")
+    parser.add_argument("--min-pt", type=float, default=5.0, help="minimum allowed Tf font size in points")
+    parser.add_argument("--json", action="store_true", help="emit machine-readable JSON")
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    if args.min_pt <= 0:
+        print("error: --min-pt must be positive", file=sys.stderr)
+        return 2
+    try:
+        data = args.pdf.read_bytes()
+    except OSError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+    if not data.startswith(b"%PDF-"):
+        print(f"error: not a PDF file: {args.pdf}", file=sys.stderr)
+        return 2
+    result = audit_pdf(data, minimum_pt=args.min_pt)
+    if args.json:
+        print(json.dumps({"pdf": str(args.pdf), **result}, indent=2, ensure_ascii=False))
+    else:
+        print(render_text(args.pdf, result))
+    if not result["auditable"]:
+        return 2
+    return 1 if result["below_minimum_count"] else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/skills/bioresearcher-plot-making/scripts/plot_helpers.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+"""Universal Scientific Plotting Helpers for bioresearcher-plot-making.
+
+Provides publication-grade layout builders, dynamic font-metric calculations,
+tight alpha-mask cropping for 3D renders, font-aware text wrapping, and
+integrated QA gate verification.
+"""
+
+import sys
+from pathlib import Path
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+import numpy as np
+from PIL import Image
+
+# Ensure bundled audit scripts are discoverable
+SCRIPTS_DIR = Path(__file__).resolve().parent
+if str(SCRIPTS_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPTS_DIR))
+
+try:
+    from audit_panel_alignment import require_matplotlib_panel_alignment
+except ImportError:
+    require_matplotlib_panel_alignment = None
+
+MM = 1 / 25.4  # Millimeter to inch conversion factor
+
+
+def make_figure(width_mm=180, height_mm=120):
+    """Initialize a publication figure sized in millimeters with Nature typography."""
+    mpl.rcParams.update({
+        "font.family": "sans-serif",
+        "font.sans-serif": ["Arial", "Helvetica", "DejaVu Sans"],
+        "font.size": 7,
+        "axes.labelsize": 7,
+        "xtick.labelsize": 6,
+        "ytick.labelsize": 6,
+        "svg.fonttype": "none",
+        "pdf.fonttype": 42,
+        "axes.linewidth": 0.7,
+        "lines.linewidth": 1.0,
+    })
+    return plt.figure(figsize=(width_mm * MM, height_mm * MM))
+
+
+def load_render_cropped(png_path, pad=(12, 12, 12, 12), return_crop=False):
+    """Load transparent 3D render and crop tightly to non-zero alpha bounding box.
+
+    Parameters
+    ----------
+    png_path : str or Path
+        Path to input transparent PNG.
+    pad : int or tuple of (left, top, right, bottom)
+        Pixel padding around the alpha bounding box.
+    return_crop : bool
+        If True, returns (cropped_image, (x0, y0, x1, y1)).
+    """
+    if isinstance(pad, (int, float)):
+        pad_l = pad_t = pad_r = pad_b = int(pad)
+    else:
+        pad_l, pad_t, pad_r, pad_b = (int(v) for v in pad)
+
+    arr = np.array(Image.open(png_path))
+    if arr.ndim < 3 or arr.shape[2] < 4:
+        return (arr, (0, 0, arr.shape[1], arr.shape[0])) if return_crop else arr
+
+    alpha = arr[..., 3]
+    ys, xs = np.where(alpha > 10)
+    if len(xs) == 0 or len(ys) == 0:
+        return (arr, (0, 0, arr.shape[1], arr.shape[0])) if return_crop else arr
+
+    x0 = max(int(xs.min()) - pad_l, 0)
+    x1 = min(int(xs.max()) + pad_r, arr.shape[1])
+    y0 = max(int(ys.min()) - pad_t, 0)
+    y1 = min(int(ys.max()) + pad_b, arr.shape[0])
+
+    cropped = arr[y0:y1, x0:x1]
+    return (cropped, (x0, y0, x1, y1)) if return_crop else cropped
+
+
+def text_width(ax, text_artist):
+    """Calculate renderer advance width in data coordinates."""
+    fig = ax.figure
+    renderer = fig.canvas.get_renderer()
+    weight = text_artist.get_fontweight()
+    is_bold = weight == "bold" or (isinstance(weight, (int, float)) and weight > 500)
+    w_adv, _, _ = renderer.get_text_width_height_descent(
+        text_artist.get_text(), text_artist.get_fontproperties(), is_bold
+    )
+    inv = ax.transData.inverted()
+    (x0_adv, _), (x1_adv, _) = inv.transform([(0, 0), (w_adv, 0)])
+    return x1_adv - x0_adv
+
+
+def right_edge(ax, text_artist):
+    """Calculate robust text right boundary taking max of ink bbox and advance width."""
+    fig = ax.figure
+    renderer = fig.canvas.get_renderer()
+    bb = text_artist.get_window_extent(renderer=renderer)
+    inv = ax.transData.inverted()
+    (_, _), (x_ink, _) = inv.transform([(bb.x0, bb.y0), (bb.x1, bb.y1)])
+    x_adv = text_artist.get_position()[0] + text_width(ax, text_artist)
+    return max(x_ink, x_adv)
+
+
+def tag_right(ax, text, y_data, size=5.5, color="#5A5A5A"):
+    """Pin provenance text flush to the panel right edge using blended transform."""
+    import matplotlib.transforms as mtransforms
+    tr = mtransforms.blended_transform_factory(ax.transAxes, ax.transData)
+    return ax.text(
+        1.0, y_data, text, transform=tr, fontsize=size,
+        color=color, ha="right", va="center", style="italic", zorder=6
+    )
+
+
+def wrap_cell_text(text, max_chars_per_line=30):
+    """Wrap string into multi-line list preserving word boundaries."""
+    words = text.split()
+    if not words:
+        return [""]
+    lines = []
+    current = []
+    curr_len = 0
+    for w in words:
+        if curr_len + len(w) + (1 if current else 0) > max_chars_per_line:
+            lines.append(" ".join(current))
+            current = [w]
+            curr_len = len(w)
+        else:
+            current.append(w)
+            curr_len += len(w) + (1 if len(current) > 1 else 0)
+    if current:
+        lines.append(" ".join(current))
+    return lines
+
+
+def export_publication_figure(
+    fig,
+    base_name,
+    panel_ids=None,
+    row_groups=None,
+    column_groups=None,
+    exemptions=None,
+    exclude_axes=None,
+    axes=None,
+    tolerance_pt=1.5,
+    dpi=300,
+    **kwargs,
+):
+    """Execute panel alignment gate and export vector PDF, SVG, and raster PNG."""
+    Path(base_name).parent.mkdir(parents=True, exist_ok=True)
+    if require_matplotlib_panel_alignment is not None:
+        opts = dict(
+            tolerance_pt=tolerance_pt,
+            strict=True,
+            panel_ids=panel_ids,
+            row_groups=row_groups,
+            column_groups=column_groups,
+            **kwargs,
+        )
+        if exemptions is not None:
+            opts["exemptions"] = exemptions
+        if exclude_axes is not None:
+            opts["exclude_axes"] = exclude_axes
+        if axes is not None:
+            opts["axes"] = axes
+        require_matplotlib_panel_alignment(
+            fig,
+            json_out=f"{base_name}.alignment.json",
+            overlay_svg=f"{base_name}.alignment.svg",
+            **opts,
+        )
+
+    fig.savefig(f"{base_name}.pdf", bbox_inches="tight")
+    fig.savefig(f"{base_name}.svg", bbox_inches="tight")
+    fig.savefig(f"{base_name}.png", dpi=dpi, bbox_inches="tight")
+    print(f"[bioresearcher-plot-making] Exported {base_name}.{{pdf,svg,png}}")

package/skills/bioresearcher-pubmed-weekly/SKILL.md

@@ -0,0 +1,223 @@
+---
+name: bioresearcher-pubmed-weekly
+description: "Downloads the past week's PubMed daily update XML.gz files from ftp.ncbi.nlm.nih.gov/pubmed/updatefiles/, streams-parses interleaved PubmedArticle and DeleteCitation records, and produces one combined Excel workbook plus a summary JSON. Use for weekly PubMed/NLM literature tracking, NCBI updatefiles retrieval, pubmedNNnNNNN.xml.gz parsing, PMID/DOI/journal/author extraction, deleted-PMID lists, or building a weekly biomedical literature table."
+license: Apache-2.0
+compatibility: "Unix-like shells (Linux, macOS, Git Bash) and Windows cmd.exe; requires uv + openpyxl and NCBI FTP access"
+metadata:
+  version: "1.0.0"
+  source: "opencode-bioresearcher-plugin@1.7.2"
+allowed-tools: Bash Read Glob
+---
+
+# PubMed Weekly Update Download and Parse
+
+This skill downloads the past week's (Monday-Sunday) PubMed daily update `xml.gz` files from `ftp://ftp.ncbi.nlm.nih.gov/pubmed/updatefiles/`, parses them with the bundled streaming scripts, and produces ONE combined Excel workbook (`combined.xlsx`) plus a summary JSON.
+
+## Workflow Overview
+
+1. **Python environment check**: ensure uv is available (install via the `bioresearcher-python-setup-uv` skill if needed)
+2. **Date-range calculation**: compute the past week's Monday-Sunday range
+3. **FTP listing + filtering**: fetch available `xml.gz` updatefiles and keep those modified within the week
+4. **Download with retry + resume**: download each filtered file (3 attempts, 2s delay, `.part` resume)
+5. **Parse + combine via scripts**: parse all downloaded files and produce one Excel workbook
+6. **Report**: verify outputs and summarize to the user
+
+## Prerequisites
+
+- Internet connection and access to the NCBI FTP server
+- uv package manager (if missing, load the `bioresearcher-python-setup-uv` skill and follow it EXACTLY, then return here)
+- openpyxl is provided at runtime via `uv run --with openpyxl`
+
+## Script Usage
+
+Replace `<skill_dir>` with the full path to this skill's directory (the scripts live in `<skill_dir>/scripts/`). Run all commands from the working directory where downloads should land.
+
+**For Unix-like shells (Git Bash / macOS / Linux):**
+```bash
+uv run --with openpyxl python <skill_dir>/scripts/pubmed_weekly.py <command> [args...]
+```
+
+**For Windows cmd.exe:**
+```bash
+uv.exe run --with openpyxl python <skill_dir>\scripts\pubmed_weekly.py <command> [args...]
+```
+
+## Steps
+
+Follow these steps EXACTLY as described.
+
+### Step 1: Check uv Prerequisite
+
+```bash
+if [ -f "uv" ] || [ -f "uv.exe" ]; then
+  echo "uv already installed"
+else
+  echo "uv not found, setting up..."
+fi
+```
+
+If uv is not installed, load the `bioresearcher-python-setup-uv` skill and follow all its steps EXACTLY, then continue with Step 2 below.
+
+### Step 2: Calculate Week Date Range
+
+Determine the date range for the past week (Monday through Sunday).
+
+**For Unix-like shells:**
+```bash
+uv run --with openpyxl python <skill_dir>/scripts/pubmed_weekly.py calculate_week
+```
+
+**For Windows cmd.exe:**
+```bash
+uv.exe run --with openpyxl python <skill_dir>\scripts\pubmed_weekly.py calculate_week
+```
+
+This outputs the week folder name in format `YYYYMMDD-YYYYMMDD`, e.g.:
+
+```
+20250217-20250223
+```
+
+### Step 3: Fetch FTP File List
+
+Fetch the list of daily update `xml.gz` files from the NCBI FTP server.
+
+**For Unix-like shells:**
+```bash
+uv run --with openpyxl python <skill_dir>/scripts/pubmed_weekly.py fetch_files
+```
+
+**For Windows cmd.exe:**
+```bash
+uv.exe run --with openpyxl python <skill_dir>\scripts\pubmed_weekly.py fetch_files
+```
+
+**Expected output (space-separated filenames):**
+```
+pubmed24n1234.xml.gz pubmed24n1235.xml.gz pubmed24n1236.xml.gz
+```
+
+### Step 4: Filter Files for Past Week
+
+Filter the file list to those modified within the week (PubMed filenames do not encode dates, so FTP modification times are used).
+
+**For Unix-like shells:**
+```bash
+uv run --with openpyxl python <skill_dir>/scripts/pubmed_weekly.py filter_files "<WEEK>" "<FILE_LIST>"
+```
+
+**For Windows cmd.exe:**
+```bash
+uv.exe run --with openpyxl python <skill_dir>\scripts\pubmed_weekly.py filter_files "<WEEK>" "<FILE_LIST>"
+```
+
+Where `<WEEK>` is the week folder name (e.g., `20250217-20250223`) and `<FILE_LIST>` is the Step 3 output (quote it). Returns the space-separated filtered list.
+
+### Step 5: Download Files with Retry and Resume
+
+Download each filtered file into `.download/pubmed-daily/<WEEK>/`.
+
+**For Unix-like shells:**
+```bash
+for file in <FILE_LIST>; do
+  uv run --with openpyxl python <skill_dir>/scripts/pubmed_weekly.py download_file "<WEEK>" "$file"
+done
+```
+
+**For Windows cmd.exe:**
+```bash
+for %f in (<FILE_LIST>) do uv.exe run --with openpyxl python <skill_dir>\scripts\pubmed_weekly.py download_file "<WEEK>" %f
+```
+
+**Download behavior:**
+- Downloads one file at a time to `<filename>.part`, renamed on success
+- Retries up to 3 times per file with 2-second delays
+- Resumes interrupted `.part` downloads; skips already-completed files
+- If a download fails after 3 retries, ask the user: "Abort remaining downloads?" ("Yes" / "No"). "Yes" stops and reports; "No" skips the failed file and continues
+
+### Step 6: Parse and Combine into One Excel
+
+Parse every downloaded `xml.gz` in the week directory and write ONE combined workbook `combined.xlsx` plus `summary.json` (recommended; done in a single command):
+
+**For Unix-like shells:**
+```bash
+uv run --with openpyxl python <skill_dir>/scripts/pubmed_weekly.py combine "<WEEK>"
+```
+
+**For Windows cmd.exe:**
+```bash
+uv.exe run --with openpyxl python <skill_dir>\scripts\pubmed_weekly.py combine "<WEEK>"
+```
+
+**Output location:**
+```
+.download/pubmed-daily/<WEEK>/combined.xlsx
+.download/pubmed-daily/<WEEK>/summary.json
+```
+
+Alternatively, invoke the parser directly (Unix; on Windows cmd.exe list files explicitly since cmd.exe does not expand `*`):
+```bash
+uv run --with openpyxl python <skill_dir>/scripts/parse_updatefiles.py \
+  .download/pubmed-daily/<WEEK>/pubmed24n1234.xml.gz \
+  .download/pubmed-daily/<WEEK>/pubmed24n1235.xml.gz \
+  -o .download/pubmed-daily/<WEEK>/combined.xlsx \
+  --summary-json .download/pubmed-daily/<WEEK>/summary.json
+```
+
+The parser accepts plain `.xml` as well as `.xml.gz` (gzip-transparent) and any number of input files; ordering follows the given file order, document order within each file.
+
+### Step 7: Verify and Report
+
+```bash
+ls -lh .download/pubmed-daily/<WEEK>/
+```
+
+Report to the user: week range, files found/downloaded/failed, download location, article count and deleted-PMID count in `combined.xlsx`, and the `combined.xlsx` / `summary.json` locations.
+
+## Frozen Output Schema
+
+`combined.xlsx` (openpyxl workbook, write_only; schema is FROZEN — do not change):
+
+**Sheet 1: `PubMed Articles`** — columns in EXACT order:
+
+| # | Column | Source |
+|---|--------|--------|
+| 1 | `PMID` | `MedlineCitation/PMID` text |
+| 2 | `DOI` | `PubmedData/ArticleIdList/ArticleId[@IdType='doi']` if present, else empty |
+| 3 | `Title` | `Article/ArticleTitle` (inline markup flattened) |
+| 4 | `Journal` | `Article/Journal/Title` |
+| 5 | `ISSN` | first `Article/Journal/ISSN` |
+| 6 | `PubDate` | `Journal/JournalIssue/PubDate` normalized: Year + Month (name or number) → `YYYY-MM`; Year only → `YYYY`; no Year → empty |
+| 7 | `FirstAuthor` | first `Article/AuthorList/Author` as `LastName Initials` (e.g., `Smith J`) |
+| 8 | `LastAuthor` | last `Article/AuthorList/Author` in the same format |
+| 9 | `PublicationTypes` | `Article/PublicationTypeList/PublicationType` values joined with `;` |
+
+**Sheet 2: `Deleted PMIDs`** — single column:
+
+| # | Column | Source |
+|---|--------|--------|
+| 1 | `PMID` | each `DeleteCitation/PMID` text |
+
+**`summary.json`** (written with `--summary-json` / by `combine`):
+
+```json
+{
+  "source_files": ["pubmed24n1234.xml.gz"],
+  "article_count": 1234,
+  "deleted_pmids": ["99999999"],
+  "first_rows": [{"PMID": "...", "DOI": "...", "Title": "...", "Journal": "...", "ISSN": "...", "PubDate": "...", "FirstAuthor": "...", "LastAuthor": "...", "PublicationTypes": "..."}]
+}
+```
+
+`first_rows` contains up to the first 3 article rows as objects keyed by the column names above.
+
+## Notes
+
+- updatefiles interleave `<PubmedArticle>` and `<DeleteCitation>` records — BOTH are handled by the parser
+- Parsing is memory-bounded streaming: `xml.etree.iterparse` with element clearing plus an openpyxl `write_only` workbook
+- No plugin tool dependencies: stdlib `urllib` for FTP, openpyxl for Excel, nothing else
+- Output ordering is deterministic: input file order (sorted within `combine`), document order within each file
+- All downloads and outputs live under `.download/pubmed-daily/<WEEK>/` in the current working directory
+- Only `.xml.gz` updatefiles are downloaded; downloads are sequential with retry + resume
+- The FTP server path is `ftp://ftp.ncbi.nlm.nih.gov/pubmed/updatefiles/`
+- Windows with Git Bash: follow Unix-like shell instructions; Windows cmd.exe: use `uv.exe ...` variants