@pikaa-ai/pikaa 0.2.4 → 0.3.0

package/skills/analytical-method-validation/scripts/compare_methods.py

@@ -0,0 +1,215 @@
+#!/usr/bin/env python3
+"""Compare two analytical procedures for a transfer, bridging, or bias study.
+
+Uses the regressions that belong to method comparison -- Deming and
+Passing-Bablok, which allow error in both measurements -- rather than ordinary
+least squares, which assumes the comparative procedure is error-free and biases
+the slope toward zero. Adds Bland-Altman agreement and, most importantly, a TOST
+equivalence test.
+
+The default reflex at a method transfer is a t test, and "p > 0.05, no
+significant difference" is then written up as evidence of equivalence. It is not:
+failing to detect a difference is not the same as showing there is none, and with
+a small transfer dataset that outcome is close to guaranteed. TOST tests the
+hypothesis that matters -- that the true difference lies inside a pre-stated
+acceptance margin.
+
+    python3 compare_methods.py --input paired.csv --margin 2
+    python3 compare_methods.py -i paired.csv --margin 2 --relative --lambda 1.0
+
+Input columns: `reference` and `test` (paired results on the same samples).
+An optional `sample` column labels the rows.
+
+Exit codes: 0 equivalence demonstrated at the stated margin, 1 not demonstrated
+or other findings, 2 bad input.
+"""
+
+from __future__ import annotations
+
+import argparse
+import math
+import sys
+
+sys.path.insert(0, str(__import__("pathlib").Path(__file__).resolve().parent))
+
+from _common import (  # noqa: E402
+    EXIT_FINDINGS,
+    EXIT_OK,
+    InputError,
+    add_common_args,
+    bland_altman,
+    deming,
+    emit,
+    finding,
+    fit_linear,
+    mean,
+    note,
+    parse_rows,
+    passing_bablok,
+    read_input,
+    require_columns,
+    run_cli,
+    t_cdf,
+    to_float,
+    tost_paired,
+)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Compare two procedures with the statistics method comparison requires."
+    )
+    parser.add_argument("--input", "-i", help="CSV/TSV/JSON with `reference` and `test`")
+    parser.add_argument("--margin", type=float, required=True,
+                        help="pre-stated equivalence margin for TOST, in the units of the "
+                             "difference (absolute, or %% when --relative is used)")
+    parser.add_argument("--relative", action="store_true",
+                        help="work in percent differences relative to the pair mean")
+    parser.add_argument("--lambda", dest="lambda_ratio", type=float, default=1.0,
+                        help="Deming error-variance ratio var(y error)/var(x error) -- test "
+                             "procedure over comparative procedure. Default 1.0 means equal "
+                             "precision. Estimate it as (SD of x replicates / SD of y "
+                             "replicates) squared")
+    parser.add_argument("--alpha", type=float, default=0.05,
+                        help="one-sided alpha for TOST (default 0.05)")
+    parser.add_argument("--slope-tolerance", type=float, default=None,
+                        help="flag when the slope CI excludes 1 +/- this amount")
+    add_common_args(parser)
+    args = parser.parse_args()
+
+    rows = parse_rows(read_input(args.input), args.input)
+    require_columns(rows, ["reference", "test"])
+    ref = [to_float(r["reference"], "reference", i) for i, r in enumerate(rows)]
+    test = [to_float(r["test"], "test", i) for i, r in enumerate(rows)]
+    n = len(ref)
+    if n < 3:
+        raise InputError("method comparison needs at least 3 paired results")
+
+    findings: list[str] = []
+
+    # Agreement.
+    ba = bland_altman(ref, test, relative=args.relative)
+    unit = "%" if args.relative else "units"
+
+    # TOST on the differences that Bland-Altman used.
+    if args.relative:
+        diffs = [
+            100.0 * (t - r) / (0.5 * (r + t)) for r, t in zip(ref, test)
+        ]
+    else:
+        diffs = [t - r for r, t in zip(ref, test)]
+    tost = tost_paired(diffs, args.margin, args.alpha)
+
+    # The naive test, computed only to show what it does not establish.
+    d_mean = mean(diffs)
+    sd = ba["sd_differences"]
+    se = sd / math.sqrt(n) if sd > 0 else float("nan")
+    t_stat = d_mean / se if se and math.isfinite(se) and se > 0 else float("nan")
+    p_naive = (
+        2.0 * (1.0 - t_cdf(abs(t_stat), n - 1)) if math.isfinite(t_stat) else float("nan")
+    )
+
+    # Regressions.
+    ols = fit_linear(ref, test)
+    dem = deming(ref, test, args.lambda_ratio)
+    try:
+        pb = passing_bablok(ref, test)
+    except InputError as exc:
+        pb = None
+        note(f"Passing-Bablok not computed: {exc}")
+
+    summary = [
+        {"statistic": "n pairs", "value": n},
+        {"statistic": f"mean difference ({unit})", "value": ba["bias"]},
+        {"statistic": "difference 95% CI", "value":
+            f"{ba['bias_ci95'][0]:.6g} to {ba['bias_ci95'][1]:.6g}"},
+        {"statistic": f"SD of differences ({unit})", "value": sd},
+        {"statistic": "limits of agreement", "value":
+            f"{ba['loa_lower']:.6g} to {ba['loa_upper']:.6g}"},
+        {"statistic": "LoA 95% CI half-width", "value": ba["loa_ci95_halfwidth"]},
+        {"statistic": "--- equivalence ---", "value": ""},
+        {"statistic": "TOST margin", "value": args.margin},
+        {"statistic": "TOST p-value", "value": tost["p_value"]},
+        {"statistic": f"{100 * (1 - 2 * args.alpha):.0f}% CI (TOST)", "value":
+            f"{tost['ci_1_minus_2alpha'][0]:.6g} to {tost['ci_1_minus_2alpha'][1]:.6g}"},
+        {"statistic": "equivalent at stated margin", "value": tost["equivalent"]},
+        {"statistic": "--- for contrast only ---", "value": ""},
+        {"statistic": "paired t-test p (NOT equivalence)", "value": p_naive},
+        {"statistic": "--- regressions ---", "value": ""},
+        {"statistic": "OLS slope (biased here)", "value": ols.slope},
+        {"statistic": "Deming slope", "value": dem["slope"]},
+        {"statistic": "Deming slope 95% CI", "value":
+            f"{dem['slope_ci95'][0]:.6g} to {dem['slope_ci95'][1]:.6g}"},
+        {"statistic": "Deming intercept", "value": dem["intercept"]},
+    ]
+    if pb:
+        summary += [
+            {"statistic": "Passing-Bablok slope", "value": pb["slope"]},
+            {"statistic": "Passing-Bablok slope 95% CI", "value":
+                f"{pb['slope_ci95'][0]:.6g} to {pb['slope_ci95'][1]:.6g}"},
+            {"statistic": "Passing-Bablok intercept", "value": pb["intercept"]},
+        ]
+
+    prop = ba.get("proportional_bias")
+    if prop and math.isfinite(prop.get("p_value", float("nan"))):
+        summary.append(
+            {"statistic": "proportional bias p (difference vs mean)", "value": prop["p_value"]}
+        )
+        if prop["p_value"] < 0.05:
+            findings.append(
+                f"the difference trends with concentration (slope {prop['slope']:.4g}, "
+                f"p={prop['p_value']:.4g}): a single mean bias does not describe the "
+                "disagreement, and limits of agreement are misleading"
+            )
+
+    if not tost["equivalent"]:
+        findings.append(
+            f"equivalence NOT demonstrated at +/-{args.margin:g} {unit}: the "
+            f"{100 * (1 - 2 * args.alpha):.0f}% CI "
+            f"({tost['ci_1_minus_2alpha'][0]:.4g}, {tost['ci_1_minus_2alpha'][1]:.4g}) is not "
+            f"contained in the margin"
+        )
+
+    if args.slope_tolerance is not None:
+        lo, hi = dem["slope_ci95"]
+        target_lo, target_hi = 1.0 - args.slope_tolerance, 1.0 + args.slope_tolerance
+        if lo < target_lo or hi > target_hi:
+            findings.append(
+                f"Deming slope 95% CI ({lo:.4g}, {hi:.4g}) is not contained in "
+                f"({target_lo:g}, {target_hi:g})"
+            )
+
+    if args.format == "json":
+        emit([{"summary": {r["statistic"]: r["value"] for r in summary
+                           if not r["statistic"].startswith("---")},
+               "bland_altman": ba, "deming": dem,
+               "passing_bablok": pb, "tost": tost,
+               "ols_slope": ols.slope, "paired_t_p_value": p_naive,
+               "findings": findings}], "json")
+    else:
+        emit(summary, args.format)
+
+    if math.isfinite(p_naive) and p_naive > 0.05 and not tost["equivalent"]:
+        note(
+            f"the paired t-test gives p={p_naive:.4g}, which would often be written up as "
+            "'no significant difference'. TOST shows equivalence is NOT established at the "
+            "stated margin. Absence of a detected difference is not evidence of equivalence"
+        )
+    note(
+        "ordinary least squares assumes the reference values carry no error, which is false in a "
+        "method comparison; the OLS slope is shown only for contrast"
+    )
+    note(
+        "the equivalence margin must be pre-stated from the specification or the analytical "
+        "target profile, never chosen after seeing the data"
+    )
+    for f in findings:
+        finding(f)
+    if not findings:
+        note("no findings against the checks that were run")
+    note("this tool does not decide that a transfer or comparison passes")
+    return EXIT_FINDINGS if findings else EXIT_OK
+
+
+if __name__ == "__main__":
+    run_cli(main)

package/skills/analytical-method-validation/scripts/plan_validation.py

@@ -0,0 +1,381 @@
+#!/usr/bin/env python3
+"""Design a validation study: which framework, which tests, what study layout.
+
+Answers the first question of any validation exercise -- what am I required to
+demonstrate, and with how much data -- before any sample is injected.
+
+    python3 plan_validation.py --framework ich-q2r2 --attribute assay --technique hplc
+    python3 plan_validation.py --framework ich-m10 --modality lba
+    python3 plan_validation.py --list-frameworks
+    python3 plan_validation.py --framework ich-q2r2 --attribute impurity --protocol > protocol.md
+
+Exit codes: 0 plan produced, 2 bad input.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+sys.path.insert(0, str(__import__("pathlib").Path(__file__).resolve().parent))
+
+from _catalog import (  # noqa: E402
+    DL_QL_APPROACHES,
+    FRAMEWORKS,
+    M10_CRITERIA,
+    Q2R2_REPORTABLE_RANGE,
+    Q2R2_STUDY_DESIGN,
+    Q2R2_TESTS_BY_ATTRIBUTE,
+    RESEARCH_DATE,
+    TECHNIQUE_NOTES,
+    resolve_attribute,
+)
+from _common import (  # noqa: E402
+    EXIT_OK,
+    InputError,
+    add_common_args,
+    emit,
+    note,
+    run_cli,
+)
+
+STATUS_TEXT = {
+    "required": "conduct",
+    "required-QL": "conduct (QL; DL too in some complex cases)",
+    "required-DL": "conduct (DL)",
+    "required-unless-reproducibility": (
+        "conduct, unless intermediate precision can be derived from a reproducibility dataset"
+    ),
+    "not-normally": "not normally conducted",
+}
+
+
+def plan_q2r2(attribute: str, technique: str | None, range_use: str | None) -> list[dict]:
+    tests = Q2R2_TESTS_BY_ATTRIBUTE[attribute]
+    rows = []
+    for characteristic, status in tests.items():
+        design = Q2R2_STUDY_DESIGN.get(characteristic, {})
+        rows.append(
+            {
+                "characteristic": characteristic,
+                "required": STATUS_TEXT[status],
+                "study_design": design.get("requirement", ""),
+                "report": design.get("report", ""),
+                "reference": design.get("reference", ""),
+            }
+        )
+    # Robustness sits in development under Q14 but belongs in the plan.
+    rows.append(
+        {
+            "characteristic": "robustness",
+            "required": "development activity (ICH Q14); available on request",
+            "study_design": Q2R2_STUDY_DESIGN["robustness"]["requirement"],
+            "report": Q2R2_STUDY_DESIGN["robustness"]["report"],
+            "reference": Q2R2_STUDY_DESIGN["robustness"]["reference"],
+        }
+    )
+    if technique:
+        tn = TECHNIQUE_NOTES.get(technique)
+        if tn:
+            rows.append(
+                {
+                    "characteristic": f"technique note ({technique})",
+                    "required": f"see Q2(R2) Annex 2 {tn['annex_table']}",
+                    "study_design": f"robustness parameters: {tn['robustness']}",
+                    "report": tn["special"],
+                    "reference": "Q2(R2) Annex 2",
+                }
+            )
+    if range_use:
+        rr = Q2R2_REPORTABLE_RANGE.get(range_use)
+        if rr:
+            rows.append(
+                {
+                    "characteristic": f"reportable range ({range_use})",
+                    "required": "confirm response, accuracy and precision across this range",
+                    "study_design": f"low: {rr['low']}",
+                    "report": f"high: {rr['high']}",
+                    "reference": "Q2(R2) 2.3, Table 2",
+                }
+            )
+    return rows
+
+
+def plan_m10(modality: str) -> list[dict]:
+    crit = M10_CRITERIA[modality]
+    rows = [
+        {
+            "item": "calibration curve",
+            "requirement": (
+                f"minimum {crit['calibration_min_levels']} concentration levels including "
+                f"the LLOQ; at least "
+                f"{crit['calibration_min_pass_fraction'] * 100:.0f}% of standards must pass"
+            ),
+            "tolerance": (
+                f"+/-{crit['calibration_tolerance_pct']:.0f}% nominal; "
+                f"+/-{crit['calibration_tolerance_lloq_pct']:.0f}% at LLOQ; "
+                f"+/-{crit['calibration_tolerance_uloq_pct']:.0f}% at ULOQ"
+            ),
+        },
+        {
+            "item": "accuracy and precision QC levels",
+            "requirement": (
+                f"{crit['qc_levels_accuracy_precision']} levels; "
+                f"{crit['ap_replicates_per_run']} replicates per level per run; "
+                f"at least {crit['ap_min_runs']} runs over at least {crit['ap_min_days']} days"
+            ),
+            "tolerance": (
+                f"accuracy +/-{crit['accuracy_tolerance_pct']:.0f}% "
+                f"(+/-{crit['accuracy_tolerance_lloq_pct']:.0f}% at {crit['limit_levels']}); "
+                f"precision CV <={crit['precision_cv_pct']:.0f}% "
+                f"(<={crit['precision_cv_lloq_pct']:.0f}% at {crit['limit_levels']})"
+            ),
+        },
+        {
+            "item": "routine run acceptance",
+            "requirement": (
+                f"{crit['qc_levels_routine_run']} QC levels; at least "
+                f"{crit['qc_run_pass_fraction'] * 100:.0f}% of all QCs and at least "
+                f"{crit['qc_run_pass_fraction_per_level'] * 100:.0f}% at each level must pass"
+            ),
+            "tolerance": f"+/-{crit['qc_run_tolerance_pct']:.0f}% nominal",
+        },
+        {
+            "item": "selectivity",
+            "requirement": f"at least {crit['selectivity_min_sources']} individual matrix sources/lots",
+            "tolerance": (
+                f"interference <={crit['carryover_blank_pct_of_lloq']:.0f}% of LLOQ analyte "
+                f"response and <={crit['carryover_blank_pct_of_is']:.0f}% of IS response"
+                if crit["carryover_blank_pct_of_lloq"] is not None
+                else "per guideline; evaluate interference in each source"
+            ),
+        },
+        {
+            "item": "dilution integrity",
+            "requirement": "validate the dilution factors used in study sample analysis",
+            "tolerance": f"mean within +/-{crit['dilution_tolerance_pct']:.0f}% nominal",
+        },
+        {
+            "item": "stability",
+            "requirement": "cover the conditions and durations study samples actually experience",
+            "tolerance": f"mean at each QC level within +/-{crit['stability_tolerance_pct']:.0f}% nominal",
+        },
+        {
+            "item": "incurred sample reanalysis",
+            "requirement": (
+                f"repeat a predefined subset in separate runs; at least "
+                f"{crit['isr_pass_fraction'] * 100:.0f}% of repeats must agree"
+            ),
+            "tolerance": f"percent difference within +/-{crit['isr_tolerance_pct']:.0f}%",
+        },
+    ]
+    if crit["total_error_pct"] is not None:
+        rows.append(
+            {
+                "item": "total error",
+                "requirement": "sum of absolute accuracy (%) and precision (%)",
+                "tolerance": (
+                    f"<={crit['total_error_pct']:.0f}%, "
+                    f"<={crit['total_error_pct_at_limits']:.0f}% at LLOQ and ULOQ"
+                ),
+            }
+        )
+    return rows
+
+
+def render_protocol(framework: str, attribute: str | None, modality: str | None,
+                    technique: str | None, range_use: str | None) -> str:
+    fw = FRAMEWORKS[framework]
+    lines = [
+        "# Analytical Procedure Validation Protocol",
+        "",
+        "> Draft skeleton. Every bracketed field is a decision the analyst and quality unit",
+        "> must make and record BEFORE data collection. Acceptance criteria set after seeing",
+        "> data are not acceptance criteria.",
+        "",
+        "## 1. Identification",
+        "",
+        "| Field | Value |",
+        "| --- | --- |",
+        "| Protocol number / version | [ ] |",
+        "| Analytical procedure identifier | [ ] |",
+        "| Product / analyte / matrix | [ ] |",
+        "| Governing framework | " + fw["title"] + " |",
+        "| Regional expectation confirmed | [ ] " + fw["effective_note"] + " |",
+        "| Author / date | [ ] |",
+        "| Reviewed / approved by (quality unit) | [ ] |",
+        "",
+        "## 2. Intended purpose and analytical target profile",
+        "",
+        "- Measurand and reporting unit: [ ]",
+        "- Decision the result supports (release, stability, in-process, clinical): [ ]",
+        "- Specification or reporting limits the procedure must serve: [ ]",
+        "- Required reportable range, derived from the specification: [ ]",
+        "- Performance characteristics and criteria (the ATP, ICH Q14 section 3): [ ]",
+        "",
+        "## 3. Pre-stated acceptance criteria",
+        "",
+        "| Characteristic | Criterion | Justification | Source |",
+        "| --- | --- | --- | --- |",
+    ]
+    if framework == "ich-m10" and modality:
+        for row in plan_m10(modality):
+            lines.append(
+                f"| {row['item']} | {row['tolerance']} | guideline default | ICH M10 |"
+            )
+    elif attribute:
+        for row in plan_q2r2(attribute, technique, range_use):
+            crit = "[ ] state a numeric criterion"
+            lines.append(
+                f"| {row['characteristic']} | {crit} | [ ] | {row['reference'] or 'ICH Q2(R2)'} |"
+            )
+    lines += [
+        "",
+        "> ICH Q2(R2) deliberately does not set numeric acceptance criteria for most",
+        "> characteristics. A criterion has to come from the specification, the ATP, product",
+        "> knowledge, or development data -- not from a remembered default.",
+        "",
+        "## 4. Study design",
+        "",
+        "| Characteristic | Levels | Replicates | Runs / days / analysts / instruments |",
+        "| --- | --- | --- | --- |",
+        "| [ ] | [ ] | [ ] | [ ] |",
+        "",
+        "- Reference materials and their documented identity/purity: [ ]",
+        "- Number of replicates matches the routine reportable result: [ ] yes / [ ] justified",
+        "- Prior knowledge or development data used in place of a test, with justification: [ ]",
+        "",
+        "## 5. Sample and solution handling",
+        "",
+        "- Preparation, storage, and solution stability window: [ ]",
+        "- Blank, placebo, and spiked matrix definitions: [ ]",
+        "",
+        "## 6. Statistical treatment",
+        "",
+        "- Calibration model and weighting, stated in advance: [ ]",
+        "- Interval to be reported with accuracy and precision (confidence level): [ ]",
+        "- Software, version, and calculation verification: [ ]",
+        "",
+        "## 7. Deviations and data integrity",
+        "",
+        "- Deviation handling and reporting: [ ]",
+        "- All results reported, including out-of-criteria values: [ ] confirmed",
+        "- Raw data location and audit trail: [ ]",
+        "",
+        "## 8. Approvals",
+        "",
+        "| Role | Name | Signature | Date |",
+        "| --- | --- | --- | --- |",
+        "| Author | | | |",
+        "| Technical reviewer | | | |",
+        "| Quality unit | | | |",
+        "",
+        f"_Framework metadata researched {RESEARCH_DATE}. Confirm the current guideline text "
+        f"and regional expectation before use: {fw['url']}_",
+    ]
+    return "\n".join(lines)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Plan an analytical procedure validation study.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("--framework", help="governing framework key")
+    parser.add_argument("--attribute", help="measured attribute (assay, impurity, identity, ...)")
+    parser.add_argument("--modality", choices=sorted(M10_CRITERIA),
+                        help="ICH M10 only: chromatographic or lba")
+    parser.add_argument("--technique", choices=sorted(TECHNIQUE_NOTES),
+                        help="analytical technique, for Annex 2 notes")
+    parser.add_argument("--range-use", choices=sorted(Q2R2_REPORTABLE_RANGE),
+                        help="reportable range example to include")
+    parser.add_argument("--protocol", action="store_true",
+                        help="emit a validation protocol skeleton in Markdown")
+    parser.add_argument("--list-frameworks", action="store_true")
+    parser.add_argument("--list-dl-ql", action="store_true",
+                        help="list the DL/QL estimation approaches")
+    add_common_args(parser)
+    args = parser.parse_args()
+
+    if args.list_frameworks:
+        rows = [
+            {
+                "key": key,
+                "title": fw["title"],
+                "adopted": fw["adopted"],
+                "text_reusable": "yes" if fw["reproducible"] else "no (paywalled)",
+                "scope": fw["scope"],
+            }
+            for key, fw in FRAMEWORKS.items()
+        ]
+        emit(rows, args.format)
+        note(f"framework metadata researched {RESEARCH_DATE}; confirm before relying on a date")
+        return EXIT_OK
+
+    if args.list_dl_ql:
+        rows = [
+            {"approach": k, "detection_limit": v["dl"], "quantitation_limit": v["ql"],
+             "note": v["note"]}
+            for k, v in DL_QL_APPROACHES.items()
+        ]
+        emit(rows, args.format)
+        note("source: ICH Q2(R2) 3.2.3")
+        return EXIT_OK
+
+    if not args.framework:
+        raise InputError("--framework is required (or use --list-frameworks)")
+    if args.framework not in FRAMEWORKS:
+        raise InputError(
+            f"unknown framework {args.framework!r}; choose from: {', '.join(FRAMEWORKS)}"
+        )
+    fw = FRAMEWORKS[args.framework]
+
+    attribute = None
+    if args.attribute:
+        try:
+            attribute = resolve_attribute(args.attribute)
+        except KeyError as exc:
+            raise InputError(str(exc)) from exc
+
+    if args.framework == "ich-m10":
+        if not args.modality:
+            raise InputError("ICH M10 needs --modality chromatographic|lba")
+        rows = plan_m10(args.modality)
+    elif args.framework == "ich-q2r2":
+        if not attribute:
+            raise InputError("ICH Q2(R2) needs --attribute (assay, impurity, identity, ...)")
+        rows = plan_q2r2(attribute, args.technique, args.range_use)
+    else:
+        note(
+            f"{fw['title']} is copyrighted and not reproduced here. This skill reports its "
+            "designation, scope, and where to obtain it; the study design must come from the "
+            "authorised text."
+        )
+        rows = [
+            {
+                "framework": args.framework,
+                "title": fw["title"],
+                "scope": fw["scope"],
+                "obtain_from": fw["url"],
+                "companion": fw["companion"],
+            }
+        ]
+
+    if args.protocol:
+        print(render_protocol(args.framework, attribute, args.modality,
+                              args.technique, args.range_use))
+        note("protocol skeleton written; every bracketed field needs a decision before data")
+        return EXIT_OK
+
+    emit(rows, args.format)
+    note(f"framework: {fw['title']}")
+    note(f"regional applicability: {fw['effective_note']}")
+    if fw["companion"]:
+        note(f"companion: {fw['companion']}")
+    note(f"catalogue researched {RESEARCH_DATE}; confirm the current text at {fw['url']}")
+    note("this tool reports requirements; it does not decide fitness for purpose")
+    return EXIT_OK
+
+
+if __name__ == "__main__":
+    run_cli(main)