ictrp-mcp-server 0.1.0

package/sidecar/vendor/ictrp_mcp/ictrp/session.py

@@ -0,0 +1,245 @@
+"""The verified three-step request chain.
+
+    GET  Default.aspx          -> harvest form state
+    POST Default.aspx          -> search, harvest results-page form state
+    POST Default.aspx          -> export CSV
+
+Each step depends on state from the previous one. This is the only part of the
+project carried over from prior verification; everything around it is new.
+
+Measured facts this module encodes (docs/MEASUREMENTS.md):
+
+* The search POST requires `__EVENTVALIDATION` and the cookie set from the GET.
+  Omitting either produces an error page rather than results.
+* The export POST must be built from the results page's own hidden inputs. It
+  works with `Button7=Export to CSV`.
+* A successful export is `200` + `application/vnd.ms-excel` +
+  `attachment;filename=IctrpResults.csv`.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import httpx
+
+from ..errors import ErrorCode, IctrpError
+from . import htmlstate
+from .export_guard import (
+    CsvPayload,
+    classify_export,
+    classify_form_page,
+    parse_reported_total,
+)
+
+BASE_URL = "https://trialsearch.who.int/Default.aspx"
+
+DEFAULT_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+        "(KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"
+    ),
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Cache-Control": "no-cache",
+    "Pragma": "no-cache",
+}
+
+#: The export button. Verified working.
+EXPORT_CONTROL = "Button7"
+
+#: The search submit button and text box on the form page.
+SEARCH_BUTTON = "Button1"
+SEARCH_TEXTBOX = "TextBox1"
+
+
+def _check_present(state: htmlstate.FormState, *, step: str) -> None:
+    missing = state.missing_state()
+    if missing:
+        raise IctrpError(
+            ErrorCode.SESSION_FAILED,
+            f"{step}: form state incomplete (missing {', '.join(missing)})",
+            detail=(
+                "The page did not carry the hidden inputs this step needs. "
+                "Either the page was an error/interstitial, or the portal changed."
+            ),
+        )
+
+
+@dataclass
+class IctrpSession:
+    """One logical search+export exchange.
+
+    A fresh session per search keeps state handling simple and avoids reusing a
+    `__VIEWSTATE` past its lifetime. Redirects are NOT followed automatically,
+    because a `302` to `/NoAccess.aspx` is a failure we must classify rather
+    than silently follow.
+    """
+
+    client: httpx.AsyncClient
+    base_url: str = BASE_URL
+    last_html: str = ""
+    form_state: htmlstate.FormState | None = None
+    results_state: htmlstate.FormState | None = None
+    reported_total: int | None = None
+    response_date: str | None = None
+    steps: list[str] = field(default_factory=list)
+
+    @classmethod
+    def create(cls, *, timeout: float = 60.0) -> "IctrpSession":
+        client = httpx.AsyncClient(
+            headers=DEFAULT_HEADERS,
+            timeout=timeout,
+            follow_redirects=False,
+        )
+        return cls(client=client)
+
+    async def __aenter__(self) -> "IctrpSession":
+        return self
+
+    async def __aexit__(self, *exc: object) -> None:
+        await self.aclose()
+
+    async def aclose(self) -> None:
+        await self.client.aclose()
+
+    # ---- step 1 -----------------------------------------------------------
+
+    async def load_form(self) -> htmlstate.FormState:
+        try:
+            response = await self.client.get(self.base_url)
+        except httpx.HTTPError as exc:
+            raise IctrpError(
+                ErrorCode.UPSTREAM_ERROR,
+                f"Could not reach the ICTRP search portal: {exc}",
+                hint="Check network connectivity, then retry.",
+            ) from exc
+
+        html = classify_form_page(status=response.status_code, body=response.content)
+        self.steps.append(f"GET {self.base_url} -> {response.status_code}")
+        state = htmlstate.extract_form_state(html)
+        _check_present(state, step="load_form")
+        self.form_state = state
+        self.last_html = html
+        return state
+
+    # ---- step 2 -----------------------------------------------------------
+
+    async def search(self, keyword: str) -> str:
+        if not keyword or not keyword.strip():
+            raise IctrpError(
+                ErrorCode.INVALID_ARGUMENT,
+                "keyword must be a non-empty string",
+            )
+        if self.form_state is None:
+            await self.load_form()
+        assert self.form_state is not None
+
+        body = self.form_state.merged_with(
+            {SEARCH_TEXTBOX: keyword, SEARCH_BUTTON: "Search"}
+        )
+        try:
+            response = await self.client.post(
+                self.base_url,
+                data=body,
+                headers={
+                    "Content-Type": "application/x-www-form-urlencoded",
+                    "Referer": self.base_url,
+                },
+            )
+        except httpx.HTTPError as exc:
+            raise IctrpError(
+                ErrorCode.UPSTREAM_ERROR,
+                f"Search request failed: {exc}",
+            ) from exc
+
+        location = response.headers.get("location")
+        if location and "/noaccess.aspx" in location.lower():
+            raise IctrpError(
+                ErrorCode.UPSTREAM_CONTRACT_DRIFT,
+                "Search POST was redirected to NoAccess.aspx",
+                upstream_status=response.status_code,
+                hint=(
+                    "The search POST was rejected. Verify that __EVENTVALIDATION "
+                    "and the GET-set cookies were replayed."
+                ),
+            )
+        if response.status_code in (403, 405, 429, 503):
+            raise IctrpError(
+                ErrorCode.UPSTREAM_BLOCKED,
+                f"Search was refused (HTTP {response.status_code})",
+                upstream_status=response.status_code,
+                hint="The portal blocked this request. This is not an empty result set.",
+            )
+
+        html = response.text
+        self.steps.append(f"POST search {keyword!r} -> {response.status_code}")
+
+        state = htmlstate.extract_form_state(html)
+        if state.missing_state():
+            raise IctrpError(
+                ErrorCode.UPSTREAM_CONTRACT_DRIFT,
+                "Results page did not carry WebForms state",
+                upstream_status=response.status_code,
+                hint=(
+                    "The search did not produce a results page. The response may be "
+                    "an error page rather than a result set."
+                ),
+            )
+        self.results_state = state
+        self.last_html = html
+        self.reported_total = parse_reported_total(html)
+        return html
+
+    # ---- step 3 -----------------------------------------------------------
+
+    async def export_csv(self) -> CsvPayload:
+        if self.results_state is None:
+            raise IctrpError(
+                ErrorCode.SESSION_FAILED,
+                "export_csv called before a successful search",
+            )
+
+        body = htmlstate.build_export_body(self.last_html, export_control=EXPORT_CONTROL)
+        # Fail loudly here rather than upstream, so the measured 302 failure mode
+        # can never be reintroduced silently.
+        htmlstate.assert_export_body_excludes_search_controls(body)
+
+        try:
+            response = await self.client.post(
+                self.base_url,
+                data=body,
+                headers={
+                    "Content-Type": "application/x-www-form-urlencoded",
+                    "Referer": self.base_url,
+                },
+            )
+        except httpx.HTTPError as exc:
+            raise IctrpError(
+                ErrorCode.UPSTREAM_ERROR,
+                f"Export request failed: {exc}",
+            ) from exc
+
+        self.steps.append(f"POST export -> {response.status_code}")
+        self.response_date = response.headers.get("date")
+
+        return classify_export(
+            status=response.status_code,
+            content_type=response.headers.get("content-type"),
+            content_disposition=response.headers.get("content-disposition"),
+            body=response.content,
+            location=response.headers.get("location"),
+            reported_total=self.reported_total,
+        )
+
+    def provenance_steps(self) -> list[str]:
+        return list(self.steps)
+
+
+async def run_chain(keyword: str, *, timeout: float = 120.0) -> tuple[CsvPayload, IctrpSession]:
+    """Run the full chain and return the payload plus the session for provenance."""
+    session = IctrpSession.create(timeout=timeout)
+    await session.load_form()
+    await session.search(keyword)
+    payload = await session.export_csv()
+    return payload, session

package/sidecar/vendor/ictrp_mcp/offline.py

@@ -0,0 +1,133 @@
+"""Offline bundle support.
+
+A packaged desktop application cannot assume a working network on first launch,
+and re-running a three-request upstream chain on every cold start is both slow and
+rude to a public service. So the service can read a pre-built snapshot instead.
+
+Resolution order for the active offline source:
+
+1. `ICTRP_BUNDLE_PATH` -- an explicit file, used by tests and by operators who
+   want to pin a specific dataset.
+2. `ICTRP_BUNDLE_DIR` -- a directory holding `<keyword-slug>.json` snapshots.
+3. The user cache directory, which is where `ictrp_snapshot` writes by default.
+
+A bundle is never silent. Every response served from one carries
+`provenance.offline_snapshot` with the snapshot's age, so a stale shipped dataset
+is visible in the output rather than hidden behind a plausible-looking result.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from pathlib import Path
+
+from .data.jsonio import Snapshot, locate_bundle
+from .errors import ErrorCode, IctrpError
+
+#: A shipped snapshot older than this is flagged loudly. WHO refreshes weekly, so
+#: four weeks is well past the point where the data should have been rebuilt.
+DEFAULT_MAX_AGE_DAYS = 28.0
+
+ENV_BUNDLE_PATH = "ICTRP_BUNDLE_PATH"
+ENV_BUNDLE_DIR = "ICTRP_BUNDLE_DIR"
+ENV_MAX_AGE_DAYS = "ICTRP_BUNDLE_MAX_AGE_DAYS"
+
+
+def snapshot_slug(keyword: str) -> str:
+    """Filesystem-safe name for a keyword."""
+    slug = re.sub(r"[^a-z0-9]+", "-", keyword.strip().lower()).strip("-")
+    return slug or "all"
+
+
+def bundle_dir() -> Path | None:
+    override = os.environ.get(ENV_BUNDLE_DIR)
+    if override:
+        return Path(override)
+    return None
+
+
+def max_age_days() -> float:
+    raw = os.environ.get(ENV_MAX_AGE_DAYS)
+    if raw:
+        try:
+            return float(raw)
+        except ValueError:
+            return DEFAULT_MAX_AGE_DAYS
+    return DEFAULT_MAX_AGE_DAYS
+
+
+def candidate_paths(keyword: str) -> list[Path]:
+    """Where a snapshot for this keyword might live, best candidate first."""
+    candidates: list[Path] = []
+
+    explicit = os.environ.get(ENV_BUNDLE_PATH)
+    if explicit:
+        # An explicit path wins outright, but only if it is for this keyword --
+        # otherwise a pinned bundle would silently answer unrelated searches.
+        path = Path(explicit)
+        if path.is_dir():
+            candidates.append(path / f"{snapshot_slug(keyword)}.json")
+        else:
+            candidates.append(path)
+
+    directory = bundle_dir()
+    if directory:
+        candidates.append(directory / f"{snapshot_slug(keyword)}.json")
+
+    return candidates
+
+
+def load_bundle(keyword: str) -> tuple[Snapshot, Path] | None:
+    """Load a snapshot for this keyword, or None when there is not one.
+
+    Returns None rather than raising: "no bundle here" is the normal case for a
+    live installation, and the caller falls through to the network.
+    """
+    path = locate_bundle(candidate_paths(keyword))
+    if path is None:
+        return None
+
+    if path.is_dir():
+        raise IctrpError(
+            ErrorCode.INVALID_ARGUMENT,
+            f"bundle path {str(path)!r} is a directory",
+            hint="Point ICTRP_BUNDLE_PATH at a .json snapshot file.",
+        )
+
+    snapshot = Snapshot.read(path)
+
+    # A single-file bundle configured by path is assumed to be for the keyword it
+    # was written for, but a mismatched keyword means it cannot answer this query.
+    if snapshot.keyword and snapshot.keyword.strip().lower() != keyword.strip().lower():
+        return None
+
+    return snapshot, path
+
+
+def snapshot_provenance(snapshot: Snapshot, path: Path) -> dict:
+    """Provenance fields describing an offline-served response."""
+    stale = snapshot.stale(max_age_days())
+    info: dict = {
+        "offline_snapshot": True,
+        "snapshot_path": str(path),
+        "snapshot_created_at": _iso(snapshot.created_at),
+        "snapshot_age_days": round(snapshot.age_days, 2),
+        "snapshot_trial_count": len(snapshot.trials),
+    }
+    if stale:
+        info["snapshot_stale"] = True
+        info["snapshot_stale_warning"] = (
+            f"This snapshot is {snapshot.age_days:.1f} days old, beyond the "
+            f"{max_age_days():.0f}-day freshness limit. WHO refreshes ICTRP weekly. "
+            "Refresh the snapshot before relying on this data."
+        )
+    else:
+        info["snapshot_stale"] = False
+    return info
+
+
+def _iso(epoch: float) -> str:
+    import time
+
+    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(epoch))

package/sidecar/vendor/ictrp_mcp/provenance.py

@@ -0,0 +1,182 @@
+"""Provenance.
+
+Every tool response carries this. The point is that a caller can always answer:
+where did this come from, when did WHO last process it, and what is missing.
+
+Two things are kept rigorously separate, because conflating them is the most
+likely way for this service to mislead:
+
+* `rows_returned` -- what we actually hold.
+* `upstream_reported_total` -- what the portal claimed matched.
+
+The export is provably incomplete (docs/MEASUREMENTS.md section 2), so neither
+number alone may be presented as "how many trials exist". When they disagree, the
+response says so explicitly.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+SOURCE_NAME = "WHO ICTRP"
+
+SOURCE_URL = "https://trialsearch.who.int/"
+
+#: Attribution wording required by the ICTRP terms of use.
+ATTRIBUTION = (
+    "Data source: WHO International Clinical Trials Registry Platform (ICTRP). "
+    "ICTRP data are publicly available for download from the ICTRP Search Portal. "
+    "WHO updates ICTRP weekly."
+)
+
+#: Stated on every response so the incompleteness can never be missed.
+INCOMPLETENESS_NOTICE = (
+    "The ICTRP CSV export is known to omit records that the search portal itself "
+    "reports as matches (measured shortfalls from 0.4% to 29% depending on query). "
+    "A record absent from a result set is therefore NOT evidence that it does not "
+    "exist. Treat counts from this service as counts of retrieved rows, not as "
+    "complete counts of matching trials."
+)
+
+
+@dataclass
+class Provenance:
+    """Where a response's data came from and how current it is."""
+
+    source: str = SOURCE_NAME
+    source_url: str = SOURCE_URL
+    retrieved_at: str = ""
+    upstream_reported_total: int | None = None
+    rows_returned: int | None = None
+    records_incomplete: bool = False
+    estimated_missing: int | None = None
+    ictrp_export_date: str | None = None
+    ictrp_last_refreshed: str | None = None
+    request_steps: list[str] = field(default_factory=list)
+    cache_hit: bool = False
+    notes: list[str] = field(default_factory=list)
+    # Provenance keys this dataclass does not model, carried through untouched.
+    #
+    # The offline path is the reason this exists. A snapshot provenance carries
+    # `offline_snapshot`, `snapshot_path`, `snapshot_age_days` and friends, and
+    # those are exactly the fields that tell a caller "this data did not come
+    # from a live query". `search()` round-trips through this class before
+    # answering, so anything not modelled here is silently dropped -- which
+    # would quietly strip the offline labelling and leave snapshot data
+    # looking indistinguishable from live data. Modelling the whole set is not
+    # an option: the snapshot shape is free to grow.
+    extra: dict[str, Any] = field(default_factory=dict)
+
+    @classmethod
+    def now(cls, **kwargs: Any) -> "Provenance":
+        return cls(retrieved_at=_utc_now(), **kwargs)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "Provenance":
+        """Rebuild from `to_dict()` output.
+
+        `to_dict()` is a presentation shape: it adds derived keys like
+        `attribution` and `incompleteness_notice`, which are not constructor
+        fields. They are accepted and ignored here so a stored provenance can be
+        round-tripped without the caller stripping them first.
+
+        Anything else this class does not model is preserved in `extra` rather
+        than discarded, so a round-trip does not lose provenance detail that
+        some other layer added.
+        """
+        known = {
+            "source", "source_url", "retrieved_at", "upstream_reported_total",
+            "rows_returned", "records_incomplete", "estimated_missing",
+            "ictrp_export_date", "ictrp_last_refreshed", "request_steps",
+            "cache_hit", "notes",
+        }
+        # Derived on every serialization, so never re-adopted from input.
+        derived = {"attribution", "incompleteness_notice"}
+        extra = {
+            k: v
+            for k, v in data.items()
+            if k not in known and k not in derived
+        }
+        return cls(**{k: v for k, v in data.items() if k in known}, extra=extra)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialized form. Attribution and the incompleteness notice always ride along."""
+        out: dict[str, Any] = {
+            "source": self.source,
+            "source_url": self.source_url,
+            "retrieved_at": self.retrieved_at,
+            "attribution": ATTRIBUTION,
+        }
+        if self.upstream_reported_total is not None:
+            out["upstream_reported_total"] = self.upstream_reported_total
+        if self.rows_returned is not None:
+            out["rows_returned"] = self.rows_returned
+        out["records_incomplete"] = self.records_incomplete
+        if self.estimated_missing is not None:
+            out["estimated_missing"] = self.estimated_missing
+        if self.ictrp_export_date:
+            out["ictrp_export_date"] = self.ictrp_export_date
+        if self.ictrp_last_refreshed:
+            out["ictrp_last_refreshed"] = self.ictrp_last_refreshed
+        if self.request_steps:
+            out["request_steps"] = list(self.request_steps)
+        out["cache_hit"] = self.cache_hit
+        if self.records_incomplete:
+            out["incompleteness_notice"] = INCOMPLETENESS_NOTICE
+        elif not self.cache_hit:
+            # Even a complete-looking response carries the caveat: we can never
+            # prove from a single export that nothing was withheld.
+            out["incompleteness_notice"] = INCOMPLETENESS_NOTICE
+        if self.notes:
+            out["notes"] = list(self.notes)
+        # Keys this class does not model, restored verbatim.
+        for key, value in self.extra.items():
+            out.setdefault(key, value)
+        return out
+
+
+def _utc_now() -> str:
+    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+
+
+def derive_set_provenance(
+    *,
+    keyword: str,
+    trials: list[dict[str, Any]],
+    reported_total: int | None,
+    response_date: str | None,
+    steps: list[str],
+) -> Provenance:
+    """Build provenance from a freshly materialized export."""
+    rows = len(trials)
+    export_dates = sorted(
+        {t.get("export_date_raw") for t in trials if t.get("export_date_raw")}
+    )
+    refreshed = sorted(
+        {t.get("last_refreshed_display") for t in trials if t.get("last_refreshed_display")}
+    )
+
+    provenance = Provenance.now(
+        upstream_reported_total=reported_total,
+        rows_returned=rows,
+        records_incomplete=bool(reported_total is not None and reported_total > rows),
+        estimated_missing=(
+            max(0, reported_total - rows) if reported_total is not None else None
+        ),
+        ictrp_export_date=export_dates[0] if export_dates else None,
+        ictrp_last_refreshed=refreshed[0] if refreshed else None,
+        request_steps=list(steps),
+        cache_hit=False,
+    )
+
+    provenance.notes.append(f"Search keyword: {keyword!r}.")
+    if refreshed:
+        provenance.notes.append(
+            f"Record-level 'Last Refreshed on' values span {len(refreshed)} distinct "
+            f"dates in this set; the earliest is shown. Per-record values are on each trial."
+        )
+    if response_date:
+        provenance.notes.append(f"Upstream response Date header: {response_date}.")
+    return provenance