ictrp-mcp-server 0.1.0

package/sidecar/vendor/ictrp_mcp/cache/store.py

@@ -0,0 +1,313 @@
+"""Caching for materialized result sets.
+
+The governing constraint is cost: a single export can be tens of megabytes, and
+the chain behind it is three round trips to a public service we should not hammer.
+So a search result is materialized once and then queried locally.
+
+Two deliberate choices:
+
+* Raw CSV bytes are written to disk instead of holding every row in memory for
+  every live set. A large set is only parsed when it is actually used.
+* Only a small number of sets stay in memory. Older ones are dropped (and can be
+  re-read from disk), which bounds the process footprint.
+
+Nothing here caches a failure as if it were data. A set only enters the store
+after it has been validated.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from ..errors import ErrorCode, IctrpError
+from ..data.normalize import to_trial
+
+#: Parsed sets held in memory at once.
+MAX_MATERIALIZED_SETS = 3
+
+#: How long a materialized set stays queryable.
+DEFAULT_SET_TTL_SECONDS = 7 * 24 * 60 * 60
+
+#: How long a single-trial lookup stays fresh.
+DEFAULT_TRIAL_TTL_SECONDS = 24 * 60 * 60
+
+
+def normalize_query(keyword: str) -> str:
+    """Collapse a keyword to a stable cache key component."""
+    return re.sub(r"\s+", " ", keyword.strip().lower())
+
+
+def query_key(keyword: str) -> str:
+    digest = hashlib.sha256(normalize_query(keyword).encode()).hexdigest()[:16]
+    return f"search:{digest}"
+
+
+def trial_key(trial_id: str) -> str:
+    return f"trial:{trial_id.strip().upper()}"
+
+
+def default_cache_dir() -> Path:
+    override = os.environ.get("ICTRP_CACHE_DIR")
+    if override:
+        return Path(override)
+    return Path.home() / ".cache" / "ictrp-mcp-service"
+
+
+@dataclass
+class MaterializedSet:
+    """One export, parsed into canonical trials."""
+
+    set_id: str
+    keyword: str
+    trials: list[dict[str, Any]]
+    header: tuple[str, ...]
+    reported_total: int | None
+    csv_bytes: int
+    created_at: float
+    provenance: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def rows_returned(self) -> int:
+        """How many rows we actually hold.
+
+        Never described as a total. The portal's own figure lives separately in
+        `reported_total`, and the two are known to differ by 0.4%-29%.
+        """
+        return len(self.trials)
+
+    @property
+    def is_incomplete(self) -> bool:
+        """True when the portal reported more matches than the export delivered."""
+        if self.reported_total is None:
+            return False
+        return self.reported_total > self.rows_returned
+
+    @property
+    def missing_estimate(self) -> int | None:
+        if self.reported_total is None:
+            return None
+        return max(0, self.reported_total - self.rows_returned)
+
+    def age_seconds(self) -> float:
+        return time.time() - self.created_at
+
+    def summary(self) -> dict[str, Any]:
+        return {
+            "set_id": self.set_id,
+            "keyword": self.keyword,
+            "rows_returned": self.rows_returned,
+            "upstream_reported_total": self.reported_total,
+            "records_incomplete": self.is_incomplete,
+            "estimated_missing": self.missing_estimate,
+            "csv_bytes": self.csv_bytes,
+            "age_seconds": round(self.age_seconds(), 1),
+            "created_at": _iso(self.created_at),
+        }
+
+
+def _iso(epoch: float) -> str:
+    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(epoch))
+
+
+class SetStore:
+    """Holds materialized sets in memory, backed by raw CSV on disk."""
+
+    def __init__(self, cache_dir: Path | None = None, ttl_seconds: int = DEFAULT_SET_TTL_SECONDS):
+        self.cache_dir = cache_dir or default_cache_dir()
+        self.ttl_seconds = ttl_seconds
+        self._sets: dict[str, MaterializedSet] = {}
+        self._order: list[str] = []
+
+    def _raw_path(self, cache_key: str) -> Path:
+        safe = re.sub(r"[^A-Za-z0-9_.-]", "_", cache_key)
+        return self.cache_dir / f"{safe}.csv"
+
+    def _meta_path(self, cache_key: str) -> Path:
+        safe = re.sub(r"[^A-Za-z0-9_.-]", "_", cache_key)
+        return self.cache_dir / f"{safe}.json"
+
+    def _ensure_dir(self) -> None:
+        try:
+            self.cache_dir.mkdir(parents=True, exist_ok=True)
+        except OSError:
+            # Fall back to a temp location rather than failing the request.
+            self.cache_dir = Path("/tmp/ictrp-mcp-service")
+            self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+    def store(
+        self,
+        *,
+        keyword: str,
+        payload_raw: bytes,
+        header: tuple[str, ...],
+        rows: list[list[str]],
+        reported_total: int | None,
+        provenance: dict[str, Any],
+    ) -> MaterializedSet:
+        """Persist and materialize a validated export."""
+        cache_key = query_key(keyword)
+        trials = [to_trial(row, header) for row in rows]
+        materialized = MaterializedSet(
+            set_id=cache_key,
+            keyword=keyword,
+            trials=trials,
+            header=header,
+            reported_total=reported_total,
+            csv_bytes=len(payload_raw),
+            created_at=time.time(),
+            provenance=dict(provenance),
+        )
+
+        self._ensure_dir()
+        try:
+            self._raw_path(cache_key).write_bytes(payload_raw)
+            self._meta_path(cache_key).write_text(
+                json.dumps(
+                    {
+                        "keyword": keyword,
+                        "header": list(header),
+                        "reported_total": reported_total,
+                        "created_at": materialized.created_at,
+                        "csv_bytes": materialized.csv_bytes,
+                        "provenance": provenance,
+                    }
+                ),
+                encoding="utf-8",
+            )
+        except OSError:
+            # Disk caching is best-effort; the in-memory set is still usable.
+            pass
+
+        self._remember(materialized)
+        return materialized
+
+    def adopt(
+        self,
+        *,
+        keyword: str,
+        trials: list[dict[str, Any]],
+        provenance: dict[str, Any],
+        created_at: float | None = None,
+        source_label: str = "snapshot",
+    ) -> MaterializedSet:
+        """Register an externally built set (e.g. from an offline snapshot).
+
+        `created_at` is carried over from the snapshot so the set's age reflects
+        when the data was retrieved, not when it was loaded. A snapshot read today
+        from a month-old file must still report as a month old.
+
+        The set is memory-resident only. Writing it back as a raw CSV would
+        fabricate an export we never received, and `_reload_from_disk` would then
+        present it as a genuine export on the next run.
+        """
+        materialized = MaterializedSet(
+            set_id=query_key(keyword),
+            keyword=keyword,
+            trials=list(trials),
+            header=(),
+            reported_total=provenance.get("upstream_reported_total"),
+            csv_bytes=0,
+            created_at=created_at if created_at is not None else time.time(),
+            provenance=dict(provenance),
+        )
+        materialized.provenance.setdefault("notes", [])
+        self._remember(materialized)
+        return materialized
+
+    def _remember(self, materialized: MaterializedSet) -> None:
+        self._sets[materialized.set_id] = materialized
+        if materialized.set_id in self._order:
+            self._order.remove(materialized.set_id)
+        self._order.append(materialized.set_id)
+        while len(self._order) > MAX_MATERIALIZED_SETS:
+            evicted = self._order.pop(0)
+            self._sets.pop(evicted, None)
+
+    def get(self, set_id: str) -> MaterializedSet:
+        """Return a set, reloading skipped ones from disk when possible."""
+        materialized = self._sets.get(set_id)
+        if materialized is not None and materialized.age_seconds() <= self.ttl_seconds:
+            return materialized
+        if materialized is not None:
+            self._sets.pop(set_id, None)
+            if set_id in self._order:
+                self._order.remove(set_id)
+
+        reloaded = self._reload_from_disk(set_id)
+        if reloaded is not None:
+            self._remember(reloaded)
+            return reloaded
+
+        raise IctrpError(
+            ErrorCode.CACHE_MISS,
+            f"result set {set_id!r} is not available",
+            hint=(
+                "Result sets are held for a limited time and only a few at once. "
+                "Run the search again to re-materialize it."
+            ),
+        )
+
+    def _reload_from_disk(self, set_id: str) -> MaterializedSet | None:
+        meta_path = self._meta_path(set_id)
+        raw_path = self._raw_path(set_id)
+        if not meta_path.exists() or not raw_path.exists():
+            return None
+        try:
+            meta = json.loads(meta_path.read_text(encoding="utf-8"))
+            raw = raw_path.read_bytes()
+        except (OSError, json.JSONDecodeError):
+            return None
+
+        created_at = float(meta.get("created_at", 0.0))
+        if time.time() - created_at > self.ttl_seconds:
+            return None
+
+        header, rows = _parse_raw(raw)
+        if not header:
+            return None
+        return MaterializedSet(
+            set_id=set_id,
+            keyword=meta.get("keyword", ""),
+            trials=[to_trial(r, header) for r in rows],
+            header=header,
+            reported_total=meta.get("reported_total"),
+            csv_bytes=meta.get("csv_bytes", len(raw)),
+            created_at=created_at,
+            provenance=meta.get("provenance", {}) or {},
+        )
+
+    def list_sets(self) -> list[dict[str, Any]]:
+        return [self._sets[s].summary() for s in self._order if s in self._sets]
+
+    def purge(self, set_id: str | None = None) -> int:
+        if set_id is None:
+            count = len(self._sets)
+            self._sets.clear()
+            self._order.clear()
+            return count
+        removed = 1 if self._sets.pop(set_id, None) is not None else 0
+        if set_id in self._order:
+            self._order.remove(set_id)
+        return removed
+
+
+def _parse_raw(raw: bytes) -> tuple[tuple[str, ...], list[list[str]]]:
+    import csv as _csv
+    import io as _io
+
+    text = raw.decode("utf-8-sig", "replace")
+    reader = _csv.reader(_io.StringIO(text))
+    try:
+        header = tuple(h.strip() for h in next(reader))
+    except StopIteration:
+        return (), []
+    rows = [r for r in reader if r and any(c.strip() for c in r)]
+    width = len(header)
+    return header, [(r + [""] * width)[:width] for r in rows]

package/sidecar/vendor/ictrp_mcp/data/columns.py

@@ -0,0 +1,108 @@
+"""The 58 ICTRP CSV column headers, as literal strings.
+
+These are reproduced VERBATIM from the live export, including three
+misspellings present in the upstream data:
+
+    'Inclusion agemin'      (not 'agemin' -> 'age_min')
+    'Inclusion agemax'
+    'Date enrollement'      (not 'enrollment')
+
+Do NOT "correct" these. The parser matches on the exact upstream strings; a
+helpful-looking normalisation here would silently produce all-null fields.
+`tests/test_columns.py` asserts the exact set so an upstream rename surfaces as
+a failing test rather than a silently degraded response.
+"""
+
+from __future__ import annotations
+
+#: Column order as returned by the live export. Order is not load-bearing for
+#: parsing (we match by name) but is asserted for drift detection.
+CSV_COLUMNS: tuple[str, ...] = (
+    "TrialID",
+    "Last Refreshed on",
+    "Public title",
+    "Scientific title",
+    "Acronym",
+    "Primary sponsor",
+    "Date registration",
+    "Date registration3",
+    "Export date",
+    "Source Register",
+    "web address",
+    "Recruitment Status",
+    "other records",
+    "Inclusion agemin",
+    "Inclusion agemax",
+    "Inclusion gender",
+    "Date enrollement",
+    "Target size",
+    "Study type",
+    "Study design",
+    "Phase",
+    "Countries",
+    "Contact Firstname",
+    "Contact Lastname",
+    "Contact Address",
+    "Contact Email",
+    "Contact Tel",
+    "Contact Affiliation",
+    "Inclusion Criteria",
+    "Exclusion Criteria",
+    "Condition",
+    "Intervention",
+    "Primary outcome",
+    "Secondary outcome",
+    "Secondary ID",
+    "Source Name",
+    "Secondary Sponsor",
+    "Ethics Status",
+    "Ethics Approval Date",
+    "Ethics Contact Name",
+    "Ethics Contact Address",
+    "Ethics Contact Phone",
+    "Ethics Contact Email",
+    "results yes no",
+    "results date posted",
+    "results url link",
+    "results url protocol",
+    "results date completed",
+    "results date first publication",
+    "results summary",
+    "results baseline char",
+    "results adverse events",
+    "results outcome measures",
+    "results ipd plan",
+    "results ipd description",
+    "Prospective registration",
+    "Bridging flag truefalse",
+    "Bridged type",
+)
+
+#: Columns without which a response cannot be treated as a valid ICTRP export.
+#: A CSV missing any of these is `UpstreamContractDrift`, never "no results".
+REQUIRED_COLUMNS: frozenset[str] = frozenset(
+    {
+        "TrialID",
+        "Source Register",
+        "Public title",
+        "Scientific title",
+        "Recruitment Status",
+        "Date registration",
+        "Last Refreshed on",
+        "Export date",
+        "Phase",
+        "Condition",
+    }
+)
+
+#: Columns where a new value indicates upstream vocabulary drift and should be
+#: surfaced rather than silently absorbed into a facet list.
+FACET_COLUMNS: frozenset[str] = frozenset(
+    {"Source Register", "Phase", "Recruitment Status", "Study type", "Inclusion gender", "Countries"}
+)
+
+EXPECTED_COLUMN_COUNT = len(CSV_COLUMNS)
+
+#: Sanity assertion at import time: the tuple is the shape we measured.
+assert EXPECTED_COLUMN_COUNT == 58, f"expected 58 columns, got {EXPECTED_COLUMN_COUNT}"
+assert REQUIRED_COLUMNS <= set(CSV_COLUMNS), "REQUIRED_COLUMNS contains an unknown column"

package/sidecar/vendor/ictrp_mcp/data/jsonio.py

@@ -0,0 +1,213 @@
+"""Canonical-JSON persistence for result sets.
+
+Why this exists
+---------------
+Two independent needs converge on the same format:
+
+1. **Distribution.** A packaged application (pi-desktop and friends) wants trials
+   available without a network round trip on first launch. That requires a file
+   the application can ship and read directly.
+2. **Durability.** The CSV cache in `cache/store.py` stores raw export bytes,
+   which are tied to one export's exact header. Canonical JSON is decoupled from
+   the upstream column set, so a snapshot stays readable across parser changes.
+
+The format is deliberately boring: one JSON object, a `trials` array of canonical
+trial dicts, and a `snapshot` block of provenance. No schema version bump will be
+silently ignored -- a reader refuses a `format_version` it does not know.
+
+Licensing note
+--------------
+WHO ICTRP terms (section 4c) state you "shall not assert any proprietary rights
+to any portion of the ICTRP database", and 4d bars commercial use of extracted
+information. A snapshot produced here is not our property, and redistributing it
+is the operator's decision, not a technical default. `snapshot.attribution` and
+`snapshot.terms_notice` are therefore written into every file so a snapshot
+cannot be separated from its terms. See docs/BUNDLE.md.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterable
+
+from ..errors import ErrorCode, IctrpError
+
+#: Bump only for a breaking change to the object shape. Readers reject unknown values.
+FORMAT_VERSION = 1
+
+#: Snapshot kind value. Distinguishes a shipped bundle from a user cache dump.
+KIND_BUNDLE = "ictrp-trial-set"
+
+TERMS_NOTICE = (
+    "Data from the WHO International Clinical Trials Registry Platform (ICTRP). "
+    "ICTRP data are publicly available for download from the ICTRP Search Portal "
+    "at no charge. WHO updates ICTRP weekly. Under the ICTRP terms of use you may "
+    "not assert proprietary rights to any portion of the ICTRP database, and may "
+    "not use the data for marketing, promotional or commercial purposes. This "
+    "snapshot is not a WHO product and is not endorsed by WHO."
+)
+
+
+@dataclass
+class Snapshot:
+    """A trial set plus the provenance needed to describe it honestly."""
+
+    trials: list[dict[str, Any]]
+    keyword: str
+    created_at: float
+    provenance: dict[str, Any]
+    format_version: int = FORMAT_VERSION
+    kind: str = KIND_BUNDLE
+
+    @property
+    def age_seconds(self) -> float:
+        return time.time() - self.created_at
+
+    @property
+    def age_days(self) -> float:
+        return self.age_seconds / 86400.0
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "kind": self.kind,
+            "format_version": self.format_version,
+            "snapshot": {
+                "keyword": self.keyword,
+                "created_at": _iso(self.created_at),
+                "created_at_epoch": self.created_at,
+                "trial_count": len(self.trials),
+                "attribution": (
+                    "Source: WHO International Clinical Trials Registry Platform (ICTRP)."
+                ),
+                "terms_notice": TERMS_NOTICE,
+                "provenance": self.provenance,
+            },
+            "trials": self.trials,
+        }
+
+    def write(self, path: Path) -> int:
+        """Write the snapshot, returning bytes written."""
+        path.parent.mkdir(parents=True, exist_ok=True)
+        body = json.dumps(self.to_dict(), ensure_ascii=False, indent=1)
+        tmp = path.with_suffix(path.suffix + ".tmp")
+        tmp.write_text(body, encoding="utf-8")
+        tmp.replace(path)
+        return len(body.encode("utf-8"))
+
+    @classmethod
+    def read(cls, path: Path) -> "Snapshot":
+        """Load a snapshot, refusing anything structurally unexpected.
+
+        Every failure raises rather than returning an empty set: an unreadable
+        bundle must never look like "no trials matched".
+        """
+        try:
+            raw = path.read_text(encoding="utf-8")
+        except OSError as exc:
+            raise IctrpError(
+                ErrorCode.CACHE_MISS,
+                f"snapshot {str(path)!r} could not be read: {exc}",
+                hint="Check the path and that the file was shipped with the application.",
+            ) from exc
+
+        try:
+            data = json.loads(raw)
+        except json.JSONDecodeError as exc:
+            raise IctrpError(
+                ErrorCode.UPSTREAM_CONTRACT_DRIFT,
+                f"snapshot {str(path)!r} is not valid JSON: {exc}",
+                hint="The file is truncated or was not produced by this service.",
+            ) from exc
+
+        if not isinstance(data, dict):
+            raise IctrpError(
+                ErrorCode.UPSTREAM_CONTRACT_DRIFT,
+                f"snapshot {str(path)!r} must be a JSON object",
+            )
+
+        version = data.get("format_version")
+        if version != FORMAT_VERSION:
+            raise IctrpError(
+                ErrorCode.UPSTREAM_CONTRACT_DRIFT,
+                f"snapshot {str(path)!r} has format_version {version!r}; "
+                f"this build reads version {FORMAT_VERSION}",
+                hint="Regenerate the snapshot with a matching build.",
+            )
+
+        trials = data.get("trials")
+        if not isinstance(trials, list):
+            raise IctrpError(
+                ErrorCode.UPSTREAM_CONTRACT_DRIFT,
+                f"snapshot {str(path)!r} has no 'trials' array",
+            )
+        for index, trial in enumerate(trials[:5]):
+            if not isinstance(trial, dict):
+                raise IctrpError(
+                    ErrorCode.UPSTREAM_CONTRACT_DRIFT,
+                    f"snapshot {str(path)!r} trial #{index} is not an object",
+                )
+
+        meta = data.get("snapshot") or {}
+        created_epoch = meta.get("created_at_epoch")
+        if not isinstance(created_epoch, (int, float)):
+            created_epoch = _parse_iso_epoch(meta.get("created_at"))
+
+        return cls(
+            trials=trials,
+            keyword=str(meta.get("keyword", "")),
+            created_at=float(created_epoch),
+            provenance=meta.get("provenance") or {},
+            format_version=FORMAT_VERSION,
+            kind=str(data.get("kind", KIND_BUNDLE)),
+        )
+
+    def stale(self, max_age_days: float) -> bool:
+        """True when the snapshot is older than the permitted age."""
+        if max_age_days <= 0:
+            return False
+        return self.age_days > max_age_days
+
+
+def snapshot_from_trials(
+    *,
+    trials: Iterable[dict[str, Any]],
+    keyword: str,
+    provenance: dict[str, Any],
+    created_at: float | None = None,
+) -> Snapshot:
+    return Snapshot(
+        trials=list(trials),
+        keyword=keyword,
+        created_at=created_at if created_at is not None else time.time(),
+        provenance=dict(provenance),
+    )
+
+
+def locate_bundle(paths: Iterable[Path]) -> Path | None:
+    """Return the first existing snapshot path, or None.
+
+    Ordered by the caller: an explicit environment override should come first, a
+    shipped bundle last.
+    """
+    for candidate in paths:
+        if candidate and candidate.is_file():
+            return candidate
+    return None
+
+
+def _iso(epoch: float) -> str:
+    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(epoch))
+
+
+def _parse_iso_epoch(value: Any) -> float:
+    if not isinstance(value, str):
+        return 0.0
+    for fmt in ("%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%dT%H:%M:%S"):
+        try:
+            return time.mktime(time.strptime(value, fmt))
+        except ValueError:
+            continue
+    return 0.0