ictrp-mcp-server 0.1.0

package/sidecar/ictrp_sidecar.py

@@ -0,0 +1,602 @@
+#!/usr/bin/env python3
+"""HTTP sidecar for the WHO ICTRP search chain.
+
+The MCP server is a Node process. The ICTRP access logic is Python. This module
+is the narrow bridge between them: a stdlib-only HTTP server that exposes the
+three-step postback chain (load form -> search -> export CSV) plus the local
+query operations over a materialized set.
+
+Why a sidecar rather than a port of the Python package to TypeScript:
+
+* The chain's hard-won details -- which HTML controls to carry, which redirect
+  means "blocked" versus "contract drift", how a legitimate empty CSV differs
+  from a failed one -- are already implemented and tested in ``ictrp_mcp``.
+  A second implementation would drift.
+* The Node layer must stay small. It is the MCP surface; the Python layer is
+  the domain.
+
+Design constraints that shape everything below:
+
+1. **Standard library only.** No Flask, no FastAPI, no uvicorn. The only
+   third-party dependency is ``httpx``, which the vendor install step provides
+   into a private venv. A sidecar that needs a web framework is a sidecar that
+   fails to start on a machine with an unusual Python.
+
+2. **A failure never returns zero rows.** The whole point of this service is
+   that ICTRP's CSV export silently omits records. An error must therefore be
+   an HTTP error carrying a machine-readable code, never a 200 with an empty
+   result list. The caller distinguishes "0 matches" from "could not ask" by
+   status code, and nothing else.
+
+3. **Sets are process state, not client state.** ``ictrp_mcp``'s materialized
+   sets live in memory keyed by ``set_id``. They do not survive a restart. The
+   sidecar keeps them for its own lifetime and says so honestly: an unknown
+   ``set_id`` is ``409 set_expired``, and the caller re-searches. Pretending
+   otherwise would silently under-report.
+
+4. **No browser, ever.** ICTRP needs no JavaScript execution to yield data.
+   There is deliberately no Playwright, no Chromium, no solver. If a check for
+   that ever appears here, it is a mistake -- see docs/DESIGN notes on the
+   measured chain.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import logging
+import os
+import sys
+import threading
+import traceback
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from typing import Any
+from urllib.parse import parse_qs, urlparse
+
+# --------------------------------------------------------------------------
+# Import the ICTRP implementation.
+#
+# Three resolutions, tried in order, because the same file runs in three
+# different layouts:
+#   1. vendored   -- ``sidecar/vendor/`` inside a published npm package. First,
+#      so a published install never silently picks up an unrelated
+#      ``ictrp_mcp`` that happens to be importable;
+#   2. development -- the repository root is the grandparent of this file and
+#      the package lives under ``src/``;
+#   3. installed  -- pip put ``ictrp_mcp`` into the private venv that
+#      ``ictrp-setup setup`` builds.
+# --------------------------------------------------------------------------
+
+_HERE = os.path.dirname(os.path.abspath(__file__))
+
+_CANDIDATES = [
+    os.path.join(_HERE, "vendor"),                                   # published
+    os.path.join(os.path.dirname(os.path.dirname(_HERE)), "src"),    # checkout
+]
+
+for _candidate in _CANDIDATES:
+    if os.path.isdir(os.path.join(_candidate, "ictrp_mcp")) and _candidate not in sys.path:
+        sys.path.insert(0, _candidate)
+        break
+
+from ictrp_mcp import offline  # noqa: E402
+from ictrp_mcp.data import query as query_mod  # noqa: E402
+from ictrp_mcp.errors import ErrorCode, IctrpError  # noqa: E402
+from ictrp_mcp.cache.store import SetStore, default_cache_dir, normalize_query  # noqa: E402
+from ictrp_mcp.ictrp.session import run_chain  # noqa: E402
+from ictrp_mcp.provenance import Provenance  # noqa: E402
+from ictrp_mcp.tools import DEFAULT_FIELDS, IctrpService  # noqa: E402
+
+LOG = logging.getLogger("ictrp-sidecar")
+
+# Repeated verbatim on every row-bearing response. The Node layer also attaches
+# it, but the sidecar must not depend on the caller remembering: anything that
+# reads this HTTP API directly deserves the same warning.
+_INCOMPLETENESS_NOTICE = (
+    "Counts describe retrieved rows only. The ICTRP CSV export is known to omit "
+    "records the portal itself reports as matches, so absence is not evidence "
+    "of nonexistence."
+)
+
+DEFAULT_HOST = "127.0.0.1"
+DEFAULT_PORT = 8849
+
+# The sidecar holds one IctrpService for its whole life. The service owns the
+# upstream session and the set store; recreating it per request would discard
+# the warm connection pool for no benefit.
+_SERVICE_LOCK = threading.Lock()
+_SERVICE: IctrpService | None = None
+
+# One event loop per worker thread. ``IctrpService`` binds an
+# ``httpx.AsyncClient`` to whatever loop first drives it; using that client
+# from a second loop raises. Since ``ThreadingHTTPServer`` spawns a thread per
+# connection, each thread gets its own loop and the client is re-created on
+# demand inside the service.
+_LOOP_LOCAL = threading.local()
+
+
+def _run_sync(coro: Any) -> Any:
+    """Drive ``coro`` to completion on this thread's event loop."""
+    loop = getattr(_LOOP_LOCAL, "loop", None)
+    if loop is None or loop.is_closed():
+        loop = asyncio.new_event_loop()
+        _LOOP_LOCAL.loop = loop
+    return loop.run_until_complete(coro)
+
+
+def get_service() -> IctrpService:
+    """Return the process-wide service, creating it on first use."""
+    global _SERVICE
+    with _SERVICE_LOCK:
+        if _SERVICE is None:
+            _SERVICE = IctrpService()
+            LOG.info("ictrp service initialized (cache=%s)", default_cache_dir())
+        return _SERVICE
+
+
+# --------------------------------------------------------------------------
+# Error mapping
+#
+# The wire contract is: HTTP status says how the caller should react; the JSON
+# body's ``error_code`` says what happened. Both are needed. A 502 tells the
+# caller "retry or report an outage"; ``upstream_blocked`` tells it "do not
+# hammer this, a human may need to look".
+# --------------------------------------------------------------------------
+
+_STATUS_FOR_CODE = {
+    ErrorCode.INVALID_ARGUMENT: 400,
+    ErrorCode.CACHE_MISS: 409,
+    ErrorCode.NO_RESULTS: 200,  # not an error -- a legitimate empty answer
+    ErrorCode.UPSTREAM_BLOCKED: 503,
+    ErrorCode.UPSTREAM_CONTRACT_DRIFT: 502,
+    ErrorCode.UPSTREAM_ERROR: 502,
+    ErrorCode.SESSION_FAILED: 502,
+}
+
+
+def error_payload(exc: IctrpError) -> dict[str, Any]:
+    return {"ok": False, **exc.to_dict()}
+
+
+def http_status_for(exc: IctrpError) -> int:
+    return _STATUS_FOR_CODE.get(exc.code, 502)
+
+
+class SetExpired(Exception):
+    """Raised when a ``set_id`` is not held by this process.
+
+    Distinct from ``CACHE_MISS`` because the remedy is different: the caller
+    must re-run the search, not fix its arguments.
+    """
+
+    def __init__(self, set_id: str) -> None:
+        super().__init__(f"set {set_id!r} is not held by this sidecar process")
+        self.set_id = set_id
+
+
+# --------------------------------------------------------------------------
+# Operations
+# --------------------------------------------------------------------------
+
+
+def op_health() -> dict[str, Any]:
+    """Liveness plus a description of what this sidecar can actually do.
+
+    Deliberately reports no upstream contact: a health check that performs a
+    search would drive a 39 MB export every time an installer polls it.
+    """
+    service = get_service()
+    sets = service.store.list_sets()
+    return {
+        "ok": True,
+        "service": "ictrp-sidecar",
+        "version": _version(),
+        "python": sys.version.split()[0],
+        "cache_dir": str(default_cache_dir()),
+        "sets_held": len(sets),
+        "upstream": "https://trialsearch.who.int/",
+        "browser_required": False,
+    }
+
+
+def _version() -> str:
+    try:
+        from ictrp_mcp import __version__  # type: ignore[attr-defined]
+
+        return str(__version__)
+    except Exception:
+        return "0.0.0"
+
+
+def op_search(
+    keyword: str,
+    limit: int = 50,
+    offset: int = 0,
+    refresh: bool = False,
+    filters: list[dict[str, Any]] | None = None,
+    sort_by: str | None = None,
+    descending: bool = False,
+    fields: list[str] | None = None,
+) -> dict[str, Any]:
+    """Run the full chain and materialize the result set.
+
+    This is the only operation that touches the network in a costly way. It
+    always materializes the *entire* result set, not just the requested page:
+    the export is a single ~39 MB response, so fetching it partially is not
+    possible and re-fetching it per page would be absurd.
+
+    ``IctrpService.search`` is the one async method on the service, because it
+    owns an ``httpx.AsyncClient``. ``ThreadingHTTPServer`` hands each request a
+    plain thread, so the coroutine is driven here on a dedicated loop.
+
+    The loop is **per thread**, not per process. A single shared loop would be
+    entered from whichever worker thread happened to serve the request, and an
+    ``httpx.AsyncClient`` bound to one loop raises when driven from another.
+    """
+    service = get_service()
+    coro = service.search(
+        keyword=keyword,
+        limit=limit,
+        offset=offset,
+        refresh=refresh,
+        filters=filters,
+        sort_by=sort_by,
+        descending=descending,
+        fields=fields,
+    )
+    return _run_sync(coro)
+
+
+def op_filter(
+    set_id: str,
+    filters: list[dict[str, Any]] | None = None,
+    sort_by: str | None = None,
+    descending: bool = False,
+    limit: int = 50,
+    offset: int = 0,
+    fields: list[str] | None = None,
+) -> dict[str, Any]:
+    return get_service().filter_set(
+        set_id=set_id,
+        filters=filters,
+        sort_by=sort_by,
+        descending=descending,
+        limit=limit,
+        offset=offset,
+        fields=fields,
+    )
+
+
+def op_field_query(
+    field: str,
+    set_id: str | None = None,
+    keyword: str | None = None,
+    limit: int = 50,
+) -> dict[str, Any]:
+    return get_service().field_query(field=field, set_id=set_id, keyword=keyword, limit=limit)
+
+
+def op_registry_summary(set_id: str) -> dict[str, Any]:
+    return get_service().registry_summary(set_id=set_id)
+
+
+def op_find_duplicates(set_id: str) -> dict[str, Any]:
+    return get_service().find_duplicates(set_id=set_id)
+
+
+def op_export(
+    set_id: str | None = None,
+    keyword: str | None = None,
+    fmt: str = "json",
+    filters: list[dict[str, Any]] | None = None,
+    fields: list[str] | None = None,
+    include_provenance_header: bool = True,
+) -> dict[str, Any]:
+    return get_service().export_records(
+        set_id=set_id,
+        keyword=keyword,
+        fmt=fmt,
+        filters=filters,
+        fields=fields,
+        include_provenance_header=include_provenance_header,
+    )
+
+
+def op_cache_status(action: str = "list", set_id: str | None = None) -> dict[str, Any]:
+    """Inspect or purge the local set cache.
+
+    ``action`` is ``"list"`` or ``"purge"`` -- the service names the operation
+    rather than taking a boolean, so the wire parameter follows it.
+    """
+    return get_service().cache_status(action=action, set_id=set_id)
+
+
+def op_snapshot(
+    keyword: str | None = None,
+    set_id: str | None = None,
+    path: str | None = None,
+    if_stale: bool = True,
+) -> dict[str, Any]:
+    return get_service().snapshot(keyword=keyword, set_id=set_id, path=path, if_stale=if_stale)
+
+
+def op_bundle_status(keyword: str | None = None) -> dict[str, Any]:
+    """Report which offline bundle, if any, serves this keyword.
+
+    ``keyword`` is optional at the HTTP boundary even though the service
+    requires it. Omitting it is a legitimate question -- "what bundles ship
+    with this install?" -- so we answer it from the bundle directory directly
+    rather than letting the service raise.
+    """
+    service = get_service()
+    if keyword:
+        return service.bundle_status(keyword=keyword)
+    return {
+        "status": "ok",
+        "keyword": None,
+        "bundle_configured": {
+            "ICTRP_BUNDLE_PATH": os.environ.get(offline.ENV_BUNDLE_PATH),
+            "ICTRP_BUNDLE_DIR": os.environ.get(offline.ENV_BUNDLE_DIR),
+            "max_age_days": offline.DEFAULT_MAX_AGE_DAYS,
+        },
+        "note": "Pass keyword=... to check whether a bundle can answer that query.",
+    }
+
+
+def op_offline_lookup(keyword: str) -> dict[str, Any]:
+    """Answer a keyword from a bundled snapshot without any network access.
+
+    Returns ``{"ok": true, "found": false}`` rather than an error when no
+    bundle matches: "this installation ships no snapshot for that keyword" is
+    the normal state of a live install, not a fault.
+    """
+    loaded = offline.load_bundle(keyword)
+    if loaded is None:
+        return {
+            "ok": True,
+            "found": False,
+            "keyword": keyword,
+            "bundle_configured": {
+                "ICTRP_BUNDLE_PATH": os.environ.get(offline.ENV_BUNDLE_PATH),
+                "ICTRP_BUNDLE_DIR": os.environ.get(offline.ENV_BUNDLE_DIR),
+                "max_age_days": offline.max_age_days(),
+            },
+        }
+    snapshot, path = loaded
+    trials = [t.to_dict() if hasattr(t, "to_dict") else t for t in snapshot.trials]
+    return {
+        "ok": True,
+        "found": True,
+        "keyword": keyword,
+        "path": str(path),
+        "rows_returned": len(trials),
+        "records_incomplete": True,
+        "incompleteness_notice": _INCOMPLETENESS_NOTICE,
+        "provenance": offline.snapshot_provenance(snapshot, path),
+        "trials": trials,
+    }
+
+
+# --------------------------------------------------------------------------
+# HTTP layer
+# --------------------------------------------------------------------------
+
+_ROUTES = {
+    "/health": op_health,
+    "/cache-status": op_cache_status,
+    "/bundle-status": op_bundle_status,
+    "/offline-lookup": op_offline_lookup,
+    "/search": op_search,
+    "/filter": op_filter,
+    "/field-query": op_field_query,
+    "/registry-summary": op_registry_summary,
+    "/find-duplicates": op_find_duplicates,
+    "/export": op_export,
+    "/snapshot": op_snapshot,
+}
+
+# Operations that must never be triggered by a bare GET poll. Search drives a
+# multi-megabyte upstream export; an installer or supervisor probing "/" must
+# not cause one.
+_EXPENSIVE = {"/search", "/snapshot"}
+
+
+def _as_bool(raw: Any, default: bool = False) -> bool:
+    if raw is None:
+        return default
+    if isinstance(raw, bool):
+        return raw
+    return str(raw).strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _as_int(raw: Any, default: int) -> int:
+    try:
+        return int(raw)
+    except (TypeError, ValueError):
+        return default
+
+
+def _json_body(value: Any) -> bytes:
+    return json.dumps(value, ensure_ascii=False, default=str).encode("utf-8")
+
+
+class Handler(BaseHTTPRequestHandler):
+    protocol_version = "HTTP/1.1"
+    server_version = "ictrp-sidecar"
+
+    # -- plumbing ---------------------------------------------------------
+
+    def log_message(self, fmt: str, *args: Any) -> None:  # noqa: A003
+        LOG.debug("%s - %s", self.address_string(), fmt % args)
+
+    def _send(self, status: int, payload: Any) -> None:
+        body = _json_body(payload)
+        self.send_response(status)
+        self.send_header("Content-Type", "application/json; charset=utf-8")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def _params(self) -> dict[str, Any]:
+        """Merge query-string and JSON-body parameters.
+
+        Both are accepted because GET is convenient for curl-based debugging
+        while POST keeps a long filter array out of the URL.
+        """
+        merged: dict[str, Any] = {}
+        parsed = urlparse(self.path)
+        for key, values in parse_qs(parsed.query).items():
+            merged[key] = values[0] if len(values) == 1 else values
+
+        length = _as_int(self.headers.get("Content-Length"), 0)
+        if length > 0:
+            raw = self.rfile.read(length)
+            try:
+                decoded = json.loads(raw.decode("utf-8"))
+            except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+                raise ValueError(f"request body is not valid JSON: {exc}") from exc
+            if not isinstance(decoded, dict):
+                raise ValueError("request body must be a JSON object")
+            merged.update(decoded)
+        return merged
+
+    # -- dispatch ---------------------------------------------------------
+
+    def do_GET(self) -> None:  # noqa: N802
+        self._dispatch("GET")
+
+    def do_POST(self) -> None:  # noqa: N802
+        self._dispatch("POST")
+
+    def _dispatch(self, method: str) -> None:
+        path = urlparse(self.path).path.rstrip("/") or "/"
+
+        if path == "/":
+            self._send(
+                200,
+                {
+                    "service": "ictrp-sidecar",
+                    "endpoints": sorted(_ROUTES),
+                    "hint": "POST /search?keyword=... starts the chain.",
+                },
+            )
+            return
+
+        handler = _ROUTES.get(path)
+        if handler is None:
+            self._send(404, {"ok": False, "error": "not found", "path": path})
+            return
+
+        if method == "GET" and path in _EXPENSIVE:
+            # Refuse rather than run: a GET that triggers a 39 MB export is
+            # almost certainly a supervisor probe, and it would be a very
+            # expensive misunderstanding.
+            self._send(
+                405,
+                {
+                    "ok": False,
+                    "error": f"{path} must be called with POST",
+                    "reason": "this operation drives a large upstream request",
+                },
+            )
+            return
+
+        try:
+            params = self._params()
+        except ValueError as exc:
+            self._send(400, {"ok": False, "error": str(exc)})
+            return
+
+        try:
+            result = handler(**params)
+        except IctrpError as exc:
+            status = http_status_for(exc)
+            LOG.info("%s -> %s %s", path, status, exc.code.value)
+            self._send(status, error_payload(exc))
+            return
+        except SetExpired as exc:
+            self._send(
+                409,
+                {
+                    "ok": False,
+                    "error_code": "set_expired",
+                    "message": str(exc),
+                    "set_id": exc.set_id,
+                    "hint": "Re-run /search; materialized sets do not survive a sidecar restart.",
+                },
+            )
+            return
+        except TypeError as exc:
+            # Almost always a caller passing an unexpected keyword. Surfacing
+            # it as 400 is far more useful than a 502 traceback.
+            self._send(
+                400,
+                {
+                    "ok": False,
+                    "error_code": "INVALID_ARGUMENT",
+                    "message": f"bad parameters for {path}: {exc}",
+                },
+            )
+            return
+        except Exception as exc:  # noqa: BLE001 - the boundary must not leak
+            LOG.error("unhandled error in %s:\n%s", path, traceback.format_exc())
+            self._send(
+                500,
+                {
+                    "ok": False,
+                    "error_code": "INTERNAL_ERROR",
+                    "message": f"{type(exc).__name__}: {exc}",
+                    "path": path,
+                    "hint": "This is a bug in the sidecar, not an upstream failure.",
+                },
+            )
+            return
+
+        # The success path. Without this the handler would compute a perfect
+        # answer and then close the connection without writing anything, which
+        # every client reports as an opaque connection failure.
+        self._send(200, result)
+
+
+def serve(host: str, port: int) -> None:
+    httpd = ThreadingHTTPServer((host, port), Handler)
+    httpd.daemon_threads = True
+    LOG.info("ictrp sidecar listening on http://%s:%d", host, port)
+    try:
+        httpd.serve_forever()
+    except KeyboardInterrupt:
+        LOG.info("interrupted; shutting down")
+    finally:
+        httpd.server_close()
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="ictrp_sidecar", description=__doc__)
+    parser.add_argument("--host", default=os.environ.get("ICTRP_SIDECAR_HOST", DEFAULT_HOST))
+    parser.add_argument(
+        "--port", type=int, default=_as_int(os.environ.get("ICTRP_SIDECAR_PORT"), DEFAULT_PORT)
+    )
+    parser.add_argument("--log-level", default=os.environ.get("ICTRP_SIDECAR_LOG", "INFO"))
+    parser.add_argument(
+        "--check",
+        action="store_true",
+        help="Import and exit. Used by the installer to validate the venv.",
+    )
+    args = parser.parse_args(argv)
+
+    logging.basicConfig(
+        level=getattr(logging, str(args.log_level).upper(), logging.INFO),
+        format="%(asctime)s %(levelname)s %(name)s %(message)s",
+    )
+
+    if args.check:
+        print(json.dumps({"ok": True, "version": _version(), "python": sys.version.split()[0]}))
+        return 0
+
+    serve(args.host, args.port)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/sidecar/vendor/ictrp_mcp/__init__.py

@@ -0,0 +1,3 @@
+"""MCP server for the WHO ICTRP clinical trials database, over plain HTTP."""
+
+__version__ = "0.1.0"