ictrp-mcp-server 0.1.0 → 0.1.2

package/CHANGELOG.md

@@ -3,6 +3,56 @@
 Notable changes to the npm package `ictrp-mcp-server`. The Python package
 `ictrp-mcp-service` is versioned separately in `pyproject.toml`.
 
+## 0.1.2
+
+**Default search view is now the clinician-triage view.** `ictrp_search` used to
+return the 10 most common columns; it now returns the columns a clinician needs
+to triage a target: title and scientific title, sponsor and secondary sponsor,
+contact (PI) name/affiliation/email, recruitment status, phase, registration
+date, condition, countries, age bounds and gender, target size, and a
+`line_of_therapy_hint`. A new derived field, `line_of_therapy_hint`, is a
+non-authoritative extraction of treatment line from the free-text
+inclusion/exclusion criteria and the scientific title. It returns
+`first-line`/`second-line`/`third-line`/`fourth-line`/`later-line`, or `null`
+when nothing is stated -- `null` is documented as "not stated", never as
+"first-line", because the export is incomplete and the column does not exist.
+
+**Line-hint fix** — the hint missed trials whose line is stated only as a number
+or only in the scientific title (e.g. "Received >=2 Prior Lines of Therapy"). It
+now scans the scientific title and recognizes numeric line statements; a minimum
+of N lines resolves to `first-line` only when N<=1, otherwise `later-line`.
+
+Documented structural limits surfaced by the wider view: ICTRP has no "PI" and
+no "participating-hospital list" column -- sponsor/PI come from
+`primary_sponsor` + `contact_*`, and "participating hospitals" degrades to
+`countries` + `contact_affiliation`. CT.gov `exclusion_criteria` is frequently
+blank upstream, which is a source gap, not a parse failure.
+
+## 0.1.1
+
+**Retry on transient stalls** — the search POST intermittently stalls past a
+minute. Measured: a `YL201` search returned `httpx.ReadTimeout` at a 60s budget
+while a plain `GET /` on the same host answered HTTP 200 in 1.4s, so the stall
+is in the POST chain rather than the host being down. Each of the three steps
+(`load_form`, `search`, `export_csv`) is now retried with exponential backoff
+and jitter, and the default per-request timeout went from 60s to 120s.
+
+Only transport failures and `SESSION_FAILED` are retried. `UPSTREAM_BLOCKED` is
+not: a refusal is information, and retrying into an active refusal both wastes
+the budget and delays the diagnosis.
+
+Two error-message fixes fell out of the same investigation:
+
+- `httpx.ReadTimeout` stringifies to an empty message, which is how a real
+  failure reached a user as `Search request failed: ` with nothing after the
+  colon. Transport errors are now named when they carry no text.
+- An exhausted retry used to repeat the per-attempt hint "then retry". It now
+  reports that the retries already happened and names `ICTRP_TIMEOUT` as the
+  next thing to raise.
+
+New environment variables: `ICTRP_TIMEOUT` (default 120),
+`ICTRP_ATTEMPTS` (default 3), `ICTRP_BACKOFF_SECONDS` (default 2, capped at 30).
+
 ## 0.1.0
 
 First release.

package/dist/index.js

@@ -21,7 +21,7 @@ import { getEnvReport } from "./runtime/env-probe.js";
 import { callSidecar, pingSidecar, SEARCH_TIMEOUT_MS, toErrorText, } from "./runtime/sidecar-client.js";
 import { ensureSidecar, installShutdownHooks, sidecarLogTail } from "./runtime/supervisor.js";
 const SERVER_NAME = "ictrp-mcp-server";
-const SERVER_VERSION = "0.1.0";
+const SERVER_VERSION = "0.1.2";
 /**
  * Attached to every payload that returns rows.
  *

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ictrp-mcp-server",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "MCP server for the WHO International Clinical Trials Registry Platform (ICTRP). Plain HTTP, no browser. Every response is explicit about the export's known incompleteness.",
   "type": "module",
   "main": "dist/index.js",

package/sidecar/vendor/ictrp_mcp/data/normalize.py

@@ -323,6 +323,18 @@ def to_trial(row: list[str], header: tuple[str, ...]) -> dict[str, Any]:
     trial["inclusion_age_min"] = _as_int(age_min)
     trial["inclusion_age_max"] = _as_int(age_max)
 
+    # ICTRP has no structured "line of therapy" column. Treatment line is stated
+    # only inside the free-text inclusion/exclusion criteria (e.g. "first-line",
+    # "previously treated", "一线") and, frequently, the scientific title
+    # ("...Received >=2 Prior Lines of Therapy"). Derive a non-authoritative hint
+    # from that text so a caller can surface it without re-parsing. It is a hint,
+    # never a fact: absence here does NOT mean the trial is treatment-naive.
+    trial["line_of_therapy_hint"] = derive_line_of_therapy_hint(
+        raw.get("inclusion_criteria"),
+        raw.get("exclusion_criteria"),
+        raw.get("scientific_title"),
+    )
+
     # `results yes no` is a flag, not content, so it is excluded from the test.
     # Measured: the substantive `results *` columns are ~0.0% populated for
     # ChiCTR records, so this is a property of the registry rather than of an
@@ -346,3 +358,66 @@ def _as_int(value: str | None) -> int | None:
         return None
     m = re.search(r"-?\d+", text)
     return int(m.group(0)) if m else None
+
+
+#: Treatment-line vocabulary, as it actually appears in live inclusion/exclusion
+#: text. Both English and Chinese markers are matched; "refractory"/"relapsed"/
+#: "previously treated" indicate a later line without naming a number.
+_LINE_PATTERNS: tuple[tuple[str, str], ...] = (
+    (r"first[-\s]?line|1st[-\s]?line|一线|treatment[-\s]?naive|previously untreated", "first-line"),
+    (r"second[-\s]?line|2nd[-\s]?line|二线", "second-line"),
+    (r"third[-\s]?line|3rd[-\s]?line|三线", "third-line"),
+    (r"fourth[-\s]?line|4th[-\s]?line|四线", "fourth-line"),
+    (r"relapsed|refractory|previously treated|prior (therapy|treatment)|已接受.*治疗|经治", "later-line"),
+)
+
+#: Numeric "line" statements, e.g. ">=2 Prior Lines of Therapy" or "2-line",
+#: where a word may sit between the number and "line" ("2 prior lines"). The
+#: captured number is the *minimum* line; ">=2" means second-line or later, never
+#: first-line, so it resolves to `later-line` rather than a specific line.
+_LINE_NUMBER = re.compile(r"(\d+)\s*(?:[-\w.,]+\s)*?(?:line|lines|线)")
+
+_WORD_NUMBERS = {
+    "one": 1, "two": 2, "three": 3, "four": 4, "five": 5,
+    "一": 1, "二": 2, "三": 3, "四": 4, "五": 5,
+}
+_WORD_NUMBER = re.compile(r"(one|two|three|four|five|一|二|三|四|五)\s*(?:line|lines|线)")
+
+
+def derive_line_of_therapy_hint(
+    inclusion: str | None, exclusion: str | None, scientific_title: str | None = None
+) -> str | None:
+    """Best-effort treatment-line label from free-text criteria.
+
+    Returns the earliest explicit line if one is named (first > second > third >
+    fourth), else "later-line" when a numeric "N lines" (N>=2) or relapse/
+    refractory language is present, else `None`. `None` is deliberate: it means
+    "not stated in the criteria we hold", which must not be read as "first-line"
+    -- the export is incomplete and the column does not exist.
+
+    The scientific title is included because the line is often stated only there
+    (e.g. "...Received >=2 Prior Lines of Therapy"), not in the criteria body.
+    """
+    text = f"{inclusion or ''} {exclusion or ''} {scientific_title or ''}"
+    if not text.strip():
+        return None
+    from .query import _coerce  # local import avoids a cycle at module load
+
+    lowered = _coerce(text)
+
+    # Explicit worded lines take precedence -- they name a specific line.
+    for pattern, label in _LINE_PATTERNS:
+        if re.search(pattern, lowered):
+            return label
+
+    # Numeric "N line(s)": a minimum of N, so only the first-line case is exact;
+    # anything >=2 is "later-line", never a specific later number.
+    for match in _LINE_NUMBER.finditer(lowered):
+        if int(match.group(1)) <= 1:
+            return "first-line"
+        return "later-line"
+    for match in _WORD_NUMBER.finditer(lowered):
+        if _WORD_NUMBERS[match.group(1)] <= 1:
+            return "first-line"
+        return "later-line"
+    return None

package/sidecar/vendor/ictrp_mcp/ictrp/session.py

@@ -19,6 +19,10 @@ Measured facts this module encodes (docs/MEASUREMENTS.md):
 
 from __future__ import annotations
 
+import asyncio
+import os
+import random
+from collections.abc import Awaitable, Callable
 from dataclasses import dataclass, field
 
 import httpx
@@ -53,6 +57,145 @@ SEARCH_BUTTON = "Button1"
 SEARCH_TEXTBOX = "TextBox1"
 
 
+#: The portal's search POST intermittently stalls past a minute. Measured: a
+#: `YL201` search returned `httpx.ReadTimeout` at a 60s budget while a plain
+#: `GET /` on the same host answered HTTP 200 in 1.4s, so the stall is in the
+#: POST chain rather than in the host being down. Widening the budget is the
+#: cheap half of the fix; see `retry` for the half that actually recovers.
+DEFAULT_TIMEOUT_SECONDS = 120.0
+ENV_TIMEOUT = "ICTRP_TIMEOUT"
+
+#: How many times a single step is attempted before its failure is surfaced.
+#: Three is enough to ride out a transient stall without turning a genuine
+#: outage into a multi-minute hang.
+DEFAULT_ATTEMPTS = 3
+ENV_ATTEMPTS = "ICTRP_ATTEMPTS"
+
+#: Base delay between attempts, doubled each time and jittered. Jitter matters
+#: because a client that retries on a fixed cadence can stay in lockstep with
+#: whatever is stalling.
+DEFAULT_BACKOFF_SECONDS = 2.0
+ENV_BACKOFF = "ICTRP_BACKOFF_SECONDS"
+
+#: The hard ceiling on a single backoff sleep, so a long search cannot turn
+#: into an unbounded wait.
+MAX_BACKOFF_SECONDS = 30.0
+
+#: Transport failures and transient portal hiccups. A 429/503 raised as
+#: UPSTREAM_BLOCKED is deliberately excluded: retrying into an active refusal
+#: makes the block worse and hides the diagnosis.
+RETRYABLE_CODES: frozenset = frozenset(
+    {ErrorCode.UPSTREAM_ERROR, ErrorCode.SESSION_FAILED}
+)
+
+
+def _env_float(name: str, default: float) -> float:
+    raw = os.environ.get(name)
+    if raw:
+        try:
+            value = float(raw)
+        except ValueError:
+            return default
+        return value if value > 0 else default
+    return default
+
+
+def _env_int(name: str, default: int) -> int:
+    raw = os.environ.get(name)
+    if raw:
+        try:
+            value = int(raw)
+        except ValueError:
+            return default
+        return value if value > 0 else default
+    return default
+
+
+def _describe(exc: httpx.HTTPError) -> str:
+    """Text for a transport error that may legitimately be empty.
+
+    `httpx.ReadTimeout` stringifies to an empty message, which is how a real
+    failure reached a user as `Search request failed: ` with nothing after the
+    colon. Name the class when there is no text, so the message is never blank.
+    """
+    text = str(exc).strip()
+    if text:
+        return text
+    return f"{type(exc).__name__} (no message)"
+
+
+def timeout_seconds() -> float:
+    """Per-request timeout, overridable for slow or poor links."""
+    return _env_float(ENV_TIMEOUT, DEFAULT_TIMEOUT_SECONDS)
+
+
+def max_attempts() -> int:
+    """Attempts per step, so a transient stall is ridden out rather than raised."""
+    return _env_int(ENV_ATTEMPTS, DEFAULT_ATTEMPTS)
+
+
+def backoff_seconds() -> float:
+    return _env_float(ENV_BACKOFF, DEFAULT_BACKOFF_SECONDS)
+
+
+def _sleep_for(attempt: int) -> float:
+    """Exponential backoff with jitter, capped.
+
+    `attempt` is 1-based: the first retry waits ~base, the second ~2x base.
+    """
+    delay = backoff_seconds() * (2 ** (attempt - 1))
+    delay = min(delay, MAX_BACKOFF_SECONDS)
+    return random.uniform(delay * 0.5, delay)
+
+
+async def retry(
+    step: str,
+    op: Callable[[], Awaitable[object]],
+    *,
+    attempts: int | None = None,
+    sleeper: Callable[[float], Awaitable[None]] | None = None,
+) -> object:
+    """Run `op`, retrying only failures that a retry could plausibly fix.
+
+    Non-retryable codes propagate on the first attempt. A refusal is
+    information -- the caller needs to see "blocked", not "slow" -- and
+    retrying it would both waste the budget and delay the diagnosis.
+    """
+    total = attempts if attempts is not None else max_attempts()
+    sleep = sleeper or asyncio.sleep
+    last: IctrpError | None = None
+
+    for index in range(1, total + 1):
+        try:
+            return await op()
+        except IctrpError as exc:
+            if exc.code not in RETRYABLE_CODES:
+                raise
+            last = exc
+            if index == total:
+                break
+            await sleep(_sleep_for(index))
+
+    assert last is not None
+    # The per-attempt hint ("then retry") is wrong by the time we get here: we
+    # have already retried. Speak to what the caller can still do.
+    hint = (
+        f"Already retried {total} time(s) with backoff; every attempt failed. "
+        f"Raise {ENV_TIMEOUT} (currently {timeout_seconds():g}s) to give each "
+        f"attempt longer, or retry later -- the portal was answering plain GETs "
+        f"while this step stalled."
+    )
+    raise IctrpError(
+        last.code,
+        f"{step}: failed after {total} attempt(s) -- {last.message}",
+        upstream_status=last.upstream_status,
+        upstream_content_type=last.upstream_content_type,
+        upstream_body_excerpt=last.upstream_body_excerpt,
+        hint=hint,
+        detail=f"Retryable failure, exhausted {total} attempts: {last.detail or ''}".strip(),
+    ) from last
+
+
 def _check_present(state: htmlstate.FormState, *, step: str) -> None:
     missing = state.missing_state()
     if missing:
@@ -86,10 +229,13 @@ class IctrpSession:
     steps: list[str] = field(default_factory=list)
 
     @classmethod
-    def create(cls, *, timeout: float = 60.0) -> "IctrpSession":
+    def create(cls, *, timeout: float | None = None) -> "IctrpSession":
+        # 60s was measured to be too tight: the search POST stalls past it while
+        # the host is answering GETs in ~1.4s. Callers that pass nothing get the
+        # current default; callers that pass a value still get that value.
         client = httpx.AsyncClient(
             headers=DEFAULT_HEADERS,
-            timeout=timeout,
+            timeout=timeout if timeout is not None else timeout_seconds(),
             follow_redirects=False,
         )
         return cls(client=client)
@@ -106,14 +252,18 @@ class IctrpSession:
     # ---- step 1 -----------------------------------------------------------
 
     async def load_form(self) -> htmlstate.FormState:
-        try:
-            response = await self.client.get(self.base_url)
-        except httpx.HTTPError as exc:
-            raise IctrpError(
-                ErrorCode.UPSTREAM_ERROR,
-                f"Could not reach the ICTRP search portal: {exc}",
-                hint="Check network connectivity, then retry.",
-            ) from exc
+        async def attempt_get():
+            try:
+                return await self.client.get(self.base_url)
+            except httpx.HTTPError as exc:
+                raise IctrpError(
+                    ErrorCode.UPSTREAM_ERROR,
+                    f"Could not reach the ICTRP search portal: {_describe(exc)}",
+                    hint="Check network connectivity, then retry.",
+                ) from exc
+
+        response = await retry("load_form", attempt_get)
+        assert isinstance(response, httpx.Response)
 
         html = classify_form_page(status=response.status_code, body=response.content)
         self.steps.append(f"GET {self.base_url} -> {response.status_code}")
@@ -138,20 +288,25 @@ class IctrpSession:
         body = self.form_state.merged_with(
             {SEARCH_TEXTBOX: keyword, SEARCH_BUTTON: "Search"}
         )
-        try:
-            response = await self.client.post(
-                self.base_url,
-                data=body,
-                headers={
-                    "Content-Type": "application/x-www-form-urlencoded",
-                    "Referer": self.base_url,
-                },
-            )
-        except httpx.HTTPError as exc:
-            raise IctrpError(
-                ErrorCode.UPSTREAM_ERROR,
-                f"Search request failed: {exc}",
-            ) from exc
+
+        async def attempt_post():
+            try:
+                return await self.client.post(
+                    self.base_url,
+                    data=body,
+                    headers={
+                        "Content-Type": "application/x-www-form-urlencoded",
+                        "Referer": self.base_url,
+                    },
+                )
+            except httpx.HTTPError as exc:
+                raise IctrpError(
+                    ErrorCode.UPSTREAM_ERROR,
+                    f"Search request failed: {_describe(exc)}",
+                ) from exc
+
+        response = await retry(f"search {keyword!r}", attempt_post)
+        assert isinstance(response, httpx.Response)
 
         location = response.headers.get("location")
         if location and "/noaccess.aspx" in location.lower():
@@ -205,20 +360,24 @@ class IctrpSession:
         # can never be reintroduced silently.
         htmlstate.assert_export_body_excludes_search_controls(body)
 
-        try:
-            response = await self.client.post(
-                self.base_url,
-                data=body,
-                headers={
-                    "Content-Type": "application/x-www-form-urlencoded",
-                    "Referer": self.base_url,
-                },
-            )
-        except httpx.HTTPError as exc:
-            raise IctrpError(
-                ErrorCode.UPSTREAM_ERROR,
-                f"Export request failed: {exc}",
-            ) from exc
+        async def attempt_export():
+            try:
+                return await self.client.post(
+                    self.base_url,
+                    data=body,
+                    headers={
+                        "Content-Type": "application/x-www-form-urlencoded",
+                        "Referer": self.base_url,
+                    },
+                )
+            except httpx.HTTPError as exc:
+                raise IctrpError(
+                    ErrorCode.UPSTREAM_ERROR,
+                    f"Export request failed: {_describe(exc)}",
+                ) from exc
+
+        response = await retry("export_csv", attempt_export)
+        assert isinstance(response, httpx.Response)
 
         self.steps.append(f"POST export -> {response.status_code}")
         self.response_date = response.headers.get("date")
@@ -236,7 +395,9 @@ class IctrpSession:
         return list(self.steps)
 
 
-async def run_chain(keyword: str, *, timeout: float = 120.0) -> tuple[CsvPayload, IctrpSession]:
+async def run_chain(
+    keyword: str, *, timeout: float | None = None
+) -> tuple[CsvPayload, IctrpSession]:
     """Run the full chain and return the payload plus the session for provenance."""
     session = IctrpSession.create(timeout=timeout)
     await session.load_form()

package/sidecar/vendor/ictrp_mcp/tools.py

@@ -35,16 +35,33 @@ from .provenance import INCOMPLETENESS_NOTICE, Provenance, derive_set_provenance
 
 #: Fields shown by default when a caller does not ask for specific ones. Chosen to
 #: be the high-coverage, decision-relevant columns rather than all 58.
+#: The default view. Deliberately NOT just the 10 most common columns: it is the
+#: view a clinician actually needs to triage a target -- name, sponsor/PI, where
+#: it runs, eligibility shape, and a treatment-line hint. Sponsor/PI come from the
+#: `primary_sponsor` + `contact_*` columns (ICTRP has no dedicated "PI" or
+#: "participating-hospital" field; hospitals exist only as `countries` and the
+#: contact's affiliation), so those are surfaced here. `line_of_therapy_hint` is a
+#: non-authoritative extraction from the free-text criteria -- see normalize.py.
 DEFAULT_FIELDS: tuple[str, ...] = (
     "trial_id",
     "source_register",
     "public_title",
+    "scientific_title",
+    "primary_sponsor",
+    "secondary_sponsor",
+    "contact_firstname",
+    "contact_lastname",
+    "contact_affiliation",
     "recruitment_status",
     "phase_code",
     "registration_date",
     "condition",
     "countries",
+    "inclusion_age_min",
+    "inclusion_age_max",
+    "inclusion_gender",
     "target_size_total",
+    "line_of_therapy_hint",
     "last_refreshed_display",
 )