ictrp-mcp-server 0.1.0 → 0.1.1

package/CHANGELOG.md

@@ -3,6 +3,31 @@
 Notable changes to the npm package `ictrp-mcp-server`. The Python package
 `ictrp-mcp-service` is versioned separately in `pyproject.toml`.
 
+## 0.1.1
+
+**Retry on transient stalls** — the search POST intermittently stalls past a
+minute. Measured: a `YL201` search returned `httpx.ReadTimeout` at a 60s budget
+while a plain `GET /` on the same host answered HTTP 200 in 1.4s, so the stall
+is in the POST chain rather than the host being down. Each of the three steps
+(`load_form`, `search`, `export_csv`) is now retried with exponential backoff
+and jitter, and the default per-request timeout went from 60s to 120s.
+
+Only transport failures and `SESSION_FAILED` are retried. `UPSTREAM_BLOCKED` is
+not: a refusal is information, and retrying into an active refusal both wastes
+the budget and delays the diagnosis.
+
+Two error-message fixes fell out of the same investigation:
+
+- `httpx.ReadTimeout` stringifies to an empty message, which is how a real
+  failure reached a user as `Search request failed: ` with nothing after the
+  colon. Transport errors are now named when they carry no text.
+- An exhausted retry used to repeat the per-attempt hint "then retry". It now
+  reports that the retries already happened and names `ICTRP_TIMEOUT` as the
+  next thing to raise.
+
+New environment variables: `ICTRP_TIMEOUT` (default 120),
+`ICTRP_ATTEMPTS` (default 3), `ICTRP_BACKOFF_SECONDS` (default 2, capped at 30).
+
 ## 0.1.0
 
 First release.

package/dist/index.js

@@ -21,7 +21,7 @@ import { getEnvReport } from "./runtime/env-probe.js";
 import { callSidecar, pingSidecar, SEARCH_TIMEOUT_MS, toErrorText, } from "./runtime/sidecar-client.js";
 import { ensureSidecar, installShutdownHooks, sidecarLogTail } from "./runtime/supervisor.js";
 const SERVER_NAME = "ictrp-mcp-server";
-const SERVER_VERSION = "0.1.0";
+const SERVER_VERSION = "0.1.1";
 /**
  * Attached to every payload that returns rows.
  *

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ictrp-mcp-server",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "MCP server for the WHO International Clinical Trials Registry Platform (ICTRP). Plain HTTP, no browser. Every response is explicit about the export's known incompleteness.",
   "type": "module",
   "main": "dist/index.js",

package/sidecar/vendor/ictrp_mcp/ictrp/session.py

@@ -19,6 +19,10 @@ Measured facts this module encodes (docs/MEASUREMENTS.md):
 
 from __future__ import annotations
 
+import asyncio
+import os
+import random
+from collections.abc import Awaitable, Callable
 from dataclasses import dataclass, field
 
 import httpx
@@ -53,6 +57,145 @@ SEARCH_BUTTON = "Button1"
 SEARCH_TEXTBOX = "TextBox1"
 
 
+#: The portal's search POST intermittently stalls past a minute. Measured: a
+#: `YL201` search returned `httpx.ReadTimeout` at a 60s budget while a plain
+#: `GET /` on the same host answered HTTP 200 in 1.4s, so the stall is in the
+#: POST chain rather than in the host being down. Widening the budget is the
+#: cheap half of the fix; see `retry` for the half that actually recovers.
+DEFAULT_TIMEOUT_SECONDS = 120.0
+ENV_TIMEOUT = "ICTRP_TIMEOUT"
+
+#: How many times a single step is attempted before its failure is surfaced.
+#: Three is enough to ride out a transient stall without turning a genuine
+#: outage into a multi-minute hang.
+DEFAULT_ATTEMPTS = 3
+ENV_ATTEMPTS = "ICTRP_ATTEMPTS"
+
+#: Base delay between attempts, doubled each time and jittered. Jitter matters
+#: because a client that retries on a fixed cadence can stay in lockstep with
+#: whatever is stalling.
+DEFAULT_BACKOFF_SECONDS = 2.0
+ENV_BACKOFF = "ICTRP_BACKOFF_SECONDS"
+
+#: The hard ceiling on a single backoff sleep, so a long search cannot turn
+#: into an unbounded wait.
+MAX_BACKOFF_SECONDS = 30.0
+
+#: Transport failures and transient portal hiccups. A 429/503 raised as
+#: UPSTREAM_BLOCKED is deliberately excluded: retrying into an active refusal
+#: makes the block worse and hides the diagnosis.
+RETRYABLE_CODES: frozenset = frozenset(
+    {ErrorCode.UPSTREAM_ERROR, ErrorCode.SESSION_FAILED}
+)
+
+
+def _env_float(name: str, default: float) -> float:
+    raw = os.environ.get(name)
+    if raw:
+        try:
+            value = float(raw)
+        except ValueError:
+            return default
+        return value if value > 0 else default
+    return default
+
+
+def _env_int(name: str, default: int) -> int:
+    raw = os.environ.get(name)
+    if raw:
+        try:
+            value = int(raw)
+        except ValueError:
+            return default
+        return value if value > 0 else default
+    return default
+
+
+def _describe(exc: httpx.HTTPError) -> str:
+    """Text for a transport error that may legitimately be empty.
+
+    `httpx.ReadTimeout` stringifies to an empty message, which is how a real
+    failure reached a user as `Search request failed: ` with nothing after the
+    colon. Name the class when there is no text, so the message is never blank.
+    """
+    text = str(exc).strip()
+    if text:
+        return text
+    return f"{type(exc).__name__} (no message)"
+
+
+def timeout_seconds() -> float:
+    """Per-request timeout, overridable for slow or poor links."""
+    return _env_float(ENV_TIMEOUT, DEFAULT_TIMEOUT_SECONDS)
+
+
+def max_attempts() -> int:
+    """Attempts per step, so a transient stall is ridden out rather than raised."""
+    return _env_int(ENV_ATTEMPTS, DEFAULT_ATTEMPTS)
+
+
+def backoff_seconds() -> float:
+    return _env_float(ENV_BACKOFF, DEFAULT_BACKOFF_SECONDS)
+
+
+def _sleep_for(attempt: int) -> float:
+    """Exponential backoff with jitter, capped.
+
+    `attempt` is 1-based: the first retry waits ~base, the second ~2x base.
+    """
+    delay = backoff_seconds() * (2 ** (attempt - 1))
+    delay = min(delay, MAX_BACKOFF_SECONDS)
+    return random.uniform(delay * 0.5, delay)
+
+
+async def retry(
+    step: str,
+    op: Callable[[], Awaitable[object]],
+    *,
+    attempts: int | None = None,
+    sleeper: Callable[[float], Awaitable[None]] | None = None,
+) -> object:
+    """Run `op`, retrying only failures that a retry could plausibly fix.
+
+    Non-retryable codes propagate on the first attempt. A refusal is
+    information -- the caller needs to see "blocked", not "slow" -- and
+    retrying it would both waste the budget and delay the diagnosis.
+    """
+    total = attempts if attempts is not None else max_attempts()
+    sleep = sleeper or asyncio.sleep
+    last: IctrpError | None = None
+
+    for index in range(1, total + 1):
+        try:
+            return await op()
+        except IctrpError as exc:
+            if exc.code not in RETRYABLE_CODES:
+                raise
+            last = exc
+            if index == total:
+                break
+            await sleep(_sleep_for(index))
+
+    assert last is not None
+    # The per-attempt hint ("then retry") is wrong by the time we get here: we
+    # have already retried. Speak to what the caller can still do.
+    hint = (
+        f"Already retried {total} time(s) with backoff; every attempt failed. "
+        f"Raise {ENV_TIMEOUT} (currently {timeout_seconds():g}s) to give each "
+        f"attempt longer, or retry later -- the portal was answering plain GETs "
+        f"while this step stalled."
+    )
+    raise IctrpError(
+        last.code,
+        f"{step}: failed after {total} attempt(s) -- {last.message}",
+        upstream_status=last.upstream_status,
+        upstream_content_type=last.upstream_content_type,
+        upstream_body_excerpt=last.upstream_body_excerpt,
+        hint=hint,
+        detail=f"Retryable failure, exhausted {total} attempts: {last.detail or ''}".strip(),
+    ) from last
+
+
 def _check_present(state: htmlstate.FormState, *, step: str) -> None:
     missing = state.missing_state()
     if missing:
@@ -86,10 +229,13 @@ class IctrpSession:
     steps: list[str] = field(default_factory=list)
 
     @classmethod
-    def create(cls, *, timeout: float = 60.0) -> "IctrpSession":
+    def create(cls, *, timeout: float | None = None) -> "IctrpSession":
+        # 60s was measured to be too tight: the search POST stalls past it while
+        # the host is answering GETs in ~1.4s. Callers that pass nothing get the
+        # current default; callers that pass a value still get that value.
         client = httpx.AsyncClient(
             headers=DEFAULT_HEADERS,
-            timeout=timeout,
+            timeout=timeout if timeout is not None else timeout_seconds(),
             follow_redirects=False,
         )
         return cls(client=client)
@@ -106,14 +252,18 @@ class IctrpSession:
     # ---- step 1 -----------------------------------------------------------
 
     async def load_form(self) -> htmlstate.FormState:
-        try:
-            response = await self.client.get(self.base_url)
-        except httpx.HTTPError as exc:
-            raise IctrpError(
-                ErrorCode.UPSTREAM_ERROR,
-                f"Could not reach the ICTRP search portal: {exc}",
-                hint="Check network connectivity, then retry.",
-            ) from exc
+        async def attempt_get():
+            try:
+                return await self.client.get(self.base_url)
+            except httpx.HTTPError as exc:
+                raise IctrpError(
+                    ErrorCode.UPSTREAM_ERROR,
+                    f"Could not reach the ICTRP search portal: {_describe(exc)}",
+                    hint="Check network connectivity, then retry.",
+                ) from exc
+
+        response = await retry("load_form", attempt_get)
+        assert isinstance(response, httpx.Response)
 
         html = classify_form_page(status=response.status_code, body=response.content)
         self.steps.append(f"GET {self.base_url} -> {response.status_code}")
@@ -138,20 +288,25 @@ class IctrpSession:
         body = self.form_state.merged_with(
             {SEARCH_TEXTBOX: keyword, SEARCH_BUTTON: "Search"}
         )
-        try:
-            response = await self.client.post(
-                self.base_url,
-                data=body,
-                headers={
-                    "Content-Type": "application/x-www-form-urlencoded",
-                    "Referer": self.base_url,
-                },
-            )
-        except httpx.HTTPError as exc:
-            raise IctrpError(
-                ErrorCode.UPSTREAM_ERROR,
-                f"Search request failed: {exc}",
-            ) from exc
+
+        async def attempt_post():
+            try:
+                return await self.client.post(
+                    self.base_url,
+                    data=body,
+                    headers={
+                        "Content-Type": "application/x-www-form-urlencoded",
+                        "Referer": self.base_url,
+                    },
+                )
+            except httpx.HTTPError as exc:
+                raise IctrpError(
+                    ErrorCode.UPSTREAM_ERROR,
+                    f"Search request failed: {_describe(exc)}",
+                ) from exc
+
+        response = await retry(f"search {keyword!r}", attempt_post)
+        assert isinstance(response, httpx.Response)
 
         location = response.headers.get("location")
         if location and "/noaccess.aspx" in location.lower():
@@ -205,20 +360,24 @@ class IctrpSession:
         # can never be reintroduced silently.
         htmlstate.assert_export_body_excludes_search_controls(body)
 
-        try:
-            response = await self.client.post(
-                self.base_url,
-                data=body,
-                headers={
-                    "Content-Type": "application/x-www-form-urlencoded",
-                    "Referer": self.base_url,
-                },
-            )
-        except httpx.HTTPError as exc:
-            raise IctrpError(
-                ErrorCode.UPSTREAM_ERROR,
-                f"Export request failed: {exc}",
-            ) from exc
+        async def attempt_export():
+            try:
+                return await self.client.post(
+                    self.base_url,
+                    data=body,
+                    headers={
+                        "Content-Type": "application/x-www-form-urlencoded",
+                        "Referer": self.base_url,
+                    },
+                )
+            except httpx.HTTPError as exc:
+                raise IctrpError(
+                    ErrorCode.UPSTREAM_ERROR,
+                    f"Export request failed: {_describe(exc)}",
+                ) from exc
+
+        response = await retry("export_csv", attempt_export)
+        assert isinstance(response, httpx.Response)
 
         self.steps.append(f"POST export -> {response.status_code}")
         self.response_date = response.headers.get("date")
@@ -236,7 +395,9 @@ class IctrpSession:
         return list(self.steps)
 
 
-async def run_chain(keyword: str, *, timeout: float = 120.0) -> tuple[CsvPayload, IctrpSession]:
+async def run_chain(
+    keyword: str, *, timeout: float | None = None
+) -> tuple[CsvPayload, IctrpSession]:
     """Run the full chain and return the payload plus the session for provenance."""
     session = IctrpSession.create(timeout=timeout)
     await session.load_form()