ictrp-mcp-server 0.1.0

package/sidecar/vendor/ictrp_mcp/errors.py

@@ -0,0 +1,123 @@
+"""Error taxonomy.
+
+The central design constraint: a failure must never be representable as a
+successful empty result. `NO_RESULTS` is reserved for exactly one situation -- a
+structurally valid CSV that arrived intact and contained zero data rows.
+
+Anything else that prevents us from returning records raises, so that callers
+receive an explicit failure rather than an empty list.
+
+See docs/MEASUREMENTS.md for the measured behaviour this encodes.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any
+
+
+class ErrorCode(str, Enum):
+    """Stable, machine-readable failure codes."""
+
+    #: The one legitimate zero. Only produced by a validated CSV with 0 data rows.
+    NO_RESULTS = "NO_RESULTS"
+
+    #: Upstream refused or intercepted the request (WAF, 403/405/429/503, block page).
+    UPSTREAM_BLOCKED = "UPSTREAM_BLOCKED"
+
+    #: The portal responded, but not in the shape we depend on: a redirect to
+    #: NoAccess.aspx, a non-CSV content type where CSV was requested, a changed
+    #: header, or a missing required column.
+    UPSTREAM_CONTRACT_DRIFT = "UPSTREAM_CONTRACT_DRIFT"
+
+    #: Transport-level failure: timeout, connection reset, 5xx.
+    UPSTREAM_ERROR = "UPSTREAM_ERROR"
+
+    #: The caller supplied something we cannot accept.
+    INVALID_ARGUMENT = "INVALID_ARGUMENT"
+
+    #: A referenced result-set id is unknown or expired.
+    CACHE_MISS = "CACHE_MISS"
+
+    #: Session state could not be established (no hidden inputs on the form page).
+    SESSION_FAILED = "SESSION_FAILED"
+
+
+#: Codes for which retrying the identical request may plausibly succeed.
+RETRYABLE: frozenset[ErrorCode] = frozenset(
+    {ErrorCode.UPSTREAM_ERROR, ErrorCode.SESSION_FAILED}
+)
+
+#: Codes that mean the upstream service did not give us data, as opposed to
+#: having told us there is none.
+FAILURE_CODES: frozenset[ErrorCode] = frozenset(
+    {
+        ErrorCode.UPSTREAM_BLOCKED,
+        ErrorCode.UPSTREAM_CONTRACT_DRIFT,
+        ErrorCode.UPSTREAM_ERROR,
+        ErrorCode.SESSION_FAILED,
+    }
+)
+
+
+class IctrpError(Exception):
+    """A failure carrying enough upstream context to be diagnosable.
+
+    `body_excerpt` is stripped of HTML and truncated; it is for diagnosis, never
+    for parsing.
+    """
+
+    def __init__(
+        self,
+        code: ErrorCode,
+        message: str,
+        *,
+        upstream_status: int | None = None,
+        upstream_content_type: str | None = None,
+        upstream_body_excerpt: str | None = None,
+        hint: str | None = None,
+        detail: str | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+        self.upstream_status = upstream_status
+        self.upstream_content_type = upstream_content_type
+        self.upstream_body_excerpt = upstream_body_excerpt
+        self.hint = hint
+        self.detail = detail
+
+    @property
+    def retryable(self) -> bool:
+        return self.code in RETRYABLE
+
+    def to_dict(self) -> dict[str, Any]:
+        out: dict[str, Any] = {
+            "status": _status_for(self.code),
+            "error_code": self.code.value,
+            "message": self.message,
+            "retryable": self.retryable,
+        }
+        if self.detail:
+            out["detail"] = self.detail
+        if self.upstream_status is not None:
+            out["upstream_status"] = self.upstream_status
+        if self.upstream_content_type:
+            out["upstream_content_type"] = self.upstream_content_type
+        if self.upstream_body_excerpt:
+            out["upstream_body_excerpt"] = self.upstream_body_excerpt
+        if self.hint:
+            out["hint"] = self.hint
+        return out
+
+
+def _status_for(code: ErrorCode) -> str:
+    return {
+        ErrorCode.NO_RESULTS: "no_results",
+        ErrorCode.UPSTREAM_BLOCKED: "upstream_blocked",
+        ErrorCode.UPSTREAM_CONTRACT_DRIFT: "upstream_contract_drift",
+        ErrorCode.UPSTREAM_ERROR: "upstream_error",
+        ErrorCode.INVALID_ARGUMENT: "invalid_argument",
+        ErrorCode.CACHE_MISS: "cache_miss",
+        ErrorCode.SESSION_FAILED: "session_failed",
+    }[code]

package/sidecar/vendor/ictrp_mcp/ictrp/export_guard.py

@@ -0,0 +1,269 @@
+"""Response classification -- applied to every upstream response before parsing.
+
+This module is the reason a WAF block page can never be mistaken for "no trials
+exist". The primary check is structural (content type + header shape), not
+textual: if we asked for a CSV and did not get a CSV with the expected header,
+that is a failure regardless of status code and regardless of body wording.
+
+Text markers exist only to improve the `hint`; they are never load-bearing,
+because the upstream block page's exact wording is not something we control or
+should depend on.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import re
+from dataclasses import dataclass
+
+from ..data.columns import REQUIRED_COLUMNS
+from ..errors import ErrorCode, IctrpError
+
+#: MIME type the portal uses for a successful export.
+CSV_CONTENT_TYPE = "application/vnd.ms-excel"
+
+#: Filename the portal advertises on a successful export.
+CSV_FILENAME = "IctrpResults.csv"
+
+#: Secondary classifier only. Used to pick a better hint, never to decide
+#: success or failure.
+_BLOCK_MARKERS = (
+    "access denied",
+    "request rejected",
+    "web application firewall",
+    "attention required",
+    "just a moment",
+    "security threat",
+    "your request has been blocked",
+    "访问被阻断",
+    "aliyunwaf",
+    "acw_sc__v2",
+)
+
+#: Marker in the redirect target that means we misused the session.
+_NOACCESS_MARKER = "/noaccess.aspx"
+
+_MAX_EXCERPT = 512
+
+
+@dataclass(frozen=True)
+class CsvPayload:
+    """A validated export response."""
+
+    header: tuple[str, ...]
+    rows: list[list[str]]
+    content_type: str | None
+    reported_total: int | None = None
+
+    @property
+    def row_count(self) -> int:
+        return len(self.rows)
+
+    @property
+    def is_empty(self) -> bool:
+        """True only for a structurally valid export with zero data rows."""
+        return len(self.rows) == 0
+
+
+def _excerpt(body: bytes | str, limit: int = _MAX_EXCERPT) -> str:
+    if isinstance(body, bytes):
+        text = body.decode("utf-8", "replace")
+    else:
+        text = body
+    text = re.sub(r"<script\b[^>]*>.*?</script>", " ", text, flags=re.S | re.I)
+    text = re.sub(r"<style\b[^>]*>.*?</style>", " ", text, flags=re.S | re.I)
+    text = re.sub(r"<[^>]+>", " ", text)
+    text = re.sub(r"\s+", " ", text).strip()
+    return text[:limit]
+
+
+def looks_like_block_page(body: bytes | str) -> bool:
+    text = _excerpt(body, 8000).lower()
+    return any(marker in text for marker in _BLOCK_MARKERS)
+
+
+def _is_successful_csv_envelope(
+    status: int, content_type: str | None, content_disposition: str | None
+) -> bool:
+    if status != 200:
+        return False
+    if not content_type:
+        return False
+    base = content_type.split(";")[0].strip().lower()
+    if base == CSV_CONTENT_TYPE or base in ("text/csv", "application/csv"):
+        return True
+    # Some responses omit the exact MIME but still advertise the filename.
+    if content_disposition and CSV_FILENAME.lower() in content_disposition.lower():
+        return True
+    return False
+
+
+def _parse_csv(body: bytes) -> tuple[tuple[str, ...], list[list[str]]]:
+    text = body.decode("utf-8-sig", "replace")
+    reader = csv.reader(io.StringIO(text))
+    try:
+        header = next(reader)
+    except StopIteration:
+        return (), []
+    header = tuple(h.strip() for h in header)
+    rows = [r for r in reader if r and any(c.strip() for c in r)]
+    return header, rows
+
+
+def _validate_header(header: tuple[str, ...]) -> None:
+    missing = REQUIRED_COLUMNS - set(header)
+    if missing:
+        raise IctrpError(
+            ErrorCode.UPSTREAM_CONTRACT_DRIFT,
+            "ICTRP export header is missing required column(s)",
+            detail=f"missing: {sorted(missing)}",
+            hint=(
+                "The portal's export format has changed. Update data/columns.py "
+                "and the field mapping before relying on results."
+            ),
+        )
+
+
+def classify_export(
+    *,
+    status: int,
+    content_type: str | None,
+    content_disposition: str | None,
+    body: bytes,
+    location: str | None = None,
+    reported_total: int | None = None,
+) -> CsvPayload:
+    """Classify an export response.
+
+    Returns a validated `CsvPayload` (possibly empty) or raises `IctrpError`.
+    An empty payload is the ONLY way a caller may report zero results.
+    """
+    if location and _NOACCESS_MARKER in location.lower():
+        raise IctrpError(
+            ErrorCode.UPSTREAM_CONTRACT_DRIFT,
+            "Portal redirected the export to NoAccess.aspx",
+            upstream_status=status,
+            upstream_content_type=content_type,
+            hint=(
+                "The export POST was rejected. Verify that the results-page hidden "
+                "inputs were used verbatim and that TextBox1/Button1 were NOT sent, "
+                "since the results page does not contain them."
+            ),
+        )
+
+    if status in (403, 405, 429, 503):
+        raise IctrpError(
+            ErrorCode.UPSTREAM_BLOCKED,
+            f"Upstream refused the request (HTTP {status})",
+            upstream_status=status,
+            upstream_content_type=content_type,
+            upstream_body_excerpt=_excerpt(body),
+            hint=(
+                "The portal blocked this request. This is not an empty result set; "
+                "do not report zero trials. Retry later, and reduce request rate."
+            ),
+        )
+
+    if status >= 500:
+        raise IctrpError(
+            ErrorCode.UPSTREAM_ERROR,
+            f"Upstream server error (HTTP {status})",
+            upstream_status=status,
+            upstream_content_type=content_type,
+            upstream_body_excerpt=_excerpt(body),
+            hint="Transient upstream failure. Retry with backoff.",
+        )
+
+    if not _is_successful_csv_envelope(status, content_type, content_disposition):
+        hint = "Response was not a CSV export."
+        if looks_like_block_page(body):
+            hint = (
+                "Response looks like an HTML block/interstitial page, not an export. "
+                "Treat as blocked, not as zero results."
+            )
+        raise IctrpError(
+            ErrorCode.UPSTREAM_CONTRACT_DRIFT,
+            "Export response was not a CSV",
+            upstream_status=status,
+            upstream_content_type=content_type,
+            upstream_body_excerpt=_excerpt(body),
+            hint=hint,
+        )
+
+    header, rows = _parse_csv(body)
+    if not header:
+        raise IctrpError(
+            ErrorCode.UPSTREAM_CONTRACT_DRIFT,
+            "Export body contained no CSV header row",
+            upstream_status=status,
+            upstream_content_type=content_type,
+            upstream_body_excerpt=_excerpt(body),
+        )
+
+    _validate_header(header)
+
+    # Pad short rows so column indexing is always safe. Upstream can emit rows
+    # with trailing empty fields omitted.
+    width = len(header)
+    rows = [(r + [""] * width)[:width] for r in rows]
+
+    return CsvPayload(
+        header=header,
+        rows=rows,
+        content_type=content_type,
+        reported_total=reported_total,
+    )
+
+
+def classify_form_page(*, status: int, body: bytes) -> str:
+    """Validate the initial GET of the search form. Returns the HTML text."""
+    text = body.decode("utf-8", "replace")
+    if status in (403, 405, 429, 503):
+        raise IctrpError(
+            ErrorCode.UPSTREAM_BLOCKED,
+            f"Search form was refused (HTTP {status})",
+            upstream_status=status,
+            upstream_body_excerpt=_excerpt(body),
+        )
+    if status >= 500:
+        raise IctrpError(
+            ErrorCode.UPSTREAM_ERROR,
+            f"Search form returned HTTP {status}",
+            upstream_status=status,
+            upstream_body_excerpt=_excerpt(body),
+        )
+    if status != 200:
+        raise IctrpError(
+            ErrorCode.UPSTREAM_CONTRACT_DRIFT,
+            f"Unexpected status for search form: HTTP {status}",
+            upstream_status=status,
+            upstream_body_excerpt=_excerpt(body),
+        )
+    if looks_like_block_page(body):
+        raise IctrpError(
+            ErrorCode.UPSTREAM_BLOCKED,
+            "Search form returned a block/interstitial page",
+            upstream_status=status,
+            upstream_body_excerpt=_excerpt(body),
+        )
+    return text
+
+
+_RECORDS_RE = re.compile(r"([\d,]+)\s*records", re.I)
+
+
+def parse_reported_total(html: str) -> int | None:
+    """Extract the portal's own 'N records' figure from a results page.
+
+    This is the portal's claim, NOT a verified count of what we received. The
+    two routinely differ -- measured shortfalls of 0.4% to 29% -- so callers must
+    surface both and never conflate them. See docs/MEASUREMENTS.md section 2.
+    """
+    match = _RECORDS_RE.search(html)
+    if not match:
+        return None
+    try:
+        return int(match.group(1).replace(",", ""))
+    except ValueError:
+        return None

package/sidecar/vendor/ictrp_mcp/ictrp/htmlstate.py

@@ -0,0 +1,143 @@
+"""HTML form-state extraction.
+
+The portal is ASP.NET WebForms. Every step of the chain depends on hidden inputs
+carried forward from the previous response, so extraction has to be exact and
+lossless -- a dropped field produces a 302 to NoAccess.aspx, which is
+indistinguishable from a block unless you check the redirect target.
+
+The extraction is deliberately dumb (regex over input tags) rather than
+DOM-based, because the pages mix well-formed and sloppy markup and we only ever
+need attributed name/value pairs.
+"""
+
+from __future__ import annotations
+
+import html as _html
+import re
+from dataclasses import dataclass
+
+#: Inputs that carry WebForms state and must be replayed on the next POST.
+STATE_FIELDS: tuple[str, ...] = (
+    "__VIEWSTATE",
+    "__VIEWSTATEGENERATOR",
+    "__VIEWSTATEENCRYPTED",
+    "__EVENTVALIDATION",
+    "__EVENTTARGET",
+    "__EVENTARGUMENT",
+    "__LASTFOCUS",
+    "ToolkitScriptManager_HiddenField",
+)
+
+_INPUT_RE = re.compile(r"<input\b[^>]*>", re.I)
+_ATTR_RE = re.compile(r"""([A-Za-z_:][-A-Za-z0-9_:.]*)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))""")
+_TAG_RE = re.compile(r"<[^>]+>")
+_WS_RE = re.compile(r"\s+")
+
+
+@dataclass(frozen=True)
+class FormState:
+    """Hidden inputs harvested from one page, ready to replay."""
+
+    fields: dict[str, str]
+
+    def merged_with(self, overrides: dict[str, str]) -> dict[str, str]:
+        """Return the state fields plus caller-supplied control values."""
+        out = dict(self.fields)
+        out.update(overrides)
+        return out
+
+    def has(self, name: str) -> bool:
+        return name in self.fields
+
+    def missing_state(self) -> list[str]:
+        """State fields we expect on a form page but did not find."""
+        required = ("__VIEWSTATE", "__EVENTVALIDATION")
+        return [f for f in required if f not in self.fields]
+
+
+def _attrs(tag: str) -> dict[str, str]:
+    got: dict[str, str] = {}
+    for m in _ATTR_RE.finditer(tag):
+        name = m.group(1).lower()
+        value = m.group(2) if m.group(2) is not None else (
+            m.group(3) if m.group(3) is not None else (m.group(4) or "")
+        )
+        got[name] = _html.unescape(value)
+    return got
+
+
+def extract_inputs(html: str) -> dict[str, str]:
+    """Return every named input's value from a page.
+
+    Includes buttons and text boxes, because the export step needs to know which
+    controls exist -- notably to *exclude* controls that the results page does
+    not carry.
+    """
+    found: dict[str, str] = {}
+    for tag in _INPUT_RE.finditer(html):
+        attrs = _attrs(tag.group(0))
+        name = attrs.get("name")
+        if not name:
+            continue
+        found[name] = attrs.get("value", "")
+    return found
+
+
+def extract_form_state(html: str) -> FormState:
+    """Extract just the WebForms state fields."""
+    all_inputs = extract_inputs(html)
+    return FormState({k: v for k, v in all_inputs.items() if k in STATE_FIELDS})
+
+
+def extract_select_options(html: str, select_name: str) -> list[tuple[str, str]]:
+    """Return (value, label) pairs for a named <select>."""
+    pattern = re.compile(
+        r"<select\b[^>]*\bname\s*=\s*[\"']?" + re.escape(select_name) + r"[\"']?[^>]*>(.*?)</select>",
+        re.I | re.S,
+    )
+    match = pattern.search(html)
+    if not match:
+        return []
+    options: list[tuple[str, str]] = []
+    for opt in re.finditer(r"<option\b([^>]*)>(.*?)</option>", match.group(1), re.I | re.S):
+        attrs = _attrs(opt.group(1))
+        label = _WS_RE.sub(" ", _TAG_RE.sub("", opt.group(2))).strip()
+        options.append((_html.unescape(attrs.get("value", "")), label))
+    return options
+
+
+def text_of(html: str) -> str:
+    """Collapse a fragment to plain text."""
+    return _WS_RE.sub(" ", _TAG_RE.sub(" ", html)).strip()
+
+
+def build_export_body(results_html: str, *, export_control: str = "Button7") -> dict[str, str]:
+    """Build the export POST body from a results page.
+
+    CRITICAL: the body is built ONLY from fields the results page actually
+    carries, plus the export button. The search controls (`TextBox1`, `Button1`)
+    are deliberately NOT sent, even when present. Measured: including controls
+    that the results page does not carry makes the portal answer `302 Found` to
+    `/NoAccess.aspx?aspxerrorpath=/Default.aspx`.
+
+    The caller is responsible for verifying the search controls are absent via
+    `assert_export_body_excludes_search_controls`.
+    """
+    state = extract_form_state(results_html)
+    body = dict(state.fields)
+    body[export_control] = "Export to CSV"
+    return body
+
+
+def assert_export_body_excludes_search_controls(body: dict[str, str]) -> None:
+    """Guard against reintroducing the measured 302-to-NoAccess failure.
+
+    `TextBox1`/`Button1` are search-form controls. Sending them on the export
+    POST is the specific mistake that produced the measured error redirect.
+    """
+    for forbidden in ("TextBox1", "Button1"):
+        if forbidden in body:
+            raise AssertionError(
+                f"export body must not contain search control {forbidden!r}; "
+                "including it produces a 302 to /NoAccess.aspx"
+            )