@gajae-code/coding-agent 0.7.1 → 0.7.2

package/vendor/insane-search/engine/url_transforms.py

@@ -0,0 +1,98 @@
+"""Generic URL transforms for the fetch grid.
+
+Transforms are domain-agnostic *rules*. They never reference a specific
+site by name. A transform either applies (returns a new URL) or is skipped
+(returns None). Callers iterate transforms in order.
+
+Empirically useful transforms (see observations/):
+  * mobile_subdomain — `www.example.com` → `m.example.com`
+    Strong win on SSR sites with mobile-first serving. Loss on SPA shells
+    (some mobile sites return tiny bootstrap HTML).
+  * am_prefix — `example.com` (no www) → `m.example.com`
+  * drop_www — occasionally unblocks hosts that gate www but not apex.
+
+Adding new transforms: prove they help on ≥2 unrelated sites first
+(cross-site validation — bias check).
+"""
+from __future__ import annotations
+
+from typing import Callable, Optional
+from urllib.parse import urlsplit, urlunsplit
+
+
+def _replace_host(url: str, new_host: str) -> str:
+    parts = urlsplit(url)
+    return urlunsplit(parts._replace(netloc=new_host))
+
+
+def _original(url: str) -> Optional[str]:
+    return url
+
+
+def _mobile_subdomain(url: str) -> Optional[str]:
+    """`https://www.example.com/a` → `https://m.example.com/a` (only if host starts with www.)."""
+    parts = urlsplit(url)
+    host = parts.hostname or ""
+    if not host.startswith("www."):
+        return None
+    new_host = "m." + host[4:]
+    if parts.port:
+        new_host = f"{new_host}:{parts.port}"
+    return _replace_host(url, new_host)
+
+
+def _am_prefix(url: str) -> Optional[str]:
+    """`https://example.com/a` → `https://m.example.com/a` (only if host has no subdomain)."""
+    parts = urlsplit(url)
+    host = parts.hostname or ""
+    if not host or host.startswith("m."):
+        return None
+    # Only apply to apex-like hosts (≤2 dot-separated labels).
+    if host.count(".") >= 2 and not host.startswith("www."):
+        return None
+    if host.startswith("www."):
+        return None  # handled by mobile_subdomain
+    return _replace_host(url, "m." + host)
+
+
+def _drop_www(url: str) -> Optional[str]:
+    parts = urlsplit(url)
+    host = parts.hostname or ""
+    if not host.startswith("www."):
+        return None
+    return _replace_host(url, host[4:])
+
+
+TRANSFORMS: dict[str, Callable[[str], Optional[str]]] = {
+    "original": _original,
+    "mobile_subdomain": _mobile_subdomain,
+    "am_prefix": _am_prefix,
+    "drop_www": _drop_www,
+}
+
+
+def apply_transform(name: str, url: str) -> Optional[str]:
+    """Apply one transform by name. Returns transformed URL or None if skipped."""
+    fn = TRANSFORMS.get(name)
+    if fn is None:
+        raise ValueError(f"Unknown transform: {name!r}. Known: {list(TRANSFORMS)}")
+    return fn(url)
+
+
+def iter_transformed(url: str, order: list[str]) -> list[tuple[str, str]]:
+    """Yield (transform_name, transformed_url) pairs for a given order.
+
+    Skips transforms that return None (not applicable) and deduplicates
+    URLs (so `original` and `drop_www` of `https://example.com` don't double-run).
+    """
+    seen: set[str] = set()
+    out: list[tuple[str, str]] = []
+    for name in order:
+        new_url = apply_transform(name, url)
+        if new_url is None:
+            continue
+        if new_url in seen:
+            continue
+        seen.add(new_url)
+        out.append((name, new_url))
+    return out

package/vendor/insane-search/engine/validators.py

@@ -0,0 +1,331 @@
+"""Generic challenge / success validator (v2).
+
+Layers (all generic, never site-specific):
+  1. HTTP status semantics (rate-limit / auth / not-found / transient / blocked)
+  2. HARD challenge markers (structural WAF containers — always decisive)
+  3. Size fingerprints (known bad BYTE sizes hinted by caller)
+  4. Content-Type / JSON awareness (small JSON APIs are NOT challenges)
+  5. Caller success_selectors (strongest positive proof for HTML)
+  6. SOFT markers + cookie sensor + tiny-body heuristics (only when no
+     positive proof is available)
+
+v2 changes vs v1 (per multi-AI review 2026-06-21):
+  * `WEAK_OK` is reserved for genuinely clean responses. Ambiguous states
+    (`_abck` unresolved, soft-block words without proof) now return the new
+    non-terminal `SUSPECT_OK` so the fetch chain keeps searching instead of
+    declaring a blocked page a success.
+  * Small valid JSON (e.g. an internal API) is no longer mislabelled
+    `CHALLENGE` — unblocks the R7 API-first route.
+  * SOFT markers (e.g. the word "captcha" buried in a script) no longer
+    override a matched success_selector.
+  * Size compared in BYTES, not unicode char count.
+  * Status codes differentiated (429/401/404/5xx) instead of one BLOCKED.
+"""
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Optional
+
+try:
+    from bs4 import BeautifulSoup
+except ImportError:  # bs4 is a soft dep: only used when selectors given
+    BeautifulSoup = None  # type: ignore
+
+
+# HARD markers: structural challenge/block containers. Decisive on their own —
+# these strings do not appear in legitimate page content. (WAF products only.)
+HARD_CHALLENGE_MARKERS: list[str] = [
+    "sec-if-cpt-container",
+    "Powered and protected by Akamai",
+    "Just a moment...",
+    "cf-chl-bypass",
+    "Attention Required! | Cloudflare",
+    "<title>Bot Challenge</title>",
+    "The requested URL was rejected",
+    "Request unsuccessful. Incapsula",
+    "Please enable JS and disable any ad blocker",
+]
+
+# SOFT markers: words that strongly suggest a challenge BUT can legitimately
+# appear in real content (scripts, articles about bots, etc). Only decisive
+# when the caller has no positive proof (success_selectors) that overrides.
+SOFT_CHALLENGE_MARKERS: list[str] = [
+    "access denied",
+    "checking your browser",
+    "datadome",
+    "captcha",
+]
+
+# Backward-compatible export (some callers import CHALLENGE_MARKERS).
+CHALLENGE_MARKERS: list[str] = HARD_CHALLENGE_MARKERS + SOFT_CHALLENGE_MARKERS
+
+# Minimum BODY BYTE size below which we suspect a stub / challenge page.
+SMALL_BODY_THRESHOLD = 3000
+
+
+class Verdict(Enum):
+    """Classification of a fetched response."""
+
+    STRONG_OK = "strong_ok"        # positive proof present → terminal success
+    WEAK_OK = "weak_ok"            # clean, no negative signal → terminal success
+    SUSPECT_OK = "suspect_ok"      # ambiguous (abck unresolved / soft) → NON-terminal
+    CHALLENGE = "challenge"        # WAF challenge (negative proof)
+    BLOCKED = "blocked"            # generic non-2xx block
+    RATE_LIMITED = "rate_limited"  # 429 — back off, do not hammer
+    AUTH_REQUIRED = "auth_required"  # 401/407 — terminal, retrying TLS won't help
+    NOT_FOUND = "not_found"        # 404/410 — terminal
+    UNKNOWN = "unknown"            # exception / dependency missing
+
+
+# Verdicts that mean "stop the grid — more TLS attempts cannot help".
+TERMINAL_NONSUCCESS = frozenset({
+    Verdict.AUTH_REQUIRED, Verdict.NOT_FOUND, Verdict.RATE_LIMITED,
+})
+
+
+@dataclass
+class ValidationResult:
+    verdict: Verdict
+    reasons: list[str] = field(default_factory=list)
+    matched_selectors: list[str] = field(default_factory=list)
+    body_size: int = 0       # bytes
+    status: int = 0
+
+    @property
+    def ok(self) -> bool:
+        """Terminal success only. SUSPECT_OK is intentionally excluded."""
+        return self.verdict in (Verdict.STRONG_OK, Verdict.WEAK_OK)
+
+    def to_dict(self) -> dict:
+        return {
+            "verdict": self.verdict.value,
+            "reasons": self.reasons,
+            "matched_selectors": self.matched_selectors,
+            "body_size": self.body_size,
+            "status": self.status,
+        }
+
+
+def _hard_marker_hits(body_lower: str) -> list[str]:
+    return [m for m in HARD_CHALLENGE_MARKERS if m.lower() in body_lower]
+
+
+def _soft_marker_hits(body_lower: str) -> list[str]:
+    return [m for m in SOFT_CHALLENGE_MARKERS if m in body_lower]
+
+
+def _abck_unresolved(cookies: dict) -> bool:
+    abck = cookies.get("_abck", "")
+    return bool(abck) and "~-1~" in abck
+
+
+def _content_type(resp) -> str:
+    try:
+        headers = {k.lower(): v for k, v in dict(getattr(resp, "headers", {}) or {}).items()}
+        return str(headers.get("content-type", "")).lower()
+    except Exception:
+        return ""
+
+
+def _looks_like_json(text: str, ctype: str) -> bool:
+    if "json" in ctype:
+        return True
+    s = text.lstrip()[:1]
+    return s in ("{", "[")
+
+
+def _json_ok(text: str) -> Optional[bool]:
+    """True if text parses as non-empty JSON, False if parses-but-empty,
+    None if not parseable."""
+    try:
+        obj = json.loads(text)
+    except Exception:
+        return None
+    if obj in (None, {}, [], ""):
+        return False
+    return True
+
+
+def _byte_size(resp, text: str) -> int:
+    content = getattr(resp, "content", None)
+    if isinstance(content, (bytes, bytearray)):
+        return len(content)
+    return len(text.encode("utf-8", "ignore"))
+
+
+def _looks_complete_content_page(text: str, lowered: str) -> bool:
+    """True when a SMALL body is still a real (short) page, not a challenge stub.
+
+    A genuine page is a COMPLETE HTML document (closes `</html>`/`</body>`) that
+    carries meaningful visible text — e.g. example.com at ~600B. A WAF interstitial
+    that slipped past the marker checks is typically script-only, empty, or an
+    incomplete fragment, so it has little visible text and returns False."""
+    if "</html>" not in lowered and "</body>" not in lowered:
+        return False
+    visible = re.sub(r"(?is)<(script|style)[^>]*>.*?</\1>", " ", text)
+    visible = re.sub(r"(?s)<[^>]+>", " ", visible)
+    visible = re.sub(r"\s+", " ", visible).strip()
+    return len(visible) >= 64
+
+
+def _selector_hits(body: str, selectors: list[str]) -> Optional[list[str]]:
+    """Return matched-selector list, or None if BS4 is unavailable."""
+    if BeautifulSoup is None:
+        return None
+    try:
+        soup = BeautifulSoup(body, "html.parser")
+    except Exception:
+        return []
+    hits: list[str] = []
+    for sel in selectors:
+        try:
+            if soup.select(sel):
+                hits.append(sel)
+        except Exception:
+            continue
+    return hits
+
+
+def validate(
+    resp,
+    *,
+    success_selectors: Optional[list[str]] = None,
+    known_bad_sizes: Optional[list[int]] = None,
+    size_tolerance: int = 20,
+) -> ValidationResult:
+    """Validate a `curl_cffi` / `requests` response (v2)."""
+    try:
+        status = int(getattr(resp, "status_code", 0) or 0)
+        text = getattr(resp, "text", "") or ""
+        size = _byte_size(resp, text)
+    except Exception as e:
+        return ValidationResult(verdict=Verdict.UNKNOWN, reasons=[f"parse_error:{e}"])
+
+    r = ValidationResult(verdict=Verdict.UNKNOWN, body_size=size, status=status)
+
+    # --- Layer 1: status semantics ----------------------------------------
+    if status == 429:
+        r.verdict = Verdict.RATE_LIMITED
+        r.reasons.append("status=429")
+        return r
+    if status in (401, 407):
+        r.verdict = Verdict.AUTH_REQUIRED
+        r.reasons.append(f"status={status}")
+        return r
+    if status in (404, 410):
+        r.verdict = Verdict.NOT_FOUND
+        r.reasons.append(f"status={status}")
+        return r
+    if 500 <= status <= 599:
+        r.verdict = Verdict.BLOCKED
+        r.reasons.append(f"status={status}")
+        return r
+    if status == 0:
+        r.verdict = Verdict.UNKNOWN
+        r.reasons.append("status=0")
+        return r
+    # 403/406/etc fall through to marker analysis (often a WAF challenge body).
+
+    lowered = text.lower()
+
+    # --- Layer 2: HARD markers (decisive) ---------------------------------
+    hard = _hard_marker_hits(lowered)
+    if hard:
+        r.verdict = Verdict.CHALLENGE
+        r.reasons.extend(f"hard:{m}" for m in hard[:3])
+        return r
+
+    # --- Layer 3: size fingerprint (bytes, tolerant) ----------------------
+    if known_bad_sizes:
+        for bad in known_bad_sizes:
+            if abs(size - bad) <= size_tolerance:
+                r.verdict = Verdict.CHALLENGE
+                r.reasons.append(f"size_fp:{size}~{bad}")
+                return r
+
+    # --- Layer 4: JSON awareness (before tiny-body heuristic) -------------
+    ctype = _content_type(resp)
+    if _looks_like_json(text, ctype):
+        j = _json_ok(text)
+        if j is True:
+            # A 2xx with non-empty parseable JSON is a successful API hit even
+            # if tiny. CSS selectors don't apply to JSON, so WEAK_OK is the
+            # ceiling here (no HTML positive-proof concept).
+            r.verdict = Verdict.WEAK_OK
+            r.reasons.append("json_ok")
+            return r
+        if j is False:
+            r.verdict = Verdict.SUSPECT_OK
+            r.reasons.append("json_empty")
+            return r
+        # j is None → not actually JSON; fall through to HTML handling.
+
+    cookies = _extract_cookies(resp)
+    abck_bad = _abck_unresolved(cookies)
+
+    # --- Layer 5: caller positive proof (HTML) ----------------------------
+    if success_selectors:
+        hits = _selector_hits(text, success_selectors)
+        if hits is None:
+            r.verdict = Verdict.UNKNOWN
+            r.reasons.append("bs4_missing")
+            return r
+        if hits:
+            r.matched_selectors = hits
+            # Selector matched → soft markers are ignored (they were likely in
+            # a script or unrelated text). But an unresolved sensor cookie
+            # still demotes us to NON-terminal SUSPECT_OK.
+            if abck_bad:
+                r.reasons.append("abck_unresolved")
+                r.verdict = Verdict.SUSPECT_OK
+                return r
+            r.verdict = Verdict.STRONG_OK
+            return r
+        # Selectors requested but none matched → challenge.
+        r.verdict = Verdict.CHALLENGE
+        r.reasons.append("no_success_selector")
+        return r
+
+    # --- Layer 6: no positive proof — heuristics --------------------------
+    soft = _soft_marker_hits(lowered)
+    if soft:
+        r.verdict = Verdict.CHALLENGE
+        r.reasons.extend(f"soft:{m}" for m in soft[:3])
+        return r
+
+    if size < SMALL_BODY_THRESHOLD:
+        # A small body is only weak evidence of a challenge stub. A COMPLETE,
+        # content-bearing HTML document that just happens to be short (e.g.
+        # example.com ~600B) is a real page → clean weak success. Only an
+        # incomplete / script-only / empty small body stays suspicious.
+        if _looks_complete_content_page(text, lowered):
+            r.verdict = Verdict.WEAK_OK
+            r.reasons.append(f"small_but_complete:{size}")
+            return r
+        r.verdict = Verdict.CHALLENGE
+        r.reasons.append(f"tiny_body:{size}")
+        return r
+
+    if abck_bad:
+        # Unresolved Akamai sensor with no positive proof: do NOT declare
+        # success. Non-terminal so the chain keeps trying.
+        r.reasons.append("abck_unresolved")
+        r.verdict = Verdict.SUSPECT_OK
+        return r
+
+    # Clean, sizeable, no negative signal, no sensor problem → terminal weak ok.
+    r.verdict = Verdict.WEAK_OK
+    return r
+
+
+def _extract_cookies(resp) -> dict:
+    try:
+        return {c.name: c.value for c in resp.cookies.jar}
+    except Exception:
+        try:
+            return dict(resp.cookies) if hasattr(resp, "cookies") else {}
+        except Exception:
+            return {}

package/vendor/insane-search/engine/waf_detector.py

@@ -0,0 +1,214 @@
+"""WAF-product detection from a live response.
+
+Returns a *ranking* of (profile_id, confidence) pairs — never a single verdict.
+Single-answer detectors cause cascading wrong plans when misfiring (Codex's
+critique). Planner consumes the ranking and tries top candidates in order.
+
+All detectors operate on WAF-vendor artifacts (cookies / headers / body
+strings) — never site hostnames. See engine/waf_profiles.yaml for the
+profile definitions.
+"""
+from __future__ import annotations
+
+import fnmatch
+import os
+import re
+from dataclasses import dataclass
+from typing import Optional
+
+try:
+    import yaml  # PyYAML
+except ImportError:
+    yaml = None  # type: ignore
+
+
+PROFILES_PATH = os.path.join(os.path.dirname(__file__), "waf_profiles.yaml")
+
+
+# In-code safety net — used when waf_profiles.yaml is missing / invalid
+# or PyYAML isn't installed. Keeps fetch() working in a degraded-but-sane
+# mode. Must stay site-agnostic (No-Site-Name Rule).
+_DEFAULT_PROFILES: dict = {
+    "unknown_challenge": {
+        "detectors": {},
+        "confidence_rules": {"strong": 0, "weak": 0},
+        "capabilities_needed": ["needs_js_exec"],
+        "tls_impersonate_candidates": [
+            ["safari", "chrome", "firefox"],
+            ["safari_ios", "chrome_android"],
+        ],
+        "referer_strategies": ["self_root", "google_search", "none"],
+        "url_transform_order": ["original", "mobile_subdomain"],
+        "fallback_when_challenge": ["playwright_mcp", "playwright_real_chrome"],
+        "notes": "in-code default — waf_profiles.yaml unavailable",
+    },
+}
+
+
+# Module-level sticky error. Readers call `last_load_error()` after each
+# `_load_profiles()` call to surface YAML problems in FetchResult.trace.
+_LAST_LOAD_ERROR: Optional[str] = None
+
+
+@dataclass
+class DetectionHit:
+    profile_id: str
+    confidence: float
+    signals: list[str]
+
+
+def last_load_error() -> Optional[str]:
+    """Return the most recent profile-loader error (or None if clean)."""
+    return _LAST_LOAD_ERROR
+
+
+def _load_profiles(path: str = PROFILES_PATH) -> dict:
+    """Load profiles with graceful fallback.
+
+    Never raises. On any failure (PyYAML missing, file missing, parse error,
+    unexpected shape) it returns a copy of `_DEFAULT_PROFILES` and stores
+    the reason in `_LAST_LOAD_ERROR` for the caller to surface.
+    """
+    global _LAST_LOAD_ERROR
+    _LAST_LOAD_ERROR = None
+
+    if yaml is None:
+        _LAST_LOAD_ERROR = "PyYAML not installed — using in-code default profile"
+        return dict(_DEFAULT_PROFILES)
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            loaded = yaml.safe_load(f) or {}
+    except FileNotFoundError:
+        _LAST_LOAD_ERROR = f"waf_profiles.yaml not found at {path}"
+        return dict(_DEFAULT_PROFILES)
+    except yaml.YAMLError as e:
+        _LAST_LOAD_ERROR = f"YAML parse error: {type(e).__name__}: {str(e)[:200]}"
+        return dict(_DEFAULT_PROFILES)
+    except Exception as e:
+        _LAST_LOAD_ERROR = f"profile loader: {type(e).__name__}: {str(e)[:200]}"
+        return dict(_DEFAULT_PROFILES)
+
+    if not isinstance(loaded, dict) or not any(k for k in loaded if not k.startswith("_")):
+        _LAST_LOAD_ERROR = f"waf_profiles.yaml has no usable profiles"
+        return dict(_DEFAULT_PROFILES)
+
+    return loaded
+
+
+def _cookies_dict(resp) -> dict:
+    try:
+        return {c.name: c.value for c in resp.cookies.jar}
+    except Exception:
+        try:
+            return dict(resp.cookies) if hasattr(resp, "cookies") else {}
+        except Exception:
+            return {}
+
+
+def _headers_dict(resp) -> dict:
+    try:
+        return {k.lower(): v for k, v in dict(resp.headers).items()}
+    except Exception:
+        return {}
+
+
+def _match_patterns(haystack_keys: list[str], patterns: list[str]) -> list[str]:
+    """Match literal names or fnmatch patterns (for wildcards like `X-Akamai-*`)."""
+    hits: list[str] = []
+    lowered_keys = [k.lower() for k in haystack_keys]
+    for pat in patterns or []:
+        pat_l = pat.lower()
+        if any(c in pat for c in "*?["):
+            for key in lowered_keys:
+                if fnmatch.fnmatchcase(key, pat_l):
+                    hits.append(pat)
+                    break
+        else:
+            if pat_l in lowered_keys:
+                hits.append(pat)
+    return hits
+
+
+def _score_profile(profile_id: str, profile: dict, resp) -> Optional[DetectionHit]:
+    """Apply profile detectors to resp. Returns hit or None."""
+    if profile_id.startswith("_"):
+        return None
+    detectors = profile.get("detectors") or {}
+    if not detectors and profile_id != "unknown_challenge":
+        return None
+
+    cookies = _cookies_dict(resp)
+    headers = _headers_dict(resp)
+    body = (getattr(resp, "text", "") or "").lower()
+    server = headers.get("server", "")
+
+    signals: list[str] = []
+
+    # Cookie detectors
+    cookie_pats = detectors.get("cookie") or []
+    for hit in _match_patterns(list(cookies.keys()), cookie_pats):
+        signals.append(f"cookie:{hit}")
+
+    # Header detectors
+    header_pats = detectors.get("header") or []
+    for hit in _match_patterns(list(headers.keys()), header_pats):
+        signals.append(f"header:{hit}")
+
+    # Server substring
+    for needle in detectors.get("server_contains") or []:
+        if needle.lower() in server:
+            signals.append(f"server:{needle}")
+
+    # Body markers
+    for needle in detectors.get("body") or []:
+        if needle.lower() in body:
+            signals.append(f"body:{needle}")
+
+    if not signals:
+        return None
+
+    rules = profile.get("confidence_rules") or {"strong": 2, "weak": 1}
+    n = len(signals)
+    if n >= rules.get("strong", 2):
+        conf = 0.9
+    elif n >= rules.get("weak", 1):
+        conf = 0.6
+    else:
+        conf = 0.3
+
+    return DetectionHit(profile_id=profile_id, confidence=conf, signals=signals)
+
+
+def detect(resp, *, profiles: Optional[dict] = None, min_confidence: float = 0.0) -> list[DetectionHit]:
+    """Return ranked list of detection hits (best first).
+
+    When nothing fires, the returned list contains a single `unknown_challenge`
+    hit with confidence 0.1 — caller can use its conservative settings.
+    """
+    if profiles is None:
+        profiles = _load_profiles()
+
+    hits: list[DetectionHit] = []
+    for profile_id, profile in profiles.items():
+        if profile_id.startswith("_"):
+            continue
+        h = _score_profile(profile_id, profile, resp)
+        if h and h.confidence >= min_confidence:
+            hits.append(h)
+
+    hits.sort(key=lambda x: x.confidence, reverse=True)
+
+    if not hits:
+        hits.append(DetectionHit(
+            profile_id="unknown_challenge",
+            confidence=0.1,
+            signals=["fallback"],
+        ))
+    return hits
+
+
+def load_profile(profile_id: str, *, profiles: Optional[dict] = None) -> dict:
+    """Get one profile by id, resolving `unknown_challenge` if missing."""
+    if profiles is None:
+        profiles = _load_profiles()
+    return profiles.get(profile_id) or profiles.get("unknown_challenge") or {}