oh-my-opencode 4.19.4 → 5.0.0-beta.1

package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/fetch_chain.py

@@ -40,6 +40,7 @@ def fetch(
     timeout: int = 25,
     max_attempts: int = 12,
     enable_playwright: bool = True,   # hook left for executor module
+    allow_surrogate_proxy: bool = False,  # opt-in: kind=proxy registry entries
 ) -> FetchResult:
     """Fetch `url` using the generic grid.
 
@@ -175,73 +176,110 @@ def fetch(
                     if att.verdict in (Verdict.STRONG_OK.value, Verdict.WEAK_OK.value):
                         return _build_result(resp, att, trace, profile_used=profile_id)
 
-    # -------- Phase 3: Playwright fallback (profile-driven order) -----------
-    if enable_playwright:
-        try:
-            from .executor import run_playwright_fallback  # lazy import
-            # Honour profile's `fallback_when_challenge` list — iterate the
-            # caller-declared order instead of capability-inferred single pick.
-            fb_profile = load_profile(profile_used or "unknown_challenge", profiles=profiles)
-            fb_order = fb_profile.get("fallback_when_challenge") or ["playwright_real_chrome"]
-            pw_attempt = None
-            pw_content = ""
-            for fb_name in fb_order:
-                if fb_name == "curl_grid_exhaust":
-                    # Already performed in Phase 2; nothing more to do here.
-                    continue
-                pw_attempt, pw_content = run_playwright_fallback(
+    # -------- Phase 2.5 + 3: fallback ladder (profile-driven order) ---------
+    # Surrogate routes are NOT browser work, so they run even when the caller
+    # disabled Playwright; only browser executors honour `enable_playwright`.
+    try:
+        # Honour profile's `fallback_when_challenge` list — iterate the
+        # caller-declared order instead of capability-inferred single pick.
+        fb_profile = load_profile(profile_used or "unknown_challenge", profiles=profiles)
+        fb_order = fb_profile.get("fallback_when_challenge") or ["playwright_real_chrome"]
+        pw_attempt = None
+        pw_content = ""
+        for fb_name in fb_order:
+            if fb_name == "curl_grid_exhaust":
+                # Already performed in Phase 2; nothing more to do here.
+                continue
+            if fb_name.startswith("surrogate_"):
+                # Phase 2.5: archive/reader/proxy routes serving a copy of
+                # the target. Cheaper than a browser; provenance-labeled.
+                from . import surrogate as surrogate_mod
+                s_atts, s_meta, s_content = surrogate_mod.run_surrogate(
                     url,
-                    profile_id=profile_used or "unknown_challenge",
+                    registry=surrogate_mod.load_surrogates(),
+                    allow_proxy=allow_surrogate_proxy,
+                    timeout=timeout,
                     success_selectors=success_selectors,
-                    device_class=device_class,
-                    force_executor=fb_name,
                 )
-                trace.append(pw_attempt)
-                if pw_attempt.verdict in (Verdict.STRONG_OK.value, Verdict.WEAK_OK.value):
+                trace.extend(s_atts)
+                s_att = s_atts[-1]
+                if s_att.verdict in (Verdict.STRONG_OK.value, Verdict.WEAK_OK.value):
                     return FetchResult(
                         ok=True,
-                        content=pw_content,
-                        final_url=pw_attempt.url,
-                        verdict=pw_attempt.verdict,
+                        content=s_content,
+                        final_url=s_att.url,
+                        verdict=s_att.verdict,
                         profile_used=profile_used,
                         trace=trace,
-                        summary=f"Playwright fallback succeeded via {fb_name}",
+                        summary=(
+                            f"surrogate {s_meta.get('surrogate')} succeeded"
+                            f" (provenance={s_meta.get('provenance')}"
+                            + (f" snapshot_ts={s_meta.get('snapshot_timestamp')})" if s_meta.get('snapshot_timestamp') else ")")
+                        ),
+                        provenance=str(s_meta.get("provenance", "live")),
+                        snapshot_timestamp=s_meta.get("snapshot_timestamp"),
+                        trust=str(s_meta.get("trust", "origin")),
                     )
-            # Synthesize a placeholder if no iteration ran (empty list).
-            if pw_attempt is None:
-                pw_attempt = Attempt(
-                    phase="fallback",
-                    executor="none",
-                    url=url,
-                    url_transform="original",
-                    impersonate=None,
-                    referer="",
-                    verdict=Verdict.UNKNOWN.value,
-                    error="profile has empty fallback_when_challenge",
+                continue
+            if not enable_playwright:
+                continue
+            from .executor import run_playwright_fallback  # lazy import
+            pw_attempt, pw_content = run_playwright_fallback(
+                url,
+                profile_id=profile_used or "unknown_challenge",
+                success_selectors=success_selectors,
+                device_class=device_class,
+                force_executor=fb_name,
+            )
+            trace.append(pw_attempt)
+            if pw_attempt.verdict in (Verdict.STRONG_OK.value, Verdict.WEAK_OK.value):
+                return FetchResult(
+                    ok=True,
+                    content=pw_content,
+                    final_url=pw_attempt.url,
+                    verdict=pw_attempt.verdict,
+                    profile_used=profile_used,
+                    trace=trace,
+                    summary=f"Playwright fallback succeeded via {fb_name}",
                 )
-                trace.append(pw_attempt)
-        except ImportError:
-            trace.append(Attempt(
+        # Synthesize a placeholder only when the profile genuinely offers no
+        # fallback route. Entries skipped because the caller disabled the
+        # browser are a caller choice, not a profile defect — no trace noise.
+        actionable = [n for n in fb_order if n != "curl_grid_exhaust"]
+        if pw_attempt is None and not actionable:
+            pw_attempt = Attempt(
                 phase="fallback",
-                executor="playwright",
+                executor="none",
                 url=url,
                 url_transform="original",
                 impersonate=None,
                 referer="",
                 verdict=Verdict.UNKNOWN.value,
-                error="executor module not available",
-            ))
-        except (RuntimeError, OSError) as e:
-            trace.append(Attempt(
-                phase="fallback",
-                executor="playwright",
-                url=url,
-                url_transform="original",
-                impersonate=None,
-                referer="",
-                verdict=Verdict.UNKNOWN.value,
-                error=f"{type(e).__name__}:{str(e)[:200]}",
-            ))
+                error="profile has empty fallback_when_challenge",
+            )
+            trace.append(pw_attempt)
+    except ImportError:
+        trace.append(Attempt(
+            phase="fallback",
+            executor="playwright",
+            url=url,
+            url_transform="original",
+            impersonate=None,
+            referer="",
+            verdict=Verdict.UNKNOWN.value,
+            error="executor module not available",
+        ))
+    except (RuntimeError, OSError) as e:
+        trace.append(Attempt(
+            phase="fallback",
+            executor="playwright",
+            url=url,
+            url_transform="original",
+            impersonate=None,
+            referer="",
+            verdict=Verdict.UNKNOWN.value,
+            error=f"{type(e).__name__}:{str(e)[:200]}",
+        ))
 
     # -------- Give up, return best we have ----------------------------------
     summary = format_summary(trace, profile_used)

package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/result_schema.py

@@ -1,7 +1,10 @@
 from __future__ import annotations
 
 from dataclasses import asdict, dataclass, field
-from typing import Optional
+from typing import Literal, Optional
+
+Provenance = Literal["live", "snapshot", "proxy"]
+Trust = Literal["origin", "archive", "untrusted"]
 
 
 @dataclass
@@ -32,6 +35,9 @@ class FetchResult:
     profile_used: Optional[str] = None
     trace: list[Attempt] = field(default_factory=list)
     summary: str = ""
+    provenance: str = "live"                       # live | snapshot | proxy
+    snapshot_timestamp: Optional[str] = None      # archive's own timestamp, when snapshot
+    trust: str = "origin"                         # origin | archive | untrusted
 
     def to_dict(self) -> dict:
         return {
@@ -41,5 +47,8 @@ class FetchResult:
             "profile_used": self.profile_used,
             "trace": [a.to_dict() for a in self.trace],
             "summary": self.summary,
+            "provenance": self.provenance,
+            "snapshot_timestamp": self.snapshot_timestamp,
+            "trust": self.trust,
             "content_length": len(self.content),
         }

package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/surrogate.py

@@ -0,0 +1,214 @@
+"""Phase 2.5 surrogate retrieval: archive / reader / proxy routes for blocked origins.
+
+When the curl grid cannot fetch the live page, this module reads a declarative
+registry (engine/surrogates.yaml) and tries third-party routes that serve a
+copy of the target instead. Provenance semantics are contractual:
+
+    kind=archive -> provenance="snapshot" (+ archive's own timestamp), trust=archive
+    kind=reader  -> provenance="live"     (server-side re-render of the live page)
+    kind=proxy   -> provenance="proxy", trust=untrusted; requires allow_proxy=True
+                    and never sends Cookie/Authorization headers (relay = MITM)
+
+Every response body passes through validators.validate() with target_url set,
+so AMP-style redirect stubs and search-engine interstitials are rejected
+rather than recorded as wins. Entries whose `last_verified` is older than
+MAX_STALE_DAYS are tried after fresh ones (routes rot faster than they are
+maintained).
+"""
+from __future__ import annotations
+
+import json
+import os
+import time
+from datetime import date
+
+try:
+    import yaml
+except ImportError:
+    yaml = None
+
+from .result_schema import Attempt
+from .validators import Verdict, validate
+
+SURROGATES_PATH = os.path.join(os.path.dirname(__file__), "surrogates.yaml")
+MAX_STALE_DAYS = 90
+
+DEFAULT_SURROGATES: dict = {
+    "wayback": {
+        "kind": "archive",
+        "trust": "archive",
+        "enabled": True,
+        "last_verified": "2026-08-09",
+        "discovery": {
+            "url": "https://archive.org/wayback/available?url={target}",
+            "flag_pointer": "archived_snapshots.closest.available",
+            "snapshot_url_pointer": "archived_snapshots.closest.url",
+            "snapshot_timestamp_pointer": "archived_snapshots.closest.timestamp",
+        },
+        "min_body_bytes": 3072,
+    },
+}
+
+
+def load_surrogates(path: str = SURROGATES_PATH) -> dict:
+    if yaml is None:
+        return dict(DEFAULT_SURROGATES)
+    try:
+        with open(path, encoding="utf-8") as fh:
+            loaded = yaml.safe_load(fh) or {}
+    except (OSError, yaml.YAMLError):
+        return dict(DEFAULT_SURROGATES)
+    if not isinstance(loaded, dict):
+        return dict(DEFAULT_SURROGATES)
+    usable = {k: v for k, v in loaded.items() if isinstance(v, dict) and k}
+    return usable or dict(DEFAULT_SURROGATES)
+
+
+def is_stale(entry: dict, max_age_days: int = MAX_STALE_DAYS, today: date | None = None) -> bool:
+    verified = entry.get("last_verified")
+    if isinstance(verified, date):
+        parsed = verified
+    elif isinstance(verified, str):
+        try:
+            parsed = date.fromisoformat(verified)
+        except ValueError:
+            return True
+    else:
+        return True
+    return ((today or date.today()) - parsed).days > max_age_days
+
+
+def _http_get(url: str, *, headers: dict | None = None, timeout: int = 25):
+    try:
+        from curl_cffi import requests as cffi_requests
+    except ImportError as exc:
+        raise RuntimeError("curl_cffi not installed") from exc
+    return cffi_requests.get(
+        url,
+        impersonate="chrome",
+        headers=headers or {"Accept": "text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8"},
+        timeout=timeout,
+        allow_redirects=True,
+    )
+
+
+def _json_pointer(obj, dotted: str):
+    current = obj
+    for part in dotted.split("."):
+        if not isinstance(current, dict):
+            return None
+        current = current.get(part)
+    return current
+
+
+def _provenance_of(kind: str) -> str:
+    if kind == "archive":
+        return "snapshot"
+    if kind == "proxy":
+        return "proxy"
+    return "live"
+
+
+def _skip_reason(name: str, entry: dict, allow_proxy: bool) -> str | None:
+    kind = entry.get("kind", "archive")
+    if kind == "proxy" and not allow_proxy:
+        return "proxy_requires_allow_proxy_flag"
+    required_env = entry.get("enabled_env")
+    if required_env and not os.environ.get(str(required_env)):
+        return f"missing_env:{required_env}"
+    if entry.get("enabled") is False and kind != "proxy":
+        return "disabled"
+    return None
+
+
+def _attempt(name: str, url: str, verdict: Verdict, error=None, status: int = 0, size: int = 0, elapsed: float = 0.0) -> Attempt:
+    return Attempt(
+        phase="surrogate",
+        executor=f"surrogate_{name}",
+        url=url,
+        url_transform="original",
+        impersonate=None,
+        referer="",
+        status=status,
+        body_size=size,
+        verdict=verdict.value,
+        error=(str(error)[:200] if error else None),
+        elapsed_s=round(elapsed, 3),
+    )
+
+
+def _resolve_target(name: str, entry: dict, target: str, timeout: int) -> tuple[str | None, str | None]:
+    discovery = entry.get("discovery")
+    if not discovery:
+        host_rotation = entry.get("host_rotation") or [None]
+        host = host_rotation[0]
+        template = entry.get("fetch")
+        if not template or host is None and "{host}" in template:
+            return None, None
+        return template.format(host=host, target=target), None
+    discovery_url = discovery.get("url", "").format(target=target)
+    resp = _http_get(discovery_url, timeout=timeout)
+    payload = resp.json()
+    if discovery.get("flag_pointer") and not _json_pointer(payload, discovery["flag_pointer"]):
+        return None, None
+    snap_url = _json_pointer(payload, discovery.get("snapshot_url_pointer", ""))
+    timestamp = _json_pointer(payload, discovery.get("snapshot_timestamp_pointer", "")) or None
+    return snap_url, timestamp
+
+
+def run_surrogate(
+    target: str,
+    *,
+    registry: dict | None = None,
+    allow_proxy: bool = False,
+    timeout: int = 25,
+    success_selectors: list[str] | None = None,
+) -> tuple[list[Attempt], dict, str]:
+    registry = dict(registry) if registry else load_surrogates()
+    ordered = sorted(registry.items(), key=lambda kv: int(is_stale(kv[1])))
+    live_meta = {"provenance": "live", "snapshot_timestamp": None, "trust": "origin", "surrogate": None}
+    attempts: list[Attempt] = []
+
+    for name, entry in ordered:
+        kind = entry.get("kind", "archive")
+        reason = _skip_reason(name, entry, allow_proxy)
+        if reason:
+            attempts.append(_attempt(name, target, Verdict.UNKNOWN, error=f"skipped:{reason}"))
+            continue
+        started = time.time()
+        try:
+            fetch_url, timestamp = _resolve_target(name, entry, target, timeout)
+        except Exception as exc:
+            attempts.append(_attempt(name, target, Verdict.UNKNOWN, error=f"{type(exc).__name__}:{exc}", elapsed=time.time() - started))
+            continue
+        if not fetch_url:
+            attempts.append(_attempt(name, target, Verdict.UNKNOWN, error="no snapshot available", elapsed=time.time() - started))
+            continue
+        try:
+            resp = _http_get(fetch_url, timeout=timeout)
+        except Exception as exc:
+            attempts.append(_attempt(name, fetch_url, Verdict.UNKNOWN, error=f"{type(exc).__name__}:{exc}", elapsed=time.time() - started))
+            continue
+        body = getattr(resp, "text", "") or ""
+        status = int(getattr(resp, "status_code", 0) or 0)
+        min_bytes = int(entry.get("min_body_bytes", 3072))
+        att = _attempt(name, fetch_url, Verdict.UNKNOWN, status=status, size=len(body), elapsed=time.time() - started)
+        attempts.append(att)
+        if status >= 400 or len(body) < min_bytes:
+            att.error = f"surrogate_unusable:status={status},size={len(body)}"
+            continue
+        vr = validate(resp, success_selectors=success_selectors, target_url=target)
+        att.verdict = vr.verdict.value
+        att.reasons = vr.reasons
+        if vr.ok:
+            meta = {
+                "provenance": _provenance_of(kind),
+                "snapshot_timestamp": timestamp if kind == "archive" else None,
+                "trust": str(entry.get("trust", "archive")),
+                "surrogate": name,
+            }
+            return attempts, meta, body
+
+    if not attempts:
+        attempts.append(_attempt("none", target, Verdict.UNKNOWN, error="no surrogate entries"))
+    return attempts, live_meta, ""

package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/surrogates.yaml

@@ -0,0 +1,60 @@
+# Surrogate routes — third-party services serving a COPY of a page when the
+# live origin blocks the fetch chain (Phase 2.5, between grid and Playwright).
+#
+# Rules (same discipline as waf_profiles.yaml):
+#   * Site-agnostic infrastructure ONLY (No-Site-Name Rule): every host here
+#     must work for any target URL. No target-site domains, selectors, brand
+#     names.
+#   * Surrogate routes ROT — the 2026-08 probe found 4 of 6 known routes dead
+#     or returning stubs. Entries must carry `last_verified` (ISO date) and a
+#     passing liveness record in the PR that adds them; refresh quarterly.
+#     Entries older than 90 days are tried after fresh ones and flagged.
+#   * kind semantics:
+#       archive — serves a dated snapshot; provenance=snapshot, trust=archive.
+#       reader  — re-renders the LIVE page server-side; provenance=live.
+#       proxy   — raw forwarding relay; MITM by construction, trust=untrusted,
+#                 skipped unless the caller passes --allow-proxy, and never
+#                 receives Cookie/Authorization headers.
+#   * keys:
+#       discovery.json_pointer — dotted path into the discovery JSON payload.
+#       host_rotation — try hostile-domain cousins in order (one may be
+#                 blocked while another answers).
+#       enabled_env — entry is inert until that env var is set (key-gated).
+#       min_body_bytes — response floor; below it the body is a stub/error.
+wayback:
+  kind: archive
+  trust: archive
+  enabled: true
+  last_verified: 2026-08-09
+  discovery:
+    url: "https://archive.org/wayback/available?url={target}"
+    flag_pointer: "archived_snapshots.closest.available"
+    snapshot_url_pointer: "archived_snapshots.closest.url"
+    snapshot_timestamp_pointer: "archived_snapshots.closest.timestamp"
+  min_body_bytes: 3072
+
+archive_today:
+  kind: archive
+  trust: archive
+  enabled: true
+  last_verified: 2026-08-09
+  host_rotation: ["archive.ph", "archive.md", "archive.li", "archive.is"]
+  fetch: "https://{host}/newest/{target}"
+  min_body_bytes: 3072
+
+jina_reader:
+  kind: reader
+  trust: archive
+  enabled: true
+  enabled_env: JINA_API_KEY
+  last_verified: 2026-08-09
+  fetch: "https://r.jina.ai/{target}"
+  min_body_bytes: 512
+
+anon_relay_example:
+  kind: proxy
+  trust: untrusted
+  enabled: false
+  last_verified: 2026-08-09
+  fetch: "https://relay.invalid/?u={target}"
+  min_body_bytes: 3072

package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/tests/fixtures/amp_redirect_stub.html

@@ -0,0 +1,7 @@
+<HTML><HEAD>
+<meta http-equiv="content-type" content="text/html;charset=utf-8">
+<TITLE>Redirecting</TITLE>
+<META HTTP-EQUIV="refresh" content="0; url=https://en.wikipedia.org/wiki/Web_archiving">
+</HEAD>
+<BODY onLoad="location.replace('https://en.wikipedia.org/wiki/Web_archiving'+document.location.hash)">
+</BODY></HTML>