oh-my-opencode 4.12.1 → 4.14.0

package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/tests/test_fetch_chain.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import sys
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from engine.fetch_chain import fetch  # noqa: E402
+from engine.result_schema import Attempt  # noqa: E402
+from engine.validators import Verdict  # noqa: E402
+
+
+class _Resp:
+    def __init__(self, text: str = "<article>ok</article>", url: str = "https://example.com/"):
+        self.text = text
+        self.url = url
+
+
+class _Hit:
+    profile_id = "cloudflare_turnstile"
+    confidence = 1.0
+    signals = ["test"]
+
+
+class FetchChain(unittest.TestCase):
+    def test_probe_success_returns_without_grid(self) -> None:
+        attempt = Attempt(
+            phase="probe",
+            executor="curl_cffi",
+            url="https://example.com",
+            url_transform="original",
+            impersonate="safari",
+            referer="self_root",
+            verdict=Verdict.WEAK_OK.value,
+        )
+
+        with patch("engine.fetch_chain._load_profiles", return_value={}), \
+                patch("engine.fetch_chain.last_load_error", return_value=None), \
+                patch("engine.fetch_chain.run_attempt", return_value=(attempt, _Resp())) as run_attempt:
+            result = fetch("https://example.com", enable_playwright=False)
+
+        self.assertTrue(result.ok)
+        self.assertEqual(result.verdict, Verdict.WEAK_OK.value)
+        self.assertEqual(len(result.trace), 1)
+        self.assertEqual(run_attempt.call_count, 1)
+
+    def test_max_attempts_stops_after_probe_before_grid(self) -> None:
+        attempt = Attempt(
+            phase="probe",
+            executor="curl_cffi",
+            url="https://example.com",
+            url_transform="original",
+            impersonate="safari",
+            referer="self_root",
+            verdict=Verdict.CHALLENGE.value,
+        )
+
+        with patch("engine.fetch_chain._load_profiles", return_value={}), \
+                patch("engine.fetch_chain.last_load_error", return_value=None), \
+                patch("engine.fetch_chain.run_attempt", return_value=(attempt, _Resp("blocked"))), \
+                patch("engine.fetch_chain.detect", return_value=[_Hit()]), \
+                patch("engine.fetch_chain.load_profile", return_value={
+                    "tls_impersonate_candidates": [["chrome"]],
+                    "referer_strategies": ["self_root"],
+                    "url_transform_order": ["original"],
+                }):
+            result = fetch("https://example.com", max_attempts=1, enable_playwright=False)
+
+        self.assertFalse(result.ok)
+        self.assertEqual(len(result.trace), 1)
+        self.assertEqual(result.trace[0].phase, "probe")
+
+
+if __name__ == "__main__":
+    unittest.main()

package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/tests/test_playwright_templates.py

@@ -0,0 +1,282 @@
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import tempfile
+import textwrap
+import unittest
+from pathlib import Path
+
+TEMPLATES_DIR = Path(__file__).resolve().parents[1] / "templates"
+TEMPLATE_NAMES = ("playwright_real_chrome.js", "playwright_mobile_chrome.js")
+JsonValue = None | bool | int | float | str | list["JsonValue"] | dict[str, "JsonValue"]
+
+
+def _write_file(path: Path, content: str) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(textwrap.dedent(content).lstrip(), encoding="utf-8")
+
+
+def _install_fake_playwright(node_modules: Path) -> None:
+    _write_file(
+        node_modules / "playwright" / "index.js",
+        """
+        const page = {
+          async goto() {},
+          async waitForTimeout() {},
+          async waitForSelector(selector) {
+            if (process.env.PW_FAKE_SELECTOR_FAIL === '1') {
+              throw new Error(`missing selector ${selector}`);
+            }
+          },
+          async reload() {},
+          async content() {
+            return '<html><article>ok</article></html>';
+          },
+        };
+
+        const context = {
+          async newPage() { return page; },
+          async close() {
+            if (process.env.PW_FAKE_CLOSE_FAIL === '1') {
+              throw new Error('close failed');
+            }
+          },
+        };
+
+        exports.chromium = {
+          use() {},
+          async launchPersistentContext() { return context; },
+        };
+        exports.devices = {
+          'iPhone 13 Pro': { viewport: { width: 390, height: 844 }, userAgent: 'fake-mobile' },
+        };
+        """,
+    )
+
+
+def _install_broken_playwright_extra(node_modules: Path) -> None:
+    _write_file(
+        node_modules / "playwright-extra" / "index.js",
+        """
+        const error = new Error('stealth init failed');
+        error.code = 'EACCES';
+        throw error;
+        """,
+    )
+    _write_file(
+        node_modules / "puppeteer-extra-plugin-stealth" / "index.js",
+        """
+        module.exports = function stealth() { return {}; };
+        """,
+    )
+
+
+def _install_working_playwright_extra(node_modules: Path) -> None:
+    _write_file(
+        node_modules / "playwright-extra" / "index.js",
+        """
+        const playwright = require('playwright');
+        exports.chromium = playwright.chromium;
+        exports.devices = playwright.devices;
+        """,
+    )
+
+
+def _install_playwright_extra_with_internal_missing_dependency(node_modules: Path) -> None:
+    _write_file(
+        node_modules / "playwright-extra" / "index.js",
+        """
+        require('transitive-stealth-runtime');
+        """,
+    )
+
+
+def _install_self_missing_optional_module(node_modules: Path, module_name: str) -> None:
+    _write_file(
+        node_modules / module_name / "index.js",
+        f"""
+        const error = new Error("Cannot find module '{module_name}'");
+        error.code = 'MODULE_NOT_FOUND';
+        throw error;
+        """,
+    )
+
+
+def _run_template(
+    template_name: str,
+    payload: dict[str, JsonValue],
+    *,
+    include_broken_extra: bool = False,
+    include_working_extra: bool = False,
+    include_internal_missing_extra: bool = False,
+    include_self_missing_module: str | None = None,
+    env_overrides: dict[str, str] | None = None,
+) -> subprocess.CompletedProcess[str]:
+    with tempfile.TemporaryDirectory(prefix="ultimate-browsing-template-test-") as tmp:
+        tmp_path = Path(tmp)
+        node_modules = tmp_path / "node_modules"
+        _install_fake_playwright(node_modules)
+        if include_working_extra:
+            _install_working_playwright_extra(node_modules)
+        if include_broken_extra:
+            _install_broken_playwright_extra(node_modules)
+        if include_internal_missing_extra:
+            _install_playwright_extra_with_internal_missing_dependency(node_modules)
+        if include_self_missing_module:
+            _install_self_missing_optional_module(node_modules, include_self_missing_module)
+
+        script_path = tmp_path / template_name
+        script_path.write_text((TEMPLATES_DIR / template_name).read_text(encoding="utf-8"), encoding="utf-8")
+
+        env = os.environ.copy()
+        env.pop("NODE_PATH", None)
+        if env_overrides:
+            env.update(env_overrides)
+
+        return subprocess.run(
+            ["node", str(script_path)],
+            input=json.dumps(payload),
+            text=True,
+            capture_output=True,
+            timeout=5,
+            env=env,
+            check=False,
+        )
+
+
+@unittest.skipUnless(shutil.which("node"), "node is required for Playwright template tests")
+class PlaywrightTemplateErrorHandling(unittest.TestCase):
+    def test_missing_playwright_extra_warns_and_falls_back_to_plain_playwright(self) -> None:
+        for template_name in TEMPLATE_NAMES:
+            with self.subTest(template_name=template_name):
+                result = _run_template(
+                    template_name,
+                    {
+                        "url": "https://example.com/article",
+                        "profileDir": "/tmp/ultimate-browsing-test-profile",
+                        "headless": True,
+                    },
+                )
+
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertIn("<article>ok</article>", result.stdout)
+                self.assertIn("best-effort optional module playwright-extra failed:", result.stderr)
+                self.assertIn("Cannot find module 'playwright-extra'", result.stderr)
+
+    def test_missing_stealth_plugin_warns_and_falls_back_to_plain_playwright(self) -> None:
+        for template_name in TEMPLATE_NAMES:
+            with self.subTest(template_name=template_name):
+                result = _run_template(
+                    template_name,
+                    {
+                        "url": "https://example.com/article",
+                        "profileDir": "/tmp/ultimate-browsing-test-profile",
+                        "headless": True,
+                    },
+                    include_working_extra=True,
+                )
+
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertIn("<article>ok</article>", result.stdout)
+                self.assertIn("best-effort optional module puppeteer-extra-plugin-stealth failed:", result.stderr)
+                self.assertIn("Cannot find module 'puppeteer-extra-plugin-stealth'", result.stderr)
+
+    def test_non_missing_stealth_dependency_error_is_not_swallowed(self) -> None:
+        for template_name in TEMPLATE_NAMES:
+            with self.subTest(template_name=template_name):
+                result = _run_template(
+                    template_name,
+                    {
+                        "url": "https://example.com/article",
+                        "profileDir": "/tmp/ultimate-browsing-test-profile",
+                        "headless": True,
+                    },
+                    include_broken_extra=True,
+                )
+
+                self.assertNotEqual(result.returncode, 0)
+                self.assertIn("stealth init failed", result.stderr)
+                self.assertNotIn("best-effort optional module", result.stderr)
+                self.assertEqual(result.stdout, "")
+
+    def test_internal_module_resolution_error_is_not_treated_as_optional_missing_dependency(self) -> None:
+        for template_name in TEMPLATE_NAMES:
+            with self.subTest(template_name=template_name):
+                result = _run_template(
+                    template_name,
+                    {
+                        "url": "https://example.com/article",
+                        "profileDir": "/tmp/ultimate-browsing-test-profile",
+                        "headless": True,
+                    },
+                    include_internal_missing_extra=True,
+                )
+
+                self.assertNotEqual(result.returncode, 0)
+                self.assertIn("Cannot find module 'transitive-stealth-runtime'", result.stderr)
+                self.assertNotIn("best-effort optional module", result.stderr)
+                self.assertEqual(result.stdout, "")
+
+    def test_present_optional_module_self_missing_error_is_not_swallowed(self) -> None:
+        cases = (("playwright-extra", False), ("puppeteer-extra-plugin-stealth", True))
+        for template_name in TEMPLATE_NAMES:
+            for module_name, include_working_extra in cases:
+                with self.subTest(template_name=template_name, module_name=module_name):
+                    result = _run_template(
+                        template_name,
+                        {
+                            "url": "https://example.com/article",
+                            "profileDir": "/tmp/ultimate-browsing-test-profile",
+                            "headless": True,
+                        },
+                        include_working_extra=include_working_extra,
+                        include_self_missing_module=module_name,
+                    )
+
+                    self.assertNotEqual(result.returncode, 0)
+                    self.assertIn(f"Cannot find module '{module_name}'", result.stderr)
+                    self.assertNotIn("best-effort optional module", result.stderr)
+                    self.assertEqual(result.stdout, "")
+
+    def test_selector_failures_are_reported_as_best_effort_warnings(self) -> None:
+        for template_name in TEMPLATE_NAMES:
+            with self.subTest(template_name=template_name):
+                result = _run_template(
+                    template_name,
+                    {
+                        "url": "https://example.com/article",
+                        "profileDir": "/tmp/ultimate-browsing-test-profile",
+                        "headless": True,
+                        "waitSelector": "article.ready",
+                    },
+                    env_overrides={"PW_FAKE_SELECTOR_FAIL": "1"},
+                )
+
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertIn("<article>ok</article>", result.stdout)
+                self.assertIn("best-effort waitSelector failed:", result.stderr)
+                self.assertNotIn("waitSelector article.ready", result.stderr)
+
+    def test_context_close_failures_are_reported_after_successful_html_output(self) -> None:
+        for template_name in TEMPLATE_NAMES:
+            with self.subTest(template_name=template_name):
+                result = _run_template(
+                    template_name,
+                    {
+                        "url": "https://example.com/article",
+                        "profileDir": "/tmp/ultimate-browsing-test-profile",
+                        "headless": True,
+                    },
+                    env_overrides={"PW_FAKE_CLOSE_FAIL": "1"},
+                )
+
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertIn("<article>ok</article>", result.stdout)
+                self.assertIn("best-effort browser context close failed:", result.stderr)
+
+
+if __name__ == "__main__":
+    unittest.main()

package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/url_transforms.py

@@ -0,0 +1,98 @@
+"""Generic URL transforms for the fetch grid.
+
+Transforms are domain-agnostic *rules*. They never reference a specific
+site by name. A transform either applies (returns a new URL) or is skipped
+(returns None). Callers iterate transforms in order.
+
+Empirically useful transforms (see observations/):
+  * mobile_subdomain — `www.example.com` → `m.example.com`
+    Strong win on SSR sites with mobile-first serving. Loss on SPA shells
+    (some mobile sites return tiny bootstrap HTML).
+  * am_prefix — `example.com` (no www) → `m.example.com`
+  * drop_www — occasionally unblocks hosts that gate www but not apex.
+
+Adding new transforms: prove they help on ≥2 unrelated sites first
+(cross-site validation — bias check).
+"""
+from __future__ import annotations
+
+from typing import Callable, Optional
+from urllib.parse import urlsplit, urlunsplit
+
+
+def _replace_host(url: str, new_host: str) -> str:
+    parts = urlsplit(url)
+    return urlunsplit(parts._replace(netloc=new_host))
+
+
+def _original(url: str) -> Optional[str]:
+    return url
+
+
+def _mobile_subdomain(url: str) -> Optional[str]:
+    """`https://www.example.com/a` → `https://m.example.com/a` (only if host starts with www.)."""
+    parts = urlsplit(url)
+    host = parts.hostname or ""
+    if not host.startswith("www."):
+        return None
+    new_host = "m." + host[4:]
+    if parts.port:
+        new_host = f"{new_host}:{parts.port}"
+    return _replace_host(url, new_host)
+
+
+def _am_prefix(url: str) -> Optional[str]:
+    """`https://example.com/a` → `https://m.example.com/a` (only if host has no subdomain)."""
+    parts = urlsplit(url)
+    host = parts.hostname or ""
+    if not host or host.startswith("m."):
+        return None
+    # Only apply to apex-like hosts (≤2 dot-separated labels).
+    if host.count(".") >= 2 and not host.startswith("www."):
+        return None
+    if host.startswith("www."):
+        return None  # handled by mobile_subdomain
+    return _replace_host(url, "m." + host)
+
+
+def _drop_www(url: str) -> Optional[str]:
+    parts = urlsplit(url)
+    host = parts.hostname or ""
+    if not host.startswith("www."):
+        return None
+    return _replace_host(url, host[4:])
+
+
+TRANSFORMS: dict[str, Callable[[str], Optional[str]]] = {
+    "original": _original,
+    "mobile_subdomain": _mobile_subdomain,
+    "am_prefix": _am_prefix,
+    "drop_www": _drop_www,
+}
+
+
+def apply_transform(name: str, url: str) -> Optional[str]:
+    """Apply one transform by name. Returns transformed URL or None if skipped."""
+    fn = TRANSFORMS.get(name)
+    if fn is None:
+        raise ValueError(f"Unknown transform: {name!r}. Known: {list(TRANSFORMS)}")
+    return fn(url)
+
+
+def iter_transformed(url: str, order: list[str]) -> list[tuple[str, str]]:
+    """Yield (transform_name, transformed_url) pairs for a given order.
+
+    Skips transforms that return None (not applicable) and deduplicates
+    URLs (so `original` and `drop_www` of `https://example.com` don't double-run).
+    """
+    seen: set[str] = set()
+    out: list[tuple[str, str]] = []
+    for name in order:
+        new_url = apply_transform(name, url)
+        if new_url is None:
+            continue
+        if new_url in seen:
+            continue
+        seen.add(new_url)
+        out.append((name, new_url))
+    return out

package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/validators.py

@@ -0,0 +1,216 @@
+"""Generic challenge / success validator.
+
+Four layers (all generic, never site-specific):
+  1. Challenge markers (WAF product strings — not site brand names)
+  2. Size fingerprints (known bad byte sizes hinted by caller)
+  3. Cookie sensor state (e.g. Akamai `_abck=~-1~`)
+  4. Caller-supplied success_selectors (strongest positive proof)
+
+Layers 1-3 are "negative proof" (fail fast).
+Layer 4 is "positive proof" — without it, HTTP 200 is only a weak success.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Optional
+
+try:
+    from bs4 import BeautifulSoup
+except ImportError:  # bs4 is a soft dep: only used when selectors given
+    BeautifulSoup = None
+
+
+# Markers are WAF-product strings only. Never include site brand / domain.
+CHALLENGE_MARKERS: list[str] = [
+    "Access Denied",
+    "sec-if-cpt-container",
+    "Powered and protected by Akamai",
+    "Just a moment...",
+    "Checking your browser",
+    "cf-chl-bypass",
+    "Attention Required! | Cloudflare",
+    "<title>Bot Challenge</title>",
+    "DataDome",
+    "captcha",
+    "Please enable JS and disable any ad blocker",
+    "The requested URL was rejected",
+    "Request unsuccessful. Incapsula",
+]
+
+# Minimum body size below which we suspect a stub / challenge page.
+# Tunable: some legitimate short JSON responses may be smaller, but callers
+# that know their response type should pass success_selectors instead.
+SMALL_BODY_THRESHOLD = 3000
+
+
+class Verdict(Enum):
+    """Three-level classification (Codex suggestion — avoid binary)."""
+
+    STRONG_OK = "strong_ok"      # passes all layers incl. success_selectors
+    WEAK_OK = "weak_ok"          # passes 1-3 but no positive proof available
+    CHALLENGE = "challenge"      # fails 1-3 (negative proof triggered)
+    BLOCKED = "blocked"          # non-200 status
+    UNKNOWN = "unknown"          # exception / malformed response
+
+
+@dataclass
+class ValidationResult:
+    verdict: Verdict
+    reasons: list[str] = field(default_factory=list)
+    matched_selectors: list[str] = field(default_factory=list)
+    body_size: int = 0
+    status: int = 0
+
+    @property
+    def ok(self) -> bool:
+        """Kept for ergonomic `if vr.ok:` use — weak_ok counts as ok."""
+        return self.verdict in (Verdict.STRONG_OK, Verdict.WEAK_OK)
+
+    def to_dict(self) -> dict:
+        return {
+            "verdict": self.verdict.value,
+            "reasons": self.reasons,
+            "matched_selectors": self.matched_selectors,
+            "body_size": self.body_size,
+            "status": self.status,
+        }
+
+
+def _marker_hits(body_lower: str) -> list[str]:
+    return [m for m in CHALLENGE_MARKERS if m.lower() in body_lower]
+
+
+def _abck_unresolved(cookies: dict) -> bool:
+    abck = cookies.get("_abck", "")
+    return bool(abck) and "~-1~" in abck
+
+
+def _selector_hits(body: str, selectors: list[str]) -> Optional[list[str]]:
+    """Return matched-selector list, or None if BS4 is unavailable.
+
+    Distinguishing None (dependency missing) from [] (nothing matched) lets
+    the caller classify as UNKNOWN vs CHALLENGE correctly (Codex review: do
+    not let dependency failure masquerade as a WAF outcome).
+    """
+    if BeautifulSoup is None:
+        return None
+    try:
+        soup = BeautifulSoup(body, "html.parser")
+    except Exception:
+        return []
+    hits: list[str] = []
+    for sel in selectors:
+        try:
+            if soup.select(sel):
+                hits.append(sel)
+        except Exception:
+            continue
+    return hits
+
+
+def validate(
+    resp,
+    *,
+    success_selectors: Optional[list[str]] = None,
+    known_bad_sizes: Optional[list[int]] = None,
+    size_tolerance: int = 20,
+) -> ValidationResult:
+    """Validate a `curl_cffi` / `requests` response.
+
+    Parameters
+    ----------
+    resp
+        Response object with `.status_code`, `.text`, and cookie-like access.
+    success_selectors
+        Caller-supplied CSS selectors. Any match promotes `weak_ok` → `strong_ok`.
+        Absence of selectors still allows `weak_ok` (no positive proof, but
+        no negative proof either).
+    known_bad_sizes
+        Byte sizes that have been empirically observed as challenge-page
+        fingerprints (caller / profile hint). NOTE: these values decay over
+        time — profiles should timestamp or refresh them.
+    """
+    try:
+        status = int(getattr(resp, "status_code", 0) or 0)
+        text = getattr(resp, "text", "") or ""
+        size = len(text)
+    except Exception as e:
+        return ValidationResult(verdict=Verdict.UNKNOWN, reasons=[f"parse_error:{e}"])
+
+    r = ValidationResult(verdict=Verdict.UNKNOWN, body_size=size, status=status)
+
+    if status == 0 or status >= 400:
+        r.verdict = Verdict.BLOCKED
+        r.reasons.append(f"status={status}")
+        return r
+
+    # --- Layer 1: challenge markers (product strings, never site brand) ---
+    lowered = text.lower()
+    markers = _marker_hits(lowered)
+    if markers:
+        r.verdict = Verdict.CHALLENGE
+        r.reasons.extend(f"marker:{m}" for m in markers[:3])
+        return r
+
+    # --- Layer 2: size fingerprints (caller hint, tolerant match) ---
+    # Fingerprint match is a strong negative signal — override even selectors.
+    if known_bad_sizes:
+        for bad in known_bad_sizes:
+            if abs(size - bad) <= size_tolerance:
+                r.verdict = Verdict.CHALLENGE
+                r.reasons.append(f"size_fp:{size}~{bad}")
+                return r
+
+    # --- Layer 4 (early): caller's positive proof overrides size heuristic ---
+    # If caller provided selectors, trust their definition of "content exists".
+    # A small page with the required selector is still success.
+    if success_selectors:
+        hits = _selector_hits(text, success_selectors)
+        if hits is None:
+            # BS4 dependency missing — can't evaluate caller's proof.
+            # Classify as UNKNOWN (not CHALLENGE) so a WAF outcome isn't faked.
+            r.verdict = Verdict.UNKNOWN
+            r.reasons.append("bs4_missing")
+            return r
+        if hits:
+            # Cookie sensor state acts as a true gate when selectors passed:
+            # unresolved `_abck` means Akamai did not accept our session even
+            # though the body has our expected selector — trust is weak.
+            cookies = _extract_cookies(resp)
+            r.matched_selectors = hits
+            if _abck_unresolved(cookies):
+                r.reasons.append("abck_unresolved")
+                r.verdict = Verdict.WEAK_OK  # demoted from STRONG_OK
+                return r
+            r.verdict = Verdict.STRONG_OK
+            return r
+        # Selectors requested but none matched → challenge regardless of size.
+        r.verdict = Verdict.CHALLENGE
+        r.reasons.append("no_success_selector")
+        return r
+
+    # No selectors: fall back to size heuristic.
+    if size < SMALL_BODY_THRESHOLD:
+        r.verdict = Verdict.CHALLENGE
+        r.reasons.append(f"tiny_body:{size}")
+        return r
+
+    # --- Layer 3: cookie sensor state (only when no selectors to decide on) ---
+    cookies = _extract_cookies(resp)
+    if _abck_unresolved(cookies):
+        r.reasons.append("abck_unresolved")
+
+    # No positive proof available — weak OK.
+    r.verdict = Verdict.WEAK_OK
+    return r
+
+
+def _extract_cookies(resp) -> dict:
+    try:
+        return {c.name: c.value for c in resp.cookies.jar}
+    except Exception:
+        try:
+            return dict(resp.cookies) if hasattr(resp, "cookies") else {}
+        except Exception:
+            return {}