oh-my-opencode 4.12.1 → 4.14.0

package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/bias_check.py

@@ -0,0 +1,174 @@
+#!/usr/bin/env python3
+"""No-Site-Name Rule checker.
+
+Run in CI / pre-commit. Scans engine/** for hard-coded site names or
+domains that would bias the generic fetch chain toward one site.
+
+Exit code 0 if clean, 1 if violations found.
+
+    python3 engine/bias_check.py
+    python3 engine/bias_check.py --strict    # also check references/*.md (usually off)
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import sys
+from pathlib import Path
+
+
+# Known brand / domain substrings that should NOT appear in engine code.
+# This is a non-exhaustive deny list. CI should treat hits as warnings that
+# require human review; false positives (e.g. "github" in comments) can be
+# whitelisted via EXPLICIT_ALLOW.
+BRAND_SUBSTRINGS = [
+    "coupang", "11st", "11번가", "musinsa", "무신사",
+    "fmkorea", "에펨코리아", "dcinside", "디시인사이드",
+    "ohou", "오늘의집", "kurly", "마켓컬리",
+    "daangn", "당근",
+    # Naver is allowed in Phase 0 references (official APIs) but not in engine code.
+    "naver.com", "blog.naver", "shopping.naver",
+    # Korean portal brand names
+    "daum.net", "kakao.com",
+]
+
+# Regex for bare URLs / domains. Used as a secondary pass to flag hardcoded
+# site hosts that slipped past the brand denylist.
+URL_PATTERN = re.compile(
+    r"https?://[\w\.-]+|[\w-]+\.(?:com|net|org|co\.kr|kr|io)\b",
+    re.IGNORECASE,
+)
+
+# Generic / neutral hosts that are allowed anywhere (examples, specs, stdlib,
+# and domains that legitimately appear as non-site-specific referrers / test
+# fixtures — Google search as a generic Referer strategy, httpbin for transport
+# tests, etc.). Anything in this set must be provably unrelated to a specific
+# target-site preference.
+URL_ALLOWLIST = {
+    "example.com", "example.org", "example.net",
+    "localhost", "127.0.0.1",
+    # Official API / documentation sources cited in code comments.
+    "curl.se", "playwright.dev", "nodejs.org", "npmjs.com",
+    # Generic Referer strategy target (used as a neutral off-site referer).
+    "www.google.com", "google.com",
+    # Generic HTTP test endpoint for infrastructure / transport tests.
+    "httpbin.org",
+}
+
+# Files / dirs that must be clean.
+SCAN_ROOTS_STRICT_OFF = ["engine"]
+SCAN_ROOTS_STRICT_ON = ["engine", "references"]
+
+# Directory names skipped during scan (third-party code, build artefacts).
+EXCLUDED_DIR_NAMES = {
+    "node_modules", "__pycache__", ".git", ".venv", "dist", "build",
+}
+
+# Comment markers within which a brand mention is OK (explanation).
+# Keyed per-extension; any line containing these is skipped.
+COMMENT_OK_MARKERS = {
+    ".py": ("# NOTE-BIAS-OK", "# EXAMPLE-ONLY"),
+    ".js": ("// NOTE-BIAS-OK", "// EXAMPLE-ONLY"),
+    ".yaml": ("# NOTE-BIAS-OK", "# EXAMPLE-ONLY"),
+    ".yml": ("# NOTE-BIAS-OK", "# EXAMPLE-ONLY"),
+    ".md": ("<!-- NOTE-BIAS-OK -->", "<!-- EXAMPLE-ONLY -->"),
+}
+
+# File paths explicitly exempted (full match against relative path from scan root).
+EXPLICIT_ALLOW_FILES = {
+    # None right now — add if needed with justification.
+}
+
+
+def _line_is_exempt(line: str, ext: str) -> bool:
+    markers = COMMENT_OK_MARKERS.get(ext, ())
+    return any(m in line for m in markers)
+
+
+def _scan_file(path: Path, root: Path) -> list[str]:
+    """Return list of violation strings for this file."""
+    rel = path.relative_to(root.parent)
+    if str(rel) in EXPLICIT_ALLOW_FILES:
+        return []
+
+    ext = path.suffix.lower()
+    try:
+        text = path.read_text(encoding="utf-8", errors="ignore")
+    except Exception as e:
+        return [f"{rel}:0 — read error: {e}"]
+
+    violations: list[str] = []
+    for lineno, line in enumerate(text.splitlines(), start=1):
+        if _line_is_exempt(line, ext):
+            continue
+        lowered = line.lower()
+        # 1) Brand / domain denylist
+        hit_brand = None
+        for brand in BRAND_SUBSTRINGS:
+            if brand.lower() in lowered:
+                hit_brand = brand
+                break
+        if hit_brand:
+            violations.append(f"{rel}:{lineno} — brand `{hit_brand}` in: {line.strip()[:120]}")
+            continue  # one violation per line
+        # 2) URL/domain regex scan — catches hosts that aren't in the denylist.
+        for match in URL_PATTERN.finditer(line):
+            host = match.group(0).lower()
+            host = host.split("//", 1)[-1].split("/", 1)[0]
+            if host in URL_ALLOWLIST:
+                continue
+            if host.endswith(".example.com") or host.endswith(".example.org"):
+                continue
+            violations.append(f"{rel}:{lineno} — hardcoded host `{host}` in: {line.strip()[:120]}")
+            break
+    return violations
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Scan engine for site-name bias")
+    parser.add_argument("--strict", action="store_true",
+                        help="Also scan references/*.md (usually noisy — off by default)")
+    parser.add_argument("--root", default=None,
+                        help="Skill root directory. Defaults to parent of this file.")
+    args = parser.parse_args(argv)
+
+    skill_root = Path(args.root) if args.root else Path(__file__).parent.parent
+    scan_roots = SCAN_ROOTS_STRICT_ON if args.strict else SCAN_ROOTS_STRICT_OFF
+
+    total_violations: list[str] = []
+    scanned = 0
+    for name in scan_roots:
+        root = skill_root / name
+        if not root.exists():
+            continue
+        for dirpath, dirnames, filenames in os.walk(root):
+            # In-place filter so os.walk skips these subtrees.
+            dirnames[:] = [d for d in dirnames if d not in EXCLUDED_DIR_NAMES]
+            for fname in filenames:
+                p = Path(dirpath) / fname
+                if p.suffix.lower() not in (".py", ".js", ".yaml", ".yml", ".md", ".ts", ".mjs"):
+                    continue
+                if p.name == "bias_check.py":
+                    continue  # self-exempt (this file lists the brands)
+                scanned += 1
+                total_violations.extend(_scan_file(p, skill_root))
+
+    print(f"[bias-check] scanned {scanned} files under {skill_root}")
+    if total_violations:
+        print(f"[bias-check] ❌ {len(total_violations)} violation(s):")
+        for v in total_violations:
+            print(f"  - {v}")
+        print()
+        print("Fix options:")
+        print("  1) Remove the brand name (preferred)")
+        print("  2) If genuinely explanatory, add '# NOTE-BIAS-OK' on the same line")
+        print("  3) If this is a Phase 0 official API reference, move it to references/*.md and rerun without --strict")
+        return 1
+
+    print("[bias-check] ✅ clean")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/curl_probe.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+import time
+from typing import Iterable, Mapping, Protocol
+
+from .referers import REFERER_STRATEGIES
+from .result_schema import Attempt
+from .validators import Verdict, validate
+
+
+class _CookieItem(Protocol):
+    name: str
+    value: str
+
+
+class _CookieJar(Protocol):
+    jar: Iterable[_CookieItem]
+
+
+class ProbeResponse(Protocol):
+    status_code: int
+    text: str
+    url: str
+    cookies: _CookieJar | Mapping[str, str]
+
+
+def _curl_probe(
+    url: str, *, impersonate: str, referer: str, timeout: int = 20
+) -> tuple[ProbeResponse | None, str | None]:
+    try:
+        from curl_cffi import requests as cffi_requests
+    except ImportError:
+        return None, "curl_cffi not installed"
+
+    headers = {
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7",
+    }
+    if referer:
+        headers["Referer"] = referer
+
+    try:
+        resp = cffi_requests.get(
+            url,
+            impersonate=impersonate,
+            headers=headers,
+            timeout=timeout,
+            allow_redirects=True,
+        )
+        return resp, None
+    except cffi_requests.exceptions.RequestException as e:
+        return None, f"{type(e).__name__}:{str(e)[:200]}"
+
+
+def run_attempt(
+    url: str,
+    *,
+    transform_name: str,
+    impersonate: str,
+    referer_name: str,
+    success_selectors: list[str] | None,
+    known_bad_sizes: list[int] | None,
+    timeout: int,
+    phase: str,
+) -> tuple[Attempt, ProbeResponse | None]:
+    referer_url = REFERER_STRATEGIES.get(referer_name, REFERER_STRATEGIES["none"])(url)
+    started_at = time.time()
+    resp, err = _curl_probe(url, impersonate=impersonate, referer=referer_url, timeout=timeout)
+    elapsed = round(time.time() - started_at, 3)
+
+    att = Attempt(
+        phase=phase,
+        executor="curl_cffi",
+        url=url,
+        url_transform=transform_name,
+        impersonate=impersonate,
+        referer=referer_name,
+        elapsed_s=elapsed,
+    )
+
+    if err or resp is None:
+        att.error = err or "no response"
+        att.verdict = Verdict.UNKNOWN.value
+        return att, None
+
+    vr = validate(resp, success_selectors=success_selectors, known_bad_sizes=known_bad_sizes)
+    att.status = vr.status
+    att.body_size = vr.body_size
+    att.verdict = vr.verdict.value
+    att.reasons = vr.reasons
+    return att, resp

package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/executor.py

@@ -0,0 +1,192 @@
+"""Capability-matched executor for fallback attempts.
+
+The fetch_chain's probe/grid phase uses curl_cffi directly. When curl can't
+punch through (JS challenge, real-TLS detection), this module routes to the
+right browser executor based on the profile's `capabilities_needed` tags:
+
+    needs_real_tls_stack + needs_js_exec  → playwright_real_chrome.js
+    needs_js_exec only                    → Playwright MCP (if available)
+    needs_mobile_context (+ real_tls)     → playwright_mobile_chrome.js
+
+The JS templates live in `engine/templates/` and accept only generic
+parameters ({{url}}, {{waitSelector}}, {{profileDir}}, {{device}}). No
+site-specific logic.
+
+Playwright MCP invocation requires caller's tool access; this module
+provides the subprocess path for local JS templates but only stubs the MCP
+path (MCP must be driven from the Claude session itself).
+"""
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import tempfile
+import time
+from typing import Optional
+
+from .validators import Verdict, validate
+from .waf_detector import load_profile
+from .result_schema import Attempt
+
+
+TEMPLATES_DIR = os.path.join(os.path.dirname(__file__), "templates")
+
+
+def _node_available() -> bool:
+    return shutil.which("node") is not None
+
+
+def _chrome_channel_available() -> bool:
+    """Heuristic: try `node -e` to import playwright. Fallback to True, let script fail loudly."""
+    if not _node_available():
+        return False
+    if shutil.which("npx") is None:
+        return False
+    return True
+
+
+def _pick_executor(capabilities: list[str], device_class: str) -> str:
+    caps = set(capabilities or [])
+    if device_class == "mobile" or "needs_mobile_context" in caps:
+        if "needs_real_tls_stack" in caps:
+            return "playwright_mobile_chrome"
+        return "playwright_mcp_mobile"
+    if "needs_real_tls_stack" in caps:
+        return "playwright_real_chrome"
+    if "needs_js_exec" in caps:
+        return "playwright_mcp"
+    return "playwright_real_chrome"  # safest general fallback
+
+
+def _run_node_template(template: str, args: dict, timeout: int = 90) -> tuple[int, str, str]:
+    """Run a Node.js template with args as JSON on stdin.
+
+    Template convention: reads `process.stdin` → JSON → runs fetch → writes
+    HTML to stdout; errors go to stderr with non-zero exit code.
+    """
+    path = os.path.join(TEMPLATES_DIR, template)
+    if not os.path.isfile(path):
+        return 127, "", f"template not found: {path}"
+    try:
+        proc = subprocess.run(
+            ["node", path],
+            input=json.dumps(args),
+            cwd=TEMPLATES_DIR,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+        return proc.returncode, proc.stdout, proc.stderr
+    except subprocess.TimeoutExpired:
+        return 124, "", f"timeout after {timeout}s"
+    except Exception as e:
+        return 1, "", f"{type(e).__name__}:{e}"
+
+
+class _FakeResp:
+    """Minimal response shim so validators.validate() works on Playwright HTML."""
+    def __init__(self, html: str, status: int = 200, final_url: str = ""):
+        self.text = html
+        self.status_code = status
+        self.url = final_url
+        self.cookies = _FakeCookies()
+        self.headers = {}
+
+
+class _FakeCookies:
+    class _Jar:
+        def __iter__(self):
+            return iter([])
+    def __init__(self):
+        self.jar = self._Jar()
+    def __iter__(self):
+        return iter([])
+
+
+def run_playwright_fallback(
+    url: str,
+    *,
+    profile_id: str,
+    success_selectors: Optional[list[str]] = None,
+    device_class: str = "auto",
+    timeout: int = 90,
+    profile_dir: Optional[str] = None,
+    force_executor: Optional[str] = None,
+) -> tuple[Attempt, str]:
+    """Invoke the appropriate Playwright executor.
+
+    force_executor: caller-specified executor name (from a profile's
+    `fallback_when_challenge` list). When set, it overrides capability-based
+    inference. Recognized values: "playwright_real_chrome",
+    "playwright_mobile_chrome", "playwright_mcp".
+
+    Returns (Attempt, html_content). Attempt.verdict reflects validation.
+    """
+    profile = load_profile(profile_id)
+    capabilities = profile.get("capabilities_needed") or []
+    choice = force_executor or _pick_executor(capabilities, device_class)
+
+    t0 = time.time()
+    att = Attempt(
+        phase="fallback",
+        executor=choice,
+        url=url,
+        url_transform="original",
+        impersonate=None,
+        referer="",
+    )
+
+    if choice.startswith("playwright_mcp"):
+        att.error = (
+            "Playwright MCP must be invoked from the Claude session — "
+            "call mcp__playwright__* tools directly instead of fetch_chain."
+        )
+        att.verdict = Verdict.UNKNOWN.value
+        att.elapsed_s = round(time.time() - t0, 3)
+        return att, ""
+
+    if not _chrome_channel_available():
+        att.error = "node/npx not available for local Playwright template"
+        att.verdict = Verdict.UNKNOWN.value
+        att.elapsed_s = round(time.time() - t0, 3)
+        return att, ""
+
+    template_map = {
+        "playwright_real_chrome": "playwright_real_chrome.js",
+        "playwright_mobile_chrome": "playwright_mobile_chrome.js",
+    }
+    template = template_map.get(choice)
+    if template is None:
+        att.error = f"no template for executor {choice}"
+        att.verdict = Verdict.UNKNOWN.value
+        att.elapsed_s = round(time.time() - t0, 3)
+        return att, ""
+
+    args: dict = {
+        "url": url,
+        "profileDir": profile_dir or os.path.join(tempfile.gettempdir(), ".insane_pw_profile"),
+        "timeout": timeout * 1000,
+    }
+    if choice == "playwright_mobile_chrome":
+        args["device"] = "iPhone 13 Pro"
+    if success_selectors:
+        args["waitSelector"] = success_selectors[0]
+
+    rc, stdout, stderr = _run_node_template(template, args, timeout=timeout + 10)
+    att.elapsed_s = round(time.time() - t0, 3)
+
+    if rc != 0 or not stdout:
+        att.error = (stderr or "no stdout")[:300]
+        att.verdict = Verdict.UNKNOWN.value
+        return att, ""
+
+    # stdout carries HTML. Validate with a shim.
+    resp = _FakeResp(stdout)
+    vr = validate(resp, success_selectors=success_selectors)
+    att.status = 200
+    att.body_size = len(stdout)
+    att.verdict = vr.verdict.value
+    att.reasons = vr.reasons
+    return att, stdout