oh-my-opencode 4.12.0 → 4.13.0

package/packages/shared-skills/skills/ultimate-browsing/engine/executor.py

@@ -0,0 +1,192 @@
+"""Capability-matched executor for fallback attempts.
+
+The fetch_chain's probe/grid phase uses curl_cffi directly. When curl can't
+punch through (JS challenge, real-TLS detection), this module routes to the
+right browser executor based on the profile's `capabilities_needed` tags:
+
+    needs_real_tls_stack + needs_js_exec  → playwright_real_chrome.js
+    needs_js_exec only                    → Playwright MCP (if available)
+    needs_mobile_context (+ real_tls)     → playwright_mobile_chrome.js
+
+The JS templates live in `engine/templates/` and accept only generic
+parameters ({{url}}, {{waitSelector}}, {{profileDir}}, {{device}}). No
+site-specific logic.
+
+Playwright MCP invocation requires caller's tool access; this module
+provides the subprocess path for local JS templates but only stubs the MCP
+path (MCP must be driven from the Claude session itself).
+"""
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import tempfile
+import time
+from typing import Optional
+
+from .validators import Verdict, validate
+from .waf_detector import load_profile
+from .result_schema import Attempt
+
+
+TEMPLATES_DIR = os.path.join(os.path.dirname(__file__), "templates")
+
+
+def _node_available() -> bool:
+    return shutil.which("node") is not None
+
+
+def _chrome_channel_available() -> bool:
+    """Heuristic: try `node -e` to import playwright. Fallback to True, let script fail loudly."""
+    if not _node_available():
+        return False
+    if shutil.which("npx") is None:
+        return False
+    return True
+
+
+def _pick_executor(capabilities: list[str], device_class: str) -> str:
+    caps = set(capabilities or [])
+    if device_class == "mobile" or "needs_mobile_context" in caps:
+        if "needs_real_tls_stack" in caps:
+            return "playwright_mobile_chrome"
+        return "playwright_mcp_mobile"
+    if "needs_real_tls_stack" in caps:
+        return "playwright_real_chrome"
+    if "needs_js_exec" in caps:
+        return "playwright_mcp"
+    return "playwright_real_chrome"  # safest general fallback
+
+
+def _run_node_template(template: str, args: dict, timeout: int = 90) -> tuple[int, str, str]:
+    """Run a Node.js template with args as JSON on stdin.
+
+    Template convention: reads `process.stdin` → JSON → runs fetch → writes
+    HTML to stdout; errors go to stderr with non-zero exit code.
+    """
+    path = os.path.join(TEMPLATES_DIR, template)
+    if not os.path.isfile(path):
+        return 127, "", f"template not found: {path}"
+    try:
+        proc = subprocess.run(
+            ["node", path],
+            input=json.dumps(args),
+            cwd=TEMPLATES_DIR,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+        return proc.returncode, proc.stdout, proc.stderr
+    except subprocess.TimeoutExpired:
+        return 124, "", f"timeout after {timeout}s"
+    except Exception as e:
+        return 1, "", f"{type(e).__name__}:{e}"
+
+
+class _FakeResp:
+    """Minimal response shim so validators.validate() works on Playwright HTML."""
+    def __init__(self, html: str, status: int = 200, final_url: str = ""):
+        self.text = html
+        self.status_code = status
+        self.url = final_url
+        self.cookies = _FakeCookies()
+        self.headers = {}
+
+
+class _FakeCookies:
+    class _Jar:
+        def __iter__(self):
+            return iter([])
+    def __init__(self):
+        self.jar = self._Jar()
+    def __iter__(self):
+        return iter([])
+
+
+def run_playwright_fallback(
+    url: str,
+    *,
+    profile_id: str,
+    success_selectors: Optional[list[str]] = None,
+    device_class: str = "auto",
+    timeout: int = 90,
+    profile_dir: Optional[str] = None,
+    force_executor: Optional[str] = None,
+) -> tuple[Attempt, str]:
+    """Invoke the appropriate Playwright executor.
+
+    force_executor: caller-specified executor name (from a profile's
+    `fallback_when_challenge` list). When set, it overrides capability-based
+    inference. Recognized values: "playwright_real_chrome",
+    "playwright_mobile_chrome", "playwright_mcp".
+
+    Returns (Attempt, html_content). Attempt.verdict reflects validation.
+    """
+    profile = load_profile(profile_id)
+    capabilities = profile.get("capabilities_needed") or []
+    choice = force_executor or _pick_executor(capabilities, device_class)
+
+    t0 = time.time()
+    att = Attempt(
+        phase="fallback",
+        executor=choice,
+        url=url,
+        url_transform="original",
+        impersonate=None,
+        referer="",
+    )
+
+    if choice.startswith("playwright_mcp"):
+        att.error = (
+            "Playwright MCP must be invoked from the Claude session — "
+            "call mcp__playwright__* tools directly instead of fetch_chain."
+        )
+        att.verdict = Verdict.UNKNOWN.value
+        att.elapsed_s = round(time.time() - t0, 3)
+        return att, ""
+
+    if not _chrome_channel_available():
+        att.error = "node/npx not available for local Playwright template"
+        att.verdict = Verdict.UNKNOWN.value
+        att.elapsed_s = round(time.time() - t0, 3)
+        return att, ""
+
+    template_map = {
+        "playwright_real_chrome": "playwright_real_chrome.js",
+        "playwright_mobile_chrome": "playwright_mobile_chrome.js",
+    }
+    template = template_map.get(choice)
+    if template is None:
+        att.error = f"no template for executor {choice}"
+        att.verdict = Verdict.UNKNOWN.value
+        att.elapsed_s = round(time.time() - t0, 3)
+        return att, ""
+
+    args: dict = {
+        "url": url,
+        "profileDir": profile_dir or os.path.join(tempfile.gettempdir(), ".insane_pw_profile"),
+        "timeout": timeout * 1000,
+    }
+    if choice == "playwright_mobile_chrome":
+        args["device"] = "iPhone 13 Pro"
+    if success_selectors:
+        args["waitSelector"] = success_selectors[0]
+
+    rc, stdout, stderr = _run_node_template(template, args, timeout=timeout + 10)
+    att.elapsed_s = round(time.time() - t0, 3)
+
+    if rc != 0 or not stdout:
+        att.error = (stderr or "no stdout")[:300]
+        att.verdict = Verdict.UNKNOWN.value
+        return att, ""
+
+    # stdout carries HTML. Validate with a shim.
+    resp = _FakeResp(stdout)
+    vr = validate(resp, success_selectors=success_selectors)
+    att.status = 200
+    att.body_size = len(stdout)
+    att.verdict = vr.verdict.value
+    att.reasons = vr.reasons
+    return att, stdout

package/packages/shared-skills/skills/ultimate-browsing/engine/fetch_chain.py

@@ -0,0 +1,268 @@
+"""Single entrypoint: insane-search generic fetch chain.
+
+    from insane_search.engine import fetch
+    result = fetch("https://example.com/path", success_selectors=["article"])
+
+Public contract:
+  * One function: `fetch(url, ...) -> FetchResult`.
+  * Internal structure preserved as explicit phases so tests & debug logs
+    can target each stage: probe → validate → detect → plan → execute → report.
+  * `FetchResult.trace` exposes every attempt (transform × impersonate ×
+    referer × executor) — callers can diagnose without re-running.
+
+No site-specific branching. Site knowledge enters only via:
+  * `success_selectors` (caller-supplied positive proof)
+  * `user_hint` (optional runtime hints; never persisted by this module)
+  * `observations/*.jsonl` (append-only log; separate concern)
+"""
+from __future__ import annotations
+
+import json
+import os
+import random
+import time
+
+from .curl_probe import run_attempt
+from .result_schema import Attempt, FetchResult
+from .summary import format_summary
+from .validators import Verdict
+from .waf_detector import DetectionHit, detect, load_profile, _load_profiles, last_load_error
+from .url_transforms import iter_transformed
+
+
+# --- Main entrypoint ---------------------------------------------------------
+def fetch(
+    url: str,
+    *,
+    success_selectors: list[str] | None = None,
+    device_class: str = "auto",      # "auto" | "desktop" | "mobile"
+    user_hint: dict | None = None,
+    timeout: int = 25,
+    max_attempts: int = 12,
+    enable_playwright: bool = True,   # hook left for executor module
+) -> FetchResult:
+    """Fetch `url` using the generic grid.
+
+    Parameters
+    ----------
+    success_selectors
+        Positive-proof CSS selectors. Presence of ≥1 match promotes verdict
+        to STRONG_OK. Without them, best outcome is WEAK_OK.
+    device_class
+        "desktop" pins curl impersonate to desktop targets (safari/chrome/firefox).
+        "mobile" pins to mobile targets (safari_ios/chrome_android) AND enables
+        mobile URL transforms.
+        "auto" (default) follows profile advice; tries desktop first, mobile on
+        persistent failure.
+    user_hint
+        Optional runtime hints, e.g. `{"impersonate_first": "safari", "referer": "..."}`.
+        Never stored. Only influences current call.
+    timeout
+        Per-attempt timeout in seconds.
+    max_attempts
+        Hard upper bound on total attempts across all phases.
+    enable_playwright
+        Placeholder — Playwright fallback invocation is delegated to
+        `engine/executor.py` (separate module, capability-matched).
+    """
+    user_hint = user_hint or {}
+    profiles = _load_profiles()
+    trace: list[Attempt] = []
+    last_resp = None
+    last_attempt: Attempt | None = None
+    profile_used: str | None = None
+
+    # Surface profile-loader failures as a trace entry so callers can see
+    # that we're running on the in-code default (YAML missing / invalid /
+    # PyYAML not installed). Never fatal by itself.
+    load_err = last_load_error()
+    if load_err:
+        trace.append(Attempt(
+            phase="probe",
+            executor="profile_loader",
+            url=url,
+            url_transform="original",
+            impersonate=None,
+            referer="",
+            verdict=Verdict.UNKNOWN.value,
+            error=f"profiles_fallback: {load_err}",
+        ))
+
+    # -------- Phase 1: probe with safe defaults ------------------------------
+    base_impersonate = user_hint.get("impersonate_first") or "safari"
+    if device_class == "mobile":
+        base_impersonate = user_hint.get("impersonate_first") or "safari_ios"
+
+    probe_attempt, probe_resp = run_attempt(
+        url,
+        transform_name="original",
+        impersonate=base_impersonate,
+        referer_name=user_hint.get("referer_strategy") or "self_root",
+        success_selectors=success_selectors,
+        known_bad_sizes=None,
+        timeout=timeout,
+        phase="probe",
+    )
+    trace.append(probe_attempt)
+    if probe_resp is not None:
+        last_resp = probe_resp
+        last_attempt = probe_attempt
+        if probe_attempt.verdict in (Verdict.STRONG_OK.value, Verdict.WEAK_OK.value):
+            return _build_result(probe_resp, probe_attempt, trace, profile_used=None)
+
+    # -------- Phase 2: detect WAF, plan grid ---------------------------------
+    if last_resp is not None:
+        hits = detect(last_resp, profiles=profiles)
+    else:
+        hits = [DetectionHit(profile_id="unknown_challenge", confidence=0.1, signals=["no_probe_response"])]
+
+    # Try top profiles by confidence.
+    attempts_used = len(trace)
+    for hit in hits[:3]:  # top 3 candidates
+        if attempts_used >= max_attempts:
+            break
+        profile_id = hit.profile_id
+        profile_used = profile_id
+        profile = load_profile(profile_id, profiles=profiles)
+
+        tls_groups: list[list[str]] = profile.get("tls_impersonate_candidates") or [["safari", "chrome"]]
+        tls_flat: list[str] = [t for group in tls_groups for t in group]
+        avoid = set((profile.get("tls_impersonate_avoid") or []))
+        tls_flat = [t for t in tls_flat if t not in avoid]
+
+        referer_order = profile.get("referer_strategies") or ["self_root"]
+        transform_order = profile.get("url_transform_order") or ["original"]
+
+        # device_class override
+        if device_class == "mobile":
+            tls_flat = [t for t in tls_flat if "ios" in t or "android" in t] or tls_flat
+            if "mobile_subdomain" not in transform_order:
+                transform_order = transform_order + ["mobile_subdomain"]
+        elif device_class == "desktop":
+            tls_flat = [t for t in tls_flat if "ios" not in t and "android" not in t] or tls_flat
+
+        known_bad_sizes = profile.get("known_bad_sizes") or None
+
+        for t_name, t_url in iter_transformed(url, transform_order):
+            for tls in tls_flat:
+                for ref in referer_order:
+                    if attempts_used >= max_attempts:
+                        break
+                    # Skip exact duplicate of probe.
+                    if (t_name == "original" and tls == base_impersonate
+                            and ref == (user_hint.get("referer_strategy") or "self_root")):
+                        continue
+                    att, resp = run_attempt(
+                        t_url,
+                        transform_name=t_name,
+                        impersonate=tls,
+                        referer_name=ref,
+                        success_selectors=success_selectors,
+                        known_bad_sizes=known_bad_sizes,
+                        timeout=timeout,
+                        phase="grid",
+                    )
+                    trace.append(att)
+                    attempts_used += 1
+                    # Jitter: politeness + IP-reputation guard. Tunable via
+                    # INSANE_JITTER_MS_MIN / INSANE_JITTER_MS_MAX env vars.
+                    _jmin = int(os.environ.get("INSANE_JITTER_MS_MIN", "150"))
+                    _jmax = int(os.environ.get("INSANE_JITTER_MS_MAX", "400"))
+                    time.sleep(random.uniform(_jmin/1000.0, _jmax/1000.0))
+                    if resp is None:
+                        continue
+                    last_resp, last_attempt = resp, att
+                    if att.verdict in (Verdict.STRONG_OK.value, Verdict.WEAK_OK.value):
+                        return _build_result(resp, att, trace, profile_used=profile_id)
+
+    # -------- Phase 3: Playwright fallback (profile-driven order) -----------
+    if enable_playwright:
+        try:
+            from .executor import run_playwright_fallback  # lazy import
+            # Honour profile's `fallback_when_challenge` list — iterate the
+            # caller-declared order instead of capability-inferred single pick.
+            fb_profile = load_profile(profile_used or "unknown_challenge", profiles=profiles)
+            fb_order = fb_profile.get("fallback_when_challenge") or ["playwright_real_chrome"]
+            pw_attempt = None
+            pw_content = ""
+            for fb_name in fb_order:
+                if fb_name == "curl_grid_exhaust":
+                    # Already performed in Phase 2; nothing more to do here.
+                    continue
+                pw_attempt, pw_content = run_playwright_fallback(
+                    url,
+                    profile_id=profile_used or "unknown_challenge",
+                    success_selectors=success_selectors,
+                    device_class=device_class,
+                    force_executor=fb_name,
+                )
+                trace.append(pw_attempt)
+                if pw_attempt.verdict in (Verdict.STRONG_OK.value, Verdict.WEAK_OK.value):
+                    return FetchResult(
+                        ok=True,
+                        content=pw_content,
+                        final_url=pw_attempt.url,
+                        verdict=pw_attempt.verdict,
+                        profile_used=profile_used,
+                        trace=trace,
+                        summary=f"Playwright fallback succeeded via {fb_name}",
+                    )
+            # Synthesize a placeholder if no iteration ran (empty list).
+            if pw_attempt is None:
+                pw_attempt = Attempt(
+                    phase="fallback",
+                    executor="none",
+                    url=url,
+                    url_transform="original",
+                    impersonate=None,
+                    referer="",
+                    verdict=Verdict.UNKNOWN.value,
+                    error="profile has empty fallback_when_challenge",
+                )
+                trace.append(pw_attempt)
+        except ImportError:
+            trace.append(Attempt(
+                phase="fallback",
+                executor="playwright",
+                url=url,
+                url_transform="original",
+                impersonate=None,
+                referer="",
+                verdict=Verdict.UNKNOWN.value,
+                error="executor module not available",
+            ))
+        except (RuntimeError, OSError) as e:
+            trace.append(Attempt(
+                phase="fallback",
+                executor="playwright",
+                url=url,
+                url_transform="original",
+                impersonate=None,
+                referer="",
+                verdict=Verdict.UNKNOWN.value,
+                error=f"{type(e).__name__}:{str(e)[:200]}",
+            ))
+
+    # -------- Give up, return best we have ----------------------------------
+    summary = format_summary(trace, profile_used)
+    return FetchResult(
+        ok=False,
+        content=getattr(last_resp, "text", "") if last_resp is not None else "",
+        final_url=getattr(last_resp, "url", url) if last_resp is not None else url,
+        verdict=last_attempt.verdict if last_attempt else Verdict.UNKNOWN.value,
+        profile_used=profile_used,
+        trace=trace,
+        summary=summary,
+    )
+
+
+def _build_result(resp, attempt: Attempt, trace: list[Attempt], profile_used: str | None) -> FetchResult:
+    return FetchResult(
+        ok=True,
+        content=getattr(resp, "text", "") or "",
+        final_url=str(getattr(resp, "url", attempt.url)),
+        verdict=attempt.verdict,
+        profile_used=profile_used,
+        trace=trace,
+        summary=f"{attempt.executor} {attempt.impersonate} + {attempt.url_transform} + referer:{attempt.referer} → {attempt.verdict}",
+    )

package/packages/shared-skills/skills/ultimate-browsing/engine/referers.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from urllib.parse import urlsplit
+
+
+def _self_root(url: str) -> str:
+    parsed = urlsplit(url)
+    return f"{parsed.scheme}://{parsed.netloc}/"
+
+
+REFERER_STRATEGIES = {
+    "self_root": _self_root,
+    "google_search": lambda _url: "https://www.google.com/",
+    "none": lambda _url: "",
+}

package/packages/shared-skills/skills/ultimate-browsing/engine/result_schema.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import Optional
+
+
+@dataclass
+class Attempt:
+    phase: str
+    executor: str
+    url: str
+    url_transform: str
+    impersonate: Optional[str]
+    referer: str
+    status: int = 0
+    body_size: int = 0
+    verdict: str = ""
+    reasons: list[str] = field(default_factory=list)
+    elapsed_s: float = 0.0
+    error: Optional[str] = None
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass
+class FetchResult:
+    ok: bool
+    content: str = ""
+    final_url: str = ""
+    verdict: str = ""
+    profile_used: Optional[str] = None
+    trace: list[Attempt] = field(default_factory=list)
+    summary: str = ""
+
+    def to_dict(self) -> dict:
+        return {
+            "ok": self.ok,
+            "final_url": self.final_url,
+            "verdict": self.verdict,
+            "profile_used": self.profile_used,
+            "trace": [a.to_dict() for a in self.trace],
+            "summary": self.summary,
+            "content_length": len(self.content),
+        }

package/packages/shared-skills/skills/ultimate-browsing/engine/summary.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from typing import Optional
+
+from .result_schema import Attempt
+from .validators import Verdict
+
+
+_R7_ELIGIBLE_PROFILES = frozenset({
+    "akamai_bot_manager",
+    "cloudflare_turnstile",
+    "datadome_probable",
+    "perimeterx_human",
+    "f5_big_ip",
+    "aws_waf",
+})
+
+R7_HINT = (
+    "💡 R7 API-first 권장: WAF가 HTML 경로를 차단 중. "
+    "Playwright MCP 사용 → browser_navigate → browser_network_requests "
+    "→ `/api/`·`/graphql`·`\\.json` 필터로 내부 엔드포인트 탐지 → "
+    "해당 URL을 `python3 -m engine <API_URL>`로 재호출. 대부분 API 레이어는 "
+    "WAF 방어가 얕아 curl_cffi만으로 수집됨."
+)
+
+
+def format_summary(trace: list[Attempt], profile: Optional[str]) -> str:
+    n = len(trace)
+    verdicts = [a.verdict for a in trace]
+    challenge_count = sum(1 for v in verdicts if v == Verdict.CHALLENGE.value)
+    base = (
+        f"failed after {n} attempts; profile={profile}; "
+        f"verdicts={','.join(v for v in verdicts[:5])}" + ("..." if n > 5 else "")
+    )
+    if profile in _R7_ELIGIBLE_PROFILES and challenge_count >= 3:
+        return base + "\n" + R7_HINT
+    return base

package/packages/shared-skills/skills/ultimate-browsing/engine/templates/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "insane-search-templates",
+  "version": "0.1.0",
+  "private": true,
+  "description": "Local deps for Playwright real-Chrome templates. npm install && npx playwright install chrome",
+  "dependencies": {
+    "playwright": "^1.61.0",
+    "playwright-extra": "^4.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2"
+  }
+}