oh-my-opencode 4.12.1 → 4.14.0

package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/fetch_chain.py

@@ -0,0 +1,268 @@
+"""Single entrypoint: insane-search generic fetch chain.
+
+    from insane_search.engine import fetch
+    result = fetch("https://example.com/path", success_selectors=["article"])
+
+Public contract:
+  * One function: `fetch(url, ...) -> FetchResult`.
+  * Internal structure preserved as explicit phases so tests & debug logs
+    can target each stage: probe → validate → detect → plan → execute → report.
+  * `FetchResult.trace` exposes every attempt (transform × impersonate ×
+    referer × executor) — callers can diagnose without re-running.
+
+No site-specific branching. Site knowledge enters only via:
+  * `success_selectors` (caller-supplied positive proof)
+  * `user_hint` (optional runtime hints; never persisted by this module)
+  * `observations/*.jsonl` (append-only log; separate concern)
+"""
+from __future__ import annotations
+
+import json
+import os
+import random
+import time
+
+from .curl_probe import run_attempt
+from .result_schema import Attempt, FetchResult
+from .summary import format_summary
+from .validators import Verdict
+from .waf_detector import DetectionHit, detect, load_profile, _load_profiles, last_load_error
+from .url_transforms import iter_transformed
+
+
+# --- Main entrypoint ---------------------------------------------------------
+def fetch(
+    url: str,
+    *,
+    success_selectors: list[str] | None = None,
+    device_class: str = "auto",      # "auto" | "desktop" | "mobile"
+    user_hint: dict | None = None,
+    timeout: int = 25,
+    max_attempts: int = 12,
+    enable_playwright: bool = True,   # hook left for executor module
+) -> FetchResult:
+    """Fetch `url` using the generic grid.
+
+    Parameters
+    ----------
+    success_selectors
+        Positive-proof CSS selectors. Presence of ≥1 match promotes verdict
+        to STRONG_OK. Without them, best outcome is WEAK_OK.
+    device_class
+        "desktop" pins curl impersonate to desktop targets (safari/chrome/firefox).
+        "mobile" pins to mobile targets (safari_ios/chrome_android) AND enables
+        mobile URL transforms.
+        "auto" (default) follows profile advice; tries desktop first, mobile on
+        persistent failure.
+    user_hint
+        Optional runtime hints, e.g. `{"impersonate_first": "safari", "referer": "..."}`.
+        Never stored. Only influences current call.
+    timeout
+        Per-attempt timeout in seconds.
+    max_attempts
+        Hard upper bound on total attempts across all phases.
+    enable_playwright
+        Placeholder — Playwright fallback invocation is delegated to
+        `engine/executor.py` (separate module, capability-matched).
+    """
+    user_hint = user_hint or {}
+    profiles = _load_profiles()
+    trace: list[Attempt] = []
+    last_resp = None
+    last_attempt: Attempt | None = None
+    profile_used: str | None = None
+
+    # Surface profile-loader failures as a trace entry so callers can see
+    # that we're running on the in-code default (YAML missing / invalid /
+    # PyYAML not installed). Never fatal by itself.
+    load_err = last_load_error()
+    if load_err:
+        trace.append(Attempt(
+            phase="probe",
+            executor="profile_loader",
+            url=url,
+            url_transform="original",
+            impersonate=None,
+            referer="",
+            verdict=Verdict.UNKNOWN.value,
+            error=f"profiles_fallback: {load_err}",
+        ))
+
+    # -------- Phase 1: probe with safe defaults ------------------------------
+    base_impersonate = user_hint.get("impersonate_first") or "safari"
+    if device_class == "mobile":
+        base_impersonate = user_hint.get("impersonate_first") or "safari_ios"
+
+    probe_attempt, probe_resp = run_attempt(
+        url,
+        transform_name="original",
+        impersonate=base_impersonate,
+        referer_name=user_hint.get("referer_strategy") or "self_root",
+        success_selectors=success_selectors,
+        known_bad_sizes=None,
+        timeout=timeout,
+        phase="probe",
+    )
+    trace.append(probe_attempt)
+    if probe_resp is not None:
+        last_resp = probe_resp
+        last_attempt = probe_attempt
+        if probe_attempt.verdict in (Verdict.STRONG_OK.value, Verdict.WEAK_OK.value):
+            return _build_result(probe_resp, probe_attempt, trace, profile_used=None)
+
+    # -------- Phase 2: detect WAF, plan grid ---------------------------------
+    if last_resp is not None:
+        hits = detect(last_resp, profiles=profiles)
+    else:
+        hits = [DetectionHit(profile_id="unknown_challenge", confidence=0.1, signals=["no_probe_response"])]
+
+    # Try top profiles by confidence.
+    attempts_used = len(trace)
+    for hit in hits[:3]:  # top 3 candidates
+        if attempts_used >= max_attempts:
+            break
+        profile_id = hit.profile_id
+        profile_used = profile_id
+        profile = load_profile(profile_id, profiles=profiles)
+
+        tls_groups: list[list[str]] = profile.get("tls_impersonate_candidates") or [["safari", "chrome"]]
+        tls_flat: list[str] = [t for group in tls_groups for t in group]
+        avoid = set((profile.get("tls_impersonate_avoid") or []))
+        tls_flat = [t for t in tls_flat if t not in avoid]
+
+        referer_order = profile.get("referer_strategies") or ["self_root"]
+        transform_order = profile.get("url_transform_order") or ["original"]
+
+        # device_class override
+        if device_class == "mobile":
+            tls_flat = [t for t in tls_flat if "ios" in t or "android" in t] or tls_flat
+            if "mobile_subdomain" not in transform_order:
+                transform_order = transform_order + ["mobile_subdomain"]
+        elif device_class == "desktop":
+            tls_flat = [t for t in tls_flat if "ios" not in t and "android" not in t] or tls_flat
+
+        known_bad_sizes = profile.get("known_bad_sizes") or None
+
+        for t_name, t_url in iter_transformed(url, transform_order):
+            for tls in tls_flat:
+                for ref in referer_order:
+                    if attempts_used >= max_attempts:
+                        break
+                    # Skip exact duplicate of probe.
+                    if (t_name == "original" and tls == base_impersonate
+                            and ref == (user_hint.get("referer_strategy") or "self_root")):
+                        continue
+                    att, resp = run_attempt(
+                        t_url,
+                        transform_name=t_name,
+                        impersonate=tls,
+                        referer_name=ref,
+                        success_selectors=success_selectors,
+                        known_bad_sizes=known_bad_sizes,
+                        timeout=timeout,
+                        phase="grid",
+                    )
+                    trace.append(att)
+                    attempts_used += 1
+                    # Jitter: politeness + IP-reputation guard. Tunable via
+                    # INSANE_JITTER_MS_MIN / INSANE_JITTER_MS_MAX env vars.
+                    _jmin = int(os.environ.get("INSANE_JITTER_MS_MIN", "150"))
+                    _jmax = int(os.environ.get("INSANE_JITTER_MS_MAX", "400"))
+                    time.sleep(random.uniform(_jmin/1000.0, _jmax/1000.0))
+                    if resp is None:
+                        continue
+                    last_resp, last_attempt = resp, att
+                    if att.verdict in (Verdict.STRONG_OK.value, Verdict.WEAK_OK.value):
+                        return _build_result(resp, att, trace, profile_used=profile_id)
+
+    # -------- Phase 3: Playwright fallback (profile-driven order) -----------
+    if enable_playwright:
+        try:
+            from .executor import run_playwright_fallback  # lazy import
+            # Honour profile's `fallback_when_challenge` list — iterate the
+            # caller-declared order instead of capability-inferred single pick.
+            fb_profile = load_profile(profile_used or "unknown_challenge", profiles=profiles)
+            fb_order = fb_profile.get("fallback_when_challenge") or ["playwright_real_chrome"]
+            pw_attempt = None
+            pw_content = ""
+            for fb_name in fb_order:
+                if fb_name == "curl_grid_exhaust":
+                    # Already performed in Phase 2; nothing more to do here.
+                    continue
+                pw_attempt, pw_content = run_playwright_fallback(
+                    url,
+                    profile_id=profile_used or "unknown_challenge",
+                    success_selectors=success_selectors,
+                    device_class=device_class,
+                    force_executor=fb_name,
+                )
+                trace.append(pw_attempt)
+                if pw_attempt.verdict in (Verdict.STRONG_OK.value, Verdict.WEAK_OK.value):
+                    return FetchResult(
+                        ok=True,
+                        content=pw_content,
+                        final_url=pw_attempt.url,
+                        verdict=pw_attempt.verdict,
+                        profile_used=profile_used,
+                        trace=trace,
+                        summary=f"Playwright fallback succeeded via {fb_name}",
+                    )
+            # Synthesize a placeholder if no iteration ran (empty list).
+            if pw_attempt is None:
+                pw_attempt = Attempt(
+                    phase="fallback",
+                    executor="none",
+                    url=url,
+                    url_transform="original",
+                    impersonate=None,
+                    referer="",
+                    verdict=Verdict.UNKNOWN.value,
+                    error="profile has empty fallback_when_challenge",
+                )
+                trace.append(pw_attempt)
+        except ImportError:
+            trace.append(Attempt(
+                phase="fallback",
+                executor="playwright",
+                url=url,
+                url_transform="original",
+                impersonate=None,
+                referer="",
+                verdict=Verdict.UNKNOWN.value,
+                error="executor module not available",
+            ))
+        except (RuntimeError, OSError) as e:
+            trace.append(Attempt(
+                phase="fallback",
+                executor="playwright",
+                url=url,
+                url_transform="original",
+                impersonate=None,
+                referer="",
+                verdict=Verdict.UNKNOWN.value,
+                error=f"{type(e).__name__}:{str(e)[:200]}",
+            ))
+
+    # -------- Give up, return best we have ----------------------------------
+    summary = format_summary(trace, profile_used)
+    return FetchResult(
+        ok=False,
+        content=getattr(last_resp, "text", "") if last_resp is not None else "",
+        final_url=getattr(last_resp, "url", url) if last_resp is not None else url,
+        verdict=last_attempt.verdict if last_attempt else Verdict.UNKNOWN.value,
+        profile_used=profile_used,
+        trace=trace,
+        summary=summary,
+    )
+
+
+def _build_result(resp, attempt: Attempt, trace: list[Attempt], profile_used: str | None) -> FetchResult:
+    return FetchResult(
+        ok=True,
+        content=getattr(resp, "text", "") or "",
+        final_url=str(getattr(resp, "url", attempt.url)),
+        verdict=attempt.verdict,
+        profile_used=profile_used,
+        trace=trace,
+        summary=f"{attempt.executor} {attempt.impersonate} + {attempt.url_transform} + referer:{attempt.referer} → {attempt.verdict}",
+    )

package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/referers.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from urllib.parse import urlsplit
+
+
+def _self_root(url: str) -> str:
+    parsed = urlsplit(url)
+    return f"{parsed.scheme}://{parsed.netloc}/"
+
+
+REFERER_STRATEGIES = {
+    "self_root": _self_root,
+    "google_search": lambda _url: "https://www.google.com/",
+    "none": lambda _url: "",
+}

package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/result_schema.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import Optional
+
+
+@dataclass
+class Attempt:
+    phase: str
+    executor: str
+    url: str
+    url_transform: str
+    impersonate: Optional[str]
+    referer: str
+    status: int = 0
+    body_size: int = 0
+    verdict: str = ""
+    reasons: list[str] = field(default_factory=list)
+    elapsed_s: float = 0.0
+    error: Optional[str] = None
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass
+class FetchResult:
+    ok: bool
+    content: str = ""
+    final_url: str = ""
+    verdict: str = ""
+    profile_used: Optional[str] = None
+    trace: list[Attempt] = field(default_factory=list)
+    summary: str = ""
+
+    def to_dict(self) -> dict:
+        return {
+            "ok": self.ok,
+            "final_url": self.final_url,
+            "verdict": self.verdict,
+            "profile_used": self.profile_used,
+            "trace": [a.to_dict() for a in self.trace],
+            "summary": self.summary,
+            "content_length": len(self.content),
+        }

package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/summary.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from typing import Optional
+
+from .result_schema import Attempt
+from .validators import Verdict
+
+
+_R7_ELIGIBLE_PROFILES = frozenset({
+    "akamai_bot_manager",
+    "cloudflare_turnstile",
+    "datadome_probable",
+    "perimeterx_human",
+    "f5_big_ip",
+    "aws_waf",
+})
+
+R7_HINT = (
+    "💡 R7 API-first 권장: WAF가 HTML 경로를 차단 중. "
+    "Playwright MCP 사용 → browser_navigate → browser_network_requests "
+    "→ `/api/`·`/graphql`·`\\.json` 필터로 내부 엔드포인트 탐지 → "
+    "해당 URL을 `python3 -m engine <API_URL>`로 재호출. 대부분 API 레이어는 "
+    "WAF 방어가 얕아 curl_cffi만으로 수집됨."
+)
+
+
+def format_summary(trace: list[Attempt], profile: Optional[str]) -> str:
+    n = len(trace)
+    verdicts = [a.verdict for a in trace]
+    challenge_count = sum(1 for v in verdicts if v == Verdict.CHALLENGE.value)
+    base = (
+        f"failed after {n} attempts; profile={profile}; "
+        f"verdicts={','.join(v for v in verdicts[:5])}" + ("..." if n > 5 else "")
+    )
+    if profile in _R7_ELIGIBLE_PROFILES and challenge_count >= 3:
+        return base + "\n" + R7_HINT
+    return base

package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/templates/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "insane-search-templates",
+  "version": "0.1.0",
+  "private": true,
+  "description": "Local deps for Playwright real-Chrome templates. npm install && npx playwright install chrome",
+  "dependencies": {
+    "playwright": "^1.61.0",
+    "playwright-extra": "^4.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2"
+  }
+}

package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/templates/playwright_mobile_chrome.js

@@ -0,0 +1,129 @@
+#!/usr/bin/env node
+/**
+ * Generic Playwright mobile fetcher — real Chrome + device emulation.
+ *
+ * Usage:
+ *   echo '{"url":"...", "device":"iPhone 13 Pro"}' | node playwright_mobile_chrome.js
+ *
+ * Device name must match playwright `devices[...]` keys (Pixel 7, iPhone 13 Pro,
+ * iPad Pro 11, etc.). When in doubt, omit `device` — default is iPhone 13 Pro.
+ *
+ * NO-SITE-NAME RULE: same as playwright_real_chrome.js — no hostname branches.
+ */
+
+async function readStdinJson() {
+  return await new Promise((resolve, reject) => {
+    let data = '';
+    process.stdin.on('data', (c) => (data += c));
+    process.stdin.on('end', () => {
+      try { resolve(JSON.parse(data || '{}')); }
+      catch (e) { reject(e); }
+    });
+    process.stdin.on('error', reject);
+  });
+}
+
+function describeError(error) {
+  if (error instanceof Error) {
+    return `${error.name}: ${error.message}`;
+  }
+  return String(error);
+}
+
+function warnBestEffort(action, error) {
+  process.stderr.write(`best-effort ${action} failed: ${describeError(error)}\n`);
+}
+
+function isMissingTopLevelModule(error, moduleName) {
+  return (
+    error instanceof Error &&
+    error.code === 'MODULE_NOT_FOUND' &&
+    typeof error.message === 'string' &&
+    error.message.includes(`Cannot find module '${moduleName}'`)
+  );
+}
+
+function requireOptionalModule(moduleName) {
+  let resolvedModule;
+  try {
+    resolvedModule = require.resolve(moduleName);
+  } catch (e) {
+    if (isMissingTopLevelModule(e, moduleName)) {
+      warnBestEffort(`optional module ${moduleName}`, e);
+      return null;
+    }
+    throw e;
+  }
+  return require(resolvedModule);
+}
+
+async function main() {
+  const args = await readStdinJson();
+  const url = args.url;
+  if (!url) {
+    process.stderr.write('missing url\n');
+    process.exitCode = 2;
+    return;
+  }
+
+  const profileDir = args.profileDir || '/tmp/.insane_pw_mobile_profile';
+  const deviceName = args.device || 'iPhone 13 Pro';
+  const waitSelector = args.waitSelector || null;
+  const timeoutMs = args.timeout || 60000;
+  const headless = args.headless ?? false;
+
+  let chromium, devices;
+  const playwrightExtra = requireOptionalModule('playwright-extra');
+  const stealthPlugin = playwrightExtra ? requireOptionalModule('puppeteer-extra-plugin-stealth') : null;
+  if (playwrightExtra && stealthPlugin) {
+    ({ chromium, devices } = playwrightExtra);
+    const stealth = stealthPlugin();
+    chromium.use(stealth);
+  } else {
+    ({ chromium, devices } = require('playwright'));
+  }
+
+  const dev = devices[deviceName];
+  if (!dev) {
+    process.stderr.write(`unknown device: ${deviceName}\n`);
+    process.exitCode = 2;
+    return;
+  }
+
+  let ctx;
+  try {
+    ctx = await chromium.launchPersistentContext(profileDir, {
+      channel: 'chrome',
+      headless,
+      ...dev,
+    });
+    const page = await ctx.newPage();
+    const navTimeout = Math.min(timeoutMs, 90000);
+    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: navTimeout });
+
+    if (waitSelector) {
+      try {
+        await page.waitForSelector(waitSelector, { timeout: Math.min(timeoutMs, 20000) });
+      } catch (e) {
+        warnBestEffort('waitSelector', e);
+      }
+    }
+
+    const html = await page.content();
+    process.stdout.write(html);
+    process.exitCode = 0;
+    return;
+  } catch (e) {
+    process.stderr.write(`${describeError(e)}\n`);
+    process.exitCode = 1;
+    return;
+  } finally {
+    try {
+      if (ctx) await ctx.close();
+    } catch (e) {
+      warnBestEffort('browser context close', e);
+    }
+  }
+}
+
+main();

package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/templates/playwright_real_chrome.js

@@ -0,0 +1,167 @@
+#!/usr/bin/env node
+/**
+ * Generic Playwright fetcher — real Chrome channel (not bundled Chromium).
+ *
+ * Usage (driven by engine/executor.py):
+ *   echo '{"url":"...", "profileDir":"/tmp/.p", "waitSelector":"article"}' | node playwright_real_chrome.js
+ *
+ * Outputs page HTML to stdout on success; errors to stderr with non-zero exit.
+ *
+ * NO-SITE-NAME RULE: this file must never branch on specific hostnames.
+ * All site specifics come from the JSON input (url, waitSelector).
+ *
+ * Dependencies (install once on target machine):
+ *   npm i -g playwright playwright-extra puppeteer-extra-plugin-stealth
+ *   npx playwright install chrome    # system Chrome binary
+ */
+
+const fs = require('fs');
+
+async function readStdinJson() {
+  return await new Promise((resolve, reject) => {
+    let data = '';
+    process.stdin.on('data', (c) => (data += c));
+    process.stdin.on('end', () => {
+      try { resolve(JSON.parse(data || '{}')); }
+      catch (e) { reject(e); }
+    });
+    process.stdin.on('error', reject);
+  });
+}
+
+function describeError(error) {
+  if (error instanceof Error) {
+    return `${error.name}: ${error.message}`;
+  }
+  return String(error);
+}
+
+function warnBestEffort(action, error) {
+  process.stderr.write(`best-effort ${action} failed: ${describeError(error)}\n`);
+}
+
+function isMissingTopLevelModule(error, moduleName) {
+  return (
+    error instanceof Error &&
+    error.code === 'MODULE_NOT_FOUND' &&
+    typeof error.message === 'string' &&
+    error.message.includes(`Cannot find module '${moduleName}'`)
+  );
+}
+
+function requireOptionalModule(moduleName) {
+  let resolvedModule;
+  try {
+    resolvedModule = require.resolve(moduleName);
+  } catch (e) {
+    if (isMissingTopLevelModule(e, moduleName)) {
+      warnBestEffort(`optional module ${moduleName}`, e);
+      return null;
+    }
+    throw e;
+  }
+  return require(resolvedModule);
+}
+
+async function main() {
+  const args = await readStdinJson();
+  const url = args.url;
+  if (!url) {
+    process.stderr.write('missing url\n');
+    process.exitCode = 2;
+    return;
+  }
+
+  const profileDir = args.profileDir || '/tmp/.insane_pw_profile';
+  const waitSelector = args.waitSelector || null;
+  const timeoutMs = args.timeout || 60000;
+  const headless = args.headless ?? false;     // Akamai/etc detect headless
+  const viewport = args.viewport || { width: 1366, height: 900 };
+
+  let chromium;
+  const playwrightExtra = requireOptionalModule('playwright-extra');
+  const stealthPlugin = playwrightExtra ? requireOptionalModule('puppeteer-extra-plugin-stealth') : null;
+  if (playwrightExtra && stealthPlugin) {
+    ({ chromium } = playwrightExtra);
+    const stealth = stealthPlugin();
+    chromium.use(stealth);
+  } else {
+    // Fallback to plain playwright (no stealth). Still uses channel:chrome.
+    ({ chromium } = require('playwright'));
+  }
+
+  let ctx;
+  try {
+    ctx = await chromium.launchPersistentContext(profileDir, {
+      channel: 'chrome',          // real Chrome, not bundled Chromium
+      headless,
+      viewport,
+    });
+    const page = await ctx.newPage();
+    const navTimeout = Math.min(timeoutMs, 90000);
+
+    // Warmup hop: visit the site root first so Akamai-style bot managers
+    // can run their JS sensor and set a resolved session cookie. Direct
+    // landing on a search/deep URL is the classic first-hit rejection pattern.
+    // Use domcontentloaded (not networkidle) — many SPAs keep analytics/xhr
+    // open indefinitely and would hit the 90s timeout.
+    try {
+      const urlObj = new URL(url);
+      const rootUrl = `${urlObj.protocol}//${urlObj.host}/`;
+      if (rootUrl !== url) {
+        await page.goto(rootUrl, { waitUntil: 'domcontentloaded', timeout: navTimeout });
+        await page.waitForTimeout(3500);   // let sensor JS finish
+      }
+    } catch (e) {
+      warnBestEffort('warmup navigation', e);
+      // warmup is best-effort; continue even if it hiccups
+    }
+
+    // Main page — DOM loaded then give the sensor a moment.
+    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: navTimeout });
+    await page.waitForTimeout(2500);
+
+    if (waitSelector) {
+      try {
+        await page.waitForSelector(waitSelector, { timeout: Math.min(timeoutMs, 20000) });
+      } catch (e) {
+        warnBestEffort('waitSelector', e);
+        // Selector still missing — try one hard reload in case the first hit
+        // landed on a challenge page and the sensor has just cleared.
+        try {
+          await page.reload({ waitUntil: 'domcontentloaded', timeout: navTimeout });
+          await page.waitForTimeout(2000);
+          try {
+            await page.waitForSelector(waitSelector, { timeout: 10000 });
+          } catch (e2) {
+            warnBestEffort('retry waitSelector', e2);
+            // Still no luck — caller validates HTML anyway.
+          }
+        } catch (e3) {
+          warnBestEffort('selector recovery reload', e3);
+          // reload failed — proceed with whatever we have
+        }
+      }
+    } else {
+      // Without a positive-proof selector, give the sensor a couple more seconds.
+      await page.waitForTimeout(2000);
+    }
+
+    const html = await page.content();
+    process.stdout.write(html);
+    process.exitCode = 0;
+    return;
+  } catch (e) {
+    process.stderr.write(`${describeError(e)}\n`);
+    process.exitCode = 1;
+    return;
+  } finally {
+    try {
+      if (ctx) await ctx.close();
+    } catch (e) {
+      warnBestEffort('browser context close', e);
+    }
+  }
+}
+
+main();