oh-my-opencode 4.12.1 → 4.14.0

package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/waf_detector.py

@@ -0,0 +1,214 @@
+"""WAF-product detection from a live response.
+
+Returns a *ranking* of (profile_id, confidence) pairs — never a single verdict.
+Single-answer detectors cause cascading wrong plans when misfiring (Codex's
+critique). Planner consumes the ranking and tries top candidates in order.
+
+All detectors operate on WAF-vendor artifacts (cookies / headers / body
+strings) — never site hostnames. See engine/waf_profiles.yaml for the
+profile definitions.
+"""
+from __future__ import annotations
+
+import fnmatch
+import os
+import re
+from dataclasses import dataclass
+from typing import Optional
+
+try:
+    import yaml  # PyYAML
+except ImportError:
+    yaml = None
+
+
+PROFILES_PATH = os.path.join(os.path.dirname(__file__), "waf_profiles.yaml")
+
+
+# In-code safety net — used when waf_profiles.yaml is missing / invalid
+# or PyYAML isn't installed. Keeps fetch() working in a degraded-but-sane
+# mode. Must stay site-agnostic (No-Site-Name Rule).
+_DEFAULT_PROFILES: dict = {
+    "unknown_challenge": {
+        "detectors": {},
+        "confidence_rules": {"strong": 0, "weak": 0},
+        "capabilities_needed": ["needs_js_exec"],
+        "tls_impersonate_candidates": [
+            ["safari", "chrome", "firefox"],
+            ["safari_ios", "chrome_android"],
+        ],
+        "referer_strategies": ["self_root", "google_search", "none"],
+        "url_transform_order": ["original", "mobile_subdomain"],
+        "fallback_when_challenge": ["playwright_mcp", "playwright_real_chrome"],
+        "notes": "in-code default — waf_profiles.yaml unavailable",
+    },
+}
+
+
+# Module-level sticky error. Readers call `last_load_error()` after each
+# `_load_profiles()` call to surface YAML problems in FetchResult.trace.
+_LAST_LOAD_ERROR: Optional[str] = None
+
+
+@dataclass
+class DetectionHit:
+    profile_id: str
+    confidence: float
+    signals: list[str]
+
+
+def last_load_error() -> Optional[str]:
+    """Return the most recent profile-loader error (or None if clean)."""
+    return _LAST_LOAD_ERROR
+
+
+def _load_profiles(path: str = PROFILES_PATH) -> dict:
+    """Load profiles with graceful fallback.
+
+    Never raises. On any failure (PyYAML missing, file missing, parse error,
+    unexpected shape) it returns a copy of `_DEFAULT_PROFILES` and stores
+    the reason in `_LAST_LOAD_ERROR` for the caller to surface.
+    """
+    global _LAST_LOAD_ERROR
+    _LAST_LOAD_ERROR = None
+
+    if yaml is None:
+        _LAST_LOAD_ERROR = "PyYAML not installed — using in-code default profile"
+        return dict(_DEFAULT_PROFILES)
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            loaded = yaml.safe_load(f) or {}
+    except FileNotFoundError:
+        _LAST_LOAD_ERROR = f"waf_profiles.yaml not found at {path}"
+        return dict(_DEFAULT_PROFILES)
+    except yaml.YAMLError as e:
+        _LAST_LOAD_ERROR = f"YAML parse error: {type(e).__name__}: {str(e)[:200]}"
+        return dict(_DEFAULT_PROFILES)
+    except Exception as e:
+        _LAST_LOAD_ERROR = f"profile loader: {type(e).__name__}: {str(e)[:200]}"
+        return dict(_DEFAULT_PROFILES)
+
+    if not isinstance(loaded, dict) or not any(k for k in loaded if not k.startswith("_")):
+        _LAST_LOAD_ERROR = f"waf_profiles.yaml has no usable profiles"
+        return dict(_DEFAULT_PROFILES)
+
+    return loaded
+
+
+def _cookies_dict(resp) -> dict:
+    try:
+        return {c.name: c.value for c in resp.cookies.jar}
+    except Exception:
+        try:
+            return dict(resp.cookies) if hasattr(resp, "cookies") else {}
+        except Exception:
+            return {}
+
+
+def _headers_dict(resp) -> dict:
+    try:
+        return {k.lower(): v for k, v in dict(resp.headers).items()}
+    except Exception:
+        return {}
+
+
+def _match_patterns(haystack_keys: list[str], patterns: list[str]) -> list[str]:
+    """Match literal names or fnmatch patterns (for wildcards like `X-Akamai-*`)."""
+    hits: list[str] = []
+    lowered_keys = [k.lower() for k in haystack_keys]
+    for pat in patterns or []:
+        pat_l = pat.lower()
+        if any(c in pat for c in "*?["):
+            for key in lowered_keys:
+                if fnmatch.fnmatchcase(key, pat_l):
+                    hits.append(pat)
+                    break
+        else:
+            if pat_l in lowered_keys:
+                hits.append(pat)
+    return hits
+
+
+def _score_profile(profile_id: str, profile: dict, resp) -> Optional[DetectionHit]:
+    """Apply profile detectors to resp. Returns hit or None."""
+    if profile_id.startswith("_"):
+        return None
+    detectors = profile.get("detectors") or {}
+    if not detectors and profile_id != "unknown_challenge":
+        return None
+
+    cookies = _cookies_dict(resp)
+    headers = _headers_dict(resp)
+    body = (getattr(resp, "text", "") or "").lower()
+    server = headers.get("server", "")
+
+    signals: list[str] = []
+
+    # Cookie detectors
+    cookie_pats = detectors.get("cookie") or []
+    for hit in _match_patterns(list(cookies.keys()), cookie_pats):
+        signals.append(f"cookie:{hit}")
+
+    # Header detectors
+    header_pats = detectors.get("header") or []
+    for hit in _match_patterns(list(headers.keys()), header_pats):
+        signals.append(f"header:{hit}")
+
+    # Server substring
+    for needle in detectors.get("server_contains") or []:
+        if needle.lower() in server:
+            signals.append(f"server:{needle}")
+
+    # Body markers
+    for needle in detectors.get("body") or []:
+        if needle.lower() in body:
+            signals.append(f"body:{needle}")
+
+    if not signals:
+        return None
+
+    rules = profile.get("confidence_rules") or {"strong": 2, "weak": 1}
+    n = len(signals)
+    if n >= rules.get("strong", 2):
+        conf = 0.9
+    elif n >= rules.get("weak", 1):
+        conf = 0.6
+    else:
+        conf = 0.3
+
+    return DetectionHit(profile_id=profile_id, confidence=conf, signals=signals)
+
+
+def detect(resp, *, profiles: Optional[dict] = None, min_confidence: float = 0.0) -> list[DetectionHit]:
+    """Return ranked list of detection hits (best first).
+
+    When nothing fires, the returned list contains a single `unknown_challenge`
+    hit with confidence 0.1 — caller can use its conservative settings.
+    """
+    if profiles is None:
+        profiles = _load_profiles()
+
+    hits: list[DetectionHit] = []
+    for profile_id, profile in profiles.items():
+        if profile_id.startswith("_"):
+            continue
+        h = _score_profile(profile_id, profile, resp)
+        if h and h.confidence >= min_confidence:
+            hits.append(h)
+
+    hits.sort(key=lambda x: x.confidence, reverse=True)
+
+    if not hits:
+        hits.append(DetectionHit(
+            profile_id="unknown_challenge",
+            confidence=0.1,
+            signals=["fallback"],
+        ))
+    return hits
+
+
+def load_profile(profile_id: str, *, profiles: Optional[dict] = None) -> dict:
+    """Get one profile by id, resolving `unknown_challenge` if missing."""
+    if profiles is None:
+        profiles = _load_profiles()
+    return profiles.get(profile_id) or profiles.get("unknown_challenge") or {}

package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/waf_profiles.yaml

@@ -0,0 +1,162 @@
+# WAF Product Profiles — never site-specific.
+#
+# NO-SITE-NAME RULE:
+#   * Key names are WAF products (akamai_bot_manager), not sites.
+#   * Detectors use product artifacts (cookies / headers / vendor strings).
+#   * Field values (markers, cookie names) must appear in any site running
+#     that WAF, not just one. If a value only fits one site, it belongs
+#     to runtime hints / observations, not this file.
+#
+# Profiles are *recommendations*, not deterministic recipes. The planner
+# treats them as priors; attempts always evaluate real responses.
+#
+# Timestamp each profile entry. If stale (> 6 months), cross-validate.
+
+_meta:
+  schema_version: 1
+  last_reviewed: "2026-04-21"
+
+akamai_bot_manager:
+  detectors:
+    cookie: ["_abck", "bm_sz", "ak_bmsc", "bm_sv", "bm_so"]
+    header: ["X-Akamai-*"]
+    server_contains: ["AkamaiGHost"]
+    body: ["sec-if-cpt-container", "Powered and protected by Akamai"]
+  confidence_rules:
+    # Multi-signal gating — single marker insufficient.
+    strong: 2     # any 2 signals from above → confidence 0.9
+    weak: 1       # 1 signal → confidence 0.6
+  capabilities_needed:
+    - needs_real_tls_stack   # Playwright Chromium (BoringSSL) is detected
+    - needs_js_exec          # 2.6KB challenge requires JS sensor
+  tls_impersonate_candidates:
+    # Every impersonate target curl_cffi supports that historically yielded
+    # at least a challenge page (i.e. IP still alive) rather than an outright
+    # TLS reject. Grouped by family; planner tries top groups first.
+    # Refresh quarterly — vendor WAFs shift which TLS fingerprints they trust.
+    - [safari, safari15_3, safari15_5, safari17_0, safari260]
+    - [safari_ios, safari17_2_ios, safari260_ios]
+    - [chrome99, chrome100, chrome101, chrome104, chrome110, chrome116, chrome119, chrome124, chrome131, chrome133a, chrome136, chrome145, chrome146]
+    - [chrome_android, chrome131_android]
+    - [edge99, edge101]
+  tls_impersonate_avoid:
+    # Empirically observed to 403 immediately (TLS fingerprint blacklisted).
+    # DO NOT hard-block — planner deprioritizes only. Refresh quarterly.
+    - safari18_0
+    - chrome107
+    - chrome120
+    - chrome123
+    - chrome145     # curl_cffi 0.11+ needed — mark avoid for older installs
+    - chrome146
+    - firefox
+    - firefox133
+    - firefox135
+  referer_strategies:
+    - self_root    # scheme://host/
+  url_transform_order:
+    - original
+    - mobile_subdomain   # www.* → m.* — strong observational win in SSR sites
+  fallback_when_challenge:
+    - curl_grid_exhaust   # try more impersonate × referer × url combos
+    - playwright_real_chrome
+  notes: |
+    DO NOT encode site-specific selectors or byte-size fingerprints here.
+    Those belong to caller's success_selectors param or observations log.
+
+cloudflare_turnstile:
+  detectors:
+    cookie: ["cf_clearance", "__cf_bm", "__cfduid"]
+    header: ["cf-ray", "cf-cache-status"]
+    server_contains: ["cloudflare"]
+    body: ["Just a moment...", "Checking your browser", "cf-chl-bypass", "Attention Required! | Cloudflare"]
+  capabilities_needed:
+    - needs_js_exec        # MCP Playwright Chromium OK — no real-TLS required
+  tls_impersonate_candidates:
+    - [chrome, chrome_android]
+  referer_strategies:
+    - google_search
+    - self_root
+  fallback_when_challenge:
+    - playwright_mcp       # MCP sufficient; Chromium TLS passes CF baseline
+    - playwright_real_chrome
+
+f5_big_ip:
+  detectors:
+    cookie: ["BigIPServer", "TS01*", "F5_*"]
+    body: ["The requested URL was rejected", "support ID is:"]
+  capabilities_needed:
+    - needs_real_tls_stack
+  tls_impersonate_candidates:
+    - [safari, chrome]
+  referer_strategies:
+    - self_root
+
+aws_waf:
+  detectors:
+    cookie: ["aws-waf-token"]
+    header: ["x-amzn-requestid", "x-amzn-errortype", "x-amzn-waf-*"]
+  capabilities_needed:
+    - needs_real_tls_stack
+  tls_impersonate_candidates:
+    - [chrome]
+  referer_strategies:
+    - self_root
+
+datadome_probable:
+  detectors:
+    cookie: ["datadome"]
+    body: ["DataDome"]
+  capabilities_needed:
+    - needs_real_tls_stack
+    - needs_js_exec
+  tls_impersonate_candidates:
+    - [safari, chrome]
+  fallback_when_challenge:
+    - playwright_real_chrome
+  notes: |
+    "_probable" suffix reminds us this is a growing attack surface — mark
+    as tentative until cross-site evidence accumulates in observations/.
+
+perimeterx_human:
+  detectors:
+    cookie: ["_px3", "_pxhd", "_px2", "pxcts"]
+    body: ["px-captcha", "Press & Hold to confirm you are a human"]
+  capabilities_needed:
+    - needs_real_tls_stack
+    - needs_js_exec
+  tls_impersonate_candidates:
+    - [safari, chrome]
+  fallback_when_challenge:
+    - playwright_real_chrome
+  notes: |
+    PerimeterX (now HUMAN Bot Defender). Distinct cookie family from
+    DataDome. Keep profiles separate so planner does not pick wrong
+    fallback strategy.
+
+# ---------------------------------------------------------------------------
+# Safety net: always-valid fallback profile.
+# ---------------------------------------------------------------------------
+unknown_challenge:
+  detectors: {}   # never matches actively — used only when no other profile fires
+  confidence_rules:
+    strong: 0
+    weak: 0
+  capabilities_needed:
+    - needs_js_exec     # conservative default
+  tls_impersonate_candidates:
+    - [safari, chrome, firefox]
+    - [safari_ios, chrome_android]
+  referer_strategies:
+    - self_root
+    - google_search
+    - none
+  url_transform_order:
+    - original
+    - mobile_subdomain
+  fallback_when_challenge:
+    - playwright_mcp
+    - playwright_real_chrome
+  notes: |
+    When detector returns low-confidence results we land here. Broad,
+    conservative grid. Evidence from these runs should feed observations/
+    for eventual profile promotion.

package/packages/omo-codex/plugin/skills/ultimate-browsing/references/agent-reach/README.md

@@ -0,0 +1,79 @@
+
+# Agent Reach — 路由器
+
+> Part of the **ultimate-browsing** skill (Tier 1.5). Routing and tier-escalation live in [../../SKILL.md](../../SKILL.md).
+> Per-category guides in this folder: [search.md](search.md) · [social.md](social.md) · [career.md](career.md) · [dev.md](dev.md) · [web.md](web.md) · [video.md](video.md).
+
+
+17 平台工具集合。根据用户意图选择对应分类。
+
+## 路由表
+
+| 用户意图 | 分类 | 详细文档 |
+|---------|------|---------|
+| 网页搜索/代码搜索 | search | [search.md](search.md) |
+| 小红书/抖音/微博/推特/B站/V2EX/Reddit | social | [social.md](social.md) |
+| 招聘/职位/LinkedIn | career | [career.md](career.md) |
+| GitHub/代码 | dev | [dev.md](dev.md) |
+| 网页/文章/公众号/RSS | web | [web.md](web.md) |
+| YouTube/B站/播客字幕 | video | [video.md](video.md) |
+
+## 零配置快速命令
+
+```bash
+# Exa 网页搜索
+mcporter call 'exa.web_search_exa(query: "query", numResults: 5)'
+
+# 通用网页阅读
+curl -s "https://r.jina.ai/URL"
+
+# GitHub 搜索
+gh search repos "query" --sort stars --limit 10
+
+# Twitter 搜索
+twitter search "query" --limit 10
+
+# YouTube/B站字幕
+yt-dlp --write-sub --skip-download -o "/tmp/%(id)s" "URL"
+
+# Reddit 搜索
+rdt search "query" --limit 10
+
+# Reddit 读帖 + 评论
+rdt read POST_ID
+
+# V2EX 热门
+curl -s "https://www.v2ex.com/api/topics/hot.json" -H "User-Agent: agent-reach/1.0"
+```
+
+## 环境检查
+
+```bash
+# 检查可用 channel
+agent-reach doctor
+
+# 查看所有 MCP 服务
+mcporter_list_servers()
+```
+
+## 工作区规则
+
+**不要在 agent workspace 创建文件。** 使用 `/tmp/` 存放临时输出。
+
+## 详细文档
+
+根据用户需求，阅读对应的详细文档：
+
+- [搜索工具](search.md) — Exa AI 搜索
+- [社交媒体](social.md) — 小红书, 抖音, Twitter, B站, V2EX, Reddit
+- [职场招聘](career.md) — LinkedIn
+- [开发工具](dev.md) — GitHub CLI
+- [网页阅读](web.md) — Jina Reader, 微信公众号, RSS
+- [视频播客](video.md) — YouTube, B站, 小宇宙
+
+## 配置渠道
+
+如果某个 channel 需要配置，获取安装指南：
+https://raw.githubusercontent.com/Panniantong/agent-reach/main/docs/install.md
+
+用户只需提供 cookies，其他配置由 agent 完成。

package/packages/omo-codex/plugin/skills/ultimate-browsing/references/agent-reach/career.md

@@ -0,0 +1,29 @@
+# 职场招聘
+
+LinkedIn。
+
+## LinkedIn
+
+```bash
+# 获取个人资料
+mcporter call 'linkedin-scraper.get_person_profile(linkedin_url: "https://linkedin.com/in/username")'
+
+# 搜索人才
+mcporter call 'linkedin-scraper.search_people(keyword: "AI engineer", limit: 10)'
+
+# 获取公司资料
+mcporter call 'linkedin-scraper.get_company_profile(linkedin_url: "https://linkedin.com/company/xxx")'
+
+# 搜索职位
+mcporter call 'linkedin-scraper.search_jobs(keyword: "software engineer", limit: 10)'
+```
+
+> **需要登录**: LinkedIn scraper 需要有效的登录态。
+
+### Fallback 方案
+
+如果 MCP 不可用，可以用 Jina Reader：
+
+```bash
+curl -s "https://r.jina.ai/https://linkedin.com/in/username"
+```

package/packages/omo-codex/plugin/skills/ultimate-browsing/references/agent-reach/dev.md

@@ -0,0 +1,62 @@
+# 开发工具
+
+GitHub CLI
+
+## GitHub (gh CLI)
+
+GitHub 官方命令行工具，用于仓库、Issue、PR、Actions、Release 以及 API 访问。
+
+```bash
+# 认证
+gh auth login
+gh auth status
+
+# 搜索
+gh search repos "query" --sort stars --limit 10
+gh search code "query" --language python
+
+# 仓库
+gh repo view owner/repo
+gh repo clone owner/repo
+gh repo create my-repo --private
+gh repo fork owner/repo
+gh repo fork owner/repo --clone
+gh repo sync owner/repo
+
+# Issues
+gh issue list -R owner/repo --state open
+gh issue view 123 -R owner/repo
+gh issue create -R owner/repo --title "Title" --body "Body"
+
+# Pull Requests
+gh pr list -R owner/repo --state open
+gh pr view 123 -R owner/repo
+gh pr create -R owner/repo --title "Title" --body "Body"
+gh pr checks 123 --repo owner/repo
+
+# Actions / CI
+gh run list --repo owner/repo --limit 10
+gh run view <run-id> --repo owner/repo
+gh run view <run-id> --repo owner/repo --log-failed
+gh workflow list --repo owner/repo
+
+# Releases
+gh release list -R owner/repo
+gh release create v1.0.0
+
+# API
+gh api /user
+gh api repos/owner/repo
+
+# JSON 输出
+gh issue list --repo owner/repo --json number,title --jq '.[] | "\(.number): \(.title)"'
+```
+
+
+## 选择指南
+
+| 工具 | 来源 | 用途 |
+|-----|------|------|
+| gh CLI | agent-reach | Git 操作 |
+| zread | my-mcp-tools | 读仓库内容 |
+| context7 | my-mcp-tools | 查技术文档 |

package/packages/omo-codex/plugin/skills/ultimate-browsing/references/agent-reach/search.md

@@ -0,0 +1,33 @@
+# 搜索工具
+
+Exa AI 搜索引擎。
+
+## Exa AI 搜索
+
+高质量 AI 搜索引擎，擅长技术和代码搜索。
+
+```bash
+mcporter call 'exa.web_search_exa(query: "query", numResults: 5)'
+mcporter call 'exa.get_code_context_exa(query: "code question", tokensNum: 3000)'
+```
+
+### 使用场景
+
+| 场景 | 参数 |
+|-----|------|
+| 网页搜索 | `web_search_exa(query: "...", numResults: 5)` |
+| 代码搜索 | `get_code_context_exa(query: "...", tokensNum: 3000)` |
+
+### 特点
+
+- 擅长英文内容和技术文档
+- 支持代码上下文搜索
+- 结果质量高
+
+## 与其他搜索工具对比
+
+| 工具 | 来源 | 适用场景 |
+|-----|------|---------|
+| Exa | agent-reach | 英文/技术/代码搜索 |
+| 智谱搜索 | my-mcp-tools | 中文搜索 |
+| GitHub 搜索 | agent-reach (dev.md) | 仓库/代码搜索 |