@geoly-ai/social-hub-cli 0.3.46 → 0.3.48

package/skills/reddit-voc-volume/references/arctic-shift.md

@@ -15,7 +15,7 @@ Pushshift 停摆后的接续者。**无需 auth**，免费公开服务，作者
 v2.12.26 修掉正文截断后，baseline 卡的长度维度只接受未截断全文。但多数网络
 环境下 reddit.com 的 `.json` 直采已被拦截（实测本机：`curl (56) CONNECT tunnel
 failed 403`，去掉代理直连亦为 `403 Blocked`）。Arctic Shift 不走 reddit.com，
-不需要 cookie / curl_cffi 伪装，且**正文不截断**。
+不需要 cookie 或指纹伪装，且**正文不截断**。
 
 ## 覆盖范围（实测，非文档转述）
 

package/skills/reddit-voc-volume/requirements.txt

@@ -1,5 +1,2 @@
 # Reddit VOC volume — Python dependencies
-# 以下两项仅供未调用的旧 Reddit HTTP 函数使用。主流程不实例化旧客户端，
-# 不读取浏览器 cookie；不安装也不影响当前模式。
-curl_cffi>=0.7.0
-browser_cookie3>=0.19.0
+# 无第三方 Python 依赖：脚本只用标准库（Arctic / Social Hub / Firecrawl 均走 urllib 或 CLI）。

package/skills/reddit-voc-volume/scripts/reddit_voc_volume.py

@@ -154,7 +154,19 @@ DEFAULT_UA = (
     "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 reddit-voc-volume/1.0"
 )
 
-IMPERSONATE = "chrome131"
+
+class _DeprecatedViaCurl(argparse.Action):
+    """`--via-curl` 自 v2.12.28 起无效果；保留一个发布周期，仅向 stderr 提示，下一版移除。"""
+
+    def __init__(self, option_strings: Any, dest: str, **kwargs: Any) -> None:
+        kwargs["nargs"] = 0
+        kwargs.setdefault("default", False)
+        kwargs.setdefault("help", "DEPRECATED：已无效果(v2.12.28 起不直连 reddit)，下一版移除")
+        super().__init__(option_strings, dest, **kwargs)
+
+    def __call__(self, parser: Any, namespace: Any, values: Any, option_string: Any = None) -> None:
+        print("DEPRECATED: --via-curl 已无效果(v2.12.28 起不直连 reddit)", file=sys.stderr)
+        setattr(namespace, self.dest, True)
 
 
 def utc_now_iso() -> str:
@@ -177,22 +189,6 @@ def sleep_with_jitter(base_sec: float) -> float:
     return jitter
 
 
-def _looks_like_html(text: str) -> bool:
-    s = (text or "").lstrip()
-    return s.startswith("<") or "<html" in s[:200].lower() or "theme-beta" in s[:500]
-
-
-def _loads_json_or_raise(text: str, *, url: str) -> Any:
-    if _looks_like_html(text):
-        raise ValueError(
-            "reddit_returned_html_not_json (likely 403/block; try proxy or browser fallback)"
-        )
-    try:
-        return json.loads(text)
-    except json.JSONDecodeError as e:
-        raise ValueError(f"invalid_json: {e}") from e
-
-
 def _loads_json_response(text: str) -> Any:
     try:
         return json.loads(text)
@@ -243,114 +239,11 @@ def resolve_proxy_url() -> str:
     return _macos_system_proxy_url()
 
 
-def _proxies_dict() -> dict[str, str]:
-    p = resolve_proxy_url()
-    return {"http": p, "https": p} if p else {}
-
-
-def _curl_proxy_args() -> list[str]:
-    proxy = resolve_proxy_url()
-    return ["--proxy", proxy] if proxy else []
-
-
 def _env_truthy(name: str) -> bool:
     raw = (os.environ.get(name) or "").strip().lower()
     return raw in ("1", "true", "yes", "on")
 
 
-def _use_browser_cookies() -> bool:
-    raw = (os.environ.get("REDDIT_USE_BROWSER_COOKIES") or "1").strip().lower()
-    return raw not in ("0", "false", "no", "off")
-
-
-class CookieStore:
-    """Load reddit.com cookies once per process and reuse for all requests."""
-
-    _instance: CookieStore | None = None
-
-    def __init__(self) -> None:
-        self._cookies: dict[str, str] = {}
-        self.source: str | None = None
-        self.loaded = False
-
-    @classmethod
-    def get(cls) -> CookieStore:
-        if cls._instance is None:
-            cls._instance = cls()
-            cls._instance.load_once()
-        return cls._instance
-
-    def load_once(self) -> dict[str, str]:
-        if self.loaded:
-            return self._cookies
-        self.loaded = True
-        raw = (os.environ.get("REDDIT_COOKIES_JSON") or "").strip()
-        if raw:
-            try:
-                obj = json.loads(raw)
-                if isinstance(obj, dict):
-                    self._cookies = {str(k): str(v) for k, v in obj.items() if k and v is not None}
-                    self.source = "env_json"
-                    return self._cookies
-            except json.JSONDecodeError:
-                pass
-        if not _use_browser_cookies():
-            self.source = None
-            return self._cookies
-        try:
-            import browser_cookie3  # type: ignore
-        except ImportError:
-            self.source = None
-            return self._cookies
-        for name in ("chrome", "safari", "firefox"):
-            fn = getattr(browser_cookie3, name, None)
-            if not callable(fn):
-                continue
-            try:
-                jar = fn(domain_name="reddit.com")
-                cookies = {c.name: c.value for c in jar if c.name and c.value}
-                if cookies:
-                    self._cookies = cookies
-                    self.source = "browser"
-                    # Share with child processes in same sync job (e.g. feishu sync)
-                    os.environ.setdefault("REDDIT_COOKIES_JSON", json.dumps(cookies))
-                    return self._cookies
-            except Exception:
-                continue
-        self.source = None
-        return self._cookies
-
-    @property
-    def cookies(self) -> dict[str, str]:
-        return self._cookies
-
-
-def _load_browser_cookies() -> dict[str, str]:
-    return CookieStore.get().cookies
-
-
-def _browser_headers(user_agent: str) -> dict[str, str]:
-    return {
-        "User-Agent": user_agent,
-        "Accept": "application/json, text/plain, */*",
-        "Accept-Language": "en-US,en;q=0.9",
-        "Referer": "https://www.reddit.com/",
-        "Origin": "https://www.reddit.com",
-        "Sec-Fetch-Dest": "empty",
-        "Sec-Fetch-Mode": "cors",
-        "Sec-Fetch-Site": "same-origin",
-    }
-
-
-def _curl_cffi_available() -> bool:
-    try:
-        import curl_cffi  # noqa: F401
-
-        return True
-    except ImportError:
-        return False
-
-
 class RequestTracker:
     def __init__(self) -> None:
         self.count = 0
@@ -358,9 +251,6 @@ class RequestTracker:
         self.rate_limited = False
         self.truncated = False
         self.last_fetch_mode = "none"
-        self.used_cookies = False
-        self.cookie_source: str | None = None
-        self.cookie_cache_reused = False
 
     def bump(self) -> None:
         self.count += 1
@@ -368,108 +258,6 @@ class RequestTracker:
             self.truncated = True
             raise RuntimeError("max_http_requests_exceeded")
 
-    def record_identity(self, mode: str, store: CookieStore) -> None:
-        self.last_fetch_mode = mode
-        if store.cookies:
-            self.used_cookies = True
-            self.cookie_source = store.source
-        if store.loaded and self.count > 1:
-            self.cookie_cache_reused = True
-
-
-class RedditHttpClient:
-    """Reuse cookies + curl_cffi session within a single CLI run."""
-
-    def __init__(self, user_agent: str, *, force_curl: bool = False) -> None:
-        self.user_agent = user_agent
-        self.force_curl = force_curl
-        self.cookie_store = CookieStore.get()
-        self._cffi_session: Any = None
-        self._warmed = False
-
-    def _ensure_cffi_session(self) -> Any:
-        if self._cffi_session is None:
-            from curl_cffi import requests as cffi_requests
-
-            self._cffi_session = cffi_requests.Session(impersonate=IMPERSONATE)
-        if not self._warmed:
-            headers = _browser_headers(self.user_agent)
-            proxies = _proxies_dict()
-            cookies = self.cookie_store.cookies
-            try:
-                self._cffi_session.get(
-                    "https://www.reddit.com/",
-                    headers=headers,
-                    proxies=proxies or None,
-                    cookies=cookies or None,
-                    timeout=30.0,
-                )
-            except Exception:
-                pass
-            self._warmed = True
-        return self._cffi_session
-
-    def fetch_via_curl_cffi(self, url: str, timeout: float = 30.0) -> tuple[Any, str]:
-        session = self._ensure_cffi_session()
-        headers = _browser_headers(self.user_agent)
-        proxies = _proxies_dict()
-        cookies = self.cookie_store.cookies
-        resp = session.get(
-            url,
-            headers=headers,
-            proxies=proxies or None,
-            cookies=cookies or None,
-            timeout=timeout,
-        )
-        if resp.status_code == 429:
-            raise RuntimeError("http_status:429")
-        if resp.status_code == 403:
-            raise RuntimeError("http_status:403")
-        if resp.status_code >= 400:
-            raise RuntimeError(f"http_status:{resp.status_code}")
-        return _loads_json_or_raise(resp.text, url=url), "curl_cffi"
-
-
-def http_get_json_via_curl(url: str, user_agent: str, timeout: int = 30) -> tuple[Any, str]:
-    curl = shutil.which("curl")
-    if not curl:
-        raise RuntimeError("curl not found in PATH")
-    cmd = [
-        curl,
-        "-sS",
-        "-L",
-        "--max-time",
-        str(int(timeout)),
-        "-A",
-        user_agent,
-        "-H",
-        "Accept: application/json, text/plain, */*",
-        "-H",
-        "Accept-Language: en-US,en;q=0.9",
-        "-H",
-        "Referer: https://www.reddit.com/",
-        *_curl_proxy_args(),
-        url,
-    ]
-    proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout + 5)
-    if proc.returncode != 0:
-        err = (proc.stderr or proc.stdout or "curl failed").strip()
-        if "429" in err:
-            raise RuntimeError("http_status:429")
-        raise RuntimeError(err)
-    return _loads_json_or_raise(proc.stdout, url=url), "curl"
-
-
-def http_get_json_urllib(url: str, user_agent: str, timeout: float = 25.0) -> tuple[Any, str]:
-    req = urllib.request.Request(url, headers=_browser_headers(user_agent), method="GET")
-    try:
-        with urllib.request.urlopen(req, timeout=timeout) as resp:
-            charset = resp.headers.get_content_charset() or "utf-8"
-            raw = resp.read().decode(charset)
-    except urllib.error.HTTPError as e:
-        raise RuntimeError(f"http_status:{e.code}") from e
-    return _loads_json_or_raise(raw, url=url), "urllib"
-
 
 def normalize_firecrawl_base_url(base_url: str | None = None) -> str:
     raw = (base_url or os.environ.get("FIRECRAWL_BASE_URL") or DEFAULT_FIRECRAWL_BASE_URL).strip()
@@ -4693,10 +4481,10 @@ def build_output(
         "request_identity": {
             "user_agent_kind": "chrome",
             "fetch_mode": tracker.last_fetch_mode,
-            "used_cookies": tracker.used_cookies,
-            "cookie_source": tracker.cookie_source,
-            "cookie_cache_reused": tracker.cookie_cache_reused,
-            "impersonation": IMPERSONATE if tracker.last_fetch_mode == "curl_cffi" else None,
+            "used_cookies": False,
+            "cookie_source": None,
+            "cookie_cache_reused": False,
+            "impersonation": None,
             "proxy_used": bool(proxy),
         },
         "notes": notes,
@@ -4833,7 +4621,7 @@ def main() -> int:
     pk.add_argument("--max-pages", type=int, default=DEFAULT_MAX_PAGES)
     pk.add_argument("--sleep-sec", type=float, default=DEFAULT_SLEEP_SEC)
     pk.add_argument("--user-agent", default=DEFAULT_UA)
-    pk.add_argument("--via-curl", action="store_true")
+    pk.add_argument("--via-curl", action=_DeprecatedViaCurl)
     pk.add_argument("--pretty", action="store_true")
     add_hub_voc_options(pk)
 
@@ -4846,7 +4634,7 @@ def main() -> int:
     ps.add_argument("--max-pages", type=int, default=DEFAULT_MAX_PAGES)
     ps.add_argument("--sleep-sec", type=float, default=DEFAULT_SLEEP_SEC)
     ps.add_argument("--user-agent", default=DEFAULT_UA)
-    ps.add_argument("--via-curl", action="store_true")
+    ps.add_argument("--via-curl", action=_DeprecatedViaCurl)
     ps.add_argument("--pretty", action="store_true")
     add_hub_voc_options(ps)
 
@@ -4855,7 +4643,7 @@ def main() -> int:
     pst.add_argument("--limit", type=int, default=10, help="Hot listing size per subreddit before filtering stickied posts")
     pst.add_argument("--sleep-sec", type=float, default=DEFAULT_SLEEP_SEC)
     pst.add_argument("--user-agent", default=DEFAULT_UA)
-    pst.add_argument("--via-curl", action="store_true")
+    pst.add_argument("--via-curl", action=_DeprecatedViaCurl)
     pst.add_argument("--pretty", action="store_true")
     add_hub_voc_options(pst)
 
@@ -4922,7 +4710,7 @@ def main() -> int:
     pl.add_argument("--file", "-f", help="File with one URL per line")
     pl.add_argument("--sleep-sec", type=float, default=DEFAULT_SLEEP_SEC)
     pl.add_argument("--user-agent", default=DEFAULT_UA)
-    pl.add_argument("--via-curl", action="store_true")
+    pl.add_argument("--via-curl", action=_DeprecatedViaCurl)
     pl.add_argument(
         "--fallback",
         choices=["none", "firecrawl", "social-hub", "auto"],
@@ -5363,7 +5151,7 @@ def main() -> int:
         errors = list(acquisition.get("errors") or [])
         if acquisition.get("should_local"):
             # Emergency Acquisition 缺省走 Arctic Shift 归档 API。
-            # 相对 .json 直采：正文不截断、不需要 cookie/curl 伪装、不受 Reddit 反爬拦截；
+            # 相对 .json 直采：正文不截断、不受 Reddit 反爬拦截；
             # 代价是 score 有约 1/3 为陈旧值（已逐条标 score_status），且无 live 语义。
             items = []
             for idx, sub in enumerate(subreddits or [None]):

package/skills/reddit-voc-volume/scripts/setup_env.sh

@@ -36,44 +36,8 @@ find_python() {
   return 1
 }
 
-ensure_curl() {
-  if command -v curl >/dev/null 2>&1; then
-    echo "  curl: $(command -v curl)"
-    return 0
-  fi
-  echo "  curl: not found — attempting install..."
-  if [[ "$SKIP_BREW" != 1 ]] && command -v brew >/dev/null 2>&1; then
-    brew install curl || true
-  fi
-  if [[ "$SKIP_APT" != 1 ]] && command -v apt-get >/dev/null 2>&1; then
-    if [[ "$ASSUME_YES" == 1 ]]; then
-      sudo apt-get update -qq && sudo apt-get install -y curl || true
-    else
-      echo "  Run: sudo apt-get install -y curl  (or re-run with --yes if sudo is OK)"
-    fi
-  fi
-  if command -v curl >/dev/null 2>&1; then
-    echo "  curl: installed at $(command -v curl)"
-    return 0
-  fi
-  echo "  curl: still missing (urllib/curl_cffi may still work; install curl manually for fallback)"
-  return 0
-}
-
-ensure_pip_packages() {
-  local py="$1"
-  echo "  pip packages: checking curl_cffi, browser_cookie3..."
-  if "$py" -c "import curl_cffi, browser_cookie3" 2>/dev/null; then
-    echo "  pip packages: OK"
-    return 0
-  fi
-  echo "  pip packages: installing from ${REQ}..."
-  "$py" -m pip install --upgrade pip >/dev/null 2>&1 || true
-  if ! "$py" -m pip install -r "$REQ"; then
-    echo "  pip install failed — try: $py -m pip install -r $REQ" >&2
-    return 1
-  fi
-  echo "  pip packages: installed"
+report_pip_packages() {
+  echo "  pip packages: none required (stdlib only; see ${REQ})"
 }
 
 main() {
@@ -84,8 +48,7 @@ main() {
     exit 1
   fi
   echo "  python: $PYTHON ($("$PYTHON" --version 2>&1))"
-  ensure_curl
-  ensure_pip_packages "$PYTHON"
+  report_pip_packages
   echo "=== setup complete ==="
 }