@geoly-ai/social-hub-cli 0.3.46 → 0.3.48

package/skills/reddit-voc-volume/tests/test_no_direct_reddit.py

@@ -0,0 +1,315 @@
+"""v2.12.28+ 契约：脚本不得直连 reddit.com，`request_identity` 输出形状保持兼容。
+
+静态扫描 + 输出契约；全部离线。成对对照组保证检查器本身不会静默变空。
+
+注意：静态部分是**启发式防回归**，不是「无直连」的完整证明——它只看 urllib 请求调用的实参
+（字面量、同函数内的局部赋值、模块级常量）是否指向 reddit.com 主机，动态拼接的 URL 抓不到。
+运行时的兜底是 test_links_fallback / 本文件中把 `urlopen` 打成「调用即失败」的用例。
+"""
+from __future__ import annotations
+
+import ast
+import contextlib
+import importlib.util
+import io
+import json
+from pathlib import Path
+import re
+import sys
+import unittest
+from unittest import mock
+
+
+SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "reddit_voc_volume.py"
+SPEC = importlib.util.spec_from_file_location("reddit_voc_no_direct_under_test", SCRIPT)
+assert SPEC is not None and SPEC.loader is not None
+voc = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(voc)
+
+FORBIDDEN_TOKENS = (
+    "curl_cffi",
+    "browser_cookie3",
+    "http_get_json",
+    "cffi_requests",
+    "RedditHttpClient",
+    "CookieStore",
+    "IMPERSONATE",
+)
+# 精确匹配 reddit.com 主机：不误报 photon-reddit.com、reddit.com.example.org。
+REDDIT_URL_RE = re.compile(r"^https?://(?:[\w-]+\.)*reddit\.com(?:[:/?#]|$)")
+# urllib 请求调用；subprocess 仅在命令里出现 curl/wget 时才算 HTTP 请求。
+URLLIB_CALLS = {"urlopen", "Request", "firecrawl_urlopen"}
+SUBPROCESS_CALLS = {"run", "Popen", "check_output"}
+HTTP_CLIENT_BINARIES = {"curl", "wget"}
+POST_URL = "https://www.reddit.com/r/example/comments/abc123/slug/"
+IDENTITY_KEYS = {
+    "user_agent_kind",
+    "fetch_mode",
+    "used_cookies",
+    "cookie_source",
+    "cookie_cache_reused",
+    "impersonation",
+    "proxy_used",
+}
+
+
+def _strings(node: ast.AST) -> list[str]:
+    return [n.value for n in ast.walk(node) if isinstance(n, ast.Constant) and isinstance(n.value, str)]
+
+
+def _has_reddit_url(node: ast.AST) -> bool:
+    return any(REDDIT_URL_RE.match(s) for s in _strings(node))
+
+
+def _call_name(call: ast.Call) -> str:
+    f = call.func
+    if isinstance(f, ast.Attribute):
+        return f.attr
+    if isinstance(f, ast.Name):
+        return f.id
+    return ""
+
+
+def _names_bound_to_reddit_url(scope: ast.AST) -> set[str]:
+    out: set[str] = set()
+    for n in ast.walk(scope):
+        if isinstance(n, ast.Assign) and _has_reddit_url(n.value):
+            out.update(t.id for t in n.targets if isinstance(t, ast.Name))
+    return out
+
+
+def _is_http_request(call: ast.Call) -> bool:
+    name = _call_name(call)
+    if name in URLLIB_CALLS:
+        return True
+    if name in SUBPROCESS_CALLS:
+        return any(s in HTTP_CLIENT_BINARIES for s in _strings(call))
+    return False
+
+
+def functions_requesting_reddit(source: str) -> tuple[list[str], list[str]]:
+    """返回 (发起 HTTP 请求的函数名, 其中请求实参指向 reddit.com 的函数名)。
+
+    只检查请求调用的实参：reddit URL 作为 permalink / 展示字段，或放进发给 Hub 的 payload，
+    都不在请求实参里，不算违规。
+    """
+    tree = ast.parse(source)
+    module_consts = _names_bound_to_reddit_url(ast.Module(body=[n for n in tree.body if isinstance(n, ast.Assign)], type_ignores=[]))
+    requesting: list[str] = []
+    offending: list[str] = []
+    for fn in ast.walk(tree):
+        if not isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            continue
+        calls = [n for n in ast.walk(fn) if isinstance(n, ast.Call) and _is_http_request(n)]
+        if not calls:
+            continue
+        requesting.append(fn.name)
+        reddit_names = module_consts | _names_bound_to_reddit_url(fn)
+        for call in calls:
+            arg_nodes = [*call.args, *(kw.value for kw in call.keywords)]
+            used = {n.id for a in arg_nodes for n in ast.walk(a) if isinstance(n, ast.Name)}
+            if any(_has_reddit_url(a) for a in arg_nodes) or used & reddit_names:
+                offending.append(fn.name)
+                break
+    return requesting, offending
+
+
+class StaticScanTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        cls.source = SCRIPT.read_text(encoding="utf-8")
+
+    def test_legacy_direct_fetch_tokens_are_gone(self) -> None:
+        for token in FORBIDDEN_TOKENS:
+            with self.subTest(token=token):
+                self.assertNotIn(token, self.source)
+
+    def test_no_function_that_sends_requests_references_reddit_com(self) -> None:
+        requesting, offending = functions_requesting_reddit(self.source)
+        # 对照：真实脚本里确实有发请求的函数（Arctic / Firecrawl / Hub），扫描器不是空转。
+        self.assertIn("arctic_get", requesting)
+        self.assertGreaterEqual(len(requesting), 3)
+        self.assertEqual(offending, [])
+
+    def test_scanner_control_flags_a_direct_reddit_request(self) -> None:
+        bad_literal = (
+            "import urllib.request\n"
+            "def fetch():\n"
+            "    return urllib.request.urlopen('https://www.reddit.com/r/x/.json')\n"
+        )
+        bad_module_const = (
+            "import urllib.request\n"
+            "REDDIT_URL = 'https://old.reddit.com/r/x/'\n"
+            "def fetch():\n"
+            "    req = urllib.request.Request(REDDIT_URL)\n"
+            "    return urllib.request.urlopen(req)\n"
+        )
+        bad_local_var = (
+            "import urllib.request\n"
+            "def fetch(sub):\n"
+            "    url = f'https://www.reddit.com/r/{sub}/about.json'\n"
+            "    return urllib.request.urlopen(url)\n"
+        )
+        bad_curl = (
+            "import subprocess\n"
+            "def fetch():\n"
+            "    return subprocess.run(['curl', 'https://www.reddit.com/.json'])\n"
+        )
+        for name, src in (
+            ("literal", bad_literal),
+            ("module_const", bad_module_const),
+            ("local_var", bad_local_var),
+            ("curl", bad_curl),
+        ):
+            with self.subTest(case=name):
+                self.assertEqual(functions_requesting_reddit(src)[1], ["fetch"])
+
+    def test_scanner_control_does_not_flag_display_payload_or_lookalike_hosts(self) -> None:
+        good = (
+            "import subprocess, urllib.request\n"
+            "REDDIT_URL = 'https://www.reddit.com/'\n"
+            "def fetch():\n"
+            "    return urllib.request.urlopen('https://arctic-shift.photon-reddit.com/api')\n"
+            "def lookalike():\n"
+            "    return urllib.request.urlopen('https://reddit.com.example.org/x')\n"
+            "def permalink(sub):\n"
+            "    return f'https://www.reddit.com/r/{sub}/'\n"
+            "def display(post):\n"
+            "    post['url'] = REDDIT_URL + 'r/x'\n"
+            "    return urllib.request.urlopen('https://arctic-shift.photon-reddit.com/api')\n"
+            "def to_hub(url):\n"
+            "    payload = {'url': 'https://www.reddit.com/r/x/', 'target': REDDIT_URL}\n"
+            "    return subprocess.run(['social-hub', 'scrape', '--json', str(payload)])\n"
+        )
+        requesting, offending = functions_requesting_reddit(good)
+        # 检查器确实扫到了这些请求函数，不是因为扫描为空才没有误报。
+        self.assertTrue({"fetch", "lookalike", "display"} <= set(requesting))
+        self.assertEqual(offending, [])
+
+    def test_requirements_have_no_legacy_packages(self) -> None:
+        req = (SCRIPT.parents[1] / "requirements.txt").read_text(encoding="utf-8")
+        lines = [ln.strip() for ln in req.splitlines() if ln.strip() and not ln.strip().startswith("#")]
+        self.assertEqual(lines, [])
+
+    def test_via_curl_is_deprecated_noop_on_all_four_subcommands(self) -> None:
+        cmds = {
+            "keyword": ["keyword", "battery"],
+            "subreddit": ["subreddit", "battery", "-r", "example"],
+            "stickies": ["stickies", "-r", "example"],
+            "links": ["links", POST_URL],
+        }
+        parser_help = io.StringIO()
+        for mode, argv in cmds.items():
+            with self.subTest(mode=mode):
+                stderr = io.StringIO()
+                with (
+                    mock.patch.object(sys, "argv", [str(SCRIPT), *argv, "--via-curl", "--help"]),
+                    contextlib.redirect_stderr(stderr),
+                    contextlib.redirect_stdout(parser_help),
+                    self.assertRaises(SystemExit) as ctx,
+                ):
+                    voc.main()
+                self.assertEqual(ctx.exception.code, 0)  # --help 正常退出，说明参数已注册
+        self.assertIn("DEPRECATED", parser_help.getvalue())
+
+    def test_links_with_via_curl_warns_and_keeps_output_shape(self) -> None:
+        def run(extra: list[str]) -> tuple[int, dict, str]:
+            stdout, stderr = io.StringIO(), io.StringIO()
+            with (
+                mock.patch.object(voc.urllib.request, "urlopen", side_effect=AssertionError("direct network call")),
+                mock.patch.object(voc, "resolve_proxy_url", return_value=""),
+                mock.patch.object(voc, "arctic_post_by_permalink", return_value=None),
+                mock.patch.object(voc, "fetch_page_scrape", return_value={"markdown": "Public page"}),
+                mock.patch.object(voc, "collect_subreddit_metrics", return_value=[]),
+                mock.patch.object(voc, "hub_writeback_items", return_value={"status": "pending"}),
+                mock.patch.object(voc, "sleep_with_jitter", return_value=0.0),
+                mock.patch.object(sys, "argv", [str(SCRIPT), "links", POST_URL, "--fallback", "auto", "--sleep-sec", "0", *extra]),
+                contextlib.redirect_stdout(stdout),
+                contextlib.redirect_stderr(stderr),
+            ):
+                code = voc.main()
+            return code, json.loads(stdout.getvalue()), stderr.getvalue()
+
+        code_a, out_a, err_a = run([])
+        code_b, out_b, err_b = run(["--via-curl"])
+        self.assertEqual((code_a, code_b), (0, 0))
+        self.assertNotIn("DEPRECATED", err_a)  # 对照：不传则无告警
+        self.assertIn("DEPRECATED: --via-curl 已无效果(v2.12.28 起不直连 reddit)", err_b)
+        self.assertEqual(set(out_a), set(out_b))
+        self.assertEqual(out_a["request_identity"], out_b["request_identity"])
+
+
+class RequestIdentityContractTests(unittest.TestCase):
+    def build(self, proxy: str, mode: str | None = None) -> dict:
+        tracker = voc.RequestTracker()
+        if mode is not None:
+            tracker.last_fetch_mode = mode
+        with mock.patch.object(voc, "resolve_proxy_url", return_value=proxy):
+            return voc.build_output(
+                ok=True,
+                mode="keyword",
+                query="q",
+                source_urls=[],
+                items=[],
+                subreddit_metrics=[],
+                requested_limit=10,
+                effective_limit=10,
+                limit_truncated=False,
+                sleep_sec=6.0,
+                max_pages=1,
+                tracker=tracker,
+                notes=[],
+                errors=[],
+            )
+
+    def test_request_identity_shape_and_fixed_values(self) -> None:
+        identity = self.build("")["request_identity"]
+        self.assertEqual(set(identity), IDENTITY_KEYS)
+        self.assertEqual(
+            identity,
+            {
+                "user_agent_kind": "chrome",
+                "fetch_mode": "none",
+                "used_cookies": False,
+                "cookie_source": None,
+                "cookie_cache_reused": False,
+                "impersonation": None,
+                "proxy_used": False,
+            },
+        )
+
+    def test_proxy_used_follows_resolved_proxy(self) -> None:
+        # 对照组：proxy_used 是唯一随环境变化的键，防止断言被写死成常量。
+        self.assertIs(self.build("http://127.0.0.1:7890")["request_identity"]["proxy_used"], True)
+        self.assertIs(self.build("")["request_identity"]["proxy_used"], False)
+
+    def test_fetch_mode_reports_tracker_mode_but_never_curl_identity(self) -> None:
+        identity = self.build("", mode="browser_fallback")["request_identity"]
+        self.assertEqual(identity["fetch_mode"], "browser_fallback")
+        self.assertIsNone(identity["impersonation"])
+        self.assertIs(identity["used_cookies"], False)
+
+    def test_links_run_emits_same_identity_without_network(self) -> None:
+        stdout = io.StringIO()
+        with (
+            mock.patch.object(voc.urllib.request, "urlopen", side_effect=AssertionError("direct network call")) as urlopen,
+            mock.patch.object(voc, "resolve_proxy_url", return_value=""),
+            mock.patch.object(voc, "arctic_post_by_permalink", return_value=None),
+            mock.patch.object(voc, "fetch_page_scrape", return_value={"markdown": "Public page"}),
+            mock.patch.object(voc, "collect_subreddit_metrics", return_value=[]),
+            mock.patch.object(voc, "hub_writeback_items", return_value={"status": "pending"}),
+            mock.patch.object(voc, "sleep_with_jitter", return_value=0.0),
+            mock.patch.object(sys, "argv", [str(SCRIPT), "links", POST_URL, "--fallback", "auto", "--sleep-sec", "0"]),
+            contextlib.redirect_stdout(stdout),
+        ):
+            code = voc.main()
+        out = json.loads(stdout.getvalue())
+        self.assertEqual(code, 0)
+        self.assertEqual(set(out["request_identity"]), IDENTITY_KEYS)
+        self.assertEqual(out["request_identity"]["fetch_mode"], "none")
+        self.assertIs(out["request_identity"]["used_cookies"], False)
+        urlopen.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()