@heretek-ai/epistemic-swarm 0.2.2 → 0.3.0

package/runner/tests/test_mcp_server.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+"""Tests for the first-party MCP server (runner/mcp_server.py) and the shared
+protocol loop (runner/mcp_protocol.py)."""
+
+import json
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from runner.mcp_protocol import StdioJsonRpcServer, ToolSpec, read_message, write_message  # noqa: E402
+from runner.mcp_server import SERVER_NAME, build_server, build_tools  # noqa: E402
+
+EXPECTED_TOOLS = [
+    "iumbtems_config",
+    "iumbtems_swarm_research",
+    "iumbtems_code_audit",
+    "iumbtems_oss_scout",
+    "iumbtems_brainstorm",
+    "iumbtems_verify_quote",
+    "iumbtems_socratic_frontier",
+    "iumbtems_reindex_claims",
+    "iumbtems_report_retraction",
+    "iumbtems_check_staleness",
+    "iumbtems_set_domain_pack",
+    "iumbtems_export_brief",
+    "iumbtems_verify_brief",
+]
+
+
+class TestMcpProtocol(unittest.TestCase):
+    def test_tool_spec_manifest_shape(self):
+        spec = ToolSpec("t", "d", {"type": "object"}, lambda a: "x")
+        self.assertEqual(spec.to_manifest(), {"name": "t", "description": "d", "inputSchema": {"type": "object"}})
+
+    def test_alias_dispatch(self):
+        calls = []
+        spec = ToolSpec("main", "d", {}, lambda a: calls.append(a) or "ok", aliases=("alt",))
+        server = StdioJsonRpcServer("s", "1", [spec])
+        res = server.handle({"jsonrpc": "2.0", "id": 1, "method": "tools/call",
+                             "params": {"name": "alt", "arguments": {"q": 1}}})
+        self.assertEqual(res["result"]["content"][0]["text"], "ok")
+        self.assertEqual(calls, [{"q": 1}])
+
+    def test_handler_exception_becomes_error_not_crash(self):
+        def boom(_):
+            raise RuntimeError("kaboom")
+
+        server = StdioJsonRpcServer("s", "1", [ToolSpec("t", "d", {}, boom)])
+        res = server.handle({"jsonrpc": "2.0", "id": 2, "method": "tools/call",
+                             "params": {"name": "t", "arguments": {}}})
+        self.assertIn("error", res)
+        self.assertEqual(res["error"]["code"], -32603)
+        self.assertIn("kaboom", res["error"]["message"])
+
+    def test_notification_gets_no_reply(self):
+        server = StdioJsonRpcServer("s", "1", [])
+        self.assertIsNone(server.handle({"jsonrpc": "2.0", "method": "notifications/initialized"}))
+
+    def test_unknown_method_is_32601(self):
+        server = StdioJsonRpcServer("s", "1", [])
+        res = server.handle({"jsonrpc": "2.0", "id": 3, "method": "bogus/x"})
+        self.assertEqual(res["error"]["code"], -32601)
+
+    def test_content_length_round_trip(self):
+        import io
+
+        body = json.dumps({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}})
+        raw = f"Content-Length: {len(body)}\r\n\r\n{body}"
+        req, use_headers = read_message(io.StringIO(raw))
+        self.assertTrue(use_headers)
+        self.assertEqual(req["method"], "initialize")
+
+        out = io.StringIO()
+        write_message(out, {"jsonrpc": "2.0", "id": 1, "result": {}}, use_headers=True)
+        self.assertTrue(out.getvalue().startswith("Content-Length:"))
+
+
+class TestMcpServerTools(unittest.TestCase):
+    def test_canonical_tool_names(self):
+        names = [t.name for t in build_tools()]
+        self.assertEqual(sorted(names), sorted(EXPECTED_TOOLS))
+
+    def test_server_meta(self):
+        server = build_server()
+        self.assertEqual(server.server_name, SERVER_NAME)
+        self.assertEqual(server.protocol_version, "2024-11-05")
+        self.assertEqual(sorted(t.name for t in server.tools), sorted(EXPECTED_TOOLS))
+
+    def test_tools_list_via_stdio_handshake(self):
+        """Subprocess handshake: initialize -> tools/list -> tools/call."""
+        with tempfile.TemporaryDirectory() as tmp:
+            tmp_path = Path(tmp)
+            # Seed a source so verify_quote has something real to check.
+            sys.path.insert(0, str(PROJECT_ROOT))
+            from skills.research_cache.hasher import SourceHasher
+
+            hasher = SourceHasher(tmp_path)
+            content = "The FPGA pipeline executes Poseidon in 184ms."
+            digest = hasher.store_source(url="https://example.test/p", content=content, title="T")
+
+            # Drive handle() directly in-process for tools/call; use the
+            # subprocess only to prove the stdio loop serves initialize/tools/list.
+            proc = subprocess.Popen(
+                [sys.executable, str(PROJECT_ROOT / "runner" / "mcp_server.py")],
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+            )
+            lines = [
+                json.dumps({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}),
+                json.dumps({"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}),
+            ]
+            out, err = proc.communicate("\n".join(lines) + "\n", timeout=20)
+            self.assertEqual(err.strip(), "", f"server stderr: {err}")
+            responses = [json.loads(l) for l in out.strip().splitlines() if l.strip()]
+            self.assertEqual(len(responses), 2)
+            init, tools_list = responses
+            self.assertEqual(init["result"]["protocolVersion"], "2024-11-05")
+            self.assertEqual(init["result"]["serverInfo"]["name"], "iumbtems")
+            self.assertEqual(
+                sorted(t["name"] for t in tools_list["result"]["tools"]),
+                sorted(EXPECTED_TOOLS),
+            )
+
+            # Round-trip verify_quote in-process against the temp cache.
+            server = build_server()
+            res = server.handle({
+                "jsonrpc": "2.0",
+                "id": 3,
+                "method": "tools/call",
+                "params": {
+                    "name": "iumbtems_verify_quote",
+                    "arguments": {
+                        "hash": digest,
+                        "quote": "Poseidon in 184ms",
+                        "base_dir": tmp,
+                    },
+                },
+            })
+            payload = json.loads(res["result"]["content"][0]["text"])
+            self.assertTrue(payload["verified"])
+            self.assertGreaterEqual(payload["confidence"], 0.95)
+
+            # And a failing quote must come back unverified.
+            res_bad = server.handle({
+                "jsonrpc": "2.0",
+                "id": 4,
+                "method": "tools/call",
+                "params": {
+                    "name": "iumbtems_verify_quote",
+                    "arguments": {
+                        "hash": digest,
+                        "quote": "this text was never cached anywhere",
+                        "base_dir": tmp,
+                    },
+                },
+            })
+            bad = json.loads(res_bad["result"]["content"][0]["text"])
+            self.assertFalse(bad["verified"])
+
+    def test_one_shot_call_mode(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            from skills.research_cache.hasher import SourceHasher
+
+            hasher = SourceHasher(Path(tmp))
+            content = "Epistemic sovereignty over parametric intuition."
+            digest = hasher.store_source(url="https://example.test/q", content=content)
+
+            proc = subprocess.run(
+                [
+                    sys.executable,
+                    str(PROJECT_ROOT / "runner" / "mcp_server.py"),
+                    "call",
+                    "iumbtems_verify_quote",
+                    json.dumps({"hash": digest, "quote": "parametric intuition", "base_dir": tmp}),
+                ],
+                capture_output=True,
+                text=True,
+                timeout=20,
+            )
+            self.assertEqual(proc.returncode, 0, proc.stderr)
+            payload = json.loads(proc.stdout)
+            self.assertTrue(payload["verified"])
+
+    def test_unknown_tool_reports_available(self):
+        proc = subprocess.run(
+            [
+                sys.executable,
+                str(PROJECT_ROOT / "runner" / "mcp_server.py"),
+                "call",
+                "nope_not_a_tool",
+                "{}",
+            ],
+            capture_output=True,
+            text=True,
+            timeout=20,
+        )
+        self.assertEqual(proc.returncode, 1)
+        payload = json.loads(proc.stdout)
+        self.assertIn("error", payload)
+        self.assertEqual(sorted(payload["available"]), sorted(EXPECTED_TOOLS))
+
+
+if __name__ == "__main__":
+    unittest.main()

package/runner/tests/test_pcrb.py

@@ -0,0 +1,240 @@
+#!/usr/bin/env python3
+"""PCRB probe test (Stream D / Gate G3 falsification criterion).
+
+[HYPOTHESIS: a standalone verifier over a 50-claim brief runs in <2s and
+flags 100% of seeded quote tampering]
+
+This test IS the probe: build a realistic 50-claim signed brief, time the
+verifier, then seed >= 20 tampering mutations across four attack classes
+(quote flips, source body edits, claim swaps, coordinated manifest rewrites)
+and assert every single one is flagged.
+"""
+
+import copy
+import json
+import sys
+import tempfile
+import time
+import unittest
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from runner.pcrb import export_brief, sha256_text, member_hash  # noqa: E402
+from runner.pcrb_verify import verify_brief  # noqa: E402
+
+TEST_KEY = "pcrb-probe-secret-key"
+N_CLAIMS = 50
+N_SOURCES = 5
+VERIFY_BUDGET_S = 2.0
+N_MUTATIONS = 20
+
+
+def build_workspace(base: Path) -> dict:
+    """A .research-shaped workspace: 5 sources x 10 claims, all quotes real."""
+    sources = base / "sources"
+    sources.mkdir(parents=True, exist_ok=True)
+    scratch = base / "scratchpads" / "scope_01"
+    scratch.mkdir(parents=True, exist_ok=True)
+
+    src_hashes = []
+    for s in range(N_SOURCES):
+        content = (
+            f"BENCHMARK REPORT {s}: The prover executes witness generation in "
+            f"{180 + s}ms under load, with peak bandwidth {40 + s} GB/s and "
+            f"validated reproducibility across {3 + s} independent runs. "
+            f"Section {s} documents the methodology and error bars in full."
+        )
+        h = sha256_text(content)
+        (sources / f"{h}.md").write_text(content, encoding="utf-8")
+        (sources / f"{h}.json").write_text(
+            json.dumps({"hash": h, "url": f"https://bench.example/{s}", "title": f"Bench {s}", "tier": "WEB_DOCUMENT"})
+        )
+        src_hashes.append(h)
+
+    alpha_claims, beta_claims = [], []
+    for i in range(N_CLAIMS):
+        s = i % N_SOURCES
+        content = (sources / f"{src_hashes[s]}.md").read_text(encoding="utf-8")
+        # Unique verbatim substring from that source.
+        quote = f"validated reproducibility across {3 + s} independent runs"
+        claim = {
+            "claim_id": f"C-{i:03d}",
+            "tag": "VERIFIED",
+            "statement": f"Claim {i}: prover behavior measured in benchmark {s}",
+            "source_hash": src_hashes[s],
+            "source_url": f"https://bench.example/{s}",
+            "verbatim_quote": quote,
+        }
+        if i % 2 == 0:
+            alpha_claims.append(claim)
+        else:
+            claim["severity"] = "LOW"
+            beta_claims.append(claim)
+
+    (scratch / "alpha_dossier.json").write_text(
+        json.dumps({"agent": "Agent Alpha", "scope_id": "scope_01", "affirmative_claims": alpha_claims, "inferred_implications": [], "negative_knowledge": []})
+    )
+    (scratch / "beta_dossier.json").write_text(
+        json.dumps({"agent": "Agent Beta", "scope_id": "scope_01", "falsification_claims": beta_claims, "methodological_critiques": [], "negative_knowledge": []})
+    )
+    (base / "final_synthesis.md").write_text("# Master Synthesis\n\nAll claims verified against benchmark sources.\n")
+    return {"src_hashes": src_hashes}
+
+
+class TestPcrbProbe(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls._tmp = tempfile.TemporaryDirectory(prefix="pcrb_probe_")
+        cls.base = Path(cls._tmp.name)
+        cls.fx = build_workspace(cls.base)
+        cls.bundle_path = export_brief(cls.base, key=TEST_KEY, objective="PCRB probe")
+        cls.bundle = json.loads(cls.bundle_path.read_text(encoding="utf-8"))
+
+    @classmethod
+    def tearDownClass(cls):
+        cls._tmp.cleanup()
+
+    def test_bundle_shape(self):
+        self.assertEqual(self.bundle["pcrb_version"], "1.0")
+        self.assertEqual(len(self.bundle["claims"]), N_CLAIMS)
+        self.assertEqual(len(self.bundle["sources"]), N_SOURCES)
+        self.assertEqual(self.bundle["signature"]["alg"], "hmac-sha256")
+        self.assertIn("manifest", self.bundle)
+
+    def test_probe_clean_brief_verifies_under_two_seconds(self):
+        """THE probe (half 1): 50-claim brief verifies in <2s."""
+        key = TEST_KEY.encode("utf-8")
+        started = time.perf_counter()
+        report = verify_brief(self.bundle, key=key)
+        elapsed = time.perf_counter() - started
+
+        self.assertTrue(report["ok"], report["failures"])
+        self.assertEqual(report["stats"]["quotes_verified"], N_CLAIMS)
+        self.assertEqual(report["stats"]["quotes_failed"], 0)
+        self.assertLess(elapsed, VERIFY_BUDGET_S, f"verify took {elapsed:.3f}s")
+        self.assertLess(report["stats"]["elapsed_s"], VERIFY_BUDGET_S)
+
+    def test_probe_twenty_tamper_mutations_all_flagged(self):
+        """THE probe (half 2): 100% of >=20 seeded mutations are caught."""
+        key = TEST_KEY.encode("utf-8")
+        mutations = self._seed_mutations()
+        self.assertGreaterEqual(len(mutations), N_MUTATIONS)
+
+        caught = []
+        missed = []
+        for name, tampered in mutations:
+            report = verify_brief(tampered, key=key)
+            if report["ok"]:
+                missed.append(name)
+            else:
+                caught.append((name, [f["check"] for f in report["failures"]]))
+
+        self.assertEqual(
+            missed, [],
+            f"MISSED tampering ({len(missed)}/{len(mutations)}): {missed}",
+        )
+        self.assertEqual(len(caught), len(mutations), "every mutation must be flagged")
+
+    def _seed_mutations(self):
+        """Four attack classes; each mutation is a fresh deep copy."""
+        out = []
+
+        # Class 1: quote character flips inside claim records (5x)
+        for i in range(5):
+            b = copy.deepcopy(self.bundle)
+            rec = b["claims"][i]
+            q = rec["verbatim_quote"]
+            rec["verbatim_quote"] = q.replace("reproducibility", "reproducib1lity", 1)
+            out.append((f"quote_flip_{i}", b))
+
+        # Class 2: source body edits (5x)
+        for i in range(5):
+            b = copy.deepcopy(self.bundle)
+            src = self.fx["src_hashes"][i % N_SOURCES]
+            b["sources"][src]["content"] = b["sources"][src]["content"] + "\nINJECTED FALSE PARAGRAPH."
+            out.append((f"source_body_edit_{i}", b))
+
+        # Class 3: claim swaps — replace a claim wholesale with a different one (5x)
+        for i in range(5):
+            b = copy.deepcopy(self.bundle)
+            donor = copy.deepcopy(b["claims"][(i + 7) % N_CLAIMS])
+            donor["claim_id"] = b["claims"][i]["claim_id"]  # same id, different content
+            b["claims"][i] = donor
+            out.append((f"claim_swap_{i}", b))
+
+        # Class 4: coordinated rewrites — attacker fixes the manifest hash too
+        # (only the HMAC signature can catch these) (3x claims + 2x sources)
+        for i in range(3):
+            b = copy.deepcopy(self.bundle)
+            rec = b["claims"][i + 10]
+            rec["statement"] = rec["statement"] + " (fabricated conclusion)"
+            rec["verbatim_quote"] = rec["verbatim_quote"].replace("independent", "independant", 1)
+            b["manifest"]["claims"][rec["claim_id"]] = member_hash(rec)  # attacker "fixes" the hash
+            out.append((f"coordinated_claim_rewrite_{i}", b))
+
+        for i in range(2):
+            b = copy.deepcopy(self.bundle)
+            src = self.fx["src_hashes"][i]
+            b["sources"][src]["content"] = b["sources"][src]["content"].replace("GB/s", "TB/s", 1)
+            b["manifest"]["sources"][src] = sha256_text(b["sources"][src]["content"])  # attacker "fixes" it
+            out.append((f"coordinated_source_rewrite_{i}", b))
+
+        return out
+
+    def test_unsigned_mode_still_catches_member_mismatches(self):
+        """alg=none: integrity catches manifest mismatches; no authenticity claim."""
+        unsigned = export_brief(self.base, out_path=self.base / "unsigned.pcrb.json", objective="unsigned probe")
+        bundle = json.loads(unsigned.read_text(encoding="utf-8"))
+        self.assertEqual(bundle["signature"]["alg"], "none")
+
+        report = verify_brief(bundle, key=None)
+        self.assertTrue(report["ok"], report["failures"])
+        self.assertEqual(report["checks"]["signature"], "unsigned")
+
+        # Member tamper still caught without a key...
+        tampered = copy.deepcopy(bundle)
+        tampered["claims"][0]["verbatim_quote"] = "totally fabricated quote text"
+        report2 = verify_brief(tampered, key=None)
+        self.assertFalse(report2["ok"])
+
+        # ...but a COORDINATED rewrite (content + manifest) is undetectable
+        # without a key. This is the honest limitation of alg=none.
+        coordinated = copy.deepcopy(bundle)
+        coordinated["claims"][0]["verbatim_quote"] = "totally fabricated quote text"
+        coordinated["manifest"]["claims"][coordinated["claims"][0]["claim_id"]] = member_hash(
+            coordinated["claims"][0]
+        )
+        report3 = verify_brief(coordinated, key=None)
+        self.assertTrue(
+            report3["ok"],
+            "alg=none cannot detect coordinated rewrites — documented limitation, not a bug",
+        )
+
+    def test_missing_key_is_reported_not_silently_passed(self):
+        report = verify_brief(self.bundle, key=None)
+        self.assertFalse(report["ok"])
+        self.assertEqual(report["checks"]["signature"], "unverifiable")
+
+    def test_wrong_key_fails(self):
+        report = verify_brief(self.bundle, key=b"wrong-key")
+        self.assertFalse(report["ok"])
+        self.assertEqual(report["checks"]["signature"], "invalid")
+
+    def test_cli_exit_codes(self):
+        import subprocess
+
+        proc = subprocess.run(
+            [sys.executable, str(PROJECT_ROOT / "runner" / "pcrb_verify.py"), str(self.bundle_path), "--json"],
+            capture_output=True, text=True, timeout=30,
+            env={**__import__("os").environ, "IUMBTEMS_PCRB_KEY": TEST_KEY},
+        )
+        self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
+        data = json.loads(proc.stdout)
+        self.assertTrue(data["ok"])
+
+
+if __name__ == "__main__":
+    unittest.main()