@heretek-ai/epistemic-swarm 0.2.2 → 0.3.0

package/runner/tests/test_domain_packs.py

@@ -0,0 +1,217 @@
+#!/usr/bin/env python3
+"""Regulated Domain Packs probe test (Stream G falsification criterion).
+
+[HYPOTHESIS: a biopharma constitution flips accept/reject on >= 25% of a
+20-claim borderline fixture set]
+
+This test IS the probe: the 20-claim borderline fixture is judged under the
+legacy default constitution and under the biopharma pack; >= 25% of per-claim
+verdicts must flip. The zero-tolerance retraction policy must downgrade
+claims with retracted sources regardless of score.
+"""
+
+import json
+import sys
+import unittest
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from runner.claim_witness import normalize_claim  # noqa: E402
+from runner.refinement import (  # noqa: E402
+    LEGACY_CONSTITUTION,
+    claim_verdict,
+    compute_epistemic_score_from_claims,
+    load_domain_pack,
+)
+
+FIXTURE = PROJECT_ROOT / "runner" / "tests" / "fixtures" / "borderline_claims.json"
+N_CLAIMS = 20
+FLIP_BAR = 0.25
+
+
+def load_fixture():
+    data = json.loads(FIXTURE.read_text(encoding="utf-8"))
+    claims = [normalize_claim(c) for c in data["claims"]]
+    return claims, set(data["retracted_hashes"])
+
+
+class TestDomainPacksProbe(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.claims, cls.retracted = load_fixture()
+        cls.biopharma = load_domain_pack("biopharma")
+
+    def test_fixture_shape(self):
+        self.assertEqual(len(self.claims), N_CLAIMS)
+
+    def test_load_all_three_packs(self):
+        for pack_id in ("biopharma", "quant", "legal"):
+            c = load_domain_pack(pack_id)
+            self.assertIsNotNone(c.tier_weights)
+            self.assertIn("PREPRINT", c.tier_weights)
+        with self.assertRaises(FileNotFoundError):
+            load_domain_pack("no-such-pack")
+
+    def test_probe_biopharma_flips_at_least_quarter(self):
+        """THE probe: >= 25% of 20 borderline verdicts flip under biopharma."""
+        default_results = [
+            claim_verdict(c, constitution=LEGACY_CONSTITUTION, retracted_hashes=self.retracted)
+            for c in self.claims
+        ]
+        bio_results = [
+            claim_verdict(c, constitution=self.biopharma, retracted_hashes=self.retracted)
+            for c in self.claims
+        ]
+
+        flips = [
+            (d["claim_id"], d["verdict"], b["verdict"])
+            for d, b in zip(default_results, bio_results)
+            if d["verdict"] != b["verdict"]
+        ]
+        flip_rate = len(flips) / N_CLAIMS
+        self.assertGreaterEqual(
+            flip_rate,
+            FLIP_BAR,
+            f"flip rate {flip_rate:.0%} ({len(flips)}/{N_CLAIMS}) below {FLIP_BAR:.0%}: {flips}",
+        )
+
+        # Default constitution accepts the whole borderline set...
+        self.assertTrue(all(r["verdict"] == "ACCEPTED" for r in default_results),
+                        [r for r in default_results if r["verdict"] != "ACCEPTED"])
+        # ...and every flip is default-ACCEPTED -> biopharma-REJECTED.
+        self.assertTrue(all(d == "ACCEPTED" and b == "REJECTED" for _c, d, b in flips))
+
+    def test_banned_domain_mechanism(self):
+        banned = [c for c in self.claims if "mirror.example" in (c.source_url or "") or "predatory-journal" in (c.source_url or "")]
+        self.assertEqual(len(banned), 6, "fixture expects 6 banned-domain claims")
+        for c in banned:
+            r = claim_verdict(c, constitution=self.biopharma)
+            self.assertEqual(r["verdict"], "REJECTED", c.claim_id)
+            self.assertTrue(any("BANNED_DOMAIN" in x for x in r["reasons"]), r["reasons"])
+
+    def test_missing_tag_mechanism(self):
+        untagged = [c for c in self.claims if not c.tag]
+        self.assertEqual(len(untagged), 2)
+        for c in untagged:
+            r = claim_verdict(c, constitution=self.biopharma)
+            self.assertEqual(r["verdict"], "REJECTED", c.claim_id)
+            self.assertTrue(any("MISSING_TAG" in x for x in r["reasons"]), r["reasons"])
+
+    def test_zero_tolerance_retraction_mechanism(self):
+        """RETRACTED source downgrades regardless of quote match or score."""
+        retracted_claims = [c for c in self.claims if c.source_hash in self.retracted]
+        self.assertEqual(len(retracted_claims), 2)
+        for c in retracted_claims:
+            # Even without passing retracted_hashes: status-based downgrade.
+            c.status = "STALE"
+            r = claim_verdict(c, constitution=self.biopharma)
+            self.assertEqual(r["verdict"], "REJECTED", c.claim_id)
+            self.assertTrue(
+                any("ZERO_TOLERANCE_RETRACTION" in x for x in r["reasons"]), r["reasons"]
+            )
+            # Standard policy does NOT apply this rule.
+            r_std = claim_verdict(c, constitution=LEGACY_CONSTITUTION)
+            self.assertEqual(r_std["verdict"], "ACCEPTED")
+
+    def test_tier_weights_move_the_score(self):
+        """Preprint downweighting lowers E(D); threshold also comes from the pack."""
+        s_default, _ = compute_epistemic_score_from_claims(self.claims, constitution=LEGACY_CONSTITUTION)
+        s_bio, br = compute_epistemic_score_from_claims(self.claims, constitution=self.biopharma)
+        self.assertLess(s_bio, s_default, (s_default, s_bio))
+        # 18 of 20 claims carry the VERIFIED tag (2 are deliberately untagged);
+        # each scores 0.3 under the biopharma preprint weight.
+        n_verified = sum(1 for c in self.claims if c.tag == "VERIFIED")
+        self.assertEqual(n_verified, 18)
+        self.assertAlmostEqual(br["verified_weight"], n_verified * 0.3, places=3)
+
+    def test_legacy_default_preserves_existing_behavior(self):
+        """No pack => flat weights, no domain/tag rules, standard retraction."""
+        self.assertEqual(LEGACY_CONSTITUTION.weight_for("PREPRINT"), 1.0)
+        self.assertEqual(LEGACY_CONSTITUTION.banned_domains, [])
+        self.assertEqual(LEGACY_CONSTITUTION.mandatory_tags, [])
+        self.assertEqual(LEGACY_CONSTITUTION.retraction_policy, "standard")
+        self.assertEqual(LEGACY_CONSTITUTION.accept_threshold, 0.65)
+
+
+class TestAuditConstitutionWiring(unittest.TestCase):
+    def test_audit_scope_default_unchanged(self):
+        """audit_scope() with no constitution must match legacy numbers exactly."""
+        import tempfile
+        from runner.auditor_engine import EpistemicAuditorEngine
+        from runner.state_machine import ResearchStateMachine
+        from skills.research_cache.hasher import SourceHasher
+
+        with tempfile.TemporaryDirectory() as tmp:
+            base = Path(tmp)
+            hasher = SourceHasher(base)
+            sm = ResearchStateMachine(base_dir=base)
+            shash = hasher.store_source(
+                "https://x", "FPGA prover executes Poseidon in 184ms.", "T"
+            )
+            sm.init_session("t")
+            sm.set_scopes([{"scope_id": "s1", "dependencies": []}])
+            sm.record_agent_completion("s1", "alpha", {
+                "affirmative_claims": [{
+                    "claim_id": "A1", "tag": "VERIFIED", "statement": "s",
+                    "source_hash": shash, "verbatim_quote": "executes Poseidon in 184ms",
+                }],
+                "inferred_implications": [],
+                "negative_knowledge": [],
+            })
+            sm.record_agent_completion("s1", "beta", {
+                "falsification_claims": [],
+                "methodological_critiques": [],
+                "negative_knowledge": [],
+            })
+
+            engine = EpistemicAuditorEngine(base_dir=base)
+            report = engine.audit_scope("s1")
+            self.assertEqual(report["summary"]["verified_passed"], 1)
+            self.assertEqual(report["summary"]["verdict"], "CERTIFIED")
+
+    def test_audit_scope_with_biopharma_uses_threshold(self):
+        import tempfile
+        from runner.auditor_engine import EpistemicAuditorEngine
+        from runner.refinement import load_domain_pack
+        from runner.state_machine import ResearchStateMachine
+        from skills.research_cache.hasher import SourceHasher
+
+        with tempfile.TemporaryDirectory() as tmp:
+            base = Path(tmp)
+            hasher = SourceHasher(base)
+            sm = ResearchStateMachine(base_dir=base)
+            shash = hasher.store_source(
+                "https://predatory-journal.example/x",
+                "Weak preprint finding about surrogate endpoints.",
+                "T", tier="PREPRINT",
+            )
+            sm.init_session("t")
+            sm.set_scopes([{"scope_id": "s1", "dependencies": []}])
+            sm.record_agent_completion("s1", "alpha", {
+                "affirmative_claims": [{
+                    "claim_id": "A1", "tag": "VERIFIED", "statement": "s",
+                    "source_hash": shash,
+                    "source_url": "https://predatory-journal.example/x",
+                    "verbatim_quote": "surrogate endpoints",
+                }],
+                "inferred_implications": [],
+                "negative_knowledge": [],
+            })
+            sm.record_agent_completion("s1", "beta", {
+                "falsification_claims": [],
+                "methodological_critiques": [],
+                "negative_knowledge": [],
+            })
+
+            engine = EpistemicAuditorEngine(base_dir=base)
+            report = engine.audit_scope("s1", constitution=load_domain_pack("biopharma"))
+            # Banned domain claim => REJECTED under biopharma.
+            self.assertEqual(report["summary"]["unverified_rejected"], 1)
+            self.assertEqual(report["summary"]["verified_passed"], 0)
+
+
+if __name__ == "__main__":
+    unittest.main()

package/runner/tests/test_fleet_seam.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""Tests for the Stream E model-diverse adversary fleet seam.
+
+The seam is config-first (per-agent backend/model), CLI-overridable, with env
+fallback. Mock mode must NEVER build or spawn these commands.
+"""
+
+import os
+import sys
+import unittest
+from pathlib import Path
+from unittest import mock
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from runner.research_swarm import SwarmRunner  # noqa: E402
+
+
+class TestFleetSeam(unittest.TestCase):
+    def test_default_backend_is_claude_with_no_model(self):
+        runner = SwarmRunner(mock_mode=True, base_dir=Path("/tmp/no-such-research-dir"))
+        cmd = runner.build_agent_cmd("prompt", role="alpha")
+        self.assertEqual(cmd[0:2], ["claude", "-p"])
+        self.assertNotIn("--model", cmd)
+        self.assertIn("prompt", cmd)
+
+    def test_model_config_adds_model_flag(self):
+        runner = SwarmRunner(
+            mock_mode=True,
+            base_dir=Path("/tmp/no-such-research-dir"),
+            agent_overrides={"alpha": {"model": "claude-opus-5"}},
+        )
+        cmd = runner.build_agent_cmd("prompt", role="alpha")
+        self.assertIn("--model", cmd)
+        self.assertEqual(cmd[cmd.index("--model") + 1], "claude-opus-5")
+
+    def test_beta_backend_command_list(self):
+        """Cross-FAMILY means non-Claude processes; backend is a cmd list."""
+        runner = SwarmRunner(
+            mock_mode=True,
+            base_dir=Path("/tmp/no-such-research-dir"),
+            agent_overrides={"beta": {"backend": ["ollama", "run", "qwen3"], "model": None}},
+        )
+        cmd = runner.build_agent_cmd("prompt", role="beta")
+        self.assertEqual(cmd[:3], ["ollama", "run", "qwen3"])
+        self.assertNotIn("--model", cmd)
+
+    def test_roles_resolve_independently(self):
+        """Asymmetry is the point: alpha and beta may use different families."""
+        runner = SwarmRunner(
+            mock_mode=True,
+            base_dir=Path("/tmp/no-such-research-dir"),
+            agent_overrides={
+                "alpha": {"model": "claude-opus-5"},
+                "beta": {"backend": ["codex", "exec"], "model": "gpt-5"},
+            },
+        )
+        cmd_a = runner.build_agent_cmd("p", role="alpha")
+        cmd_b = runner.build_agent_cmd("p", role="beta")
+        self.assertEqual(cmd_a[0:2], ["claude", "-p"])
+        self.assertIn("--model", cmd_a)
+        self.assertEqual(cmd_b[:2], ["codex", "exec"])
+        self.assertIn("--model", cmd_b)
+        self.assertEqual(cmd_b[cmd_b.index("--model") + 1], "gpt-5")
+
+    def test_env_fallback(self):
+        with mock.patch.dict(
+            os.environ,
+            {"IUMBTEMS_MODEL_ALPHA": "env-model", "IUMBTEMS_BACKEND_BETA": "gemini -p"},
+            clear=False,
+        ):
+            runner = SwarmRunner(mock_mode=True, base_dir=Path("/tmp/no-such-research-dir"))
+            cmd_a = runner.build_agent_cmd("p", role="alpha")
+            cmd_b = runner.build_agent_cmd("p", role="beta")
+            self.assertIn("--model", cmd_a)
+            self.assertEqual(cmd_a[cmd_a.index("--model") + 1], "env-model")
+            self.assertEqual(cmd_b[:2], ["gemini", "-p"])
+
+    def test_cli_overrides_beat_config(self):
+        with mock.patch.dict(os.environ, {"IUMBTEMS_MODEL_ALPHA": "env-model"}, clear=False):
+            runner = SwarmRunner(
+                mock_mode=True,
+                base_dir=Path("/tmp/no-such-research-dir"),
+                agent_overrides={"alpha": {"model": "cli-model"}},
+            )
+            cmd = runner.build_agent_cmd("p", role="alpha")
+            self.assertEqual(cmd[cmd.index("--model") + 1], "cli-model")
+
+    def test_mock_mode_never_spawns(self):
+        """mock_mode short-circuits before cmd construction."""
+        runner = SwarmRunner(
+            mock_mode=True,
+            base_dir=Path("/tmp/no-such-research-dir"),
+            agent_overrides={"alpha": {"model": "should-not-be-used"}},
+        )
+        with mock.patch("subprocess.run") as spawn:
+            out = runner.run_claude_process("Orchestrator: build manifest.json")
+            spawn.assert_not_called()
+            # Canned orchestrator response is scope-decomposition JSON.
+            self.assertIn("scopes", out)
+
+    def test_system_prompt_appended_when_present(self):
+        runner = SwarmRunner(mock_mode=True, base_dir=Path("/tmp/no-such-research-dir"))
+        existing = PROJECT_ROOT / "prompts" / "agent_alpha_thesis.md"
+        cmd = runner.build_agent_cmd("p", system_prompt_file=existing, role="alpha")
+        self.assertIn("--system-prompt", cmd)
+        self.assertEqual(cmd[cmd.index("--system-prompt") + 1], str(existing))
+
+
+if __name__ == "__main__":
+    unittest.main()

package/runner/tests/test_living_dossiers.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+"""Living Dossiers probe test (Stream C falsification criterion).
+
+[HYPOTHESIS: injecting a simulated retraction into 5 completed dossiers
+marks >= 4 dependent claim sets STALE within one auditor pass]
+
+This test IS the probe: 5 scope dossiers cite shared source hashes; a
+simulated retraction event is injected; exactly one degradation pass runs;
+>= 4 claim sets must end STALE (or SUSPECT for REVISED events).
+"""
+
+import json
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from runner.claim_witness import (  # noqa: E402
+    STATUS_LIVE,
+    STATUS_STALE,
+    STATUS_SUSPECT,
+)
+from runner.living_dossiers import (  # noqa: E402
+    EVENT_RETRACTED,
+    EVENT_REVISED,
+    RetractionEvent,
+    apply_degradation,
+    check_staleness,
+    load_requeue,
+    load_retractions,
+    write_retraction,
+)
+
+
+def build_fixture(base: Path, n_scopes: int = 5) -> dict:
+    """5 completed scopes; scopes 0-3 cite SHARED_A, scope 4 cites SHARED_B."""
+    hashes = {}
+    sources = base / "sources"
+    sources.mkdir(parents=True, exist_ok=True)
+    for name, content in (
+        ("SHARED_A", "Poseidon prover executes the round constraints in 184ms."),
+        ("SHARED_B", "Independent replication confirmed 150k ops/sec throughput."),
+    ):
+        (sources / f"{name}.json").write_text(
+            json.dumps({"hash": name, "url": f"https://x/{name}", "title": name})
+        )
+        (sources / f"{name}.md").write_text(content)
+        hashes[name] = content
+
+    scratch = base / "scratchpads"
+    for i in range(n_scopes):
+        scope = scratch / f"scope_{i:02d}"
+        scope.mkdir(parents=True, exist_ok=True)
+        src = "SHARED_A" if i < 4 else "SHARED_B"
+        quote = "184ms" if src == "SHARED_A" else "150k ops/sec"
+        (scope / "alpha_dossier.json").write_text(
+            json.dumps(
+                {
+                    "agent": "Agent Alpha",
+                    "scope_id": scope.name,
+                    "affirmative_claims": [
+                        {
+                            "claim_id": f"A-{i:02d}",
+                            "tag": "VERIFIED",
+                            "statement": f"scope {i} affirmative",
+                            "source_hash": src,
+                            "verbatim_quote": quote,
+                        }
+                    ],
+                    "inferred_implications": [],
+                    "negative_knowledge": [],
+                }
+            )
+        )
+        (scope / "beta_dossier.json").write_text(
+            json.dumps(
+                {
+                    "agent": "Agent Beta",
+                    "scope_id": scope.name,
+                    "falsification_claims": [
+                        {
+                            "claim_id": f"B-{i:02d}",
+                            "tag": "VERIFIED",
+                            "statement": f"scope {i} falsification",
+                            "source_hash": src,
+                            "verbatim_quote": quote,
+                            "severity": "LOW",
+                        }
+                    ],
+                    "methodological_critiques": [],
+                    "negative_knowledge": [],
+                }
+            )
+        )
+    return hashes
+
+
+class TestLivingDossiersProbe(unittest.TestCase):
+    def test_probe_retraction_marks_four_claim_sets_stale(self):
+        """THE probe: 5 dossiers, simulated retraction, >=4 claim sets STALE."""
+        with tempfile.TemporaryDirectory() as tmp:
+            base = Path(tmp)
+            build_fixture(base, n_scopes=5)
+
+            # Simulate retraction of SHARED_A (cited by scopes 0-3 = 4 sets).
+            write_retraction(
+                base,
+                "SHARED_A",
+                EVENT_RETRACTED,
+                note="simulated retraction: paper withdrawn by authors",
+            )
+
+            # Exactly one auditor/degradation pass.
+            summary = check_staleness(base)
+
+            degraded = set(summary["degraded_scopes"])
+            # >= 4 claim sets (scopes) must degrade to STALE.
+            self.assertGreaterEqual(
+                len(degraded), 4, f"expected >=4 degraded scopes, got {degraded}"
+            )
+            self.assertEqual(degraded, {"scope_00", "scope_01", "scope_02", "scope_03"})
+
+            # Scope 04 (SHARED_B) must remain LIVE.
+            self.assertNotIn("scope_04", degraded)
+
+            # Ledger is append-only and records the transitions.
+            ledger_path = base / "ledger" / "claim_status.json"
+            self.assertTrue(ledger_path.exists())
+            ledger = json.loads(ledger_path.read_text())
+            self.assertGreaterEqual(len(ledger), 8)  # 4 scopes x (alpha+beta claims)
+            self.assertTrue(all(e["to_status"] == STATUS_STALE for e in ledger))
+
+            # Requeue sidecar lists exactly the degraded scopes.
+            requeue = load_requeue(base)
+            self.assertEqual(
+                sorted(e["scope_id"] for e in requeue),
+                ["scope_00", "scope_01", "scope_02", "scope_03"],
+            )
+
+            # Dossiers themselves are NEVER mutated (historical record).
+            d = json.loads((base / "scratchpads" / "scope_00" / "alpha_dossier.json").read_text())
+            self.assertNotIn("status", d["affirmative_claims"][0])
+            self.assertEqual(d["affirmative_claims"][0]["claim_id"], "A-00")
+
+    def test_revised_marks_suspect_not_stale(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            base = Path(tmp)
+            build_fixture(base, n_scopes=2)
+            write_retraction(base, "SHARED_A", EVENT_REVISED, note="benchmark table updated")
+            summary = check_staleness(base)
+            self.assertIn("scope_00", summary["degraded_scopes"])
+
+            # apply_degradation unit-level: SUSPECT for REVISED
+            from runner.claim_witness import normalize_claim
+
+            claim = normalize_claim(
+                {"claim_id": "C1", "tag": "VERIFIED", "source_hash": "SHARED_A", "verbatim_quote": "184ms"}
+            )
+            retr = load_retractions(base)
+            degraded, events = apply_degradation([claim], retr, scope_id="s")
+            self.assertEqual(degraded[0].status, STATUS_SUSPECT)
+            self.assertEqual(events[0].to_status, STATUS_SUSPECT)
+
+    def test_no_retractions_no_degradation(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            base = Path(tmp)
+            build_fixture(base, n_scopes=5)
+            summary = check_staleness(base)
+            self.assertEqual(summary["retractions"], 0)
+            self.assertEqual(summary["degraded_scopes"], [])
+            self.assertEqual(summary["status_events"], 0)
+
+    def test_retraction_event_validation(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            base = Path(tmp)
+            with self.assertRaises(ValueError):
+                write_retraction(base, "H", "EXPLODED", note="bad event")
+
+    def test_apply_degradation_idempotent(self):
+        """Second pass over already-STALE claims produces no new events."""
+        from runner.claim_witness import normalize_claim
+
+        retr = {
+            "SHARED_A": RetractionEvent(
+                hash="SHARED_A", event=EVENT_RETRACTED, at="t", note="n"
+            )
+        }
+        claim = normalize_claim(
+            {"claim_id": "C1", "tag": "VERIFIED", "source_hash": "SHARED_A", "verbatim_quote": "q"}
+        )
+        d1, ev1 = apply_degradation([claim], retr, scope_id="s")
+        self.assertEqual(len(ev1), 1)
+        self.assertEqual(d1[0].status, STATUS_STALE)
+        d2, ev2 = apply_degradation(d1, retr, scope_id="s")
+        self.assertEqual(len(ev2), 0, "already-degraded claims produce no new events")
+        self.assertEqual(d2[0].status, STATUS_STALE)
+
+
+if __name__ == "__main__":
+    unittest.main()