@heretek-ai/epistemic-swarm 0.7.11 → 0.7.13

package/runner/retrieval_log.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+"""Append-only retrieval telemetry for a research workspace.
+
+Agents invoke the skill scripts (`skills/epistemic_search/scripts/search.py`,
+`skills/research_cache/hasher.py`, `webcache.py`) as *subprocesses*, so a shared
+JSONL log is the only place their retrieval activity is visible to the runner.
+Without it a run that never searched looks identical to one whose cache writes
+failed — the exact confusion in Heretek-AI/IUMBTEMS#3.
+
+`.research/retrieval.jsonl`, one event per line::
+
+    {"kind": "query", "query": "...", "results": 5, "ts": "..."}
+    {"kind": "cache", "url": "...", "hash": "...", "ts": "..."}
+
+Telemetry is best-effort: it must never break the caller.
+"""
+
+import json
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+LOG_NAME = "retrieval.jsonl"
+RESEARCH_DIR_ENV = "IUMBTEMS_RESEARCH_DIR"
+
+
+def default_base_dir() -> Path:
+    """Workspace for a skill subprocess: env override, else cwd/.research."""
+    override = os.environ.get(RESEARCH_DIR_ENV, "").strip()
+    if override:
+        return Path(override)
+    return Path.cwd() / ".research"
+
+
+def log_path(base_dir) -> Path:
+    return Path(base_dir) / LOG_NAME
+
+
+def log_event(base_dir, kind: str, **fields: Any) -> None:
+    """Append one telemetry event. Never raises."""
+    try:
+        path = log_path(base_dir)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        event = {"kind": kind, "ts": datetime.now(timezone.utc).isoformat()}
+        event.update(fields)
+        with open(path, "a", encoding="utf-8") as f:
+            f.write(json.dumps(event) + "\n")
+    except OSError:
+        pass
+
+
+def current_offset(base_dir) -> int:
+    """Byte offset to pass as `since` for a per-run delta."""
+    try:
+        return log_path(base_dir).stat().st_size
+    except OSError:
+        return 0
+
+
+def summarize(base_dir, since: Optional[int] = None) -> Dict[str, int]:
+    """Count query/cache events, optionally only those after a byte offset."""
+    counts = {"queries": 0, "results": 0, "cached": 0, "events": 0}
+    path = log_path(base_dir)
+    if not path.exists():
+        return counts
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            if since:
+                f.seek(since)
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                counts["events"] += 1
+                try:
+                    event = json.loads(line)
+                except ValueError:
+                    continue
+                if event.get("kind") == "query":
+                    counts["queries"] += 1
+                    try:
+                        counts["results"] += int(event.get("results") or 0)
+                    except (TypeError, ValueError):
+                        pass
+                elif event.get("kind") == "cache":
+                    counts["cached"] += 1
+    except OSError:
+        pass
+    return counts

package/runner/state_machine.py

@@ -219,6 +219,41 @@ class ResearchStateMachine:
     def get_scope_dir(self, scope_id: str) -> Path:
         return self.scratchpads_dir / scope_id
 
+    def find_orphan_scopes(self) -> List[Dict[str, Any]]:
+        """Scope dirs not referenced by any manifest — marked, never deleted.
+
+        A dir may belong to a concurrently running session, so this reports
+        rather than prunes (issue #5 layer 6: 14 manifest-only leftovers).
+        """
+        referenced: set = set()
+        manifests = [self.manifest_file, *self.base_dir.glob("manifest.*.json")]
+        for manifest_path in manifests:
+            if not manifest_path.exists():
+                continue
+            try:
+                data = json.loads(manifest_path.read_text(encoding="utf-8"))
+            except (OSError, ValueError):
+                continue
+            for scope in data.get("scopes") or []:
+                if isinstance(scope, dict) and scope.get("scope_id"):
+                    referenced.add(scope["scope_id"])
+
+        orphans: List[Dict[str, Any]] = []
+        if not self.scratchpads_dir.is_dir():
+            return orphans
+        for scope_dir in sorted(self.scratchpads_dir.iterdir()):
+            if not scope_dir.is_dir() or scope_dir.name in referenced:
+                continue
+            orphans.append(
+                {
+                    "scope_id": scope_dir.name,
+                    "path": str(scope_dir),
+                    "alpha_dossier": (scope_dir / "alpha_dossier.json").exists(),
+                    "beta_dossier": (scope_dir / "beta_dossier.json").exists(),
+                }
+            )
+        return orphans
+
     def _read_scope_unlocked(self, scope_id: str) -> Dict[str, Any]:
         scope_manifest_file = self.get_scope_dir(scope_id) / MANIFEST_FILENAME
         if not scope_manifest_file.exists():

package/runner/tests/test_darkharvest_audit.py

@@ -0,0 +1,290 @@
+#!/usr/bin/env python3
+"""Darkharvest audit regressions (issue #5).
+
+Covers the three layers that made darkharvest ungrounded by construction:
+claim-schema normalization (auditor could not see darkharvest claims), the
+alpha/beta license cross-check (MIT vs NOASSERTION reached a human as
+divergence 0), and self-reported verification fields that must never count.
+"""
+
+import json
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from runner.auditor_engine import EpistemicAuditorEngine  # noqa: E402
+from runner.darkharvest_claims import (  # noqa: E402
+    RepoValidator,
+    cross_check_repos,
+    normalize_dossier_claims,
+    sanitize_self_reported,
+)
+from runner.research_swarm import SwarmRunner  # noqa: E402
+from skills.research_cache.hasher import SourceHasher  # noqa: E402
+
+
+class FakeValidator:
+    def __init__(self, result):
+        self.result = result
+        self.calls = []
+
+    def validate(self, owner_repo):
+        self.calls.append(owner_repo)
+        return self.result
+
+
+def _candidate(license_, risk, policy, shash, quote):
+    return {
+        "repo_id": "DH-01",
+        "name": "getpaseo/paseo",
+        "url": "https://github.com/getpaseo/paseo",
+        "source_hash": shash,
+        "verbatim_quote": quote,
+        "license": license_,
+        "license_risk": risk,
+        "harvest_policy": policy,
+        "verdicts": [
+            {"feature": "worktree isolation", "verdict": "vendor", "reason": "r"}
+        ],
+    }
+
+
+class TestNormalization(unittest.TestCase):
+    def test_darkharvest_shape_becomes_standard_claims(self):
+        dossier = {
+            "candidate_repositories": [
+                _candidate("MIT", "SAFE", "depend-or-vendor", "abc", "q")
+            ]
+        }
+        claims = normalize_dossier_claims(dossier, "alpha")
+        self.assertEqual(len(claims), 1)
+        self.assertEqual(claims[0]["source_hash"], "abc")
+        self.assertEqual(claims[0]["verbatim_quote"], "q")
+
+    def test_standard_keys_win_when_present(self):
+        dossier = {
+            "affirmative_claims": [{"claim_id": "C1", "statement": "s"}],
+            "candidate_repositories": [
+                _candidate("MIT", "SAFE", "depend-or-vendor", "abc", "q")
+            ],
+        }
+        claims = normalize_dossier_claims(dossier, "alpha")
+        self.assertEqual([c["claim_id"] for c in claims], ["C1"])
+
+
+class TestSelfReported(unittest.TestCase):
+    def test_audit_shaped_keys_are_renamed_not_counted(self):
+        dossier = {
+            "epistemic_audit": {"verified_claims": 15},
+            "confidence": "HIGH",
+            "scope_id": "s1",
+        }
+        dossier, renamed = sanitize_self_reported(dossier)
+        self.assertIn("epistemic_audit", renamed)
+        self.assertIn("confidence", renamed)
+        self.assertNotIn("epistemic_audit", dossier)
+        self.assertEqual(
+            dossier["self_reported_epistemic_audit"]["verified_claims"], 15
+        )
+        # Nothing audit-shaped remains to be mistaken for verified counts.
+        self.assertFalse(set(dossier) & {"epistemic_audit", "confidence"})
+
+
+class TestLicenseCrossCheck(unittest.TestCase):
+    def test_mit_vs_noassertion_is_blocking_and_conservative(self):
+        from runner.auditor_engine import _repo_validation_enabled  # noqa: F401
+
+        alpha = {
+            "candidate_repositories": [
+                _candidate("MIT", "SAFE", "depend-or-vendor", "a", "q")
+            ]
+        }
+        beta = {
+            "candidate_repositories": [
+                _candidate(
+                    "NOASSERTION", "PROHIBITIVE", "clean-room-rebuild-only", "a", "q"
+                )
+            ]
+        }
+        findings = cross_check_repos(alpha, beta)
+        kinds = {f["kind"] for f in findings}
+        self.assertIn("LICENSE_DISAGREEMENT", kinds)
+        self.assertTrue(all(f["severity"] == "BLOCKING" for f in findings))
+        self.assertEqual(findings[0]["resolution"], "clean-room-rebuild-only")
+
+    def test_agreement_yields_no_findings(self):
+        alpha = {
+            "candidate_repositories": [
+                _candidate("MIT", "SAFE", "depend-or-vendor", "a", "q")
+            ]
+        }
+        beta = {
+            "candidate_repositories": [
+                _candidate("MIT", "SAFE", "depend-or-vendor", "a", "q")
+            ]
+        }
+        self.assertEqual(cross_check_repos(alpha, beta), [])
+
+    def test_404_is_an_unresolvable_blocking_finding(self):
+        alpha = {
+            "candidate_repositories": [
+                _candidate("MIT", "SAFE", "depend-or-vendor", "a", "q")
+            ]
+        }
+        findings = cross_check_repos(
+            alpha, {}, validator=FakeValidator({"exists": False})
+        )
+        self.assertTrue(any(f["kind"] == "REPO_UNRESOLVABLE" for f in findings))
+
+    def test_offline_fails_open(self):
+        alpha = {
+            "candidate_repositories": [
+                _candidate("MIT", "SAFE", "depend-or-vendor", "a", "q")
+            ]
+        }
+        findings = cross_check_repos(
+            alpha,
+            {},
+            validator=FakeValidator(
+                {"exists": None, "license": None, "archived": None}
+            ),
+        )
+        self.assertEqual(findings, [])
+
+    def test_validator_fail_open_on_error(self):
+        """Network failure must yield exists=None, never raise (offline-safe)."""
+        import urllib.error
+        from unittest import mock
+
+        v = RepoValidator(enabled=True, timeout=0.001)
+        with mock.patch(
+            "urllib.request.urlopen", side_effect=urllib.error.URLError("offline")
+        ):
+            self.assertIsNone(v.validate("owner/repo")["exists"])
+
+    def test_validator_404_is_not_exists(self):
+        import urllib.error
+        from unittest import mock
+
+        v = RepoValidator(enabled=True, timeout=0.001)
+        err = urllib.error.HTTPError("u", 404, "not found", {}, None)
+        with mock.patch("urllib.request.urlopen", side_effect=err):
+            self.assertFalse(v.validate("owner/missing")["exists"])
+
+
+class TestDarkharvestAuditIntegration(unittest.TestCase):
+    def _runner_and_scope(self):
+        tmp = tempfile.mkdtemp()
+        base = Path(tmp) / ".research"
+        runner = SwarmRunner(base_dir=base, mock_mode=True, mode="darkharvest")
+        sm = runner.state_machine
+        sm.init_session("arena")
+        sm.set_scopes([{"scope_id": "s1", "title": "t", "objective": "o"}])
+        hasher = SourceHasher(base_dir=base)
+        shash = hasher.store_source(
+            "https://github.com/getpaseo/paseo",
+            "Paseo archives a clean worktree with no confirmation.",
+            "paseo",
+        )
+        quote = "Paseo archives a clean worktree with no confirmation."
+        scope_dir = sm.get_scope_dir("s1")
+        (scope_dir / "alpha_dossier.json").write_text(
+            json.dumps(
+                {
+                    "scope_id": "s1",
+                    "candidate_repositories": [
+                        _candidate("MIT", "SAFE", "depend-or-vendor", shash, quote)
+                    ],
+                    "epistemic_audit": {"verified_claims": 15, "confidence": "HIGH"},
+                }
+            ),
+            encoding="utf-8",
+        )
+        (scope_dir / "beta_dossier.json").write_text(
+            json.dumps(
+                {
+                    "scope_id": "s1",
+                    "candidate_repositories": [
+                        _candidate(
+                            "NOASSERTION",
+                            "PROHIBITIVE",
+                            "clean-room-rebuild-only",
+                            shash,
+                            quote,
+                        )
+                    ],
+                }
+            ),
+            encoding="utf-8",
+        )
+        return runner, base
+
+    def test_auditor_sees_claims_and_blocks_license_conflict(self):
+        runner, base = self._runner_and_scope()
+        auditor = EpistemicAuditorEngine(
+            base_dir=base,
+            repo_validator=FakeValidator(
+                {"exists": True, "license": "NOASSERTION", "archived": False}
+            ),
+        )
+        report = auditor.audit_scope("s1", mode="darkharvest")
+        summary = report["summary"]
+
+        self.assertGreater(summary["total_claims_audited"], 0)
+        self.assertGreaterEqual(summary["verified_passed"], 1)
+        self.assertEqual(summary["verdict"], "WARNING_LICENSE_CONFLICT")
+        self.assertTrue(
+            any(f["kind"] == "LICENSE_DISAGREEMENT" for f in report["harvest_findings"])
+        )
+
+    def test_synthesis_section_no_longer_claims_one_hundred_percent(self):
+        from runner.auditor_engine import EpistemicAuditorEngine as AE
+
+        lines = AE._render_rejected_section(
+            [], {"verified_passed": 0, "verdict": "WARNING_LOW_GROUNDING"}
+        )
+        text = "\n".join(lines)
+        self.assertIn("0 claims were verified", text)
+        self.assertNotIn("100%", text)
+
+    def test_synthesis_section_reports_license_conflict(self):
+        from runner.auditor_engine import EpistemicAuditorEngine as AE
+
+        lines = AE._render_rejected_section(
+            [],
+            {
+                "verified_passed": 0,
+                "verdict": "WARNING_LICENSE_CONFLICT",
+                "blocking_findings": 2,
+            },
+        )
+        self.assertIn("blocking license/ground-truth finding", "\n".join(lines))
+
+
+class TestOrphanScopes(unittest.TestCase):
+    def test_orphan_detected_and_not_deleted(self):
+        tmp = tempfile.mkdtemp()
+        base = Path(tmp) / ".research"
+        runner = SwarmRunner(base_dir=base, mock_mode=True, mode="research")
+        sm = runner.state_machine
+        sm.init_session("x")
+        sm.set_scopes([{"scope_id": "kept", "title": "t", "objective": "o"}])
+        stray = sm.get_scope_dir("abandoned")
+        stray.mkdir(parents=True, exist_ok=True)
+        (stray / "manifest.json").write_text("{}", encoding="utf-8")
+
+        orphans = sm.find_orphan_scopes()
+        ids = {o["scope_id"] for o in orphans}
+        self.assertIn("abandoned", ids)
+        self.assertNotIn("kept", ids)
+        # Marked, never deleted.
+        self.assertTrue(stray.exists())
+
+
+if __name__ == "__main__":
+    unittest.main()

package/runner/tests/test_grounding.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+"""Grounding regressions (issue #3): the summary must match the audit, brainstorm
+scores speculation rather than web quotes, retrieval is observable, and dossiers
+are treated as untrusted input.
+"""
+
+import json
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from runner.claim_witness import (  # noqa: E402
+    KIND_HYPOTHESIS,
+    KIND_INFERENCE,
+    TAG_HYPOTHESIS,
+    TAG_INFERRED,
+    ClaimWitness,
+)
+from runner.refinement import (  # noqa: E402
+    BRAINSTORM_MODE_THRESHOLD,
+    compute_brainstorm_score_from_claims,
+    compute_epistemic_score_from_claims,
+)
+from runner.research_swarm import SwarmRunner  # noqa: E402
+from runner.retrieval_log import log_event, summarize  # noqa: E402
+from runner.state_machine import ResearchStateMachine  # noqa: E402
+
+
+def _hypothesis(i: int, falsification: str = "measurable test") -> ClaimWitness:
+    return ClaimWitness(
+        claim_id=f"H{i}",
+        kind=KIND_HYPOTHESIS,
+        tag=TAG_HYPOTHESIS,
+        statement=f"speculative vector {i}",
+        falsification=falsification,
+    )
+
+
+class TestRetrievalTelemetry(unittest.TestCase):
+    def test_events_summarize_with_run_offset(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            base = Path(tmp) / ".research"
+            log_event(base, "query", query="a", results=5)
+            log_event(base, "cache", url="u", hash="h")
+
+            self.assertEqual(
+                summarize(base), {"queries": 1, "results": 5, "cached": 1, "events": 2}
+            )
+
+            offset = (base / "retrieval.jsonl").stat().st_size
+            log_event(base, "query", query="b", results=2)
+            delta = summarize(base, since=offset)
+            self.assertEqual(delta["queries"], 1)
+            self.assertEqual(delta["results"], 2)
+            self.assertEqual(delta["cached"], 0)
+
+    def test_missing_log_is_zero(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            self.assertEqual(
+                summarize(Path(tmp) / "nope"),
+                {"queries": 0, "results": 0, "cached": 0, "events": 0},
+            )
+
+
+class TestSummaryMatchesAudit(unittest.TestCase):
+    def _runner_with_scope(self, verified: int, verdict: str):
+        tmp = tempfile.mkdtemp()
+        base = Path(tmp) / ".research"
+        runner = SwarmRunner(base_dir=base, mock_mode=True, mode="research")
+        sm: ResearchStateMachine = runner.state_machine
+        sm.init_session("objective")
+        sm.set_scopes([{"scope_id": "s1", "title": "t", "objective": "o"}])
+        scope_dir = sm.get_scope_dir("s1")
+        (scope_dir / "audit_report.json").write_text(
+            json.dumps(
+                {
+                    "summary": {
+                        "verified_passed": verified,
+                        "unverified_rejected": 2,
+                        "divergence_score": 0.0,
+                        "verdict": verdict,
+                    }
+                }
+            ),
+            encoding="utf-8",
+        )
+        (scope_dir / "scope_synthesis.md").write_text(
+            "# scope synthesis", encoding="utf-8"
+        )
+        return runner, base
+
+    def test_zero_verified_summary_warns_and_does_not_claim_certainty(self):
+        runner, base = self._runner_with_scope(0, "WARNING_LOW_GROUNDING")
+        report = runner._compile_master_synthesis("objective").read_text(
+            encoding="utf-8"
+        )
+        self.assertIn("⚠️ **WARNING_LOW_GROUNDING**", report)
+        self.assertNotIn("Every factual statement carries", report)
+        self.assertIn("retrieval:", report)
+
+    def test_grounded_summary_does_not_warn(self):
+        runner, _ = self._runner_with_scope(7, "CERTIFIED")
+        report = runner._compile_master_synthesis("objective").read_text(
+            encoding="utf-8"
+        )
+        self.assertNotIn("⚠️ **WARNING_LOW_GROUNDING**", report)
+        self.assertIn("7 primary citation(s) verified", report)
+
+
+class TestBrainstormScoring(unittest.TestCase):
+    def test_well_formed_hypotheses_pass_brainstorm(self):
+        claims = [_hypothesis(i) for i in range(4)]
+        score, breakdown = compute_brainstorm_score_from_claims(claims)
+        self.assertGreaterEqual(score, BRAINSTORM_MODE_THRESHOLD)
+        self.assertEqual(breakdown["well_formed_count"], 4)
+
+    def test_research_formula_would_fail_same_dossier(self):
+        claims = [_hypothesis(i) for i in range(4)]
+        research_score, _ = compute_epistemic_score_from_claims(claims)
+        self.assertLess(research_score, BRAINSTORM_MODE_THRESHOLD)
+
+    def test_hypotheses_without_falsification_score_lower(self):
+        good = [_hypothesis(i) for i in range(4)]
+        weak = [_hypothesis(i, falsification="") for i in range(4)]
+        self.assertGreater(
+            compute_brainstorm_score_from_claims(good)[0],
+            compute_brainstorm_score_from_claims(weak)[0],
+        )
+
+    def test_inference_without_parents_is_not_credited(self):
+        with_parents = ClaimWitness(
+            claim_id="I1",
+            kind=KIND_INFERENCE,
+            tag=TAG_INFERRED,
+            statement="because",
+            parent_claims=["H1"],
+        )
+        without = ClaimWitness(
+            claim_id="I2", kind=KIND_INFERENCE, tag=TAG_INFERRED, statement="because"
+        )
+        self.assertGreater(
+            compute_brainstorm_score_from_claims([with_parents])[0],
+            compute_brainstorm_score_from_claims([without])[0],
+        )
+
+
+class TestDossierIsUntrusted(unittest.TestCase):
+    def test_scope_id_mismatch_is_rejected(self):
+        tmp = tempfile.mkdtemp()
+        base = Path(tmp) / ".research"
+        runner = SwarmRunner(base_dir=base, mock_mode=True, mode="research")
+        scope_dir = runner.state_machine.get_scope_dir("s1")
+        scope_dir.mkdir(parents=True, exist_ok=True)
+        (scope_dir / "beta_dossier.json").write_text(
+            json.dumps({"scope_id": "some_other_scope", "claims": []}), encoding="utf-8"
+        )
+        with self.assertRaises(FileNotFoundError):
+            runner._load_agent_dossier(
+                scope_dir / "beta_dossier.json", "s1", "beta", None
+            )
+
+    def test_matching_scope_id_is_accepted(self):
+        tmp = tempfile.mkdtemp()
+        base = Path(tmp) / ".research"
+        runner = SwarmRunner(base_dir=base, mock_mode=True, mode="research")
+        scope_dir = runner.state_machine.get_scope_dir("s1")
+        scope_dir.mkdir(parents=True, exist_ok=True)
+        (scope_dir / "beta_dossier.json").write_text(
+            json.dumps({"scope_id": "s1", "falsification_claims": []}), encoding="utf-8"
+        )
+        data = runner._load_agent_dossier(
+            scope_dir / "beta_dossier.json", "s1", "beta", None
+        )
+        self.assertEqual(data["scope_id"], "s1")
+
+
+if __name__ == "__main__":
+    unittest.main()