@agentikos/omega-os 0.1.0 → 0.19.5

package/omega/Agentik_Engine/tests/test_audits_pipeline.py

@@ -0,0 +1,348 @@
+"""Tests for the audit pipeline + batcher + history + generator.
+
+This is the "100% mega powerful audit" layer: smart batching of findings,
+capped fix-worker dispatch, re-audit with score comparison, persistent
+history. Each test exercises one piece in isolation; the integration test
+at the end runs the full pipeline end-to-end against a registered audit.
+
+We use the MockProvider (returns a structured forensic verdict) and a
+synthetic audit registered in a temp tree, so the whole suite runs with
+no network and no real linter binaries.
+"""
+from __future__ import annotations
+
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+import yaml
+
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE.parent))
+
+from omega_engine.audit_arsenal import (  # noqa: E402
+    ArsenalFinding,
+    AuditRegistry,
+)
+from omega_engine.audits import (  # noqa: E402
+    AuditPipeline,
+    Batch,
+    Finding,
+    batch_findings,
+)
+from omega_engine.audits.dispatcher import dispatch_fixes  # noqa: E402
+from omega_engine.audits.generator import (  # noqa: E402
+    generate_skill,
+    write_all,
+)
+from omega_engine.audits.history import (  # noqa: E402
+    AuditRun,
+    latest_score,
+    list_runs,
+    make_run_id,
+    now,
+    record_run,
+    trend,
+)
+from omega_engine.provider import MockProvider  # noqa: E402
+from omega_engine.router import ModelRouter  # noqa: E402
+
+
+# ---------------------------------------------------------------------------
+# Batcher
+# ---------------------------------------------------------------------------
+
+
+class TestBatcher(unittest.TestCase):
+    def test_groups_by_file_into_disjoint_batches(self):
+        findings = [
+            Finding(audit="x", phase="p", severity="high",
+                    file="a.py", line=10, message="bug A"),
+            Finding(audit="x", phase="p", severity="medium",
+                    file="a.py", line=22, message="bug A2"),
+            Finding(audit="x", phase="p", severity="critical",
+                    file="b.py", line=5, message="bug B"),
+            Finding(audit="x", phase="p", severity="low",
+                    file="c.py", line=1, message="nit"),
+        ]
+        plan = batch_findings(findings, max_batches=3)
+        # No file appears in two different batches.
+        files_per_batch = [set(b.files) for b in plan.batches]
+        for i, fs in enumerate(files_per_batch):
+            for fs2 in files_per_batch[i + 1:]:
+                self.assertEqual(fs & fs2, set(), msg=f"{fs} ∩ {fs2}")
+        # Every finding ends up in some batch.
+        total = sum(len(b.findings) for b in plan.batches)
+        self.assertEqual(total, len(findings))
+
+    def test_min_severity_filters_low(self):
+        findings = [
+            Finding(audit="x", phase="p", severity="low",
+                    file="a.py", line=1, message="nit"),
+            Finding(audit="x", phase="p", severity="high",
+                    file="b.py", line=1, message="real"),
+        ]
+        plan = batch_findings(findings, min_severity="high")
+        all_findings = [f for b in plan.batches for f in b.findings]
+        self.assertEqual(len(all_findings), 1)
+        self.assertEqual(all_findings[0].file, "b.py")
+
+    def test_global_findings_go_into_one_batch(self):
+        findings = [
+            Finding(audit="x", phase="arch", severity="high",
+                    file=None, line=None, message="architectural"),
+            Finding(audit="x", phase="arch", severity="medium",
+                    file=None, line=None, message="another"),
+        ]
+        plan = batch_findings(findings)
+        non_empty = [b for b in plan.batches if b.findings]
+        self.assertEqual(len(non_empty), 1)
+        self.assertEqual(len(non_empty[0].findings), 2)
+
+    def test_from_arsenal_finding(self):
+        af = ArsenalFinding(
+            audit="codeaudit", phase="phantoms", severity="high",
+            location="src/util.py:42",
+            claim="returns user id", reality="returns None",
+        )
+        f = Finding.from_arsenal(af)
+        self.assertEqual(f.file, "src/util.py")
+        self.assertEqual(f.line, 42)
+        self.assertEqual(f.audit, "codeaudit")
+
+    def test_batch_intent_renders_findings(self):
+        batch = Batch(
+            id="b1",
+            findings=[
+                Finding(audit="x", phase="p", severity="high",
+                        file="a.py", line=10, message="bug A"),
+                Finding(audit="x", phase="p", severity="medium",
+                        file="a.py", line=22, message="bug A2"),
+            ],
+            files=["a.py"],
+        )
+        intent = batch.to_intent("codeaudit")
+        self.assertIn("codeaudit", intent)
+        self.assertIn("a.py:10", intent)
+        self.assertIn("bug A", intent)
+        self.assertIn("a.py", intent)
+
+
+# ---------------------------------------------------------------------------
+# History
+# ---------------------------------------------------------------------------
+
+
+class TestHistory(unittest.TestCase):
+    def test_round_trip(self):
+        with tempfile.TemporaryDirectory() as td:
+            run = AuditRun(
+                run_id="ar-test1", audit_id="codeaudit", scope=".",
+                timestamp=now(), score=88, verified=True,
+                finding_count=2, fix_workers_dispatched=1, reaudit_score=92,
+                verdict={"findings": [], "score": 88},
+            )
+            record_run(td, run)
+            runs = list_runs(td, audit_id="codeaudit")
+            self.assertEqual(len(runs), 1)
+            self.assertEqual(runs[0].score, 88)
+            self.assertEqual(runs[0].reaudit_score, 92)
+            self.assertTrue(runs[0].verified)
+
+    def test_latest_score(self):
+        with tempfile.TemporaryDirectory() as td:
+            base = now()
+            for i, s in enumerate((70, 80, 90)):
+                record_run(td, AuditRun(
+                    run_id=make_run_id(), audit_id="codeaudit",
+                    scope=".", timestamp=base + i, score=s,
+                    verified=s >= 85, finding_count=0,
+                    verdict={"score": s},
+                ))
+            self.assertEqual(latest_score(td, "codeaudit"), 90)
+
+    def test_trend_detects_improvement(self):
+        with tempfile.TemporaryDirectory() as td:
+            for i, s in enumerate((60, 65, 75, 88, 90)):
+                record_run(td, AuditRun(
+                    run_id=f"ar-{i}", audit_id="codeaudit",
+                    scope=".", timestamp=now() + i, score=s,
+                    verified=False, finding_count=0,
+                    verdict={"score": s},
+                ))
+            self.assertEqual(trend(td, "codeaudit"), "↑")
+
+    def test_trend_detects_degradation(self):
+        with tempfile.TemporaryDirectory() as td:
+            for i, s in enumerate((90, 80, 70, 60, 50)):
+                record_run(td, AuditRun(
+                    run_id=f"ar-{i}", audit_id="codeaudit",
+                    scope=".", timestamp=now() + i, score=s,
+                    verified=False, finding_count=0,
+                    verdict={"score": s},
+                ))
+            self.assertEqual(trend(td, "codeaudit"), "↓")
+
+
+# ---------------------------------------------------------------------------
+# Dispatcher
+# ---------------------------------------------------------------------------
+
+
+class TestDispatcher(unittest.TestCase):
+    def test_dispatch_dry_run_returns_intent_without_executor(self):
+        from omega_engine.audits.batcher import BatchPlan
+        plan = BatchPlan(batches=[
+            Batch(
+                id="b1",
+                findings=[Finding(audit="x", phase="p", severity="high",
+                                   file="a.py", line=1, message="m")],
+                files=["a.py"],
+            ),
+        ], audit_id="codeaudit")
+        results = dispatch_fixes(plan, audit_id="codeaudit", dry_run=True)
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0].status, "skipped")
+        self.assertIn("a.py", results[0].intent)
+
+    def test_dispatch_with_executor_runs_each_batch(self):
+        from omega_engine.audits.batcher import BatchPlan
+
+        class _FakeOutcome:
+            mission_id = "m-test1"
+            final_state = type("S", (), {"value": "completed"})()
+            verified = True
+
+        class _FakeExecutor:
+            def __init__(self): self.calls: list[str] = []
+            def run_mission(self, intent, root_role="oracle"):
+                self.calls.append(intent)
+                return _FakeOutcome()
+
+        plan = BatchPlan(batches=[
+            Batch(id="b1", findings=[Finding(
+                audit="x", phase="p", severity="high",
+                file="a.py", line=1, message="m1",
+            )], files=["a.py"]),
+            Batch(id="b2", findings=[Finding(
+                audit="x", phase="p", severity="critical",
+                file="b.py", line=1, message="m2",
+            )], files=["b.py"]),
+        ], audit_id="codeaudit")
+        ex = _FakeExecutor()
+        results = dispatch_fixes(plan, audit_id="codeaudit", executor=ex)
+        self.assertEqual(len(results), 2)
+        self.assertEqual(len(ex.calls), 2)
+        for r in results:
+            self.assertEqual(r.status, "dispatched")
+            self.assertEqual(r.mission_id, "m-test1")
+
+
+# ---------------------------------------------------------------------------
+# Generator
+# ---------------------------------------------------------------------------
+
+
+class TestGenerator(unittest.TestCase):
+    def test_generate_skill_produces_valid_frontmatter(self):
+        with tempfile.TemporaryDirectory() as td:
+            audits_dir = Path(td) / "Agentik_SSOT" / "audits"
+            audits_dir.mkdir(parents=True)
+            (audits_dir / "test.yaml").write_text(yaml.safe_dump({
+                "id": "testaudit",
+                "domain": "test",
+                "question": "Does it work?",
+                "threshold": 80,
+                "phases": [
+                    {"id": "p1", "checks": "Check one thing"},
+                    {"id": "p2", "checks": "Check another"},
+                ],
+                "falsification": "Bias toward FAIL.",
+            }))
+            sf = generate_skill(audits_dir / "test.yaml")
+            self.assertEqual(sf.audit_id, "testaudit")
+            self.assertIn("name: testaudit", sf.body)
+            self.assertIn("Does it work?", sf.body)
+            self.assertIn("p1", sf.body)
+            self.assertIn("Bias toward FAIL", sf.body)
+            # Frontmatter is valid YAML between the leading --- markers.
+            parts = sf.body.split("---", 2)
+            self.assertGreaterEqual(len(parts), 3)
+            fm = yaml.safe_load(parts[1])
+            self.assertEqual(fm["name"], "testaudit")
+            self.assertIn("Bash", fm["allowed-tools"])
+
+    def test_write_all_creates_skills_dir(self):
+        with tempfile.TemporaryDirectory() as td:
+            ssot = Path(td) / "Agentik_SSOT"
+            audits = ssot / "audits"; audits.mkdir(parents=True)
+            (audits / "a.yaml").write_text(yaml.safe_dump({
+                "id": "a", "domain": "code", "question": "q?",
+                "phases": [{"id": "x", "checks": "y"}],
+            }))
+            (audits / "b.yaml").write_text(yaml.safe_dump({
+                "id": "b", "domain": "ui", "question": "q?",
+                "phases": [{"id": "x", "checks": "y"}],
+            }))
+            result = write_all(audits)
+            self.assertEqual(result["written"], 2)
+            self.assertTrue((ssot / "skills" / "a" / "SKILL.md").exists())
+            self.assertTrue((ssot / "skills" / "b" / "SKILL.md").exists())
+            # Re-running is idempotent.
+            result2 = write_all(audits)
+            self.assertEqual(result2["written"], 0)
+            self.assertEqual(result2["skipped"], 2)
+
+
+# ---------------------------------------------------------------------------
+# Pipeline integration
+# ---------------------------------------------------------------------------
+
+
+def _seed_audit(home: Path) -> AuditRegistry:
+    audits = home / "Agentik_SSOT" / "audits"
+    audits.mkdir(parents=True, exist_ok=True)
+    (audits / "demoaudit.yaml").write_text(yaml.safe_dump({
+        "id": "demoaudit",
+        "domain": "demo",
+        "question": "Is the demo solid?",
+        "weight": 1.0,
+        "threshold": 85,
+        "applies_to": {"roles": ["worker"], "changed": ["*"]},
+        "gather": [],
+        "phases": [{"id": "claims", "checks": "Every label is a promise."}],
+        "falsification": "Bias toward FAIL.",
+    }))
+    return AuditRegistry.load(audits)
+
+
+class TestPipelineReadOnly(unittest.TestCase):
+    def test_pipeline_records_history(self):
+        with tempfile.TemporaryDirectory() as td:
+            home = Path(td)
+            registry = _seed_audit(home)
+            router = ModelRouter.single(MockProvider())
+            pipeline = AuditPipeline(registry, router, omega_home=home)
+            outcome = pipeline.run("demoaudit", scope=str(home), read_only=True)
+            self.assertEqual(outcome.audit_id, "demoaudit")
+            self.assertTrue(outcome.verdict.verified)
+            # History row was persisted.
+            runs = list_runs(home, audit_id="demoaudit")
+            self.assertEqual(len(runs), 1)
+            self.assertEqual(runs[0].run_id, outcome.run_id)
+
+    def test_pipeline_unknown_audit_raises(self):
+        with tempfile.TemporaryDirectory() as td:
+            home = Path(td)
+            registry = _seed_audit(home)
+            router = ModelRouter.single(MockProvider())
+            pipeline = AuditPipeline(registry, router, omega_home=home)
+            with self.assertRaises(KeyError):
+                pipeline.run("doesnotexist", scope=str(home))
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)