mcp-agentic-pipelines 1.0.1

package/vendors/piste/pipeline/stage2/test_stage2.py

@@ -0,0 +1,265 @@
+# Copyright (c) 2026 Jinan Kordab
+# SPDX-License-Identifier: MIT
+
+"""
+Unit Tests — Stage 2: Blind Retrieval
+=======================================
+Tests SearchDecisionGenerator [J1], BlindRetriever [J2],
+CredibilityScorer [J1b], QueryRefiner [J8c], CanonicalEvidenceMapper [C6].
+
+Run: pytest pipeline/stage2/test_stage2.py -v
+"""
+
+import pytest
+from unittest.mock import patch, MagicMock, AsyncMock
+
+
+# ============================================================
+# Canonical Evidence Mapper Tests [C6]
+# ============================================================
+
+class TestCanonicalEvidenceMapper:
+    """Test evidence normalization from all provider formats."""
+
+    @pytest.fixture
+    def mapper(self):
+        from pipeline.stage2.canonical_mapper import CanonicalEvidenceMapper
+        return CanonicalEvidenceMapper()
+
+    @pytest.fixture
+    def tavily_result(self):
+        from pipeline.stage2.blind_retriever import RawSearchResult
+        return RawSearchResult(
+            provider="tavily",
+            query_used="price of oil June 2008",
+            title="Oil reaches record high in 2008",
+            url="https://reuters.com/oil-2008",
+            snippet="Crude oil prices reached a record $145 per barrel...",
+            domain="reuters.com",
+        )
+
+    def test_map_tavily_result(self, mapper, tavily_result):
+        """Tavily result → CanonicalEvidence with correct fields."""
+        import asyncio
+        results = asyncio.run(
+            mapper.map_results([tavily_result])
+        )
+        assert len(results) == 1
+        ev = results[0]
+        assert ev.title == "Oil reaches record high in 2008"
+        assert ev.url == "https://reuters.com/oil-2008"
+        assert ev.source_domain == "reuters.com"
+        assert ev.provider == "tavily"
+        assert ev.query_used == "price of oil June 2008"
+
+    def test_map_results_sorts_by_credibility(self, mapper):
+        """Results are sorted by credibility score descending."""
+        from pipeline.stage2.blind_retriever import RawSearchResult
+        import asyncio
+
+        raw = [
+            RawSearchResult("tavily", "q", "Low cred", "http://a.com", "...", "low.com"),
+            RawSearchResult("serper", "q", "High cred", "http://b.com", "...", "high.com"),
+            RawSearchResult("tavily", "q", "Mid cred", "http://c.com", "...", "mid.com"),
+        ]
+
+        # Mock credibility scorer
+        class MockScorer:
+            async def score_domain(self, domain):
+                return {"high.com": 0.95, "mid.com": 0.60, "low.com": 0.25}[domain]
+
+        results = asyncio.run(
+            mapper.map_results(raw, MockScorer())
+        )
+        assert results[0].source_domain == "high.com"
+        assert results[1].source_domain == "mid.com"
+        assert results[2].source_domain == "low.com"
+
+    def test_to_dict_from_dict_roundtrip(self, mapper):
+        """CanonicalEvidence serializes/deserializes for JSONB storage."""
+        from pipeline.stage2.canonical_mapper import CanonicalEvidence
+
+        ev = CanonicalEvidence(
+            title="Test",
+            url="https://example.com",
+            excerpt="An excerpt.",
+            source_domain="example.com",
+            credibility_score=0.85,
+            query_used="test query",
+            provider="tavily",
+        )
+        d = mapper.to_dict(ev)
+        restored = mapper.from_dict(d)
+        assert restored.title == "Test"
+        assert restored.credibility_score == 0.85
+        assert restored.source_domain == "example.com"
+
+
+# ============================================================
+# Credibility Scorer Tests [J1b]
+# ============================================================
+
+class TestCredibilityScorer:
+    """Test per-domain credibility scoring."""
+
+    def test_default_score_for_unknown_domain(self):
+        from pipeline.stage2.credibility_scorer import CredibilityScorer
+        scorer = CredibilityScorer(db=None)
+        # Without DB, always returns default
+        assert scorer.DEFAULT_SCORE == 0.5
+
+    def test_cache_hit_after_first_lookup(self):
+        from pipeline.stage2.credibility_scorer import CredibilityScorer
+        import asyncio
+        scorer = CredibilityScorer(db=None)
+        scorer._cache["test.com"] = 0.88
+        score = asyncio.run(scorer.score_domain("test.com"))
+        assert score == 0.88
+
+    def test_is_reliable_threshold(self):
+        from pipeline.stage2.credibility_scorer import CredibilityScorer
+        scorer = CredibilityScorer(db=None)
+        scorer._cache["good.com"] = 0.92
+        scorer._cache["bad.com"] = 0.30
+        assert asyncio.run(scorer.is_reliable("good.com")) is True
+        assert asyncio.run(scorer.is_reliable("bad.com")) is False
+
+
+# ============================================================
+# Search Decision Generator Tests [J1]
+# ============================================================
+
+class TestSearchDecisionGenerator:
+    """Test search decision logic."""
+
+    def test_known_fact_skips_search(self):
+        """Simple, well-known facts should skip search."""
+        from pipeline.stage2.search_decision import SearchDecisionGenerator
+        gen = SearchDecisionGenerator()
+        # The actual decision depends on LLM, but the module structure
+        # ensures needs_search is a bool and queries are strings
+        assert gen.decide is not None
+        assert gen.generate_queries is not None
+
+
+# ============================================================
+# Query Refiner Tests [J8c]
+# ============================================================
+
+class TestQueryRefiner:
+    """Test intelligent query refinement — Loop 1."""
+
+    @pytest.fixture
+    def refiner(self):
+        from pipeline.stage2.query_refiner import QueryRefiner
+        return QueryRefiner()
+
+    def test_analyze_no_results(self, refiner):
+        """No results → specific message."""
+        reason = refiner.analyze_insufficiency([], "test claim")
+        assert "No search results" in reason
+
+    def test_analyze_low_credibility(self, refiner):
+        """Majority low-credibility sources → flagged."""
+        # Create mock results with low credibility scores
+        class MockResult:
+            credibility_score = 0.2
+        results = [MockResult() for _ in range(5)]
+        reason = refiner.analyze_insufficiency(results, "test claim")
+        assert "low-credibility" in reason.lower()
+
+    def test_analyze_few_results(self, refiner):
+        """Few results → insufficient warning."""
+        class MockResult:
+            credibility_score = 0.8
+        results = [MockResult(), MockResult()]
+        reason = refiner.analyze_insufficiency(results, "test claim")
+        assert "Only 2 results" in reason
+
+    def test_max_refined_queries_capped(self, refiner):
+        """Refined queries are capped at 3."""
+        # The forward method caps at 3; verify the module exists
+        assert refiner.max_retries > 0
+        assert refiner.refine is not None
+
+
+# ============================================================
+# Blind Retriever Tests [J2]
+# ============================================================
+
+class TestBlindRetriever:
+    """Test blind retrieval architecture."""
+
+    def test_domain_extraction(self):
+        """Domain extraction from URLs works correctly."""
+        from pipeline.stage2.blind_retriever import BlindRetriever
+        retriever = BlindRetriever()
+
+        assert retriever._extract_domain("https://www.bbc.com/news") == "bbc.com"
+        assert retriever._extract_domain("https://reuters.com/article/1") == "reuters.com"
+        assert retriever._extract_domain("http://sub.domain.co.uk/path") == "sub.domain.co.uk"
+
+    def test_deduplication_by_url(self):
+        """Duplicate URLs are removed from results."""
+        import asyncio
+        from pipeline.stage2.blind_retriever import BlindRetriever, RawSearchResult
+
+        retriever = BlindRetriever()
+        results = [
+            RawSearchResult("tavily", "q", "A", "https://example.com", "...", "example.com"),
+            RawSearchResult("serper", "q", "A dup", "https://example.com", "...", "example.com"),
+            RawSearchResult("tavily", "q", "B", "https://other.com", "...", "other.com"),
+        ]
+
+        # Simulate dedup logic
+        seen = set()
+        unique = []
+        for r in results:
+            norm = r.url.lower().rstrip("/")
+            if norm not in seen:
+                seen.add(norm)
+                unique.append(r)
+
+        assert len(unique) == 2
+        assert unique[0].url == "https://example.com"
+        assert unique[1].url == "https://other.com"
+
+
+# ============================================================
+# Stage2Result Tests
+# ============================================================
+
+class TestStage2Result:
+    """Test Stage2Result dataclass."""
+
+    def test_skipped_search_result(self):
+        from pipeline.stage2.orchestrator import Stage2Result
+        result = Stage2Result(
+            atomic_claim="Water boils at 100°C.",
+            needs_search=False,
+            search_queries=[],
+            search_reasoning="Well-known scientific fact.",
+            skipped_search=True,
+        )
+        assert result.needs_search is False
+        assert result.skipped_search is True
+        assert len(result.canonical_evidence) == 0
+
+    def test_search_with_retry_result(self):
+        from pipeline.stage2.orchestrator import Stage2Result
+        from pipeline.stage2.canonical_mapper import CanonicalEvidence
+        result = Stage2Result(
+            atomic_claim="Complex claim.",
+            needs_search=True,
+            search_queries=["neutral query"],
+            search_reasoning="Requires external evidence.",
+            canonical_evidence=[
+                CanonicalEvidence("S1", "http://a.com", "...", "a.com", 0.9),
+            ],
+            retry_count=1,
+            retry_queries=["refined query"],
+            insufficient_reason="Initial results insufficient.",
+        )
+        assert result.retry_count == 1
+        assert len(result.retry_queries) == 1
+        assert len(result.canonical_evidence) == 1

package/vendors/piste/pipeline/stage3/__init__.py

@@ -0,0 +1,20 @@
+# Copyright (c) 2026 Jinan Kordab
+# SPDX-License-Identifier: MIT
+
+# Stage 3 — Per-Source Classification [J3][J8]
+#   Classifier: pipeline/stage3/classifier.py
+#   Orchestrator: pipeline/stage3/orchestrator.py (asyncio.gather parallelism)
+
+from pipeline.stage3.classifier import SourceClassifier, source_classifier
+from pipeline.stage3.orchestrator import (
+    Stage3Orchestrator, Stage3Result, ClassificationResult, stage3_orchestrator,
+)
+
+__all__ = [
+    "SourceClassifier",
+    "source_classifier",
+    "Stage3Orchestrator",
+    "Stage3Result",
+    "ClassificationResult",
+    "stage3_orchestrator",
+]

package/vendors/piste/pipeline/stage3/classifier.py

@@ -0,0 +1,79 @@
+# Copyright (c) 2026 Jinan Kordab
+# SPDX-License-Identifier: MIT
+
+"""
+Stage 3: Per-Source Classification [J3][J8]
+==============================================
+Aletheia's jewel: each evidence source is independently classified
+as SUPPORTS, REFUTES, or UNRELATED to the claim BEFORE aggregation.
+
+Run in parallel via asyncio.gather — N classifiers = N simultaneous LLM calls.
+This provides:
+  - Full audit trail: every source's contribution is explicit
+  - Debuggability: errors isolated to single classifications
+  - Parallelism: independent evaluations run concurrently
+"""
+
+import dspy
+from typing import Tuple
+from pipeline.signatures.signatures import SourceClassificationSignature
+from pipeline.stage2.canonical_mapper import CanonicalEvidence
+
+
+class SourceClassifier(dspy.Module):
+    """
+    DSPy module that classifies a single evidence source relative to a claim.
+
+    Jewel [J3] — Aletheia's structured per-source classification:
+    - Each source evaluated independently
+    - Label: SUPPORTS / REFUTES / UNRELATED
+    - Returns confidence + rationale for audit trail [C5]
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.classify = dspy.ChainOfThought(SourceClassificationSignature)
+
+    def forward(
+        self,
+        claim: str,
+        evidence: CanonicalEvidence,
+        locale: str = "en",
+    ) -> Tuple[str, float, str]:
+        """
+        Classify one source against the claim.
+
+        Args:
+            claim: The atomic claim being verified.
+            evidence: A single CanonicalEvidence source.
+            locale: Language locale for the response (en, fr).
+
+        Returns:
+            label: "SUPPORTS", "REFUTES", or "UNRELATED"
+            confidence: 0.0–1.0
+            rationale: Explanation of the classification
+        """
+        result = self.classify(
+            claim=claim,
+            evidence_title=evidence.title,
+            evidence_excerpt=evidence.excerpt,
+            source_domain=evidence.source_domain,
+            credibility_score=evidence.credibility_score,
+            locale=locale,
+        )
+
+        label = result.label.strip().upper()
+        # Normalize to valid labels
+        if label not in ("SUPPORTS", "REFUTES", "UNRELATED"):
+            if "SUPPORT" in label:
+                label = "SUPPORTS"
+            elif "REFUT" in label:
+                label = "REFUTES"
+            else:
+                label = "UNRELATED"
+
+        return label, float(result.confidence), result.rationale
+
+
+# Singleton
+source_classifier = SourceClassifier()

package/vendors/piste/pipeline/stage3/orchestrator.py

@@ -0,0 +1,225 @@
+# Copyright (c) 2026 Jinan Kordab
+# SPDX-License-Identifier: MIT
+
+"""
+Stage 3 Orchestrator — Per-Source Classification
+===================================================
+Runs N classifiers in parallel via asyncio.gather.
+Each source independently evaluated → SUPPORTS/REFUTES/UNRELATED.
+
+Emits SSE events per classification for real-time frontend updates.
+Writes APPEND-ONLY classification records to PostgreSQL [C5].
+"""
+
+import asyncio
+import time
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from typing import Optional, List
+from dataclasses import dataclass, field
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import StageRecord, Classification
+from pipeline.stage3.classifier import source_classifier
+from pipeline.stage2.canonical_mapper import CanonicalEvidence
+
+
+@dataclass
+class ClassificationResult:
+    """Output of a single source classification."""
+    source_index: int
+    source_url: str
+    source_domain: str
+    label: str  # SUPPORTS | REFUTES | UNRELATED
+    confidence: float
+    rationale: str
+    credibility_score: float
+
+
+@dataclass
+class Stage3Result:
+    """Output of Stage 3 — Per-Source Classification."""
+    atomic_claim: str
+    classifications: list[ClassificationResult]
+    support_count: int = 0
+    refute_count: int = 0
+    unrelated_count: int = 0
+    total_sources: int = 0
+
+
+class Stage3Orchestrator:
+    """
+    Orchestrates Stage 3 — parallel per-source classification.
+
+    Jewel [J3][J8] — Aletheia's structured evaluation:
+    Each source classified independently BEFORE aggregation.
+    Parallel execution via asyncio.gather reduces latency.
+    """
+
+    _semaphore: asyncio.Semaphore = asyncio.Semaphore(200)
+    _executor: ThreadPoolExecutor = ThreadPoolExecutor(max_workers=200)
+
+    def __init__(self, sse_callback: Optional[callable] = None):
+        self.sse_callback = sse_callback
+
+    async def process(
+        self,
+        atomic_claim: str,
+        evidence_list: List[CanonicalEvidence],
+        db: Optional[AsyncSession] = None,
+        run_id: Optional[uuid.UUID] = None,
+        locale: str = "en",
+    ) -> Stage3Result:
+        """
+        Classify all evidence sources in parallel against one atomic claim.
+
+        Args:
+            atomic_claim: The atomic claim to verify.
+            evidence_list: CanonicalEvidence sources from Stage 2.
+            db: Optional DB session for audit ledger writes.
+            run_id: UUID of the analysis run for audit trail [C5].
+
+        Returns:
+            Stage3Result with all per-source classifications.
+        """
+        await self._emit("stage_3_start", {
+            "atomic_claim": atomic_claim,
+            "sources_count": len(evidence_list),
+        })
+
+        if not evidence_list:
+            await self._emit("stage_3_complete", {
+                "atomic_claim": atomic_claim,
+                "support": 0, "refute": 0, "unrelated": 0,
+            })
+            return Stage3Result(
+                atomic_claim=atomic_claim,
+                classifications=[],
+                total_sources=0,
+            )
+
+        # --- Run N classifiers in parallel ---
+        t0 = time.monotonic()
+
+        async def _classify_with_limit(
+            idx: int, claim: str, ev: CanonicalEvidence, loc: str
+        ) -> ClassificationResult:
+            async with self._semaphore:
+                return await self._classify_single(idx, claim, ev, loc)
+
+        tasks = [
+            _classify_with_limit(i, atomic_claim, evidence, locale)
+            for i, evidence in enumerate(evidence_list)
+        ]
+        results: List[ClassificationResult] = await asyncio.gather(*tasks)
+
+        latency_total = (time.monotonic() - t0) * 1000
+
+        # --- Tally ---
+        support_count = sum(1 for r in results if r.label == "SUPPORTS")
+        refute_count = sum(1 for r in results if r.label == "REFUTES")
+        unrelated_count = sum(1 for r in results if r.label == "UNRELATED")
+
+        # --- Write to PostgreSQL (append-only) ---
+        if db:
+            for r in results:
+                # Stage 2 attached the Source row PK to the in-memory evidence
+                # at insertion time; reuse it here as the FK.
+                ev = evidence_list[r.source_index]
+                db.add(Classification(
+                    run_id=run_id or uuid.UUID("00000000-0000-0000-0000-000000000000"),
+                    source_id=ev.db_id,
+                    label=r.label,
+                    confidence=r.confidence,
+                    rationale=r.rationale,
+                    model_used="dspy/source_classifier",
+                ))
+
+            db.add(StageRecord(
+                run_id=run_id or uuid.UUID("00000000-0000-0000-0000-000000000000"),
+                stage_name="stage_3",
+                input_snapshot={
+                    "atomic_claim": atomic_claim,
+                    "sources_count": len(evidence_list),
+                },
+                output_snapshot={
+                    "support": support_count,
+                    "refute": refute_count,
+                    "unrelated": unrelated_count,
+                    "classifications": [
+                        {"label": r.label, "confidence": r.confidence}
+                        for r in results
+                    ],
+                },
+                model_used="dspy/source_classifier",
+                latency_ms=latency_total,
+                retry_attempt=0,
+            ))
+
+        # --- Emit SSE events ---
+        for r in results:
+            await self._emit("source_classified", {
+                "source_index": r.source_index,
+                "source_url": r.source_url,
+                "source_domain": r.source_domain,
+                "label": r.label,
+                "confidence": r.confidence,
+                "credibility_score": r.credibility_score,
+            })
+
+        await self._emit("stage_3_complete", {
+            "atomic_claim": atomic_claim,
+            "support": support_count,
+            "refute": refute_count,
+            "unrelated": unrelated_count,
+        })
+
+        return Stage3Result(
+            atomic_claim=atomic_claim,
+            classifications=results,
+            support_count=support_count,
+            refute_count=refute_count,
+            unrelated_count=unrelated_count,
+            total_sources=len(evidence_list),
+        )
+
+    async def _classify_single(
+        self,
+        index: int,
+        atomic_claim: str,
+        evidence: CanonicalEvidence,
+        locale: str = "en",
+    ) -> ClassificationResult:
+        """Classify a single source (runs in parallel via asyncio.gather).
+
+        The DSPy source_classifier is synchronous and would block the asyncio
+        event loop on every LLM call, forcing serial execution despite the
+        gather.  We off-load each call to a thread so the event loop stays
+        free and N classifications truly run concurrently.
+        """
+        loop = asyncio.get_running_loop()
+        label, confidence, rationale = await loop.run_in_executor(
+            self._executor,
+            source_classifier,
+            atomic_claim,
+            evidence,
+            locale,
+        )
+        return ClassificationResult(
+            source_index=index,
+            source_url=evidence.url,
+            source_domain=evidence.source_domain,
+            label=label,
+            confidence=confidence,
+            rationale=rationale,
+            credibility_score=evidence.credibility_score,
+        )
+
+    async def _emit(self, event_type: str, data: dict):
+        if self.sse_callback:
+            await self.sse_callback(event_type, data)
+
+
+# Singleton
+stage3_orchestrator = Stage3Orchestrator()

package/vendors/piste/pipeline/stage3/test_stage3.py

@@ -0,0 +1,101 @@
+# Copyright (c) 2026 Jinan Kordab
+# SPDX-License-Identifier: MIT
+
+"""
+Unit Tests — Stage 3: Per-Source Classification
+=================================================
+Tests SourceClassifier [J3], parallel orchestrator, ClassificationResult.
+
+Run: pytest pipeline/stage3/test_stage3.py -v
+"""
+
+import pytest
+
+
+class TestClassificationResult:
+    """Test ClassificationResult dataclass."""
+
+    def test_supports_classification(self):
+        from pipeline.stage3.orchestrator import ClassificationResult
+        r = ClassificationResult(
+            source_index=0,
+            source_url="https://reuters.com/article",
+            source_domain="reuters.com",
+            label="SUPPORTS",
+            confidence=0.92,
+            rationale="The article confirms the claim with primary data.",
+            credibility_score=0.95,
+        )
+        assert r.label == "SUPPORTS"
+        assert r.credibility_score == 0.95
+
+    def test_refutes_classification(self):
+        from pipeline.stage3.orchestrator import ClassificationResult
+        r = ClassificationResult(
+            source_index=1,
+            source_url="https://example.com",
+            source_domain="example.com",
+            label="REFUTES",
+            confidence=0.78,
+            rationale="Data contradicts the claim.",
+            credibility_score=0.45,
+        )
+        assert r.label == "REFUTES"
+
+    def test_unrelated_classification(self):
+        from pipeline.stage3.orchestrator import ClassificationResult
+        r = ClassificationResult(
+            source_index=2,
+            source_url="https://other.com",
+            source_domain="other.com",
+            label="UNRELATED",
+            confidence=0.95,
+            rationale="Source is about a different topic.",
+            credibility_score=0.60,
+        )
+        assert r.label == "UNRELATED"
+
+
+class TestStage3Result:
+    """Test Stage3Result tallying."""
+
+    def test_tally_counts(self):
+        from pipeline.stage3.orchestrator import Stage3Result, ClassificationResult
+
+        classifications = [
+            ClassificationResult(0, "a.com", "a.com", "SUPPORTS", 0.9, "...", 0.9),
+            ClassificationResult(1, "b.com", "b.com", "SUPPORTS", 0.8, "...", 0.8),
+            ClassificationResult(2, "c.com", "c.com", "REFUTES", 0.7, "...", 0.7),
+            ClassificationResult(3, "d.com", "d.com", "UNRELATED", 0.9, "...", 0.5),
+        ]
+
+        # Manual tally
+        support = sum(1 for c in classifications if c.label == "SUPPORTS")
+        refute = sum(1 for c in classifications if c.label == "REFUTES")
+        unrelated = sum(1 for c in classifications if c.label == "UNRELATED")
+
+        assert support == 2
+        assert refute == 1
+        assert unrelated == 1
+
+    def test_empty_evidence(self):
+        from pipeline.stage3.orchestrator import Stage3Result
+        result = Stage3Result(
+            atomic_claim="Test",
+            classifications=[],
+            total_sources=0,
+        )
+        assert result.total_sources == 0
+        assert result.support_count == 0
+
+
+class TestStage3Orchestrator:
+    """Test parallel classification orchestration."""
+
+    def test_sse_callback_stored(self):
+        from pipeline.stage3.orchestrator import Stage3Orchestrator
+        calls = []
+        async def cb(event, data):
+            calls.append((event, data))
+        orch = Stage3Orchestrator(sse_callback=cb)
+        assert orch.sse_callback is not None